From 374e1947abcd3e127a2a613aff73ecffdb9199ea Mon Sep 17 00:00:00 2001 From: Apollon Oikonomopoulos Date: Thu, 14 Jan 2016 00:10:06 +0200 Subject: Imported Upstream version 2.6.11 --- src/SConscript.client | 372 +- src/mongo/SConscript | 1012 +- src/mongo/base/SConscript | 36 +- src/mongo/base/configuration_variable_manager.cpp | 86 - src/mongo/base/configuration_variable_manager.h | 121 - .../base/configuration_variable_manager_test.cpp | 112 - src/mongo/base/counter.h | 12 + src/mongo/base/counter_test.cpp | 6 +- src/mongo/base/error_codes.err | 80 +- src/mongo/base/generate_error_codes.py | 34 +- src/mongo/base/init.cpp | 5 - src/mongo/base/init.h | 66 +- src/mongo/base/initializer.cpp | 36 +- src/mongo/base/initializer.h | 32 +- src/mongo/base/initializer_context.cpp | 5 +- src/mongo/base/initializer_context.h | 6 +- .../base/initializer_dependency_graph_test.cpp | 61 +- src/mongo/base/initializer_test.cpp | 19 +- src/mongo/base/make_string_vector.cpp | 6 +- src/mongo/base/make_string_vector.h | 9 +- src/mongo/base/owned_pointer_map.h | 26 +- src/mongo/base/owned_pointer_map_test.cpp | 29 + src/mongo/base/owned_pointer_vector.h | 100 +- src/mongo/base/owned_pointer_vector_test.cpp | 268 +- src/mongo/base/parse_number.cpp | 75 +- src/mongo/base/parse_number.h | 3 +- src/mongo/base/parse_number_test.cpp | 85 + src/mongo/base/status-inl.h | 86 + src/mongo/base/status.cpp | 74 +- src/mongo/base/status.h | 66 +- src/mongo/base/status_test.cpp | 33 +- src/mongo/base/status_with.h | 78 + src/mongo/base/string_data.cpp | 25 +- src/mongo/base/string_data.h | 11 +- src/mongo/base/string_data_test.cpp | 93 + src/mongo/base/validate_locale.cpp | 54 + src/mongo/bson/bson-inl.h | 103 +- src/mongo/bson/bson.h | 77 +- src/mongo/bson/bson_db.h | 4 +- src/mongo/bson/bson_field.h | 2 +- src/mongo/bson/bson_obj_test.cpp | 30 + src/mongo/bson/bson_validate.cpp | 112 +- src/mongo/bson/bson_validate_test.cpp | 76 + src/mongo/bson/bsondemo/bsondemo.cpp | 3 +- src/mongo/bson/bsonelement.h | 45 +- src/mongo/bson/bsonmisc.h | 39 +- src/mongo/bson/bsonobj.h | 112 +- src/mongo/bson/bsonobjbuilder.h | 68 +- src/mongo/bson/bsonobjbuilder_test.cpp | 264 + src/mongo/bson/bsonobjiterator.h | 2 + src/mongo/bson/mutable/algorithm.h | 289 + src/mongo/bson/mutable/const_element-inl.h | 229 + src/mongo/bson/mutable/const_element.h | 125 + src/mongo/bson/mutable/damage_vector.h | 43 + src/mongo/bson/mutable/document-inl.h | 67 + src/mongo/bson/mutable/document.cpp | 2639 +++ src/mongo/bson/mutable/document.h | 509 + src/mongo/bson/mutable/element-inl.h | 144 + src/mongo/bson/mutable/element.cpp | 173 + src/mongo/bson/mutable/element.h | 609 + src/mongo/bson/mutable/mutable_bson.cpp | 1035 - src/mongo/bson/mutable/mutable_bson.h | 343 - src/mongo/bson/mutable/mutable_bson_algo.h | 72 - src/mongo/bson/mutable/mutable_bson_algo_test.cpp | 307 +- src/mongo/bson/mutable/mutable_bson_builder.cpp | 298 - src/mongo/bson/mutable/mutable_bson_builder.h | 47 - .../bson/mutable/mutable_bson_builder_test.cpp | 63 - src/mongo/bson/mutable/mutable_bson_heap.cpp | 167 - src/mongo/bson/mutable/mutable_bson_heap.h | 116 - src/mongo/bson/mutable/mutable_bson_heap_test.cpp | 37 - src/mongo/bson/mutable/mutable_bson_internal.cpp | 25 - src/mongo/bson/mutable/mutable_bson_internal.h | 215 - src/mongo/bson/mutable/mutable_bson_test.cpp | 2993 ++- src/mongo/bson/mutable/mutable_bson_test_utils.cpp | 206 + src/mongo/bson/mutable/mutable_bson_test_utils.h | 124 + src/mongo/bson/oid.cpp | 13 +- src/mongo/bson/oid.h | 1 - src/mongo/bson/optime.cpp | 105 + src/mongo/bson/optime.h | 160 + src/mongo/bson/ordering.h | 3 + src/mongo/bson/util/bson_extract.cpp | 63 +- src/mongo/bson/util/bson_extract.h | 41 + src/mongo/bson/util/bson_extract_test.cpp | 41 + src/mongo/bson/util/builder.h | 38 +- src/mongo/bson/util/misc.h | 76 +- src/mongo/client/auth_helpers.cpp | 56 + src/mongo/client/auth_helpers.h | 36 + src/mongo/client/authlevel.h | 42 - src/mongo/client/clientAndShell.cpp | 15 +- src/mongo/client/clientOnly-private.h | 7 +- src/mongo/client/connection_factory.cpp | 54 - src/mongo/client/connpool.cpp | 81 +- src/mongo/client/connpool.h | 114 +- src/mongo/client/dbclient.cpp | 371 +- src/mongo/client/dbclient.h | 5 +- src/mongo/client/dbclient_rs.cpp | 1875 +- src/mongo/client/dbclient_rs.h | 507 +- src/mongo/client/dbclient_rs_test.cpp | 104 +- src/mongo/client/dbclientcursor.cpp | 49 +- src/mongo/client/dbclientcursor.h | 31 +- src/mongo/client/dbclientinterface.h | 244 +- src/mongo/client/dbclientmockcursor.h | 5 +- src/mongo/client/distlock.cpp | 1159 - src/mongo/client/distlock.h | 350 - src/mongo/client/distlock_test.cpp | 491 - src/mongo/client/examples/authTest.cpp | 46 +- src/mongo/client/examples/clientTest.cpp | 6 + src/mongo/client/examples/first.cpp | 9 +- src/mongo/client/examples/httpClientTest.cpp | 47 +- src/mongo/client/examples/insert_demo.cpp | 13 +- src/mongo/client/examples/mongoperf.cpp | 2 +- src/mongo/client/examples/rs.cpp | 7 + src/mongo/client/examples/second.cpp | 18 +- src/mongo/client/examples/tail.cpp | 4 +- src/mongo/client/examples/tutorial.cpp | 34 +- src/mongo/client/examples/whereExample.cpp | 33 +- src/mongo/client/export_macros.h | 52 + src/mongo/client/gridfs.cpp | 13 +- src/mongo/client/gridfs.h | 10 +- src/mongo/client/include_bsonh_test.cpp | 20 + src/mongo/client/include_dbclienth_test.cpp | 20 + src/mongo/client/init.cpp | 66 + src/mongo/client/init.h | 55 + src/mongo/client/model.cpp | 141 - src/mongo/client/model.h | 61 - src/mongo/client/mongo_client_lib.cpp | 108 - src/mongo/client/parallel.cpp | 477 +- src/mongo/client/parallel.h | 295 +- src/mongo/client/redef_macros.h | 6 +- src/mongo/client/replica_set_monitor.cpp | 1092 + src/mongo/client/replica_set_monitor.h | 326 + src/mongo/client/replica_set_monitor_internal.h | 199 + src/mongo/client/replica_set_monitor_test.cpp | 964 + src/mongo/client/sasl_client_authenticate.cpp | 4 +- src/mongo/client/sasl_client_authenticate.h | 51 +- src/mongo/client/sasl_client_authenticate_impl.cpp | 64 +- src/mongo/client/sasl_client_session.cpp | 29 +- src/mongo/client/sasl_client_session.h | 3 +- src/mongo/client/sasl_sspi.cpp | 529 + src/mongo/client/scoped_db_conn_test.cpp | 106 +- src/mongo/client/simple_client_demo.cpp | 4 - src/mongo/client/syncclusterconnection.cpp | 106 +- src/mongo/client/syncclusterconnection.h | 57 +- src/mongo/client/undef_macros.h | 4 +- src/mongo/db/SConscript | 64 +- src/mongo/db/audit.cpp | 228 + src/mongo/db/audit.h | 404 + src/mongo/db/auth/SConscript | 89 +- src/mongo/db/auth/action_set.cpp | 43 +- src/mongo/db/auth/action_set.h | 21 +- src/mongo/db/auth/action_set_test.cpp | 45 + src/mongo/db/auth/action_types.txt | 83 +- src/mongo/db/auth/auth_external_state.cpp | 93 - src/mongo/db/auth/auth_external_state.h | 74 - src/mongo/db/auth/auth_external_state_d.cpp | 52 - src/mongo/db/auth/auth_external_state_d.h | 45 - src/mongo/db/auth/auth_external_state_mock.h | 54 - src/mongo/db/auth/auth_external_state_s.cpp | 60 - src/mongo/db/auth/auth_external_state_s.h | 43 - .../db/auth/auth_external_state_server_common.cpp | 64 - .../db/auth/auth_external_state_server_common.h | 50 - src/mongo/db/auth/auth_index_d.cpp | 132 +- src/mongo/db/auth/auth_index_d.h | 37 +- src/mongo/db/auth/auth_server_parameters.cpp | 12 + src/mongo/db/auth/authorization_manager.cpp | 1885 +- src/mongo/db/auth/authorization_manager.h | 627 +- src/mongo/db/auth/authorization_manager_global.cpp | 94 + src/mongo/db/auth/authorization_manager_global.h | 47 + src/mongo/db/auth/authorization_manager_test.cpp | 1052 +- src/mongo/db/auth/authorization_session.cpp | 532 + src/mongo/db/auth/authorization_session.h | 232 + src/mongo/db/auth/authorization_session_test.cpp | 545 + src/mongo/db/auth/authz_documents_update_guard.cpp | 57 + src/mongo/db/auth/authz_documents_update_guard.h | 69 + src/mongo/db/auth/authz_manager_external_state.cpp | 195 + src/mongo/db/auth/authz_manager_external_state.h | 264 + .../db/auth/authz_manager_external_state_d.cpp | 263 + src/mongo/db/auth/authz_manager_external_state_d.h | 91 + .../db/auth/authz_manager_external_state_local.cpp | 392 + .../db/auth/authz_manager_external_state_local.h | 116 + .../db/auth/authz_manager_external_state_mock.cpp | 370 + .../db/auth/authz_manager_external_state_mock.h | 128 + .../db/auth/authz_manager_external_state_s.cpp | 357 + src/mongo/db/auth/authz_manager_external_state_s.h | 116 + src/mongo/db/auth/authz_session_external_state.cpp | 46 + src/mongo/db/auth/authz_session_external_state.h | 76 + .../db/auth/authz_session_external_state_d.cpp | 58 + src/mongo/db/auth/authz_session_external_state_d.h | 53 + .../db/auth/authz_session_external_state_mock.h | 62 + .../db/auth/authz_session_external_state_s.cpp | 50 + src/mongo/db/auth/authz_session_external_state_s.h | 51 + .../authz_session_external_state_server_common.cpp | 79 + .../authz_session_external_state_server_common.h | 63 + src/mongo/db/auth/principal.cpp | 49 - src/mongo/db/auth/principal.h | 70 - src/mongo/db/auth/principal_name.h | 76 - src/mongo/db/auth/principal_set.cpp | 82 - src/mongo/db/auth/principal_set.h | 106 - src/mongo/db/auth/principal_set_test.cpp | 97 - src/mongo/db/auth/privilege.cpp | 27 +- src/mongo/db/auth/privilege.h | 31 +- src/mongo/db/auth/privilege_parser.cpp | 453 + src/mongo/db/auth/privilege_parser.h | 196 + src/mongo/db/auth/privilege_parser_test.cpp | 210 + src/mongo/db/auth/privilege_set.cpp | 145 - src/mongo/db/auth/privilege_set.h | 139 - src/mongo/db/auth/privilege_set_test.cpp | 211 - src/mongo/db/auth/resource_pattern.cpp | 64 + src/mongo/db/auth/resource_pattern.h | 210 + src/mongo/db/auth/role_graph.cpp | 561 + src/mongo/db/auth/role_graph.h | 313 + src/mongo/db/auth/role_graph_builtin_roles.cpp | 789 + src/mongo/db/auth/role_graph_test.cpp | 732 + src/mongo/db/auth/role_graph_update.cpp | 337 + src/mongo/db/auth/role_name.cpp | 55 + src/mongo/db/auth/role_name.h | 186 + src/mongo/db/auth/security_key.cpp | 91 +- src/mongo/db/auth/security_key.h | 31 + src/mongo/db/auth/user.cpp | 169 + src/mongo/db/auth/user.h | 237 + src/mongo/db/auth/user_cache_invalidator_job.cpp | 141 + src/mongo/db/auth/user_cache_invalidator_job.h | 44 + src/mongo/db/auth/user_document_parser.cpp | 505 + src/mongo/db/auth/user_document_parser.h | 76 + src/mongo/db/auth/user_document_parser_test.cpp | 388 + .../db/auth/user_management_commands_parser.cpp | 749 + .../db/auth/user_management_commands_parser.h | 242 + src/mongo/db/auth/user_name.cpp | 42 + src/mongo/db/auth/user_name.h | 150 + src/mongo/db/auth/user_name_hash.h | 31 + src/mongo/db/auth/user_set.cpp | 117 + src/mongo/db/auth/user_set.h | 107 + src/mongo/db/auth/user_set_test.cpp | 99 + src/mongo/db/background.cpp | 179 + src/mongo/db/background.h | 40 +- src/mongo/db/btree.cpp | 1982 -- src/mongo/db/btree.h | 1026 - src/mongo/db/btree_stats.cpp | 78 - src/mongo/db/btree_stats.h | 74 - src/mongo/db/btreebuilder.cpp | 168 - src/mongo/db/btreebuilder.h | 67 - src/mongo/db/btreecursor.cpp | 425 - src/mongo/db/btreecursor.h | 252 - src/mongo/db/btreeposition.cpp | 97 - src/mongo/db/btreeposition.h | 105 - src/mongo/db/cap.cpp | 494 - src/mongo/db/catalog/collection.cpp | 465 + src/mongo/db/catalog/collection.h | 251 + src/mongo/db/catalog/collection_cursor_cache.cpp | 496 + src/mongo/db/catalog/collection_cursor_cache.h | 139 + src/mongo/db/catalog/collection_info_cache.cpp | 131 + src/mongo/db/catalog/collection_info_cache.h | 112 + src/mongo/db/catalog/database.cpp | 848 + src/mongo/db/catalog/database.h | 258 + src/mongo/db/catalog/database_holder.cpp | 127 + src/mongo/db/catalog/database_holder.h | 140 + src/mongo/db/catalog/index_catalog.cpp | 1424 ++ src/mongo/db/catalog/index_catalog.h | 349 + src/mongo/db/catalog/index_catalog_entry.cpp | 189 + src/mongo/db/catalog/index_catalog_entry.h | 154 + src/mongo/db/catalog/index_create.cpp | 390 + src/mongo/db/catalog/index_create.h | 86 + src/mongo/db/catalog/index_key_validate.cpp | 111 + src/mongo/db/catalog/index_key_validate.h | 40 + src/mongo/db/catalog/index_pregen.cpp | 137 + src/mongo/db/catalog/index_pregen.h | 126 + src/mongo/db/cc_by_loc.h | 67 - src/mongo/db/client.cpp | 223 +- src/mongo/db/client.h | 79 +- src/mongo/db/client_basic.cpp | 30 +- src/mongo/db/client_basic.h | 22 +- src/mongo/db/clientcursor.cpp | 970 +- src/mongo/db/clientcursor.h | 545 +- src/mongo/db/clientlistplugin.cpp | 96 + src/mongo/db/cloner.cpp | 781 +- src/mongo/db/cloner.h | 59 +- src/mongo/db/cmdline.cpp | 533 - src/mongo/db/cmdline.h | 220 - src/mongo/db/cmdline_test.cpp | 224 - src/mongo/db/collection.h | 31 - src/mongo/db/commands.cpp | 639 +- src/mongo/db/commands.h | 121 +- src/mongo/db/commands/apply_ops.cpp | 167 + src/mongo/db/commands/auth_schema_upgrade_d.cpp | 158 + src/mongo/db/commands/authentication_commands.cpp | 296 +- src/mongo/db/commands/authentication_commands.h | 48 +- src/mongo/db/commands/cleanup_orphaned_cmd.cpp | 267 + src/mongo/db/commands/collection_to_capped.cpp | 256 + src/mongo/db/commands/compact.cpp | 180 + src/mongo/db/commands/connection_status.cpp | 29 +- src/mongo/db/commands/copydb.h | 50 + src/mongo/db/commands/copydb_common.cpp | 100 + src/mongo/db/commands/cpuprofile.cpp | 14 +- src/mongo/db/commands/create_indexes.cpp | 234 + src/mongo/db/commands/dbhash.cpp | 212 + src/mongo/db/commands/dbhash.h | 69 + src/mongo/db/commands/distinct.cpp | 151 +- src/mongo/db/commands/drop_indexes.cpp | 263 + src/mongo/db/commands/fail_point_cmd.cpp | 16 +- src/mongo/db/commands/find_and_modify.cpp | 128 +- src/mongo/db/commands/find_and_modify.h | 22 +- src/mongo/db/commands/find_and_modify_common.cpp | 23 +- src/mongo/db/commands/find_cmd.cpp | 146 + src/mongo/db/commands/fsync.cpp | 14 +- src/mongo/db/commands/fsync.h | 14 + src/mongo/db/commands/geonear.cpp | 306 + src/mongo/db/commands/get_last_error.cpp | 284 + src/mongo/db/commands/group.cpp | 131 +- src/mongo/db/commands/hashcmd.cpp | 14 +- src/mongo/db/commands/index_filter_commands.cpp | 369 + src/mongo/db/commands/index_filter_commands.h | 169 + .../db/commands/index_filter_commands_test.cpp | 319 + src/mongo/db/commands/index_stats.cpp | 14 +- src/mongo/db/commands/isself.cpp | 100 +- src/mongo/db/commands/merge_chunks_cmd.cpp | 179 + src/mongo/db/commands/mr.cpp | 423 +- src/mongo/db/commands/mr.h | 40 +- src/mongo/db/commands/mr_common.cpp | 36 +- src/mongo/db/commands/oplog_note.cpp | 82 + src/mongo/db/commands/parallel_collection_scan.cpp | 254 + src/mongo/db/commands/parameters.cpp | 265 +- src/mongo/db/commands/pipeline_command.cpp | 395 +- src/mongo/db/commands/plan_cache_commands.cpp | 419 + src/mongo/db/commands/plan_cache_commands.h | 169 + src/mongo/db/commands/plan_cache_commands_test.cpp | 383 + src/mongo/db/commands/rename_collection.cpp | 313 + src/mongo/db/commands/rename_collection.h | 20 +- src/mongo/db/commands/rename_collection_common.cpp | 84 +- src/mongo/db/commands/server_status.cpp | 46 +- src/mongo/db/commands/server_status.h | 12 + src/mongo/db/commands/shutdown.cpp | 52 + src/mongo/db/commands/shutdown.h | 66 + src/mongo/db/commands/storage_details.cpp | 52 +- src/mongo/db/commands/test_commands.cpp | 213 + src/mongo/db/commands/touch.cpp | 84 +- src/mongo/db/commands/user_management_commands.cpp | 3025 +++ src/mongo/db/commands/user_management_commands.h | 50 + src/mongo/db/commands/validate.cpp | 422 + .../db/commands/write_commands/batch_executor.cpp | 1272 ++ .../db/commands/write_commands/batch_executor.h | 197 + .../db/commands/write_commands/write_commands.cpp | 188 + .../db/commands/write_commands/write_commands.h | 118 + .../write_commands/write_commands_common.cpp | 99 + .../write_commands/write_commands_common.h | 49 + src/mongo/db/common.cpp | 46 - src/mongo/db/compact.cpp | 425 - src/mongo/db/conn_pool_options.cpp | 75 + src/mongo/db/conn_pool_options.h | 54 + src/mongo/db/connection_factory.cpp | 60 - src/mongo/db/curop-inl.h | 14 +- src/mongo/db/curop.cpp | 223 +- src/mongo/db/curop.h | 291 +- src/mongo/db/curop_test.cpp | 91 + src/mongo/db/cursor.cpp | 178 - src/mongo/db/cursor.h | 319 - src/mongo/db/d_concurrency.cpp | 56 +- src/mongo/db/d_concurrency.h | 23 +- src/mongo/db/d_globals.cpp | 22 +- src/mongo/db/d_globals.h | 12 + src/mongo/db/database.cpp | 502 - src/mongo/db/database.h | 152 - src/mongo/db/databaseholder.h | 128 - src/mongo/db/db.cpp | 1257 +- src/mongo/db/db.h | 68 +- src/mongo/db/db_10.sln | 38 - src/mongo/db/dbcommands.cpp | 1556 +- src/mongo/db/dbcommands_admin.cpp | 339 +- src/mongo/db/dbcommands_generic.cpp | 129 +- src/mongo/db/dbeval.cpp | 49 +- src/mongo/db/dbhelpers.cpp | 517 +- src/mongo/db/dbhelpers.h | 152 +- src/mongo/db/dbmessage.cpp | 106 +- src/mongo/db/dbmessage.h | 190 +- src/mongo/db/dbmessage_test.cpp | 143 + src/mongo/db/dbwebserver.cpp | 699 +- src/mongo/db/dbwebserver.h | 43 +- src/mongo/db/diskloc.h | 37 +- src/mongo/db/diskloc_test.cpp | 12 + src/mongo/db/driverHelpers.cpp | 29 +- src/mongo/db/dur.cpp | 153 +- src/mongo/db/dur.h | 34 +- src/mongo/db/dur_commitjob.cpp | 33 +- src/mongo/db/dur_commitjob.h | 27 +- src/mongo/db/dur_journal.cpp | 71 +- src/mongo/db/dur_journal.h | 12 + src/mongo/db/dur_journalformat.h | 12 + src/mongo/db/dur_journalimpl.h | 16 +- src/mongo/db/dur_preplogbuffer.cpp | 53 +- src/mongo/db/dur_recover.cpp | 136 +- src/mongo/db/dur_recover.h | 22 +- src/mongo/db/dur_stats.h | 12 + src/mongo/db/dur_writetodatafiles.cpp | 22 +- src/mongo/db/durop.cpp | 37 +- src/mongo/db/durop.h | 18 +- src/mongo/db/exec/2d.cpp | 287 + src/mongo/db/exec/2d.h | 181 + src/mongo/db/exec/2dcommon.cpp | 680 + src/mongo/db/exec/2dcommon.h | 273 + src/mongo/db/exec/2dnear.cpp | 530 + src/mongo/db/exec/2dnear.h | 184 + src/mongo/db/exec/SConscript | 97 + src/mongo/db/exec/and_common-inl.h | 95 + src/mongo/db/exec/and_hash.cpp | 515 + src/mongo/db/exec/and_hash.h | 132 + src/mongo/db/exec/and_sorted.cpp | 311 + src/mongo/db/exec/and_sorted.h | 105 + src/mongo/db/exec/collection_scan.cpp | 206 + src/mongo/db/exec/collection_scan.h | 91 + src/mongo/db/exec/collection_scan_common.h | 62 + src/mongo/db/exec/count.cpp | 204 + src/mongo/db/exec/count.h | 115 + src/mongo/db/exec/distinct_scan.cpp | 239 + src/mongo/db/exec/distinct_scan.h | 151 + src/mongo/db/exec/fetch.cpp | 230 + src/mongo/db/exec/fetch.h | 89 + src/mongo/db/exec/filter.h | 171 + src/mongo/db/exec/index_scan.cpp | 368 + src/mongo/db/exec/index_scan.h | 173 + src/mongo/db/exec/keep_mutations.cpp | 131 + src/mongo/db/exec/keep_mutations.h | 86 + src/mongo/db/exec/limit.cpp | 106 + src/mongo/db/exec/limit.h | 69 + src/mongo/db/exec/merge_sort.cpp | 255 + src/mongo/db/exec/merge_sort.h | 152 + src/mongo/db/exec/mock_stage.cpp | 67 + src/mongo/db/exec/mock_stage.h | 89 + src/mongo/db/exec/oplogstart.cpp | 222 + src/mongo/db/exec/oplogstart.h | 125 + src/mongo/db/exec/or.cpp | 184 + src/mongo/db/exec/or.h | 87 + src/mongo/db/exec/plan_stage.h | 225 + src/mongo/db/exec/plan_stats.cpp | 49 + src/mongo/db/exec/plan_stats.h | 431 + src/mongo/db/exec/plan_stats_test.cpp | 107 + src/mongo/db/exec/projection.cpp | 255 + src/mongo/db/exec/projection.h | 143 + src/mongo/db/exec/projection_exec.cpp | 551 + src/mongo/db/exec/projection_exec.h | 197 + src/mongo/db/exec/projection_exec_test.cpp | 219 + src/mongo/db/exec/s2near.cpp | 444 + src/mongo/db/exec/s2near.h | 165 + src/mongo/db/exec/shard_filter.cpp | 107 + src/mongo/db/exec/shard_filter.h | 103 + src/mongo/db/exec/skip.cpp | 110 + src/mongo/db/exec/skip.h | 68 + src/mongo/db/exec/sort.cpp | 562 + src/mongo/db/exec/sort.h | 252 + src/mongo/db/exec/sort_test.cpp | 241 + src/mongo/db/exec/stagedebug_cmd.cpp | 390 + src/mongo/db/exec/text.cpp | 362 + src/mongo/db/exec/text.h | 171 + src/mongo/db/exec/working_set.cpp | 183 + src/mongo/db/exec/working_set.h | 260 + src/mongo/db/exec/working_set_common.cpp | 123 + src/mongo/db/exec/working_set_common.h | 82 + src/mongo/db/exec/working_set_computed_data.h | 99 + src/mongo/db/exec/working_set_test.cpp | 150 + src/mongo/db/explain.cpp | 277 - src/mongo/db/explain.h | 188 - src/mongo/db/extsort.cpp | 371 +- src/mongo/db/extsort.h | 141 +- src/mongo/db/field_parser-inl.h | 367 + src/mongo/db/field_parser.cpp | 411 + src/mongo/db/field_parser.h | 310 + src/mongo/db/field_parser_test.cpp | 444 + src/mongo/db/field_ref.cpp | 218 +- src/mongo/db/field_ref.h | 109 +- src/mongo/db/field_ref_set.cpp | 155 + src/mongo/db/field_ref_set.h | 116 + src/mongo/db/field_ref_set_test.cpp | 132 + src/mongo/db/field_ref_test.cpp | 173 +- src/mongo/db/filever.h | 12 + src/mongo/db/flushtest.cpp | 20 +- src/mongo/db/fts/SConscript | 33 +- src/mongo/db/fts/fts_command.cpp | 22 +- src/mongo/db/fts/fts_command.h | 15 + src/mongo/db/fts/fts_command_mongod.cpp | 170 +- src/mongo/db/fts/fts_command_mongos.cpp | 28 +- src/mongo/db/fts/fts_element_iterator.cpp | 186 + src/mongo/db/fts/fts_element_iterator.h | 173 + src/mongo/db/fts/fts_element_iterator_test.cpp | 310 + src/mongo/db/fts/fts_enabled.cpp | 56 +- src/mongo/db/fts/fts_enabled.h | 25 - src/mongo/db/fts/fts_index.cpp | 104 - src/mongo/db/fts/fts_index.h | 69 - src/mongo/db/fts/fts_index_format.cpp | 101 +- src/mongo/db/fts/fts_index_format.h | 20 +- src/mongo/db/fts/fts_index_format_test.cpp | 95 + src/mongo/db/fts/fts_language.cpp | 262 + src/mongo/db/fts/fts_language.h | 128 + src/mongo/db/fts/fts_language_test.cpp | 137 + src/mongo/db/fts/fts_matcher.cpp | 166 +- src/mongo/db/fts/fts_matcher.h | 26 +- src/mongo/db/fts/fts_matcher_test.cpp | 48 +- src/mongo/db/fts/fts_query.cpp | 47 +- src/mongo/db/fts/fts_query.h | 31 +- src/mongo/db/fts/fts_query_test.cpp | 112 +- src/mongo/db/fts/fts_search.cpp | 177 - src/mongo/db/fts/fts_search.h | 103 - src/mongo/db/fts/fts_spec.cpp | 392 +- src/mongo/db/fts/fts_spec.h | 116 +- src/mongo/db/fts/fts_spec_legacy.cpp | 320 + src/mongo/db/fts/fts_spec_test.cpp | 467 +- src/mongo/db/fts/fts_util.cpp | 12 + src/mongo/db/fts/fts_util.h | 20 +- src/mongo/db/fts/fts_util_test.cpp | 12 + src/mongo/db/fts/generate_stop_words.py | 6 +- src/mongo/db/fts/stemmer.cpp | 20 +- src/mongo/db/fts/stemmer.h | 18 +- src/mongo/db/fts/stemmer_test.cpp | 19 +- src/mongo/db/fts/stop_words.cpp | 27 +- src/mongo/db/fts/stop_words.h | 15 +- src/mongo/db/fts/stop_words_test.cpp | 19 +- src/mongo/db/fts/tokenizer.cpp | 17 +- src/mongo/db/fts/tokenizer.h | 15 +- src/mongo/db/fts/tokenizer_test.cpp | 25 +- src/mongo/db/geo/2d.cpp | 2732 --- src/mongo/db/geo/2d.h | 23 - src/mongo/db/geo/core.h | 90 +- src/mongo/db/geo/geoconstants.h | 36 + src/mongo/db/geo/geonear.cpp | 132 - src/mongo/db/geo/geonear.h | 33 - src/mongo/db/geo/geoparser.cpp | 801 +- src/mongo/db/geo/geoparser.h | 84 +- src/mongo/db/geo/geoparser_test.cpp | 306 +- src/mongo/db/geo/geoquery.cpp | 979 +- src/mongo/db/geo/geoquery.h | 159 +- src/mongo/db/geo/hash.cpp | 22 +- src/mongo/db/geo/hash.h | 28 + src/mongo/db/geo/hash_test.cpp | 14 +- src/mongo/db/geo/haystack.cpp | 336 +- src/mongo/db/geo/s2.h | 45 + src/mongo/db/geo/s2common.cpp | 142 +- src/mongo/db/geo/s2common.h | 51 +- src/mongo/db/geo/s2cursor.cpp | 184 - src/mongo/db/geo/s2cursor.h | 92 - src/mongo/db/geo/s2index.cpp | 467 - src/mongo/db/geo/s2index.h | 22 - src/mongo/db/geo/s2nearcursor.cpp | 394 - src/mongo/db/geo/s2nearcursor.h | 157 - src/mongo/db/geo/shapes.cpp | 23 +- src/mongo/db/geo/shapes.h | 97 +- src/mongo/db/hasher.cpp | 12 + src/mongo/db/hasher.h | 21 +- src/mongo/db/hasher_test.cpp | 385 + src/mongo/db/hashindex.cpp | 170 - src/mongo/db/hashindex.h | 125 - src/mongo/db/index.cpp | 492 - src/mongo/db/index.h | 287 - src/mongo/db/index/2d_access_method.cpp | 63 + src/mongo/db/index/2d_access_method.h | 113 + src/mongo/db/index/2d_common.h | 44 + src/mongo/db/index/2d_key_generator.cpp | 149 + src/mongo/db/index/2d_key_generator.h | 59 + src/mongo/db/index/SConscript | 36 + src/mongo/db/index/btree_access_method.cpp | 73 + src/mongo/db/index/btree_access_method.h | 69 + src/mongo/db/index/btree_based_access_method.cpp | 595 + src/mongo/db/index/btree_based_access_method.h | 142 + src/mongo/db/index/btree_index_cursor.cpp | 282 + src/mongo/db/index/btree_index_cursor.h | 137 + src/mongo/db/index/btree_interface.cpp | 235 + src/mongo/db/index/btree_interface.h | 156 + src/mongo/db/index/btree_key_generator.cpp | 332 + src/mongo/db/index/btree_key_generator.h | 115 + src/mongo/db/index/expression_index.h | 162 + src/mongo/db/index/expression_params.h | 190 + src/mongo/db/index/external_key_generator.cpp | 209 + src/mongo/db/index/external_key_generator.h | 46 + src/mongo/db/index/fts_access_method.cpp | 45 + src/mongo/db/index/fts_access_method.h | 55 + src/mongo/db/index/fts_key_generator.cpp | 42 + src/mongo/db/index/fts_key_generator.h | 50 + src/mongo/db/index/hash_access_method.cpp | 64 + src/mongo/db/index/hash_access_method.h | 74 + src/mongo/db/index/hash_key_generator.cpp | 70 + src/mongo/db/index/hash_key_generator.h | 60 + src/mongo/db/index/haystack_access_method.cpp | 131 + src/mongo/db/index/haystack_access_method.h | 77 + .../db/index/haystack_access_method_internal.h | 81 + src/mongo/db/index/haystack_key_generator.cpp | 128 + src/mongo/db/index/haystack_key_generator.h | 65 + src/mongo/db/index/index_access_method.h | 238 + src/mongo/db/index/index_cursor.h | 148 + src/mongo/db/index/index_descriptor.cpp | 85 + src/mongo/db/index/index_descriptor.h | 205 + src/mongo/db/index/key_generator.cpp | 40 + src/mongo/db/index/key_generator.h | 48 + src/mongo/db/index/s2_access_method.cpp | 114 + src/mongo/db/index/s2_access_method.h | 65 + src/mongo/db/index/s2_key_generator.cpp | 272 + src/mongo/db/index/s2_key_generator.h | 56 + src/mongo/db/index_builder.cpp | 141 + src/mongo/db/index_builder.h | 88 + src/mongo/db/index_insertion_continuation.h | 70 - src/mongo/db/index_legacy.cpp | 96 + src/mongo/db/index_legacy.h | 80 + src/mongo/db/index_names.cpp | 95 + src/mongo/db/index_names.h | 96 + src/mongo/db/index_rebuilder.cpp | 177 +- src/mongo/db/index_rebuilder.h | 35 +- src/mongo/db/index_set.cpp | 134 - src/mongo/db/index_set.h | 48 - src/mongo/db/index_set_test.cpp | 78 - src/mongo/db/index_update.cpp | 704 - src/mongo/db/index_update.h | 99 - src/mongo/db/indexkey.cpp | 518 - src/mongo/db/indexkey.h | 267 - src/mongo/db/initialize_server_global_state.cpp | 264 +- src/mongo/db/initialize_server_global_state.h | 31 +- src/mongo/db/instance.cpp | 683 +- src/mongo/db/instance.h | 44 +- src/mongo/db/interrupt_status.h | 18 +- src/mongo/db/interrupt_status_mongod.cpp | 20 +- src/mongo/db/interrupt_status_mongod.h | 21 +- src/mongo/db/intervalbtreecursor.cpp | 202 - src/mongo/db/intervalbtreecursor.h | 145 - src/mongo/db/introspect.cpp | 100 +- src/mongo/db/introspect.h | 28 +- src/mongo/db/invalidation_type.h | 42 + src/mongo/db/javatest.cpp | 12 + src/mongo/db/jsobj.cpp | 192 +- src/mongo/db/jsobj.h | 25 +- src/mongo/db/jsobjmanipulator.h | 39 +- src/mongo/db/json.cpp | 403 +- src/mongo/db/json.h | 58 +- src/mongo/db/key.cpp | 683 - src/mongo/db/key.h | 118 - src/mongo/db/keypattern.cpp | 23 + src/mongo/db/keypattern.h | 22 +- src/mongo/db/kill_current_op.cpp | 104 +- src/mongo/db/kill_current_op.h | 12 + src/mongo/db/lasterror.cpp | 14 +- src/mongo/db/lasterror.h | 23 +- src/mongo/db/lockstat.cpp | 18 +- src/mongo/db/lockstat.h | 14 +- src/mongo/db/lockstate.cpp | 30 +- src/mongo/db/lockstate.h | 36 +- src/mongo/db/log_process_details.cpp | 67 + src/mongo/db/log_process_details.h | 44 + src/mongo/db/matcher.cpp | 1369 -- src/mongo/db/matcher.h | 338 +- src/mongo/db/matcher/expression.cpp | 72 + src/mongo/db/matcher/expression.h | 257 + src/mongo/db/matcher/expression_array.cpp | 231 + src/mongo/db/matcher/expression_array.h | 155 + src/mongo/db/matcher/expression_array_test.cpp | 483 + src/mongo/db/matcher/expression_geo.cpp | 144 + src/mongo/db/matcher/expression_geo.h | 86 + src/mongo/db/matcher/expression_geo_test.cpp | 75 + src/mongo/db/matcher/expression_leaf.cpp | 529 + src/mongo/db/matcher/expression_leaf.h | 385 + src/mongo/db/matcher/expression_leaf_test.cpp | 1653 ++ src/mongo/db/matcher/expression_parser.cpp | 821 + src/mongo/db/matcher/expression_parser.h | 157 + .../db/matcher/expression_parser_array_test.cpp | 647 + src/mongo/db/matcher/expression_parser_geo.cpp | 87 + .../db/matcher/expression_parser_geo_test.cpp | 78 + .../db/matcher/expression_parser_leaf_test.cpp | 538 + src/mongo/db/matcher/expression_parser_test.cpp | 109 + src/mongo/db/matcher/expression_parser_text.cpp | 82 + .../db/matcher/expression_parser_text_test.cpp | 62 + src/mongo/db/matcher/expression_parser_tree.cpp | 112 + .../db/matcher/expression_parser_tree_test.cpp | 188 + src/mongo/db/matcher/expression_test.cpp | 107 + src/mongo/db/matcher/expression_text.cpp | 88 + src/mongo/db/matcher/expression_text.h | 61 + src/mongo/db/matcher/expression_tree.cpp | 171 + src/mongo/db/matcher/expression_tree.h | 186 + src/mongo/db/matcher/expression_tree_test.cpp | 571 + src/mongo/db/matcher/expression_where.cpp | 202 + src/mongo/db/matcher/match_details.cpp | 72 + src/mongo/db/matcher/match_details.h | 74 + src/mongo/db/matcher/matchable.cpp | 45 + src/mongo/db/matcher/matchable.h | 105 + src/mongo/db/matcher/matcher.cpp | 62 + src/mongo/db/matcher/matcher.h | 64 + src/mongo/db/matcher/path.cpp | 317 + src/mongo/db/matcher/path.h | 168 + src/mongo/db/matcher/path_internal.cpp | 87 + src/mongo/db/matcher/path_internal.h | 48 + src/mongo/db/matcher/path_test.cpp | 485 + src/mongo/db/matcher_covered.cpp | 132 - src/mongo/db/max_time.h | 38 + src/mongo/db/memconcept.cpp | 126 - src/mongo/db/memconcept.h | 68 - src/mongo/db/minilex.h | 12 + src/mongo/db/module.cpp | 68 - src/mongo/db/module.h | 70 - src/mongo/db/mongod.vcxproj | 2049 +- src/mongo/db/mongod.vcxproj.filters | 4154 ++-- src/mongo/db/mongod_options.cpp | 1124 + src/mongo/db/mongod_options.h | 88 + src/mongo/db/mongod_options_init.cpp | 80 + src/mongo/db/mongod_sm.sln | 58 - src/mongo/db/mongod_sm.vcxproj | 2959 --- src/mongo/db/mongod_sm.vcxproj.filters | 3378 --- src/mongo/db/mongommf.cpp | 244 - src/mongo/db/mongommf.h | 146 - src/mongo/db/namespace-inl.h | 72 - src/mongo/db/namespace.cpp | 33 - src/mongo/db/namespace.h | 62 - src/mongo/db/namespace_details-inl.h | 104 - src/mongo/db/namespace_details.cpp | 992 - src/mongo/db/namespace_details.h | 692 - src/mongo/db/namespace_string-inl.h | 200 + src/mongo/db/namespace_string.h | 231 + src/mongo/db/namespace_string_test.cpp | 187 + src/mongo/db/namespacestring.h | 231 - src/mongo/db/namespacestring_test.cpp | 102 - src/mongo/db/oplog.cpp | 1020 - src/mongo/db/oplog.h | 173 - src/mongo/db/oplogreader.h | 137 - src/mongo/db/ops/SConscript | 230 + src/mongo/db/ops/count.cpp | 206 +- src/mongo/db/ops/count.h | 29 +- src/mongo/db/ops/delete.cpp | 190 +- src/mongo/db/ops/delete.h | 53 +- src/mongo/db/ops/delete_executor.cpp | 198 + src/mongo/db/ops/delete_executor.h | 106 + src/mongo/db/ops/delete_request.h | 69 + src/mongo/db/ops/field_checker.cpp | 84 + src/mongo/db/ops/field_checker.h | 59 + src/mongo/db/ops/field_checker_test.cpp | 94 + src/mongo/db/ops/insert.cpp | 206 + src/mongo/db/ops/insert.h | 57 + src/mongo/db/ops/log_builder.cpp | 173 + src/mongo/db/ops/log_builder.h | 122 + src/mongo/db/ops/log_builder_test.cpp | 266 + src/mongo/db/ops/modifier_add_to_set.cpp | 432 + src/mongo/db/ops/modifier_add_to_set.h | 86 + src/mongo/db/ops/modifier_add_to_set_test.cpp | 393 + src/mongo/db/ops/modifier_base.h | 128 - src/mongo/db/ops/modifier_bit.cpp | 300 + src/mongo/db/ops/modifier_bit.h | 100 + src/mongo/db/ops/modifier_bit_test.cpp | 736 + src/mongo/db/ops/modifier_compare.cpp | 192 + src/mongo/db/ops/modifier_compare.h | 111 + src/mongo/db/ops/modifier_compare_test.cpp | 296 + src/mongo/db/ops/modifier_current_date.cpp | 277 + src/mongo/db/ops/modifier_current_date.h | 90 + src/mongo/db/ops/modifier_current_date_test.cpp | 365 + src/mongo/db/ops/modifier_inc.cpp | 297 + src/mongo/db/ops/modifier_inc.h | 101 + src/mongo/db/ops/modifier_inc_test.cpp | 524 + src/mongo/db/ops/modifier_interface.h | 195 + src/mongo/db/ops/modifier_object_replace.cpp | 194 + src/mongo/db/ops/modifier_object_replace.h | 97 + src/mongo/db/ops/modifier_object_replace_test.cpp | 309 + src/mongo/db/ops/modifier_pop.cpp | 213 + src/mongo/db/ops/modifier_pop.h | 89 + src/mongo/db/ops/modifier_pop_test.cpp | 316 + src/mongo/db/ops/modifier_pull.cpp | 299 + src/mongo/db/ops/modifier_pull.h | 88 + src/mongo/db/ops/modifier_pull_all.cpp | 252 + src/mongo/db/ops/modifier_pull_all.h | 88 + src/mongo/db/ops/modifier_pull_all_test.cpp | 250 + src/mongo/db/ops/modifier_pull_test.cpp | 576 + src/mongo/db/ops/modifier_push.cpp | 721 + src/mongo/db/ops/modifier_push.h | 136 + src/mongo/db/ops/modifier_push_sorter.h | 77 + src/mongo/db/ops/modifier_push_sorter_test.cpp | 179 + src/mongo/db/ops/modifier_push_test.cpp | 1455 ++ src/mongo/db/ops/modifier_rename.cpp | 316 + src/mongo/db/ops/modifier_rename.h | 99 + src/mongo/db/ops/modifier_rename_test.cpp | 395 + src/mongo/db/ops/modifier_set.cpp | 289 + src/mongo/db/ops/modifier_set.h | 115 + src/mongo/db/ops/modifier_set_test.cpp | 756 + src/mongo/db/ops/modifier_table.cpp | 176 + src/mongo/db/ops/modifier_table.h | 70 + src/mongo/db/ops/modifier_table_test.cpp | 59 + src/mongo/db/ops/modifier_unset.cpp | 185 + src/mongo/db/ops/modifier_unset.h | 104 + src/mongo/db/ops/modifier_unset_test.cpp | 451 + src/mongo/db/ops/path_support.cpp | 264 + src/mongo/db/ops/path_support.h | 98 + src/mongo/db/ops/path_support_test.cpp | 450 + src/mongo/db/ops/query.cpp | 1060 - src/mongo/db/ops/query.h | 322 - src/mongo/db/ops/update.cpp | 1244 +- src/mongo/db/ops/update.h | 100 +- src/mongo/db/ops/update_driver.cpp | 448 + src/mongo/db/ops/update_driver.h | 192 + src/mongo/db/ops/update_driver_test.cpp | 173 + src/mongo/db/ops/update_executor.cpp | 123 + src/mongo/db/ops/update_executor.h | 125 + src/mongo/db/ops/update_internal.cpp | 1496 -- src/mongo/db/ops/update_internal.h | 780 - src/mongo/db/ops/update_lifecycle.h | 69 + src/mongo/db/ops/update_lifecycle_impl.cpp | 81 + src/mongo/db/ops/update_lifecycle_impl.h | 65 + src/mongo/db/ops/update_request.h | 180 + src/mongo/db/ops/update_result.h | 86 + src/mongo/db/pagefault.cpp | 31 +- src/mongo/db/pagefault.h | 12 + src/mongo/db/pdfile.cpp | 2113 +- src/mongo/db/pdfile.h | 647 +- src/mongo/db/pdfile_private.h | 12 + src/mongo/db/pdfile_version.h | 17 +- src/mongo/db/pipeline/accumulator.cpp | 78 - src/mongo/db/pipeline/accumulator.h | 265 +- src/mongo/db/pipeline/accumulator_add_to_set.cpp | 88 +- src/mongo/db/pipeline/accumulator_avg.cpp | 101 +- src/mongo/db/pipeline/accumulator_first.cpp | 52 +- src/mongo/db/pipeline/accumulator_last.cpp | 45 +- src/mongo/db/pipeline/accumulator_min_max.cpp | 67 +- src/mongo/db/pipeline/accumulator_push.cpp | 73 +- src/mongo/db/pipeline/accumulator_single_value.cpp | 32 - src/mongo/db/pipeline/accumulator_sum.cpp | 68 +- src/mongo/db/pipeline/builder.cpp | 132 - src/mongo/db/pipeline/builder.h | 101 - src/mongo/db/pipeline/dependencies.cpp | 175 + src/mongo/db/pipeline/dependencies.h | 77 + src/mongo/db/pipeline/doc_mem_monitor.cpp | 68 - src/mongo/db/pipeline/doc_mem_monitor.h | 94 - src/mongo/db/pipeline/document.cpp | 87 +- src/mongo/db/pipeline/document.h | 176 +- src/mongo/db/pipeline/document_internal.h | 34 + src/mongo/db/pipeline/document_source.cpp | 183 +- src/mongo/db/pipeline/document_source.h | 988 +- .../db/pipeline/document_source_bson_array.cpp | 85 +- .../db/pipeline/document_source_command_shards.cpp | 121 +- src/mongo/db/pipeline/document_source_cursor.cpp | 358 +- src/mongo/db/pipeline/document_source_filter.cpp | 104 - .../db/pipeline/document_source_filter_base.cpp | 83 - src/mongo/db/pipeline/document_source_geo_near.cpp | 98 +- src/mongo/db/pipeline/document_source_group.cpp | 525 +- src/mongo/db/pipeline/document_source_limit.cpp | 66 +- src/mongo/db/pipeline/document_source_match.cpp | 336 +- .../db/pipeline/document_source_merge_cursors.cpp | 180 + src/mongo/db/pipeline/document_source_out.cpp | 175 +- src/mongo/db/pipeline/document_source_project.cpp | 109 +- src/mongo/db/pipeline/document_source_redact.cpp | 169 + src/mongo/db/pipeline/document_source_skip.cpp | 94 +- src/mongo/db/pipeline/document_source_sort.cpp | 352 +- src/mongo/db/pipeline/document_source_unwind.cpp | 153 +- src/mongo/db/pipeline/expression.cpp | 2411 +- src/mongo/db/pipeline/expression.h | 1113 +- src/mongo/db/pipeline/expression_context.cpp | 57 - src/mongo/db/pipeline/expression_context.h | 104 +- src/mongo/db/pipeline/field_path.cpp | 20 +- src/mongo/db/pipeline/field_path.h | 12 + src/mongo/db/pipeline/pipeline.cpp | 528 +- src/mongo/db/pipeline/pipeline.h | 194 +- src/mongo/db/pipeline/pipeline_d.cpp | 315 +- src/mongo/db/pipeline/pipeline_d.h | 54 +- src/mongo/db/pipeline/pipeline_optimizations.h | 124 + src/mongo/db/pipeline/value.cpp | 293 +- src/mongo/db/pipeline/value.h | 60 +- src/mongo/db/pipeline/value_internal.h | 19 +- src/mongo/db/prefetch.cpp | 70 +- src/mongo/db/prefetch.h | 16 +- src/mongo/db/projection.cpp | 6 +- src/mongo/db/projection.h | 7 +- src/mongo/db/query/SConscript | 242 + src/mongo/db/query/cached_plan_runner.cpp | 215 + src/mongo/db/query/cached_plan_runner.h | 126 + src/mongo/db/query/canonical_query.cpp | 771 + src/mongo/db/query/canonical_query.h | 187 + src/mongo/db/query/canonical_query_test.cpp | 586 + src/mongo/db/query/eof_runner.cpp | 106 + src/mongo/db/query/eof_runner.h | 89 + src/mongo/db/query/explain_plan.cpp | 670 + src/mongo/db/query/explain_plan.h | 96 + src/mongo/db/query/find_constants.h | 35 + src/mongo/db/query/get_runner.cpp | 877 + src/mongo/db/query/get_runner.h | 147 + src/mongo/db/query/get_runner_test.cpp | 144 + src/mongo/db/query/idhack_runner.cpp | 315 + src/mongo/db/query/idhack_runner.h | 136 + src/mongo/db/query/index_bounds.cpp | 577 + src/mongo/db/query/index_bounds.h | 241 + src/mongo/db/query/index_bounds_builder.cpp | 991 + src/mongo/db/query/index_bounds_builder.h | 195 + src/mongo/db/query/index_bounds_builder_test.cpp | 1195 + src/mongo/db/query/index_bounds_test.cpp | 850 + src/mongo/db/query/index_entry.h | 132 + src/mongo/db/query/index_tag.cpp | 115 + src/mongo/db/query/index_tag.h | 132 + src/mongo/db/query/indexability.h | 133 + src/mongo/db/query/internal_plans.h | 118 + src/mongo/db/query/internal_runner.cpp | 136 + src/mongo/db/query/internal_runner.h | 103 + src/mongo/db/query/interval.cpp | 316 + src/mongo/db/query/interval.h | 196 + src/mongo/db/query/interval_test.cpp | 296 + src/mongo/db/query/lite_parsed_query.cpp | 414 + src/mongo/db/query/lite_parsed_query.h | 162 + src/mongo/db/query/lite_parsed_query_test.cpp | 258 + src/mongo/db/query/lru_key_value.h | 222 + src/mongo/db/query/lru_key_value_test.cpp | 197 + src/mongo/db/query/multi_plan_runner.cpp | 666 + src/mongo/db/query/multi_plan_runner.h | 198 + src/mongo/db/query/new_find.cpp | 857 + src/mongo/db/query/new_find.h | 53 + src/mongo/db/query/parsed_projection.cpp | 317 + src/mongo/db/query/parsed_projection.h | 127 + src/mongo/db/query/parsed_projection_test.cpp | 214 + src/mongo/db/query/plan_cache.cpp | 513 + src/mongo/db/query/plan_cache.h | 417 + src/mongo/db/query/plan_cache_test.cpp | 1036 + src/mongo/db/query/plan_enumerator.cpp | 1259 + src/mongo/db/query/plan_enumerator.h | 447 + src/mongo/db/query/plan_executor.cpp | 198 + src/mongo/db/query/plan_executor.h | 111 + src/mongo/db/query/plan_ranker.cpp | 272 + src/mongo/db/query/plan_ranker.h | 132 + src/mongo/db/query/planner_access.cpp | 1295 ++ src/mongo/db/query/planner_access.h | 404 + src/mongo/db/query/planner_analysis.cpp | 725 + src/mongo/db/query/planner_analysis.h | 108 + src/mongo/db/query/planner_analysis_test.cpp | 165 + src/mongo/db/query/planner_ixselect.cpp | 731 + src/mongo/db/query/planner_ixselect.h | 182 + src/mongo/db/query/planner_ixselect_test.cpp | 269 + src/mongo/db/query/qlog.cpp | 49 + src/mongo/db/query/qlog.h | 43 + src/mongo/db/query/query_knobs.cpp | 65 + src/mongo/db/query/query_knobs.h | 95 + src/mongo/db/query/query_planner.cpp | 1003 + src/mongo/db/query/query_planner.h | 129 + src/mongo/db/query/query_planner_common.h | 132 + src/mongo/db/query/query_planner_params.h | 107 + src/mongo/db/query/query_planner_test.cpp | 4778 ++++ src/mongo/db/query/query_planner_test_lib.cpp | 483 + src/mongo/db/query/query_planner_test_lib.h | 61 + src/mongo/db/query/query_planner_text_test.cpp | 579 + src/mongo/db/query/query_settings.cpp | 166 + src/mongo/db/query/query_settings.h | 145 + src/mongo/db/query/query_solution.cpp | 900 + src/mongo/db/query/query_solution.h | 751 + src/mongo/db/query/runner.h | 223 + src/mongo/db/query/runner_yield_policy.h | 130 + src/mongo/db/query/single_solution_runner.cpp | 133 + src/mongo/db/query/single_solution_runner.h | 96 + src/mongo/db/query/stage_builder.cpp | 343 + src/mongo/db/query/stage_builder.h | 54 + src/mongo/db/query/stage_types.h | 76 + src/mongo/db/query/subplan_runner.cpp | 501 + src/mongo/db/query/subplan_runner.h | 140 + src/mongo/db/query/type_explain.cpp | 805 + src/mongo/db/query/type_explain.h | 289 + src/mongo/db/queryoptimizer.cpp | 2038 -- src/mongo/db/queryoptimizer.h | 870 - src/mongo/db/queryoptimizercursor.h | 147 - src/mongo/db/queryoptimizercursorimpl.cpp | 542 - src/mongo/db/queryoptimizercursorimpl.h | 298 - src/mongo/db/querypattern.cpp | 117 - src/mongo/db/querypattern.h | 118 - src/mongo/db/queryutil.cpp | 163 +- src/mongo/db/queryutil.h | 244 +- src/mongo/db/range_deleter.cpp | 623 + src/mongo/db/range_deleter.h | 305 + src/mongo/db/range_deleter_db_env.cpp | 184 + src/mongo/db/range_deleter_db_env.h | 66 + src/mongo/db/range_deleter_mock_env.cpp | 154 + src/mongo/db/range_deleter_mock_env.h | 171 + src/mongo/db/range_deleter_service.cpp | 49 + src/mongo/db/range_deleter_service.h | 40 + src/mongo/db/range_deleter_stat_test.cpp | 322 + src/mongo/db/range_deleter_stats.cpp | 55 + src/mongo/db/range_deleter_stats.h | 115 + src/mongo/db/range_deleter_test.cpp | 516 + src/mongo/db/range_preserver.h | 68 + src/mongo/db/record.cpp | 595 - src/mongo/db/record.h | 35 - src/mongo/db/repair_database.cpp | 469 + src/mongo/db/repair_database.h | 52 + src/mongo/db/repl.cpp | 1640 -- src/mongo/db/repl.h | 197 - src/mongo/db/repl/bgsync.cpp | 148 +- src/mongo/db/repl/bgsync.h | 16 +- src/mongo/db/repl/connections.h | 30 +- src/mongo/db/repl/consensus.cpp | 48 +- src/mongo/db/repl/health.cpp | 93 +- src/mongo/db/repl/health.h | 12 + src/mongo/db/repl/heartbeat.cpp | 107 +- src/mongo/db/repl/is_master.h | 92 + src/mongo/db/repl/manager.cpp | 48 +- src/mongo/db/repl/master_slave.cpp | 1300 ++ src/mongo/db/repl/master_slave.h | 179 + src/mongo/db/repl/multicmd.h | 22 +- src/mongo/db/repl/oplog.cpp | 720 + src/mongo/db/repl/oplog.h | 96 + src/mongo/db/repl/oplogreader.cpp | 191 + src/mongo/db/repl/oplogreader.h | 166 + src/mongo/db/repl/repl_reads_ok.cpp | 64 + src/mongo/db/repl/repl_reads_ok.h | 38 + src/mongo/db/repl/repl_start.cpp | 63 + src/mongo/db/repl/repl_start.h | 37 + src/mongo/db/repl/replication_server_status.cpp | 210 + src/mongo/db/repl/replication_server_status.h | 99 + src/mongo/db/repl/replset_commands.cpp | 180 +- src/mongo/db/repl/replset_web_handler.cpp | 132 + src/mongo/db/repl/resync.cpp | 116 + src/mongo/db/repl/rs.cpp | 193 +- src/mongo/db/repl/rs.h | 102 +- src/mongo/db/repl/rs_config.cpp | 67 +- src/mongo/db/repl/rs_config.h | 21 +- src/mongo/db/repl/rs_exception.h | 12 + src/mongo/db/repl/rs_initialsync.cpp | 156 +- src/mongo/db/repl/rs_initiate.cpp | 43 +- src/mongo/db/repl/rs_member.h | 17 +- src/mongo/db/repl/rs_optime.h | 27 - src/mongo/db/repl/rs_rollback.cpp | 157 +- src/mongo/db/repl/rs_sync.cpp | 317 +- src/mongo/db/repl/rs_sync.h | 71 +- src/mongo/db/repl/sync.cpp | 136 + src/mongo/db/repl/sync.h | 52 + src/mongo/db/repl/sync_source_feedback.cpp | 364 + src/mongo/db/repl/sync_source_feedback.h | 195 + src/mongo/db/repl/write_concern.cpp | 364 + src/mongo/db/repl/write_concern.h | 59 + src/mongo/db/repl_block.cpp | 311 - src/mongo/db/repl_block.h | 43 - src/mongo/db/replutil.h | 89 - src/mongo/db/resource.h | 12 + src/mongo/db/restapi.cpp | 81 +- src/mongo/db/restapi.h | 19 +- src/mongo/db/scanandorder.cpp | 122 - src/mongo/db/scanandorder.h | 126 - src/mongo/db/server_extra_log_context.cpp | 34 +- src/mongo/db/server_options.cpp | 28 + src/mongo/db/server_options.h | 130 + src/mongo/db/server_options_helpers.cpp | 815 + src/mongo/db/server_options_helpers.h | 74 + src/mongo/db/server_options_test.cpp | 330 + src/mongo/db/server_parameters.cpp | 52 +- src/mongo/db/server_parameters.h | 12 + src/mongo/db/server_parameters_inline.h | 55 +- src/mongo/db/server_parameters_test.cpp | 12 + src/mongo/db/sort_phase_one.h | 33 +- src/mongo/db/sorter/SConscript | 3 + src/mongo/db/sorter/sorter.cpp | 888 + src/mongo/db/sorter/sorter.h | 237 + src/mongo/db/sorter/sorter_test.cpp | 549 + src/mongo/db/startup_warnings.cpp | 236 + src/mongo/db/startup_warnings.h | 32 + src/mongo/db/stats/counters.cpp | 56 +- src/mongo/db/stats/counters.h | 34 +- src/mongo/db/stats/fine_clock.h | 12 + src/mongo/db/stats/service_stats.cpp | 14 +- src/mongo/db/stats/service_stats.h | 14 +- src/mongo/db/stats/snapshots.cpp | 114 +- src/mongo/db/stats/snapshots.h | 16 +- src/mongo/db/stats/snapshots_webplugins.cpp | 128 + src/mongo/db/stats/timer_stats.cpp | 5 +- src/mongo/db/stats/top.cpp | 20 +- src/mongo/db/stats/top.h | 3 +- src/mongo/db/storage/data_file.cpp | 235 + src/mongo/db/storage/data_file.h | 141 + src/mongo/db/storage/durable_mapped_file.cpp | 212 + src/mongo/db/storage/durable_mapped_file.h | 152 + src/mongo/db/storage/extent.cpp | 238 + src/mongo/db/storage/extent.h | 124 + src/mongo/db/storage/extent_manager.cpp | 617 + src/mongo/db/storage/extent_manager.h | 195 + src/mongo/db/storage/record.cpp | 623 + src/mongo/db/storage/record.h | 240 + src/mongo/db/storage/storage_engine_metadata.cpp | 253 + src/mongo/db/storage/storage_engine_metadata.h | 110 + .../db/storage/storage_engine_metadata_test.cpp | 331 + src/mongo/db/storage_options.cpp | 71 + src/mongo/db/storage_options.h | 116 + src/mongo/db/structure/btree/btree.cpp | 2003 ++ src/mongo/db/structure/btree/btree.h | 1099 + src/mongo/db/structure/btree/btree_stats.cpp | 90 + src/mongo/db/structure/btree/btree_stats.h | 86 + src/mongo/db/structure/btree/btreebuilder.cpp | 197 + src/mongo/db/structure/btree/btreebuilder.h | 85 + src/mongo/db/structure/btree/key.cpp | 695 + src/mongo/db/structure/btree/key.h | 130 + src/mongo/db/structure/catalog/cap.cpp | 498 + src/mongo/db/structure/catalog/hashtab.h | 176 + src/mongo/db/structure/catalog/index_details.cpp | 66 + src/mongo/db/structure/catalog/index_details.h | 187 + src/mongo/db/structure/catalog/namespace-inl.h | 74 + src/mongo/db/structure/catalog/namespace.cpp | 45 + src/mongo/db/structure/catalog/namespace.h | 92 + .../db/structure/catalog/namespace_details-inl.h | 44 + .../db/structure/catalog/namespace_details.cpp | 762 + src/mongo/db/structure/catalog/namespace_details.h | 437 + src/mongo/db/structure/catalog/namespace_index.cpp | 227 + src/mongo/db/structure/catalog/namespace_index.h | 87 + src/mongo/db/structure/catalog/namespace_test.cpp | 67 + src/mongo/db/structure/collection_compact.cpp | 324 + src/mongo/db/structure/collection_iterator.cpp | 318 + src/mongo/db/structure/collection_iterator.h | 150 + src/mongo/db/structure/record_store.cpp | 272 + src/mongo/db/structure/record_store.h | 128 + src/mongo/db/taskqueue.h | 12 + src/mongo/db/tests.cpp | 12 + src/mongo/db/ttl.cpp | 52 +- src/mongo/db/ttl.h | 12 + src/mongo/db/update_index_data.cpp | 175 + src/mongo/db/update_index_data.h | 86 + src/mongo/db/update_index_data_test.cpp | 116 + src/mongo/db/wire_version.h | 65 + src/mongo/db/write_concern.cpp | 225 + src/mongo/db/write_concern.h | 82 + src/mongo/db/write_concern_options.cpp | 81 + src/mongo/db/write_concern_options.h | 65 + src/mongo/dbtests/accumulatortests.cpp | 496 +- src/mongo/dbtests/background_job_test.cpp | 110 - src/mongo/dbtests/basictests.cpp | 166 +- src/mongo/dbtests/btreebuildertests.cpp | 80 +- src/mongo/dbtests/btreepositiontests.cpp | 327 - src/mongo/dbtests/btreetests.cpp | 40 +- src/mongo/dbtests/btreetests.inl | 177 +- src/mongo/dbtests/chunk_manager_targeter_test.cpp | 43 + src/mongo/dbtests/chunktests.cpp | 19 +- src/mongo/dbtests/clienttests.cpp | 22 +- src/mongo/dbtests/commandtests.cpp | 19 +- src/mongo/dbtests/config_server_fixture.cpp | 22 +- src/mongo/dbtests/config_server_fixture.h | 40 + src/mongo/dbtests/config_upgrade_tests.cpp | 164 +- src/mongo/dbtests/counttests.cpp | 61 +- src/mongo/dbtests/cursortests.cpp | 807 - src/mongo/dbtests/d_chunk_manager_tests.cpp | 478 - src/mongo/dbtests/dbclient_multi_command_test.cpp | 42 + src/mongo/dbtests/dbhelper_tests.cpp | 233 + src/mongo/dbtests/dbtests.cpp | 49 +- src/mongo/dbtests/dbtests.h | 16 + src/mongo/dbtests/directclienttests.cpp | 35 +- src/mongo/dbtests/documentsourcetests.cpp | 854 +- src/mongo/dbtests/documenttests.cpp | 332 +- src/mongo/dbtests/expressiontests.cpp | 1253 +- src/mongo/dbtests/extsorttests.cpp | 92 +- src/mongo/dbtests/framework.cpp | 196 +- src/mongo/dbtests/framework.h | 14 +- src/mongo/dbtests/framework_options.cpp | 230 + src/mongo/dbtests/framework_options.h | 70 + src/mongo/dbtests/framework_options_init.cpp | 60 + src/mongo/dbtests/gle_test.cpp | 12 + src/mongo/dbtests/gridfstest.cpp | 12 + src/mongo/dbtests/histogram_test.cpp | 18 +- src/mongo/dbtests/indexcatalogtests.cpp | 141 + src/mongo/dbtests/indexupdatetests.cpp | 529 +- src/mongo/dbtests/intervalbtreecursortests.cpp | 660 - src/mongo/dbtests/jsobjhashingtests.cpp | 170 - src/mongo/dbtests/jsobjtests.cpp | 195 +- src/mongo/dbtests/jsontests.cpp | 1264 +- src/mongo/dbtests/jstests.cpp | 807 +- src/mongo/dbtests/keypatterntests.cpp | 12 + src/mongo/dbtests/matchertests.cpp | 310 +- src/mongo/dbtests/merge_chunk_tests.cpp | 421 + src/mongo/dbtests/mmaptests.cpp | 43 +- src/mongo/dbtests/mock/mock_conn_registry.cpp | 14 +- src/mongo/dbtests/mock/mock_conn_registry.h | 12 + .../dbtests/mock/mock_dbclient_connection.cpp | 2 +- src/mongo/dbtests/mock/mock_dbclient_connection.h | 2 +- src/mongo/dbtests/namespacetests.cpp | 828 +- src/mongo/dbtests/oplogstarttests.cpp | 381 + src/mongo/dbtests/pdfiletests.cpp | 495 +- src/mongo/dbtests/perf/btreeperf.cpp | 2 +- src/mongo/dbtests/perf/perftest.cpp | 33 +- src/mongo/dbtests/perftests.cpp | 291 +- src/mongo/dbtests/pipelinetests.cpp | 198 +- src/mongo/dbtests/plan_ranking.cpp | 812 + src/mongo/dbtests/profile_test.cpp | 12 + src/mongo/dbtests/query_multi_plan_runner.cpp | 164 + src/mongo/dbtests/query_single_solution_runner.cpp | 447 + src/mongo/dbtests/query_stage_and.cpp | 1375 ++ src/mongo/dbtests/query_stage_collscan.cpp | 692 + src/mongo/dbtests/query_stage_count.cpp | 643 + src/mongo/dbtests/query_stage_distinct.cpp | 242 + src/mongo/dbtests/query_stage_fetch.cpp | 440 + src/mongo/dbtests/query_stage_ixscan.cpp | 124 + src/mongo/dbtests/query_stage_keep.cpp | 236 + src/mongo/dbtests/query_stage_limit_skip.cpp | 104 + src/mongo/dbtests/query_stage_merge_sort.cpp | 599 + src/mongo/dbtests/query_stage_sort.cpp | 384 + src/mongo/dbtests/query_stage_tests.cpp | 221 + src/mongo/dbtests/query_subplan_runner.cpp | 120 + src/mongo/dbtests/queryoptimizercursortests.cpp | 4968 ---- src/mongo/dbtests/queryoptimizertests.cpp | 1109 - src/mongo/dbtests/queryoptimizertests2.cpp | 831 - src/mongo/dbtests/querytests.cpp | 385 +- src/mongo/dbtests/queryutiltests.cpp | 277 +- src/mongo/dbtests/removetests.cpp | 88 - src/mongo/dbtests/replica_set_monitor_test.cpp | 1146 +- src/mongo/dbtests/replsettests.cpp | 100 +- src/mongo/dbtests/repltests.cpp | 413 +- src/mongo/dbtests/runner_registry.cpp | 306 + src/mongo/dbtests/sharding.cpp | 23 +- src/mongo/dbtests/socktests.cpp | 19 +- src/mongo/dbtests/spin_lock_test.cpp | 114 - src/mongo/dbtests/stacktests.cpp | 19 +- src/mongo/dbtests/test.vcxproj | 1617 +- src/mongo/dbtests/test.vcxproj.filters | 4174 ++-- src/mongo/dbtests/test_sm.sln | 39 - src/mongo/dbtests/test_sm.vcxproj | 3287 --- src/mongo/dbtests/test_sm.vcxproj.filters | 3523 --- src/mongo/dbtests/threadedtests.cpp | 69 +- src/mongo/dbtests/updatetests.cpp | 906 +- src/mongo/installer/msi/SConscript | 140 + src/mongo/installer/msi/wxs/BinaryFragment.wxs | 143 + src/mongo/installer/msi/wxs/FeatureFragment.wxs | 81 + src/mongo/installer/msi/wxs/Installer.wxs | 48 + src/mongo/installer/msi/wxs/Installer_64.wxs | 67 + src/mongo/installer/msi/wxs/LicensingFragment.wxs | 44 + src/mongo/logger/SConscript | 34 + src/mongo/logger/appender.h | 44 + src/mongo/logger/console.cpp | 242 + src/mongo/logger/console.h | 47 + src/mongo/logger/console_appender.h | 54 + src/mongo/logger/console_test.cpp | 71 + src/mongo/logger/encoder.h | 37 + src/mongo/logger/labeled_level.h | 64 + src/mongo/logger/log_domain-impl.h | 101 + src/mongo/logger/log_domain.h | 144 + src/mongo/logger/log_manager.cpp | 47 + src/mongo/logger/log_manager.h | 57 + src/mongo/logger/log_severity-inl.h | 42 + src/mongo/logger/log_severity.cpp | 55 + src/mongo/logger/log_severity.h | 123 + src/mongo/logger/log_test.cpp | 147 + src/mongo/logger/logger.cpp | 54 + src/mongo/logger/logger.h | 43 + src/mongo/logger/logstream_builder.cpp | 134 + src/mongo/logger/logstream_builder.h | 141 + src/mongo/logger/message_event.h | 57 + src/mongo/logger/message_event_utf8_encoder.cpp | 95 + src/mongo/logger/message_event_utf8_encoder.h | 73 + src/mongo/logger/message_log_domain.cpp | 28 + src/mongo/logger/message_log_domain.h | 32 + src/mongo/logger/ramlog.cpp | 247 + src/mongo/logger/ramlog.h | 176 + src/mongo/logger/rotatable_file_appender.h | 61 + src/mongo/logger/rotatable_file_manager.cpp | 70 + src/mongo/logger/rotatable_file_manager.h | 77 + src/mongo/logger/rotatable_file_writer.cpp | 279 + src/mongo/logger/rotatable_file_writer.h | 116 + src/mongo/logger/rotatable_file_writer_test.cpp | 132 + src/mongo/logger/syslog_appender.h | 71 + src/mongo/logger/tee.h | 35 + src/mongo/pch.cpp | 2 +- src/mongo/pch.h | 13 +- src/mongo/platform/SConscript | 4 +- src/mongo/platform/atomic_intrinsics.h | 6 +- src/mongo/platform/atomic_intrinsics_gcc.h | 166 - src/mongo/platform/atomic_intrinsics_gcc_generic.h | 70 + src/mongo/platform/atomic_intrinsics_gcc_intel.h | 166 + src/mongo/platform/atomic_intrinsics_win32.h | 129 +- src/mongo/platform/basic.h | 14 - src/mongo/platform/bits.h | 4 +- src/mongo/platform/compiler.h | 91 +- src/mongo/platform/compiler_gcc.h | 68 +- src/mongo/platform/compiler_msvc.h | 60 +- src/mongo/platform/hash_namespace.h | 64 + src/mongo/platform/process_id.cpp | 63 + src/mongo/platform/process_id.h | 112 + src/mongo/platform/process_id_test.cpp | 56 + src/mongo/platform/random.h | 16 + src/mongo/platform/strtoll.h | 12 + src/mongo/platform/unordered_map.h | 13 +- src/mongo/platform/unordered_set.h | 13 +- src/mongo/platform/windows_basic.h | 59 +- src/mongo/s/SConscript | 361 +- src/mongo/s/balance.cpp | 157 +- src/mongo/s/balance.h | 16 +- src/mongo/s/balancer_policy.cpp | 288 +- src/mongo/s/balancer_policy.h | 45 +- src/mongo/s/balancer_policy_tests.cpp | 284 +- src/mongo/s/bson_serializable.h | 68 + src/mongo/s/chunk.cpp | 371 +- src/mongo/s/chunk.h | 24 +- src/mongo/s/chunk_diff-inl.cpp | 250 + src/mongo/s/chunk_diff.h | 15 +- src/mongo/s/chunk_diff.hpp | 303 - src/mongo/s/chunk_diff_test.cpp | 12 + src/mongo/s/chunk_manager_targeter.cpp | 737 + src/mongo/s/chunk_manager_targeter.h | 154 + src/mongo/s/chunk_version.h | 189 +- src/mongo/s/chunk_version_test.cpp | 41 + src/mongo/s/client_info.cpp | 395 +- src/mongo/s/client_info.h | 94 +- src/mongo/s/cluster_client_internal.cpp | 147 +- src/mongo/s/cluster_client_internal.h | 12 + src/mongo/s/cluster_write.cpp | 465 + src/mongo/s/cluster_write.h | 125 + src/mongo/s/collection_manager.cpp | 337 - src/mongo/s/collection_manager.h | 175 - src/mongo/s/collection_manager_test.cpp | 617 - src/mongo/s/collection_metadata.cpp | 730 + src/mongo/s/collection_metadata.h | 317 + src/mongo/s/collection_metadata_test.cpp | 1390 ++ src/mongo/s/commands/auth_schema_upgrade_s.cpp | 175 + src/mongo/s/commands/cluster_index_filter_cmd.cpp | 183 + src/mongo/s/commands/cluster_merge_chunks_cmd.cpp | 196 + src/mongo/s/commands/cluster_plan_cache_cmd.cpp | 186 + src/mongo/s/commands/cluster_write_cmd.cpp | 252 + src/mongo/s/commands_admin.cpp | 645 +- src/mongo/s/commands_public.cpp | 967 +- src/mongo/s/config.cpp | 423 +- src/mongo/s/config.h | 32 +- src/mongo/s/config_server_checker_service.cpp | 12 + src/mongo/s/config_server_checker_service.h | 12 + src/mongo/s/config_server_tests.cpp | 81 + src/mongo/s/config_upgrade.cpp | 58 +- src/mongo/s/config_upgrade.h | 31 +- src/mongo/s/config_upgrade_helpers.cpp | 312 +- src/mongo/s/config_upgrade_helpers.h | 60 +- src/mongo/s/config_upgrade_v0_to_v4.cpp | 84 - src/mongo/s/config_upgrade_v0_to_v5.cpp | 94 + src/mongo/s/config_upgrade_v3_to_v4.cpp | 765 - src/mongo/s/config_upgrade_v4_to_v5.cpp | 105 + src/mongo/s/cursors.cpp | 201 +- src/mongo/s/cursors.h | 38 +- src/mongo/s/d_chunk_manager.cpp | 456 - src/mongo/s/d_chunk_manager.h | 172 - src/mongo/s/d_logic.cpp | 38 +- src/mongo/s/d_logic.h | 234 +- src/mongo/s/d_merge.cpp | 487 + src/mongo/s/d_merge.h | 60 + src/mongo/s/d_migrate.cpp | 1036 +- src/mongo/s/d_split.cpp | 416 +- src/mongo/s/d_state.cpp | 940 +- src/mongo/s/d_writeback.cpp | 27 +- src/mongo/s/d_writeback.h | 12 + src/mongo/s/dbclient_multi_command.cpp | 277 + src/mongo/s/dbclient_multi_command.h | 89 + src/mongo/s/dbclient_shard_resolver.cpp | 108 + src/mongo/s/dbclient_shard_resolver.h | 71 + src/mongo/s/default_version.cpp | 16 +- src/mongo/s/distlock.cpp | 1216 + src/mongo/s/distlock.h | 362 + src/mongo/s/distlock_test.cpp | 493 + src/mongo/s/field_parser-inl.h | 127 - src/mongo/s/field_parser.cpp | 285 - src/mongo/s/field_parser.h | 137 - src/mongo/s/field_parser_test.cpp | 413 - src/mongo/s/grid.cpp | 157 +- src/mongo/s/grid.h | 18 +- src/mongo/s/interrupt_status_mongos.cpp | 22 +- src/mongo/s/interrupt_status_mongos.h | 21 +- src/mongo/s/metadata_loader.cpp | 420 +- src/mongo/s/metadata_loader.h | 139 +- src/mongo/s/metadata_loader_test.cpp | 957 +- src/mongo/s/mock_multi_write_command.h | 172 + src/mongo/s/mock_ns_targeter.h | 231 + src/mongo/s/mock_shard_resolver.h | 55 + src/mongo/s/mongo_version_range.cpp | 12 + src/mongo/s/mongo_version_range.h | 12 + src/mongo/s/mongo_version_range_test.cpp | 12 + src/mongo/s/mongos.vcxproj | 1051 +- src/mongo/s/mongos.vcxproj.filters | 943 +- src/mongo/s/mongos_options.cpp | 268 + src/mongo/s/mongos_options.h | 86 + src/mongo/s/mongos_options_init.cpp | 80 + src/mongo/s/mongos_persistence_stubs.cpp | 46 + src/mongo/s/mongos_sm.sln | 59 - src/mongo/s/mongos_sm.vcxproj | 3012 --- src/mongo/s/mongos_sm.vcxproj.filters | 2493 -- src/mongo/s/multi_command_dispatch.h | 87 + src/mongo/s/multi_host_query.cpp | 405 + src/mongo/s/multi_host_query.h | 330 + src/mongo/s/multi_host_query_test.cpp | 757 + src/mongo/s/ns_targeter.h | 188 + src/mongo/s/range_arithmetic.cpp | 138 + src/mongo/s/range_arithmetic.h | 157 + src/mongo/s/range_arithmetic_test.cpp | 161 + src/mongo/s/request.cpp | 110 +- src/mongo/s/request.h | 42 +- src/mongo/s/s_only.cpp | 91 +- src/mongo/s/scc_fast_query_handler.cpp | 248 + src/mongo/s/scc_fast_query_handler.h | 76 + src/mongo/s/server.cpp | 517 +- src/mongo/s/server.h | 17 +- src/mongo/s/shard.cpp | 209 +- src/mongo/s/shard.h | 32 +- src/mongo/s/shard_conn_test.cpp | 34 +- src/mongo/s/shard_key_pattern.cpp | 66 + src/mongo/s/shard_key_pattern.h | 47 + src/mongo/s/shard_resolver.h | 61 + src/mongo/s/shard_test.cpp | 31 +- src/mongo/s/shardconnection.cpp | 89 +- src/mongo/s/shardkey.cpp | 63 +- src/mongo/s/shardkey.h | 35 +- src/mongo/s/stale_exception.h | 12 + src/mongo/s/strategy.cpp | 565 +- src/mongo/s/strategy.h | 76 +- src/mongo/s/strategy_shard.cpp | 1331 -- src/mongo/s/strategy_single.cpp | 225 - src/mongo/s/type_changelog.cpp | 14 +- src/mongo/s/type_changelog.h | 12 + src/mongo/s/type_changelog_test.cpp | 14 +- src/mongo/s/type_chunk.cpp | 14 +- src/mongo/s/type_chunk.h | 12 + src/mongo/s/type_chunk_test.cpp | 14 +- src/mongo/s/type_collection.cpp | 14 +- src/mongo/s/type_collection.h | 14 +- src/mongo/s/type_collection_test.cpp | 14 +- src/mongo/s/type_config_version.cpp | 14 +- src/mongo/s/type_config_version.h | 16 +- src/mongo/s/type_config_version_test.cpp | 12 + src/mongo/s/type_database.cpp | 14 +- src/mongo/s/type_database.h | 12 + src/mongo/s/type_database_test.cpp | 14 +- src/mongo/s/type_lockpings.cpp | 14 +- src/mongo/s/type_lockpings.h | 12 + src/mongo/s/type_lockpings_test.cpp | 16 +- src/mongo/s/type_locks.cpp | 14 +- src/mongo/s/type_locks.h | 14 +- src/mongo/s/type_locks_test.cpp | 12 + src/mongo/s/type_mongos.cpp | 14 +- src/mongo/s/type_mongos.h | 16 +- src/mongo/s/type_mongos_test.cpp | 14 +- src/mongo/s/type_settings.cpp | 14 +- src/mongo/s/type_settings.h | 12 + src/mongo/s/type_settings_test.cpp | 12 + src/mongo/s/type_shard.cpp | 20 +- src/mongo/s/type_shard.h | 12 + src/mongo/s/type_shard_test.cpp | 23 + src/mongo/s/type_tags.cpp | 14 +- src/mongo/s/type_tags.h | 12 + src/mongo/s/type_tags_test.cpp | 12 + src/mongo/s/version_manager.cpp | 76 +- src/mongo/s/version_manager.h | 12 + src/mongo/s/version_mongos.cpp | 64 + src/mongo/s/version_mongos.h | 32 + src/mongo/s/write_ops/batch_downconvert.cpp | 574 + src/mongo/s/write_ops/batch_downconvert.h | 162 + src/mongo/s/write_ops/batch_downconvert_test.cpp | 735 + src/mongo/s/write_ops/batch_upconvert.cpp | 238 + src/mongo/s/write_ops/batch_upconvert.h | 65 + src/mongo/s/write_ops/batch_upconvert_test.cpp | 148 + src/mongo/s/write_ops/batch_write_exec.cpp | 386 + src/mongo/s/write_ops/batch_write_exec.h | 134 + src/mongo/s/write_ops/batch_write_exec_test.cpp | 310 + src/mongo/s/write_ops/batch_write_op.cpp | 913 + src/mongo/s/write_ops/batch_write_op.h | 303 + src/mongo/s/write_ops/batch_write_op_test.cpp | 1833 ++ src/mongo/s/write_ops/batched_command_request.cpp | 390 + src/mongo/s/write_ops/batched_command_request.h | 243 + src/mongo/s/write_ops/batched_command_response.cpp | 553 + src/mongo/s/write_ops/batched_command_response.h | 201 + .../s/write_ops/batched_command_response_test.cpp | 87 + src/mongo/s/write_ops/batched_delete_document.cpp | 154 + src/mongo/s/write_ops/batched_delete_document.h | 101 + src/mongo/s/write_ops/batched_delete_request.cpp | 300 + src/mongo/s/write_ops/batched_delete_request.h | 148 + .../s/write_ops/batched_delete_request_test.cpp | 88 + src/mongo/s/write_ops/batched_insert_request.cpp | 304 + src/mongo/s/write_ops/batched_insert_request.h | 143 + .../s/write_ops/batched_insert_request_test.cpp | 150 + src/mongo/s/write_ops/batched_request_metadata.cpp | 174 + src/mongo/s/write_ops/batched_request_metadata.h | 94 + .../s/write_ops/batched_request_metadata_test.cpp | 53 + src/mongo/s/write_ops/batched_update_document.cpp | 225 + src/mongo/s/write_ops/batched_update_document.h | 121 + src/mongo/s/write_ops/batched_update_request.cpp | 301 + src/mongo/s/write_ops/batched_update_request.h | 148 + .../s/write_ops/batched_update_request_test.cpp | 91 + src/mongo/s/write_ops/batched_upsert_detail.cpp | 158 + src/mongo/s/write_ops/batched_upsert_detail.h | 101 + src/mongo/s/write_ops/config_coordinator.cpp | 429 + src/mongo/s/write_ops/config_coordinator.h | 58 + src/mongo/s/write_ops/config_coordinator_test.cpp | 52 + src/mongo/s/write_ops/dbclient_safe_writer.cpp | 125 + src/mongo/s/write_ops/dbclient_safe_writer.h | 64 + src/mongo/s/write_ops/wc_error_detail.cpp | 180 + src/mongo/s/write_ops/wc_error_detail.h | 111 + src/mongo/s/write_ops/write_error_detail.cpp | 216 + src/mongo/s/write_ops/write_error_detail.h | 121 + src/mongo/s/write_ops/write_op.cpp | 288 + src/mongo/s/write_ops/write_op.h | 244 + src/mongo/s/write_ops/write_op_test.cpp | 398 + src/mongo/s/writeback_listener.cpp | 304 +- src/mongo/s/writeback_listener.h | 48 +- src/mongo/scripting/bench.cpp | 100 +- src/mongo/scripting/bson_template_evaluator.cpp | 12 + .../scripting/bson_template_evaluator_test.cpp | 12 + src/mongo/scripting/engine.cpp | 191 +- src/mongo/scripting/engine.h | 21 +- src/mongo/scripting/engine_none.cpp | 12 + src/mongo/scripting/engine_spidermonkey.cpp | 1872 -- src/mongo/scripting/engine_spidermonkey.h | 104 - src/mongo/scripting/engine_spidermonkey_internal.h | 338 - src/mongo/scripting/engine_v8.cpp | 301 +- src/mongo/scripting/engine_v8.h | 60 +- src/mongo/scripting/sm_db.cpp | 1789 -- src/mongo/scripting/utils.cpp | 12 + src/mongo/scripting/v8_db.cpp | 183 +- src/mongo/scripting/v8_db.h | 11 +- src/mongo/scripting/v8_deadline_monitor_test.cpp | 12 + src/mongo/scripting/v8_profiler.cpp | 12 + src/mongo/scripting/v8_utils.cpp | 87 +- src/mongo/server.h | 14 +- src/mongo/shell/assert.js | 273 +- src/mongo/shell/bulk_api.js | 1194 + src/mongo/shell/collection.js | 507 +- src/mongo/shell/createCPPfromJavaScriptFiles.js | 4 +- src/mongo/shell/db.js | 1097 +- src/mongo/shell/dbshell.cpp | 492 +- src/mongo/shell/mongo.js | 79 +- src/mongo/shell/mongo.vcxproj | 368 +- src/mongo/shell/mongo.vcxproj.filters | 369 +- src/mongo/shell/mongo_sm.sln | 39 - src/mongo/shell/mongo_sm.vcxproj | 1456 -- src/mongo/shell/mongo_sm.vcxproj.filters | 1493 -- src/mongo/shell/query.js | 99 +- src/mongo/shell/replsetbridge.js | 14 +- src/mongo/shell/replsettest.js | 369 +- src/mongo/shell/servers.js | 196 +- src/mongo/shell/servers_misc.js | 71 +- src/mongo/shell/shardingtest.js | 11 +- src/mongo/shell/shell_options.cpp | 293 + src/mongo/shell/shell_options.h | 91 + src/mongo/shell/shell_options_init.cpp | 60 + src/mongo/shell/shell_utils.cpp | 76 +- src/mongo/shell/shell_utils.h | 4 + src/mongo/shell/shell_utils_extended.cpp | 56 +- src/mongo/shell/shell_utils_launcher.cpp | 167 +- src/mongo/shell/shell_utils_launcher.h | 31 +- src/mongo/shell/types.js | 45 +- src/mongo/shell/upgrade_check.js | 195 + src/mongo/shell/utils.js | 576 +- src/mongo/shell/utils_sh.js | 6 +- src/mongo/targetver.h | 29 - src/mongo/tools/bridge.cpp | 102 +- src/mongo/tools/bsondump.cpp | 88 +- src/mongo/tools/bsondump_options.cpp | 108 + src/mongo/tools/bsondump_options.h | 60 + src/mongo/tools/bsondump_options_init.cpp | 60 + src/mongo/tools/docgenerator.cpp | 60 - src/mongo/tools/docgenerator.h | 80 - src/mongo/tools/docgeneratormain.cpp | 150 - src/mongo/tools/dump.cpp | 349 +- src/mongo/tools/export.cpp | 143 +- src/mongo/tools/files.cpp | 133 +- src/mongo/tools/import.cpp | 463 +- src/mongo/tools/loadgenerator.cpp | 417 - src/mongo/tools/mongobridge_options.cpp | 93 + src/mongo/tools/mongobridge_options.h | 70 + src/mongo/tools/mongobridge_options_init.cpp | 61 + src/mongo/tools/mongodump_options.cpp | 157 + src/mongo/tools/mongodump_options.h | 64 + src/mongo/tools/mongodump_options_init.cpp | 60 + src/mongo/tools/mongoexport_options.cpp | 157 + src/mongo/tools/mongoexport_options.h | 68 + src/mongo/tools/mongoexport_options_init.cpp | 60 + src/mongo/tools/mongofiles_options.cpp | 128 + src/mongo/tools/mongofiles_options.h | 63 + src/mongo/tools/mongofiles_options_init.cpp | 60 + src/mongo/tools/mongoimport_options.cpp | 162 + src/mongo/tools/mongoimport_options.h | 68 + src/mongo/tools/mongoimport_options_init.cpp | 60 + src/mongo/tools/mongooplog_options.cpp | 109 + src/mongo/tools/mongooplog_options.h | 61 + src/mongo/tools/mongooplog_options_init.cpp | 60 + src/mongo/tools/mongorestore_options.cpp | 185 + src/mongo/tools/mongorestore_options.h | 69 + src/mongo/tools/mongorestore_options_init.cpp | 60 + src/mongo/tools/mongostat_options.cpp | 168 + src/mongo/tools/mongostat_options.h | 66 + src/mongo/tools/mongostat_options_init.cpp | 60 + src/mongo/tools/mongotop_options.cpp | 96 + src/mongo/tools/mongotop_options.h | 60 + src/mongo/tools/mongotop_options_init.cpp | 60 + src/mongo/tools/oplog.cpp | 87 +- src/mongo/tools/restore.cpp | 744 +- src/mongo/tools/sniffer.cpp | 97 +- src/mongo/tools/stat.cpp | 211 +- src/mongo/tools/stat_util.cpp | 22 +- src/mongo/tools/stat_util.h | 12 + src/mongo/tools/tool.cpp | 420 +- src/mongo/tools/tool.h | 111 +- src/mongo/tools/tool_logger.cpp | 144 + src/mongo/tools/tool_logger.h | 58 + src/mongo/tools/tool_options.cpp | 400 + src/mongo/tools/tool_options.h | 120 + src/mongo/tools/top.cpp | 86 +- src/mongo/unittest/SConscript | 21 +- src/mongo/unittest/crutch.cpp | 46 +- src/mongo/unittest/temp_dir.cpp | 109 + src/mongo/unittest/temp_dir.h | 67 + src/mongo/unittest/temp_dir_test.cpp | 89 + src/mongo/unittest/unittest-inl.h | 12 + src/mongo/unittest/unittest.cpp | 89 +- src/mongo/unittest/unittest.h | 51 +- src/mongo/unittest/unittest_main.cpp | 2 + src/mongo/util/admin_access.h | 21 +- src/mongo/util/alignedbuilder.cpp | 17 +- src/mongo/util/alignedbuilder.h | 12 + src/mongo/util/allocator.h | 10 +- src/mongo/util/assert_util.cpp | 56 +- src/mongo/util/assert_util.h | 140 +- src/mongo/util/background.cpp | 346 +- src/mongo/util/background.h | 91 +- src/mongo/util/background_job_test.cpp | 140 + src/mongo/util/base64.cpp | 5 +- src/mongo/util/bufreader.h | 56 +- src/mongo/util/checksum.h | 12 + src/mongo/util/cmdline_utils/SConscript | 10 + src/mongo/util/cmdline_utils/censor_cmdline.cpp | 172 + src/mongo/util/cmdline_utils/censor_cmdline.h | 49 + .../util/cmdline_utils/censor_cmdline_test.cpp | 311 + src/mongo/util/compress.cpp | 14 +- src/mongo/util/compress.h | 12 + src/mongo/util/concurrency/SConscript | 8 + src/mongo/util/concurrency/list.h | 12 + src/mongo/util/concurrency/mapsf.h | 12 + src/mongo/util/concurrency/msg.h | 15 + src/mongo/util/concurrency/mutex.h | 15 +- src/mongo/util/concurrency/mutexdebugger.cpp | 11 +- src/mongo/util/concurrency/mutexdebugger.h | 12 + src/mongo/util/concurrency/qlock.h | 16 +- src/mongo/util/concurrency/race.h | 16 +- src/mongo/util/concurrency/rwlock.h | 29 +- src/mongo/util/concurrency/rwlockimpl.cpp | 18 +- src/mongo/util/concurrency/rwlockimpl.h | 14 +- src/mongo/util/concurrency/simplerwlock.h | 14 +- src/mongo/util/concurrency/spin_lock.cpp | 7 +- src/mongo/util/concurrency/spin_lock.h | 12 + src/mongo/util/concurrency/spin_lock_test.cpp | 112 + src/mongo/util/concurrency/synchronization.cpp | 3 +- src/mongo/util/concurrency/task.cpp | 14 +- src/mongo/util/concurrency/task.h | 14 +- src/mongo/util/concurrency/thread_name.cpp | 77 + src/mongo/util/concurrency/thread_name.h | 36 + src/mongo/util/concurrency/thread_pool.cpp | 7 +- src/mongo/util/concurrency/thread_pool.h | 2 + src/mongo/util/concurrency/threadlocal.h | 18 +- src/mongo/util/concurrency/ticketholder.h | 5 +- src/mongo/util/concurrency/value.h | 12 + src/mongo/util/debug_util.cpp | 10 +- src/mongo/util/descriptive_stats_test.cpp | 12 + src/mongo/util/elapsed_tracker.cpp | 19 +- src/mongo/util/elapsed_tracker.h | 14 +- src/mongo/util/exception_filter_win32.cpp | 71 +- src/mongo/util/exception_filter_win32.h | 12 + src/mongo/util/fail_point.cpp | 12 + src/mongo/util/fail_point.h | 12 + src/mongo/util/fail_point_registry.cpp | 13 +- src/mongo/util/fail_point_registry.h | 12 + src/mongo/util/fail_point_service.cpp | 16 +- src/mongo/util/fail_point_service.h | 12 + src/mongo/util/fail_point_test.cpp | 14 +- src/mongo/util/file.cpp | 6 +- src/mongo/util/file.h | 3 +- src/mongo/util/file_allocator.cpp | 97 +- src/mongo/util/file_allocator.h | 2 + src/mongo/util/gcov.h | 36 + src/mongo/util/goodies.h | 55 +- src/mongo/util/hashtab.h | 176 - src/mongo/util/heapcheck.h | 12 + src/mongo/util/hex.cpp | 54 + src/mongo/util/hex.h | 2 + src/mongo/util/histogram.cpp | 12 + src/mongo/util/histogram.h | 12 + src/mongo/util/intrusive_counter.cpp | 26 +- src/mongo/util/intrusive_counter.h | 20 + src/mongo/util/lame_stacktrace_test.cpp | 14 - src/mongo/util/log.cpp | 396 +- src/mongo/util/log.h | 477 +- src/mongo/util/logfile.cpp | 12 + src/mongo/util/logfile.h | 16 +- src/mongo/util/lruishmap.h | 2 +- src/mongo/util/md5.hpp | 8 +- src/mongo/util/md5main.cpp | 6 +- src/mongo/util/mmap.cpp | 54 +- src/mongo/util/mmap.h | 85 +- src/mongo/util/mmap_mm.cpp | 7 +- src/mongo/util/mmap_posix.cpp | 129 +- src/mongo/util/mmap_win.cpp | 82 +- src/mongo/util/mongoutils/html.h | 12 + src/mongo/util/mongoutils/str.h | 26 +- src/mongo/util/net/hostandport.h | 31 +- src/mongo/util/net/httpclient.cpp | 39 +- src/mongo/util/net/httpclient.h | 12 +- src/mongo/util/net/listen.cpp | 122 +- src/mongo/util/net/listen.h | 66 +- src/mongo/util/net/message.cpp | 13 +- src/mongo/util/net/message.h | 36 +- src/mongo/util/net/message_port.cpp | 110 +- src/mongo/util/net/message_port.h | 40 +- src/mongo/util/net/message_server.h | 1 + src/mongo/util/net/message_server_asio.cpp | 6 +- src/mongo/util/net/message_server_port.cpp | 33 +- src/mongo/util/net/miniwebserver.cpp | 18 +- src/mongo/util/net/miniwebserver.h | 9 +- src/mongo/util/net/sock.cpp | 329 +- src/mongo/util/net/sock.h | 120 +- src/mongo/util/net/sock_test.cpp | 42 +- src/mongo/util/net/socket_poll.cpp | 62 + src/mongo/util/net/socket_poll.h | 48 + src/mongo/util/net/ssl_manager.cpp | 1020 +- src/mongo/util/net/ssl_manager.h | 134 +- src/mongo/util/net/ssl_options.cpp | 305 + src/mongo/util/net/ssl_options.h | 96 + src/mongo/util/ntservice.cpp | 41 +- src/mongo/util/ntservice.h | 10 +- src/mongo/util/optime.h | 155 - src/mongo/util/options_parser/SConscript | 32 + src/mongo/util/options_parser/constraints.cpp | 129 + src/mongo/util/options_parser/constraints.h | 170 + src/mongo/util/options_parser/environment.cpp | 372 + src/mongo/util/options_parser/environment.h | 231 + src/mongo/util/options_parser/environment_test.cpp | 160 + .../util/options_parser/option_description.cpp | 243 + src/mongo/util/options_parser/option_description.h | 223 + src/mongo/util/options_parser/option_section.cpp | 698 + src/mongo/util/options_parser/option_section.h | 150 + src/mongo/util/options_parser/options_parser.cpp | 959 + src/mongo/util/options_parser/options_parser.h | 110 + .../util/options_parser/options_parser_init.cpp | 55 + .../util/options_parser/options_parser_test.cpp | 3851 ++++ .../util/options_parser/startup_option_init.cpp | 59 + .../util/options_parser/startup_option_init.h | 118 + src/mongo/util/options_parser/startup_options.cpp | 41 + src/mongo/util/options_parser/startup_options.h | 66 + .../options_parser/test_config_files/empty.json | 0 .../options_parser/test_config_files/good.conf | 1 + .../options_parser/test_config_files/good.json | 3 + src/mongo/util/options_parser/value.cpp | 228 + src/mongo/util/options_parser/value.h | 170 + src/mongo/util/password.cpp | 6 +- src/mongo/util/password.h | 34 - src/mongo/util/password_digest.cpp | 53 + src/mongo/util/password_digest.h | 46 + src/mongo/util/paths.cpp | 68 + src/mongo/util/paths.h | 43 +- src/mongo/util/platform_init.cpp | 18 + src/mongo/util/processinfo.cpp | 19 +- src/mongo/util/processinfo.h | 32 +- src/mongo/util/processinfo_darwin.cpp | 8 +- src/mongo/util/processinfo_freebsd.cpp | 6 +- src/mongo/util/processinfo_linux2.cpp | 42 +- src/mongo/util/processinfo_none.cpp | 8 +- src/mongo/util/processinfo_sunos5.cpp | 2 +- src/mongo/util/processinfo_test.cpp | 14 + src/mongo/util/processinfo_win32.cpp | 175 +- src/mongo/util/progress_meter.cpp | 19 +- src/mongo/util/progress_meter.h | 21 +- src/mongo/util/ramlog.cpp | 191 - src/mongo/util/ramlog.h | 51 +- src/mongo/util/safe_num-inl.h | 115 + src/mongo/util/safe_num.cpp | 266 +- src/mongo/util/safe_num.h | 63 +- src/mongo/util/safe_num_test.cpp | 266 + src/mongo/util/sequence_util.h | 6 +- src/mongo/util/signal_handlers.cpp | 284 +- src/mongo/util/signal_handlers.h | 28 +- src/mongo/util/signal_win32.cpp | 44 + src/mongo/util/signal_win32.h | 39 + src/mongo/util/stack_introspect.cpp | 112 +- src/mongo/util/stacktrace.cpp | 12 +- src/mongo/util/stacktrace.h | 10 +- src/mongo/util/startup_test.cpp | 12 + src/mongo/util/string_map.h | 6 +- src/mongo/util/string_map_internal.h | 34 - src/mongo/util/string_map_test.cpp | 42 - src/mongo/util/string_writer.h | 28 - src/mongo/util/stringutils.cpp | 4 +- src/mongo/util/stringutils_test.cpp | 15 + src/mongo/util/systeminfo.h | 41 - src/mongo/util/systeminfo_linux2.cpp | 47 - src/mongo/util/systeminfo_none.cpp | 26 - src/mongo/util/systeminfo_win32.cpp | 48 - src/mongo/util/tcmalloc_server_status_section.cpp | 82 + src/mongo/util/text.cpp | 62 +- src/mongo/util/text.h | 9 +- src/mongo/util/time_support.cpp | 610 +- src/mongo/util/time_support.h | 83 +- src/mongo/util/time_support_test.cpp | 762 + src/mongo/util/timer.cpp | 3 +- src/mongo/util/timer.h | 4 +- src/mongo/util/touch_pages.cpp | 72 +- src/mongo/util/touch_pages.h | 9 +- src/mongo/util/trace.cpp | 14 +- src/mongo/util/trace.h | 14 +- src/mongo/util/unordered_fast_key_table.h | 4 + src/mongo/util/unordered_fast_key_table_internal.h | 9 + src/mongo/util/util.cpp | 79 +- src/mongo/util/version.cpp | 245 +- src/mongo/util/version.h | 26 +- src/mongo/util/version_reporting.cpp | 111 + src/mongo/util/version_reporting.h | 56 + src/mongo/util/version_test.cpp | 49 + src/mongo/util/winutil.h | 14 +- src/mongodb.sln | 92 - src/mongodb_sm.sln | 119 - src/third_party/SConscript | 104 +- src/third_party/boost/SConscript | 9 +- .../boost/boost/config/compiler/visualc.hpp | 2 +- .../boost/libs/thread/src/win32/thread.cpp | 5 + src/third_party/gperftools-2.0/SConscript | 24 +- src/third_party/gperftools-2.0/src/tcmalloc.cc | 6 +- src/third_party/js-1.7/Makefile.in | 388 - src/third_party/js-1.7/Makefile.ref | 375 - src/third_party/js-1.7/README.html | 826 - src/third_party/js-1.7/SConscript | 74 - .../js-1.7/SpiderMonkey-prebuild.vcxproj | 164 - .../js-1.7/SpiderMonkey-prebuild.vcxproj.filters | 31 - src/third_party/js-1.7/Y.js | 19 - src/third_party/js-1.7/config.mk | 186 - src/third_party/js-1.7/config/AIX4.1.mk | 65 - src/third_party/js-1.7/config/AIX4.2.mk | 64 - src/third_party/js-1.7/config/AIX4.3.mk | 65 - src/third_party/js-1.7/config/CVS/Entries | 36 - src/third_party/js-1.7/config/CVS/Repository | 1 - src/third_party/js-1.7/config/CVS/Root | 1 - src/third_party/js-1.7/config/CVS/Tag | 1 - src/third_party/js-1.7/config/Darwin.mk | 83 - src/third_party/js-1.7/config/Darwin1.3.mk | 81 - src/third_party/js-1.7/config/Darwin1.4.mk | 41 - src/third_party/js-1.7/config/Darwin5.2.mk | 81 - src/third_party/js-1.7/config/Darwin5.3.mk | 81 - src/third_party/js-1.7/config/HP-UXB.10.10.mk | 77 - src/third_party/js-1.7/config/HP-UXB.10.20.mk | 77 - src/third_party/js-1.7/config/HP-UXB.11.00.mk | 80 - src/third_party/js-1.7/config/IRIX.mk | 87 - src/third_party/js-1.7/config/IRIX5.3.mk | 44 - src/third_party/js-1.7/config/IRIX6.1.mk | 44 - src/third_party/js-1.7/config/IRIX6.2.mk | 44 - src/third_party/js-1.7/config/IRIX6.3.mk | 44 - src/third_party/js-1.7/config/IRIX6.5.mk | 44 - src/third_party/js-1.7/config/Linux_All.mk | 103 - src/third_party/js-1.7/config/Mac_OS10.0.mk | 82 - src/third_party/js-1.7/config/OSF1V4.0.mk | 72 - src/third_party/js-1.7/config/OSF1V5.0.mk | 69 - src/third_party/js-1.7/config/SunOS4.1.4.mk | 101 - src/third_party/js-1.7/config/SunOS5.3.mk | 91 - src/third_party/js-1.7/config/SunOS5.4.mk | 92 - src/third_party/js-1.7/config/SunOS5.5.1.mk | 44 - src/third_party/js-1.7/config/SunOS5.5.mk | 87 - src/third_party/js-1.7/config/SunOS5.6.mk | 89 - src/third_party/js-1.7/config/SunOS5.7.mk | 44 - src/third_party/js-1.7/config/SunOS5.8.mk | 44 - src/third_party/js-1.7/config/SunOS5.9.mk | 44 - src/third_party/js-1.7/config/WINNT4.0.mk | 117 - src/third_party/js-1.7/config/WINNT5.0.mk | 117 - src/third_party/js-1.7/config/WINNT5.1.mk | 117 - src/third_party/js-1.7/config/WINNT5.2.mk | 117 - src/third_party/js-1.7/config/dgux.mk | 64 - src/third_party/js-1.7/fdlibm/.cvsignore | 7 - src/third_party/js-1.7/fdlibm/CVS/Entries | 87 - src/third_party/js-1.7/fdlibm/CVS/Repository | 1 - src/third_party/js-1.7/fdlibm/CVS/Root | 1 - src/third_party/js-1.7/fdlibm/CVS/Tag | 1 - src/third_party/js-1.7/fdlibm/Makefile.in | 127 - src/third_party/js-1.7/fdlibm/Makefile.ref | 192 - src/third_party/js-1.7/fdlibm/e_acos.c | 147 - src/third_party/js-1.7/fdlibm/e_acosh.c | 105 - src/third_party/js-1.7/fdlibm/e_asin.c | 156 - src/third_party/js-1.7/fdlibm/e_atan2.c | 165 - src/third_party/js-1.7/fdlibm/e_atanh.c | 110 - src/third_party/js-1.7/fdlibm/e_cosh.c | 133 - src/third_party/js-1.7/fdlibm/e_exp.c | 202 - src/third_party/js-1.7/fdlibm/e_fmod.c | 184 - src/third_party/js-1.7/fdlibm/e_gamma.c | 71 - src/third_party/js-1.7/fdlibm/e_gamma_r.c | 70 - src/third_party/js-1.7/fdlibm/e_hypot.c | 173 - src/third_party/js-1.7/fdlibm/e_j0.c | 524 - src/third_party/js-1.7/fdlibm/e_j1.c | 523 - src/third_party/js-1.7/fdlibm/e_jn.c | 315 - src/third_party/js-1.7/fdlibm/e_lgamma.c | 71 - src/third_party/js-1.7/fdlibm/e_lgamma_r.c | 347 - src/third_party/js-1.7/fdlibm/e_log.c | 184 - src/third_party/js-1.7/fdlibm/e_log10.c | 134 - src/third_party/js-1.7/fdlibm/e_pow.c | 386 - src/third_party/js-1.7/fdlibm/e_rem_pio2.c | 222 - src/third_party/js-1.7/fdlibm/e_remainder.c | 120 - src/third_party/js-1.7/fdlibm/e_scalb.c | 89 - src/third_party/js-1.7/fdlibm/e_sinh.c | 122 - src/third_party/js-1.7/fdlibm/e_sqrt.c | 497 - src/third_party/js-1.7/fdlibm/fdlibm.h | 273 - src/third_party/js-1.7/fdlibm/fdlibm.mak | 1453 -- src/third_party/js-1.7/fdlibm/fdlibm.mdp | Bin 42143 -> 0 bytes src/third_party/js-1.7/fdlibm/k_cos.c | 135 - src/third_party/js-1.7/fdlibm/k_rem_pio2.c | 354 - src/third_party/js-1.7/fdlibm/k_sin.c | 114 - src/third_party/js-1.7/fdlibm/k_standard.c | 785 - src/third_party/js-1.7/fdlibm/k_tan.c | 170 - src/third_party/js-1.7/fdlibm/s_asinh.c | 101 - src/third_party/js-1.7/fdlibm/s_atan.c | 175 - src/third_party/js-1.7/fdlibm/s_cbrt.c | 133 - src/third_party/js-1.7/fdlibm/s_ceil.c | 120 - src/third_party/js-1.7/fdlibm/s_copysign.c | 72 - src/third_party/js-1.7/fdlibm/s_cos.c | 118 - src/third_party/js-1.7/fdlibm/s_erf.c | 356 - src/third_party/js-1.7/fdlibm/s_expm1.c | 267 - src/third_party/js-1.7/fdlibm/s_fabs.c | 70 - src/third_party/js-1.7/fdlibm/s_finite.c | 71 - src/third_party/js-1.7/fdlibm/s_floor.c | 121 - src/third_party/js-1.7/fdlibm/s_frexp.c | 99 - src/third_party/js-1.7/fdlibm/s_ilogb.c | 85 - src/third_party/js-1.7/fdlibm/s_isnan.c | 74 - src/third_party/js-1.7/fdlibm/s_ldexp.c | 66 - src/third_party/js-1.7/fdlibm/s_lib_version.c | 73 - src/third_party/js-1.7/fdlibm/s_log1p.c | 211 - src/third_party/js-1.7/fdlibm/s_logb.c | 79 - src/third_party/js-1.7/fdlibm/s_matherr.c | 64 - src/third_party/js-1.7/fdlibm/s_modf.c | 132 - src/third_party/js-1.7/fdlibm/s_nextafter.c | 124 - src/third_party/js-1.7/fdlibm/s_rint.c | 131 - src/third_party/js-1.7/fdlibm/s_scalbn.c | 107 - src/third_party/js-1.7/fdlibm/s_signgam.c | 40 - src/third_party/js-1.7/fdlibm/s_significand.c | 68 - src/third_party/js-1.7/fdlibm/s_sin.c | 118 - src/third_party/js-1.7/fdlibm/s_tan.c | 112 - src/third_party/js-1.7/fdlibm/s_tanh.c | 122 - src/third_party/js-1.7/fdlibm/w_acos.c | 78 - src/third_party/js-1.7/fdlibm/w_acosh.c | 78 - src/third_party/js-1.7/fdlibm/w_asin.c | 80 - src/third_party/js-1.7/fdlibm/w_atan2.c | 79 - src/third_party/js-1.7/fdlibm/w_atanh.c | 81 - src/third_party/js-1.7/fdlibm/w_cosh.c | 77 - src/third_party/js-1.7/fdlibm/w_exp.c | 88 - src/third_party/js-1.7/fdlibm/w_fmod.c | 78 - src/third_party/js-1.7/fdlibm/w_gamma.c | 85 - src/third_party/js-1.7/fdlibm/w_gamma_r.c | 81 - src/third_party/js-1.7/fdlibm/w_hypot.c | 78 - src/third_party/js-1.7/fdlibm/w_j0.c | 105 - src/third_party/js-1.7/fdlibm/w_j1.c | 106 - src/third_party/js-1.7/fdlibm/w_jn.c | 128 - src/third_party/js-1.7/fdlibm/w_lgamma.c | 85 - src/third_party/js-1.7/fdlibm/w_lgamma_r.c | 81 - src/third_party/js-1.7/fdlibm/w_log.c | 78 - src/third_party/js-1.7/fdlibm/w_log10.c | 81 - src/third_party/js-1.7/fdlibm/w_pow.c | 99 - src/third_party/js-1.7/fdlibm/w_remainder.c | 77 - src/third_party/js-1.7/fdlibm/w_scalb.c | 95 - src/third_party/js-1.7/fdlibm/w_sinh.c | 77 - src/third_party/js-1.7/fdlibm/w_sqrt.c | 77 - src/third_party/js-1.7/js.c | 3181 --- src/third_party/js-1.7/js.mak | 4344 ---- src/third_party/js-1.7/js.mdp | Bin 17922 -> 0 bytes src/third_party/js-1.7/js.msg | 301 - src/third_party/js-1.7/js.pkg | 2 - src/third_party/js-1.7/js3240.rc | 79 - src/third_party/js-1.7/jsOS240.def | 654 - src/third_party/js-1.7/jsapi.c | 5011 ---- src/third_party/js-1.7/jsapi.h | 2220 -- src/third_party/js-1.7/jsarena.c | 502 - src/third_party/js-1.7/jsarena.h | 303 - src/third_party/js-1.7/jsarray.c | 1864 -- src/third_party/js-1.7/jsarray.h | 95 - src/third_party/js-1.7/jsatom.c | 999 - src/third_party/js-1.7/jsatom.h | 456 - src/third_party/js-1.7/jsbit.h | 195 - src/third_party/js-1.7/jsbool.c | 227 - src/third_party/js-1.7/jsbool.h | 76 - src/third_party/js-1.7/jsclist.h | 139 - src/third_party/js-1.7/jscntxt.c | 1229 - src/third_party/js-1.7/jscntxt.h | 1013 - src/third_party/js-1.7/jscompat.h | 57 - src/third_party/js-1.7/jsconfig.h | 208 - src/third_party/js-1.7/jsconfig.mk | 181 - src/third_party/js-1.7/jscpucfg.c | 380 - src/third_party/js-1.7/jscpucfg.h | 212 - src/third_party/js-1.7/jsdate.c | 2371 -- src/third_party/js-1.7/jsdate.h | 120 - src/third_party/js-1.7/jsdbgapi.c | 1439 -- src/third_party/js-1.7/jsdbgapi.h | 406 - src/third_party/js-1.7/jsdhash.c | 826 - src/third_party/js-1.7/jsdhash.h | 581 - src/third_party/js-1.7/jsdtoa.c | 3132 --- src/third_party/js-1.7/jsdtoa.h | 130 - src/third_party/js-1.7/jsemit.c | 6845 ------ src/third_party/js-1.7/jsemit.h | 743 - src/third_party/js-1.7/jsexn.c | 1348 -- src/third_party/js-1.7/jsexn.h | 96 - src/third_party/js-1.7/jsfile.c | 2735 --- src/third_party/js-1.7/jsfile.h | 56 - src/third_party/js-1.7/jsfile.msg | 90 - src/third_party/js-1.7/jsfun.c | 2330 -- src/third_party/js-1.7/jsfun.h | 170 - src/third_party/js-1.7/jsgc.c | 3201 --- src/third_party/js-1.7/jsgc.h | 368 - src/third_party/js-1.7/jshash.c | 483 - src/third_party/js-1.7/jshash.h | 151 - src/third_party/js-1.7/jsify.pl | 485 - src/third_party/js-1.7/jsinterp.c | 6216 ----- src/third_party/js-1.7/jsinterp.h | 361 - src/third_party/js-1.7/jsiter.c | 1080 - src/third_party/js-1.7/jsiter.h | 114 - src/third_party/js-1.7/jskeyword.tbl | 124 - src/third_party/js-1.7/jskwgen.c | 460 - src/third_party/js-1.7/jslibmath.h | 266 - src/third_party/js-1.7/jslock.c | 1303 -- src/third_party/js-1.7/jslock.h | 266 - src/third_party/js-1.7/jslocko.asm | 60 - src/third_party/js-1.7/jslog2.c | 94 - src/third_party/js-1.7/jslong.c | 281 - src/third_party/js-1.7/jslong.h | 437 - src/third_party/js-1.7/jsmath.c | 514 - src/third_party/js-1.7/jsmath.h | 57 - src/third_party/js-1.7/jsnum.c | 1147 - src/third_party/js-1.7/jsnum.h | 268 - src/third_party/js-1.7/jsobj.c | 5035 ---- src/third_party/js-1.7/jsobj.h | 596 - src/third_party/js-1.7/jsopcode.c | 4794 ---- src/third_party/js-1.7/jsopcode.h | 318 - src/third_party/js-1.7/jsopcode.tbl | 478 - src/third_party/js-1.7/jsosdep.h | 115 - src/third_party/js-1.7/jsotypes.h | 202 - src/third_party/js-1.7/jsparse.c | 6547 ------ src/third_party/js-1.7/jsparse.h | 438 - src/third_party/js-1.7/jsprf.c | 1266 - src/third_party/js-1.7/jsprf.h | 150 - src/third_party/js-1.7/jsproto.tbl | 116 - src/third_party/js-1.7/jsprvtd.h | 202 - src/third_party/js-1.7/jspubtd.h | 667 - src/third_party/js-1.7/jsregexp.c | 4206 ---- src/third_party/js-1.7/jsregexp.h | 183 - src/third_party/js-1.7/jsscan.c | 2101 -- src/third_party/js-1.7/jsscan.h | 389 - src/third_party/js-1.7/jsscope.c | 1776 -- src/third_party/js-1.7/jsscope.h | 407 - src/third_party/js-1.7/jsscript.c | 1717 -- src/third_party/js-1.7/jsscript.h | 225 - src/third_party/js-1.7/jsshell.msg | 50 - src/third_party/js-1.7/jsstddef.h | 83 - src/third_party/js-1.7/jsstr.c | 4818 ---- src/third_party/js-1.7/jsstr.h | 500 - src/third_party/js-1.7/jstypes.h | 464 - src/third_party/js-1.7/jsutil.c | 198 - src/third_party/js-1.7/jsutil.h | 106 - src/third_party/js-1.7/jsxdrapi.c | 835 - src/third_party/js-1.7/jsxdrapi.h | 223 - src/third_party/js-1.7/jsxml.c | 8357 ------- src/third_party/js-1.7/jsxml.h | 332 - src/third_party/js-1.7/lock_SunOS.s | 114 - src/third_party/js-1.7/perfect.js | 39 - src/third_party/js-1.7/plify_jsdhash.sed | 33 - src/third_party/js-1.7/prmjtime.c | 439 - src/third_party/js-1.7/prmjtime.h | 95 - src/third_party/js-1.7/resource.h | 15 - src/third_party/js-1.7/rules.mk | 193 - src/third_party/js-1.7/win32.order | 391 - src/third_party/libstemmer_c/SConscript | 2 +- src/third_party/murmurhash3/SConscript | 2 +- src/third_party/pcre-8.30/SConscript | 2 +- src/third_party/pcre-8.36/132html | 313 + src/third_party/pcre-8.36/AUTHORS | 45 + src/third_party/pcre-8.36/CMakeLists.txt | 991 + src/third_party/pcre-8.36/COPYING | 5 + src/third_party/pcre-8.36/ChangeLog | 5575 +++++ src/third_party/pcre-8.36/CheckMan | 67 + src/third_party/pcre-8.36/CleanTxt | 113 + src/third_party/pcre-8.36/Detrail | 35 + src/third_party/pcre-8.36/HACKING | 528 + src/third_party/pcre-8.36/INSTALL | 370 + src/third_party/pcre-8.36/LICENCE | 92 + src/third_party/pcre-8.36/Makefile.am | 894 + src/third_party/pcre-8.36/Makefile.in | 3283 +++ src/third_party/pcre-8.36/NEWS | 700 + src/third_party/pcre-8.36/NON-AUTOTOOLS-BUILD | 764 + src/third_party/pcre-8.36/NON-UNIX-USE | 7 + src/third_party/pcre-8.36/PrepareRelease | 258 + src/third_party/pcre-8.36/README | 993 + src/third_party/pcre-8.36/RunGrepTest | 571 + src/third_party/pcre-8.36/RunTest | 1010 + src/third_party/pcre-8.36/RunTest.bat | 616 + src/third_party/pcre-8.36/SConscript | 68 + src/third_party/pcre-8.36/aclocal.m4 | 1431 ++ src/third_party/pcre-8.36/ar-lib | 270 + src/third_party/pcre-8.36/build_posix/config.h | 349 + src/third_party/pcre-8.36/build_solaris/config.h | 349 + src/third_party/pcre-8.36/build_windows/config.h | 57 + src/third_party/pcre-8.36/compile | 347 + src/third_party/pcre-8.36/config-cmake.h.in | 57 + src/third_party/pcre-8.36/config.guess | 1568 ++ src/third_party/pcre-8.36/config.h.generic | 350 + src/third_party/pcre-8.36/config.h.in | 348 + src/third_party/pcre-8.36/config.sub | 1793 ++ src/third_party/pcre-8.36/configure | 22282 ++++++++++++++++++ src/third_party/pcre-8.36/configure.ac | 1111 + src/third_party/pcre-8.36/depcomp | 791 + src/third_party/pcre-8.36/dftables.c | 212 + src/third_party/pcre-8.36/install-sh | 527 + src/third_party/pcre-8.36/libpcre.pc.in | 12 + src/third_party/pcre-8.36/libpcre16.pc.in | 12 + src/third_party/pcre-8.36/libpcre32.pc.in | 12 + src/third_party/pcre-8.36/libpcrecpp.pc.in | 12 + src/third_party/pcre-8.36/libpcreposix.pc.in | 13 + src/third_party/pcre-8.36/ltmain.sh | 9655 ++++++++ src/third_party/pcre-8.36/makevp.bat | 66 + src/third_party/pcre-8.36/makevp_c.txt | 21 + src/third_party/pcre-8.36/makevp_l.txt | 21 + src/third_party/pcre-8.36/missing | 215 + src/third_party/pcre-8.36/pcre-config.in | 133 + src/third_party/pcre-8.36/pcre.h | 677 + src/third_party/pcre-8.36/pcre.h.generic | 677 + src/third_party/pcre-8.36/pcre.h.in | 677 + src/third_party/pcre-8.36/pcre16_byte_order.c | 45 + src/third_party/pcre-8.36/pcre16_chartables.c | 45 + src/third_party/pcre-8.36/pcre16_compile.c | 45 + src/third_party/pcre-8.36/pcre16_config.c | 45 + src/third_party/pcre-8.36/pcre16_dfa_exec.c | 45 + src/third_party/pcre-8.36/pcre16_exec.c | 45 + src/third_party/pcre-8.36/pcre16_fullinfo.c | 45 + src/third_party/pcre-8.36/pcre16_get.c | 45 + src/third_party/pcre-8.36/pcre16_globals.c | 45 + src/third_party/pcre-8.36/pcre16_jit_compile.c | 45 + src/third_party/pcre-8.36/pcre16_maketables.c | 45 + src/third_party/pcre-8.36/pcre16_newline.c | 45 + src/third_party/pcre-8.36/pcre16_ord2utf16.c | 90 + src/third_party/pcre-8.36/pcre16_printint.c | 45 + src/third_party/pcre-8.36/pcre16_refcount.c | 45 + src/third_party/pcre-8.36/pcre16_string_utils.c | 45 + src/third_party/pcre-8.36/pcre16_study.c | 45 + src/third_party/pcre-8.36/pcre16_tables.c | 45 + src/third_party/pcre-8.36/pcre16_ucd.c | 45 + src/third_party/pcre-8.36/pcre16_utf16_utils.c | 130 + src/third_party/pcre-8.36/pcre16_valid_utf16.c | 137 + src/third_party/pcre-8.36/pcre16_version.c | 45 + src/third_party/pcre-8.36/pcre16_xclass.c | 45 + src/third_party/pcre-8.36/pcre32_byte_order.c | 45 + src/third_party/pcre-8.36/pcre32_chartables.c | 45 + src/third_party/pcre-8.36/pcre32_compile.c | 45 + src/third_party/pcre-8.36/pcre32_config.c | 45 + src/third_party/pcre-8.36/pcre32_dfa_exec.c | 45 + src/third_party/pcre-8.36/pcre32_exec.c | 45 + src/third_party/pcre-8.36/pcre32_fullinfo.c | 45 + src/third_party/pcre-8.36/pcre32_get.c | 45 + src/third_party/pcre-8.36/pcre32_globals.c | 45 + src/third_party/pcre-8.36/pcre32_jit_compile.c | 45 + src/third_party/pcre-8.36/pcre32_maketables.c | 45 + src/third_party/pcre-8.36/pcre32_newline.c | 45 + src/third_party/pcre-8.36/pcre32_ord2utf32.c | 82 + src/third_party/pcre-8.36/pcre32_printint.c | 45 + src/third_party/pcre-8.36/pcre32_refcount.c | 45 + src/third_party/pcre-8.36/pcre32_string_utils.c | 45 + src/third_party/pcre-8.36/pcre32_study.c | 45 + src/third_party/pcre-8.36/pcre32_tables.c | 45 + src/third_party/pcre-8.36/pcre32_ucd.c | 45 + src/third_party/pcre-8.36/pcre32_utf32_utils.c | 141 + src/third_party/pcre-8.36/pcre32_valid_utf32.c | 124 + src/third_party/pcre-8.36/pcre32_version.c | 45 + src/third_party/pcre-8.36/pcre32_xclass.c | 45 + src/third_party/pcre-8.36/pcre_byte_order.c | 319 + src/third_party/pcre-8.36/pcre_chartables.c | 198 + src/third_party/pcre-8.36/pcre_chartables.c.dist | 198 + src/third_party/pcre-8.36/pcre_compile.c | 9515 ++++++++ src/third_party/pcre-8.36/pcre_config.c | 190 + src/third_party/pcre-8.36/pcre_dfa_exec.c | 3673 +++ src/third_party/pcre-8.36/pcre_exec.c | 7163 ++++++ src/third_party/pcre-8.36/pcre_fullinfo.c | 245 + src/third_party/pcre-8.36/pcre_get.c | 662 + src/third_party/pcre-8.36/pcre_globals.c | 86 + src/third_party/pcre-8.36/pcre_internal.h | 2781 +++ src/third_party/pcre-8.36/pcre_jit_compile.c | 10704 +++++++++ src/third_party/pcre-8.36/pcre_jit_test.c | 1729 ++ src/third_party/pcre-8.36/pcre_maketables.c | 156 + src/third_party/pcre-8.36/pcre_newline.c | 210 + src/third_party/pcre-8.36/pcre_ord2utf8.c | 94 + src/third_party/pcre-8.36/pcre_printint.c | 834 + src/third_party/pcre-8.36/pcre_refcount.c | 92 + src/third_party/pcre-8.36/pcre_scanner.cc | 199 + src/third_party/pcre-8.36/pcre_scanner.h | 172 + src/third_party/pcre-8.36/pcre_scanner_unittest.cc | 161 + src/third_party/pcre-8.36/pcre_string_utils.c | 211 + src/third_party/pcre-8.36/pcre_stringpiece.cc | 43 + src/third_party/pcre-8.36/pcre_stringpiece.h | 180 + src/third_party/pcre-8.36/pcre_stringpiece.h.in | 180 + .../pcre-8.36/pcre_stringpiece_unittest.cc | 152 + src/third_party/pcre-8.36/pcre_study.c | 1643 ++ src/third_party/pcre-8.36/pcre_tables.c | 727 + src/third_party/pcre-8.36/pcre_ucd.c | 3630 +++ src/third_party/pcre-8.36/pcre_valid_utf8.c | 301 + src/third_party/pcre-8.36/pcre_version.c | 98 + src/third_party/pcre-8.36/pcre_xclass.c | 268 + src/third_party/pcre-8.36/pcrecpp.cc | 924 + src/third_party/pcre-8.36/pcrecpp.h | 710 + src/third_party/pcre-8.36/pcrecpp_internal.h | 71 + src/third_party/pcre-8.36/pcrecpp_unittest.cc | 1291 ++ src/third_party/pcre-8.36/pcrecpparg.h | 174 + src/third_party/pcre-8.36/pcrecpparg.h.in | 174 + src/third_party/pcre-8.36/pcredemo.c | 406 + src/third_party/pcre-8.36/pcregexp.pas | 845 + src/third_party/pcre-8.36/pcregrep.c | 3290 +++ src/third_party/pcre-8.36/pcreposix.c | 430 + src/third_party/pcre-8.36/pcreposix.h | 146 + src/third_party/pcre-8.36/pcretest.c | 5709 +++++ src/third_party/pcre-8.36/perltest.pl | 242 + src/third_party/pcre-8.36/test-driver | 139 + src/third_party/pcre-8.36/ucp.h | 224 + src/third_party/pcre-8.37/132html | 313 + src/third_party/pcre-8.37/AUTHORS | 45 + src/third_party/pcre-8.37/CMakeLists.txt | 991 + src/third_party/pcre-8.37/COPYING | 5 + src/third_party/pcre-8.37/ChangeLog | 5742 +++++ src/third_party/pcre-8.37/CheckMan | 67 + src/third_party/pcre-8.37/CleanTxt | 113 + src/third_party/pcre-8.37/Detrail | 35 + src/third_party/pcre-8.37/HACKING | 528 + src/third_party/pcre-8.37/INSTALL | 370 + src/third_party/pcre-8.37/LICENCE | 93 + src/third_party/pcre-8.37/Makefile.am | 894 + src/third_party/pcre-8.37/Makefile.in | 3296 +++ src/third_party/pcre-8.37/NEWS | 708 + src/third_party/pcre-8.37/NON-AUTOTOOLS-BUILD | 772 + src/third_party/pcre-8.37/NON-UNIX-USE | 7 + src/third_party/pcre-8.37/PrepareRelease | 258 + src/third_party/pcre-8.37/README | 1002 + src/third_party/pcre-8.37/RunGrepTest | 576 + src/third_party/pcre-8.37/RunTest | 1010 + src/third_party/pcre-8.37/RunTest.bat | 616 + src/third_party/pcre-8.37/SConscript | 68 + src/third_party/pcre-8.37/aclocal.m4 | 1434 ++ src/third_party/pcre-8.37/ar-lib | 270 + src/third_party/pcre-8.37/build_posix/config.h | 349 + src/third_party/pcre-8.37/build_solaris/config.h | 349 + src/third_party/pcre-8.37/build_windows/config.h | 57 + src/third_party/pcre-8.37/compile | 347 + src/third_party/pcre-8.37/config-cmake.h.in | 57 + src/third_party/pcre-8.37/config.guess | 1421 ++ src/third_party/pcre-8.37/config.h.generic | 349 + src/third_party/pcre-8.37/config.h.in | 347 + src/third_party/pcre-8.37/config.sub | 1807 ++ src/third_party/pcre-8.37/configure | 22813 +++++++++++++++++++ src/third_party/pcre-8.37/configure.ac | 1111 + src/third_party/pcre-8.37/depcomp | 791 + src/third_party/pcre-8.37/dftables.c | 212 + src/third_party/pcre-8.37/install-sh | 501 + src/third_party/pcre-8.37/libpcre.pc.in | 13 + src/third_party/pcre-8.37/libpcre16.pc.in | 13 + src/third_party/pcre-8.37/libpcre32.pc.in | 13 + src/third_party/pcre-8.37/libpcrecpp.pc.in | 12 + src/third_party/pcre-8.37/libpcreposix.pc.in | 13 + src/third_party/pcre-8.37/ltmain.sh | 11147 +++++++++ src/third_party/pcre-8.37/makevp.bat | 66 + src/third_party/pcre-8.37/makevp_c.txt | 21 + src/third_party/pcre-8.37/makevp_l.txt | 21 + src/third_party/pcre-8.37/missing | 215 + src/third_party/pcre-8.37/pcre-config.in | 133 + src/third_party/pcre-8.37/pcre.h | 677 + src/third_party/pcre-8.37/pcre.h.generic | 677 + src/third_party/pcre-8.37/pcre.h.in | 677 + src/third_party/pcre-8.37/pcre16_byte_order.c | 45 + src/third_party/pcre-8.37/pcre16_chartables.c | 45 + src/third_party/pcre-8.37/pcre16_compile.c | 45 + src/third_party/pcre-8.37/pcre16_config.c | 45 + src/third_party/pcre-8.37/pcre16_dfa_exec.c | 45 + src/third_party/pcre-8.37/pcre16_exec.c | 45 + src/third_party/pcre-8.37/pcre16_fullinfo.c | 45 + src/third_party/pcre-8.37/pcre16_get.c | 45 + src/third_party/pcre-8.37/pcre16_globals.c | 45 + src/third_party/pcre-8.37/pcre16_jit_compile.c | 45 + src/third_party/pcre-8.37/pcre16_maketables.c | 45 + src/third_party/pcre-8.37/pcre16_newline.c | 45 + src/third_party/pcre-8.37/pcre16_ord2utf16.c | 90 + src/third_party/pcre-8.37/pcre16_printint.c | 45 + src/third_party/pcre-8.37/pcre16_refcount.c | 45 + src/third_party/pcre-8.37/pcre16_string_utils.c | 45 + src/third_party/pcre-8.37/pcre16_study.c | 45 + src/third_party/pcre-8.37/pcre16_tables.c | 45 + src/third_party/pcre-8.37/pcre16_ucd.c | 45 + src/third_party/pcre-8.37/pcre16_utf16_utils.c | 130 + src/third_party/pcre-8.37/pcre16_valid_utf16.c | 137 + src/third_party/pcre-8.37/pcre16_version.c | 45 + src/third_party/pcre-8.37/pcre16_xclass.c | 45 + src/third_party/pcre-8.37/pcre32_byte_order.c | 45 + src/third_party/pcre-8.37/pcre32_chartables.c | 45 + src/third_party/pcre-8.37/pcre32_compile.c | 45 + src/third_party/pcre-8.37/pcre32_config.c | 45 + src/third_party/pcre-8.37/pcre32_dfa_exec.c | 45 + src/third_party/pcre-8.37/pcre32_exec.c | 45 + src/third_party/pcre-8.37/pcre32_fullinfo.c | 45 + src/third_party/pcre-8.37/pcre32_get.c | 45 + src/third_party/pcre-8.37/pcre32_globals.c | 45 + src/third_party/pcre-8.37/pcre32_jit_compile.c | 45 + src/third_party/pcre-8.37/pcre32_maketables.c | 45 + src/third_party/pcre-8.37/pcre32_newline.c | 45 + src/third_party/pcre-8.37/pcre32_ord2utf32.c | 82 + src/third_party/pcre-8.37/pcre32_printint.c | 45 + src/third_party/pcre-8.37/pcre32_refcount.c | 45 + src/third_party/pcre-8.37/pcre32_string_utils.c | 45 + src/third_party/pcre-8.37/pcre32_study.c | 45 + src/third_party/pcre-8.37/pcre32_tables.c | 45 + src/third_party/pcre-8.37/pcre32_ucd.c | 45 + src/third_party/pcre-8.37/pcre32_utf32_utils.c | 141 + src/third_party/pcre-8.37/pcre32_valid_utf32.c | 124 + src/third_party/pcre-8.37/pcre32_version.c | 45 + src/third_party/pcre-8.37/pcre32_xclass.c | 45 + src/third_party/pcre-8.37/pcre_byte_order.c | 319 + src/third_party/pcre-8.37/pcre_chartables.c | 198 + src/third_party/pcre-8.37/pcre_chartables.c.dist | 198 + src/third_party/pcre-8.37/pcre_compile.c | 9596 ++++++++ src/third_party/pcre-8.37/pcre_config.c | 190 + src/third_party/pcre-8.37/pcre_dfa_exec.c | 3674 +++ src/third_party/pcre-8.37/pcre_exec.c | 7172 ++++++ src/third_party/pcre-8.37/pcre_fullinfo.c | 245 + src/third_party/pcre-8.37/pcre_get.c | 662 + src/third_party/pcre-8.37/pcre_globals.c | 86 + src/third_party/pcre-8.37/pcre_internal.h | 2789 +++ src/third_party/pcre-8.37/pcre_jit_compile.c | 10690 +++++++++ src/third_party/pcre-8.37/pcre_jit_test.c | 1737 ++ src/third_party/pcre-8.37/pcre_maketables.c | 156 + src/third_party/pcre-8.37/pcre_newline.c | 210 + src/third_party/pcre-8.37/pcre_ord2utf8.c | 94 + src/third_party/pcre-8.37/pcre_printint.c | 834 + src/third_party/pcre-8.37/pcre_refcount.c | 92 + src/third_party/pcre-8.37/pcre_scanner.cc | 199 + src/third_party/pcre-8.37/pcre_scanner.h | 172 + src/third_party/pcre-8.37/pcre_scanner_unittest.cc | 161 + src/third_party/pcre-8.37/pcre_string_utils.c | 211 + src/third_party/pcre-8.37/pcre_stringpiece.cc | 43 + src/third_party/pcre-8.37/pcre_stringpiece.h | 180 + src/third_party/pcre-8.37/pcre_stringpiece.h.in | 180 + .../pcre-8.37/pcre_stringpiece_unittest.cc | 152 + src/third_party/pcre-8.37/pcre_study.c | 1679 ++ src/third_party/pcre-8.37/pcre_tables.c | 727 + src/third_party/pcre-8.37/pcre_ucd.c | 3630 +++ src/third_party/pcre-8.37/pcre_valid_utf8.c | 301 + src/third_party/pcre-8.37/pcre_version.c | 98 + src/third_party/pcre-8.37/pcre_xclass.c | 268 + src/third_party/pcre-8.37/pcrecpp.cc | 924 + src/third_party/pcre-8.37/pcrecpp.h | 710 + src/third_party/pcre-8.37/pcrecpp_internal.h | 71 + src/third_party/pcre-8.37/pcrecpp_unittest.cc | 1291 ++ src/third_party/pcre-8.37/pcrecpparg.h | 174 + src/third_party/pcre-8.37/pcrecpparg.h.in | 174 + src/third_party/pcre-8.37/pcredemo.c | 406 + src/third_party/pcre-8.37/pcregexp.pas | 845 + src/third_party/pcre-8.37/pcregrep.c | 3325 +++ src/third_party/pcre-8.37/pcreposix.c | 430 + src/third_party/pcre-8.37/pcreposix.h | 146 + src/third_party/pcre-8.37/pcretest.c | 5742 +++++ src/third_party/pcre-8.37/perltest.pl | 242 + src/third_party/pcre-8.37/test-driver | 148 + src/third_party/pcre-8.37/ucp.h | 224 + src/third_party/s2/SConscript | 11 +- src/third_party/s2/base/SConscript | 2 +- src/third_party/s2/base/logging.cc | 29 +- src/third_party/s2/base/logging.h | 50 +- src/third_party/s2/base/port.h | 2 + src/third_party/s2/base/strtoint.cc | 6 + src/third_party/s2/hash.h | 11 +- src/third_party/s2/s2.cc | 30 +- src/third_party/s2/s2polygonbuilder.cc | 2 +- src/third_party/s2/s2regionunion.cc | 2 + src/third_party/s2/s2regionunion.h | 3 +- src/third_party/s2/strings/SConscript | 2 +- src/third_party/s2/util/coding/SConscript | 2 +- src/third_party/s2/util/endian/endian.h | 4 +- src/third_party/s2/util/math/SConscript | 8 +- src/third_party/shim_spidermonkey.cpp | 3 - src/third_party/shim_stemmer.cpp | 3 + src/third_party/shim_tz.cpp | 2 + src/third_party/shim_yaml.cpp | 3 + src/third_party/snappy/SConscript | 2 +- src/third_party/tz/SConscript | 3 + src/third_party/tz/timegm.c | 713 + src/third_party/tz/timegm_private.h | 181 + src/third_party/tz/timegm_tzfile.h | 93 + src/third_party/v8/SConscript | 21 +- src/third_party/v8/src/spaces-inl.h | 2 +- src/third_party/v8/src/spaces.h | 3 +- src/third_party/v8/src/win32-math.cc | 2 + src/third_party/v8/src/win32-math.h | 4 + src/third_party/yaml-cpp-0.5.1/SConscript | 43 + .../yaml-cpp-0.5.1/include/yaml-cpp/anchor.h | 16 + .../yaml-cpp-0.5.1/include/yaml-cpp/binary.h | 62 + .../include/yaml-cpp/contrib/anchordict.h | 42 + .../include/yaml-cpp/contrib/graphbuilder.h | 133 + .../yaml-cpp-0.5.1/include/yaml-cpp/dll.h | 28 + .../include/yaml-cpp/emitfromevents.h | 45 + .../yaml-cpp-0.5.1/include/yaml-cpp/emitter.h | 209 + .../yaml-cpp-0.5.1/include/yaml-cpp/emitterdef.h | 13 + .../yaml-cpp-0.5.1/include/yaml-cpp/emittermanip.h | 149 + .../yaml-cpp-0.5.1/include/yaml-cpp/eventhandler.h | 36 + .../yaml-cpp-0.5.1/include/yaml-cpp/exceptions.h | 208 + .../yaml-cpp-0.5.1/include/yaml-cpp/mark.h | 26 + .../yaml-cpp-0.5.1/include/yaml-cpp/node/convert.h | 280 + .../include/yaml-cpp/node/detail/bool_type.h | 26 + .../include/yaml-cpp/node/detail/impl.h | 168 + .../include/yaml-cpp/node/detail/iterator.h | 64 + .../include/yaml-cpp/node/detail/iterator_fwd.h | 27 + .../include/yaml-cpp/node/detail/memory.h | 39 + .../include/yaml-cpp/node/detail/node.h | 130 + .../include/yaml-cpp/node/detail/node_data.h | 110 + .../include/yaml-cpp/node/detail/node_iterator.h | 139 + .../include/yaml-cpp/node/detail/node_ref.h | 69 + .../yaml-cpp-0.5.1/include/yaml-cpp/node/emit.h | 23 + .../yaml-cpp-0.5.1/include/yaml-cpp/node/impl.h | 451 + .../include/yaml-cpp/node/iterator.h | 28 + .../yaml-cpp-0.5.1/include/yaml-cpp/node/node.h | 116 + .../yaml-cpp-0.5.1/include/yaml-cpp/node/parse.h | 28 + .../yaml-cpp-0.5.1/include/yaml-cpp/node/ptr.h | 29 + .../yaml-cpp-0.5.1/include/yaml-cpp/node/type.h | 14 + .../yaml-cpp-0.5.1/include/yaml-cpp/noncopyable.h | 25 + .../yaml-cpp-0.5.1/include/yaml-cpp/null.h | 25 + .../include/yaml-cpp/ostream_wrapper.h | 69 + .../yaml-cpp-0.5.1/include/yaml-cpp/parser.h | 47 + .../yaml-cpp-0.5.1/include/yaml-cpp/stlemitter.h | 51 + .../yaml-cpp-0.5.1/include/yaml-cpp/traits.h | 57 + .../yaml-cpp-0.5.1/include/yaml-cpp/yaml.h | 21 + src/third_party/yaml-cpp-0.5.1/license.txt | 19 + src/third_party/yaml-cpp-0.5.1/src/binary.cpp | 93 + .../yaml-cpp-0.5.1/src/collectionstack.h | 35 + src/third_party/yaml-cpp-0.5.1/src/convert.cpp | 83 + src/third_party/yaml-cpp-0.5.1/src/directives.cpp | 24 + src/third_party/yaml-cpp-0.5.1/src/directives.h | 29 + src/third_party/yaml-cpp-0.5.1/src/emit.cpp | 29 + .../yaml-cpp-0.5.1/src/emitfromevents.cpp | 105 + src/third_party/yaml-cpp-0.5.1/src/emitter.cpp | 951 + .../yaml-cpp-0.5.1/src/emitterstate.cpp | 384 + src/third_party/yaml-cpp-0.5.1/src/emitterstate.h | 190 + .../yaml-cpp-0.5.1/src/emitterutils.cpp | 424 + src/third_party/yaml-cpp-0.5.1/src/emitterutils.h | 36 + src/third_party/yaml-cpp-0.5.1/src/exp.cpp | 113 + src/third_party/yaml-cpp-0.5.1/src/exp.h | 196 + .../yaml-cpp-0.5.1/src/graphbuilder.cpp | 16 + .../yaml-cpp-0.5.1/src/graphbuilderadapter.cpp | 96 + .../yaml-cpp-0.5.1/src/graphbuilderadapter.h | 73 + src/third_party/yaml-cpp-0.5.1/src/indentation.h | 38 + src/third_party/yaml-cpp-0.5.1/src/memory.cpp | 29 + src/third_party/yaml-cpp-0.5.1/src/node.cpp | 14 + src/third_party/yaml-cpp-0.5.1/src/node_data.cpp | 295 + src/third_party/yaml-cpp-0.5.1/src/nodebuilder.cpp | 138 + src/third_party/yaml-cpp-0.5.1/src/nodebuilder.h | 58 + src/third_party/yaml-cpp-0.5.1/src/nodeevents.cpp | 99 + src/third_party/yaml-cpp-0.5.1/src/nodeevents.h | 57 + src/third_party/yaml-cpp-0.5.1/src/null.cpp | 6 + .../yaml-cpp-0.5.1/src/ostream_wrapper.cpp | 57 + src/third_party/yaml-cpp-0.5.1/src/parse.cpp | 68 + src/third_party/yaml-cpp-0.5.1/src/parser.cpp | 141 + src/third_party/yaml-cpp-0.5.1/src/ptr_stack.h | 49 + src/third_party/yaml-cpp-0.5.1/src/ptr_vector.h | 47 + src/third_party/yaml-cpp-0.5.1/src/regex.cpp | 60 + src/third_party/yaml-cpp-0.5.1/src/regex.h | 67 + src/third_party/yaml-cpp-0.5.1/src/regeximpl.h | 186 + src/third_party/yaml-cpp-0.5.1/src/scanner.cpp | 394 + src/third_party/yaml-cpp-0.5.1/src/scanner.h | 133 + src/third_party/yaml-cpp-0.5.1/src/scanscalar.cpp | 214 + src/third_party/yaml-cpp-0.5.1/src/scanscalar.h | 45 + src/third_party/yaml-cpp-0.5.1/src/scantag.cpp | 84 + src/third_party/yaml-cpp-0.5.1/src/scantag.h | 20 + src/third_party/yaml-cpp-0.5.1/src/scantoken.cpp | 439 + src/third_party/yaml-cpp-0.5.1/src/setting.h | 105 + src/third_party/yaml-cpp-0.5.1/src/simplekey.cpp | 139 + .../yaml-cpp-0.5.1/src/singledocparser.cpp | 394 + .../yaml-cpp-0.5.1/src/singledocparser.h | 65 + src/third_party/yaml-cpp-0.5.1/src/stream.cpp | 447 + src/third_party/yaml-cpp-0.5.1/src/stream.h | 79 + .../yaml-cpp-0.5.1/src/streamcharsource.h | 48 + src/third_party/yaml-cpp-0.5.1/src/stringsource.h | 47 + src/third_party/yaml-cpp-0.5.1/src/tag.cpp | 52 + src/third_party/yaml-cpp-0.5.1/src/tag.h | 28 + src/third_party/yaml-cpp-0.5.1/src/token.h | 85 + src/vs2010_mongodb.sln | 92 + 2446 files changed, 537335 insertions(+), 261620 deletions(-) delete mode 100644 src/mongo/base/configuration_variable_manager.cpp delete mode 100644 src/mongo/base/configuration_variable_manager.h delete mode 100644 src/mongo/base/configuration_variable_manager_test.cpp create mode 100644 src/mongo/base/status-inl.h create mode 100644 src/mongo/base/status_with.h create mode 100644 src/mongo/base/validate_locale.cpp create mode 100644 src/mongo/bson/bson_obj_test.cpp create mode 100644 src/mongo/bson/bsonobjbuilder_test.cpp create mode 100644 src/mongo/bson/mutable/algorithm.h create mode 100644 src/mongo/bson/mutable/const_element-inl.h create mode 100644 src/mongo/bson/mutable/const_element.h create mode 100644 src/mongo/bson/mutable/damage_vector.h create mode 100644 src/mongo/bson/mutable/document-inl.h create mode 100644 src/mongo/bson/mutable/document.cpp create mode 100644 src/mongo/bson/mutable/document.h create mode 100644 src/mongo/bson/mutable/element-inl.h create mode 100644 src/mongo/bson/mutable/element.cpp create mode 100644 src/mongo/bson/mutable/element.h delete mode 100644 src/mongo/bson/mutable/mutable_bson.cpp delete mode 100644 src/mongo/bson/mutable/mutable_bson.h delete mode 100644 src/mongo/bson/mutable/mutable_bson_algo.h delete mode 100644 src/mongo/bson/mutable/mutable_bson_builder.cpp delete mode 100644 src/mongo/bson/mutable/mutable_bson_builder.h delete mode 100644 src/mongo/bson/mutable/mutable_bson_builder_test.cpp delete mode 100644 src/mongo/bson/mutable/mutable_bson_heap.cpp delete mode 100644 src/mongo/bson/mutable/mutable_bson_heap.h delete mode 100644 src/mongo/bson/mutable/mutable_bson_heap_test.cpp delete mode 100644 src/mongo/bson/mutable/mutable_bson_internal.cpp delete mode 100644 src/mongo/bson/mutable/mutable_bson_internal.h create mode 100644 src/mongo/bson/mutable/mutable_bson_test_utils.cpp create mode 100644 src/mongo/bson/mutable/mutable_bson_test_utils.h create mode 100644 src/mongo/bson/optime.cpp create mode 100644 src/mongo/bson/optime.h create mode 100644 src/mongo/client/auth_helpers.cpp create mode 100644 src/mongo/client/auth_helpers.h delete mode 100644 src/mongo/client/authlevel.h delete mode 100644 src/mongo/client/connection_factory.cpp delete mode 100644 src/mongo/client/distlock.cpp delete mode 100644 src/mongo/client/distlock.h delete mode 100644 src/mongo/client/distlock_test.cpp create mode 100644 src/mongo/client/export_macros.h create mode 100644 src/mongo/client/include_bsonh_test.cpp create mode 100644 src/mongo/client/include_dbclienth_test.cpp create mode 100644 src/mongo/client/init.cpp create mode 100644 src/mongo/client/init.h delete mode 100644 src/mongo/client/model.cpp delete mode 100644 src/mongo/client/model.h delete mode 100644 src/mongo/client/mongo_client_lib.cpp create mode 100644 src/mongo/client/replica_set_monitor.cpp create mode 100644 src/mongo/client/replica_set_monitor.h create mode 100644 src/mongo/client/replica_set_monitor_internal.h create mode 100644 src/mongo/client/replica_set_monitor_test.cpp create mode 100644 src/mongo/client/sasl_sspi.cpp create mode 100644 src/mongo/db/audit.cpp create mode 100644 src/mongo/db/audit.h delete mode 100644 src/mongo/db/auth/auth_external_state.cpp delete mode 100644 src/mongo/db/auth/auth_external_state.h delete mode 100644 src/mongo/db/auth/auth_external_state_d.cpp delete mode 100644 src/mongo/db/auth/auth_external_state_d.h delete mode 100644 src/mongo/db/auth/auth_external_state_mock.h delete mode 100644 src/mongo/db/auth/auth_external_state_s.cpp delete mode 100644 src/mongo/db/auth/auth_external_state_s.h delete mode 100644 src/mongo/db/auth/auth_external_state_server_common.cpp delete mode 100644 src/mongo/db/auth/auth_external_state_server_common.h create mode 100644 src/mongo/db/auth/authorization_manager_global.cpp create mode 100644 src/mongo/db/auth/authorization_manager_global.h create mode 100644 src/mongo/db/auth/authorization_session.cpp create mode 100644 src/mongo/db/auth/authorization_session.h create mode 100644 src/mongo/db/auth/authorization_session_test.cpp create mode 100644 src/mongo/db/auth/authz_documents_update_guard.cpp create mode 100644 src/mongo/db/auth/authz_documents_update_guard.h create mode 100644 src/mongo/db/auth/authz_manager_external_state.cpp create mode 100644 src/mongo/db/auth/authz_manager_external_state.h create mode 100644 src/mongo/db/auth/authz_manager_external_state_d.cpp create mode 100644 src/mongo/db/auth/authz_manager_external_state_d.h create mode 100644 src/mongo/db/auth/authz_manager_external_state_local.cpp create mode 100644 src/mongo/db/auth/authz_manager_external_state_local.h create mode 100644 src/mongo/db/auth/authz_manager_external_state_mock.cpp create mode 100644 src/mongo/db/auth/authz_manager_external_state_mock.h create mode 100644 src/mongo/db/auth/authz_manager_external_state_s.cpp create mode 100644 src/mongo/db/auth/authz_manager_external_state_s.h create mode 100644 src/mongo/db/auth/authz_session_external_state.cpp create mode 100644 src/mongo/db/auth/authz_session_external_state.h create mode 100644 src/mongo/db/auth/authz_session_external_state_d.cpp create mode 100644 src/mongo/db/auth/authz_session_external_state_d.h create mode 100644 src/mongo/db/auth/authz_session_external_state_mock.h create mode 100644 src/mongo/db/auth/authz_session_external_state_s.cpp create mode 100644 src/mongo/db/auth/authz_session_external_state_s.h create mode 100644 src/mongo/db/auth/authz_session_external_state_server_common.cpp create mode 100644 src/mongo/db/auth/authz_session_external_state_server_common.h delete mode 100644 src/mongo/db/auth/principal.cpp delete mode 100644 src/mongo/db/auth/principal.h delete mode 100644 src/mongo/db/auth/principal_name.h delete mode 100644 src/mongo/db/auth/principal_set.cpp delete mode 100644 src/mongo/db/auth/principal_set.h delete mode 100644 src/mongo/db/auth/principal_set_test.cpp create mode 100644 src/mongo/db/auth/privilege_parser.cpp create mode 100644 src/mongo/db/auth/privilege_parser.h create mode 100644 src/mongo/db/auth/privilege_parser_test.cpp delete mode 100644 src/mongo/db/auth/privilege_set.cpp delete mode 100644 src/mongo/db/auth/privilege_set.h delete mode 100644 src/mongo/db/auth/privilege_set_test.cpp create mode 100644 src/mongo/db/auth/resource_pattern.cpp create mode 100644 src/mongo/db/auth/resource_pattern.h create mode 100644 src/mongo/db/auth/role_graph.cpp create mode 100644 src/mongo/db/auth/role_graph.h create mode 100644 src/mongo/db/auth/role_graph_builtin_roles.cpp create mode 100644 src/mongo/db/auth/role_graph_test.cpp create mode 100644 src/mongo/db/auth/role_graph_update.cpp create mode 100644 src/mongo/db/auth/role_name.cpp create mode 100644 src/mongo/db/auth/role_name.h create mode 100644 src/mongo/db/auth/user.cpp create mode 100644 src/mongo/db/auth/user.h create mode 100644 src/mongo/db/auth/user_cache_invalidator_job.cpp create mode 100644 src/mongo/db/auth/user_cache_invalidator_job.h create mode 100644 src/mongo/db/auth/user_document_parser.cpp create mode 100644 src/mongo/db/auth/user_document_parser.h create mode 100644 src/mongo/db/auth/user_document_parser_test.cpp create mode 100644 src/mongo/db/auth/user_management_commands_parser.cpp create mode 100644 src/mongo/db/auth/user_management_commands_parser.h create mode 100644 src/mongo/db/auth/user_name.cpp create mode 100644 src/mongo/db/auth/user_name.h create mode 100644 src/mongo/db/auth/user_name_hash.h create mode 100644 src/mongo/db/auth/user_set.cpp create mode 100644 src/mongo/db/auth/user_set.h create mode 100644 src/mongo/db/auth/user_set_test.cpp create mode 100644 src/mongo/db/background.cpp delete mode 100644 src/mongo/db/btree.cpp delete mode 100644 src/mongo/db/btree.h delete mode 100644 src/mongo/db/btree_stats.cpp delete mode 100644 src/mongo/db/btree_stats.h delete mode 100644 src/mongo/db/btreebuilder.cpp delete mode 100644 src/mongo/db/btreebuilder.h delete mode 100644 src/mongo/db/btreecursor.cpp delete mode 100644 src/mongo/db/btreecursor.h delete mode 100644 src/mongo/db/btreeposition.cpp delete mode 100644 src/mongo/db/btreeposition.h delete mode 100644 src/mongo/db/cap.cpp create mode 100644 src/mongo/db/catalog/collection.cpp create mode 100644 src/mongo/db/catalog/collection.h create mode 100644 src/mongo/db/catalog/collection_cursor_cache.cpp create mode 100644 src/mongo/db/catalog/collection_cursor_cache.h create mode 100644 src/mongo/db/catalog/collection_info_cache.cpp create mode 100644 src/mongo/db/catalog/collection_info_cache.h create mode 100644 src/mongo/db/catalog/database.cpp create mode 100644 src/mongo/db/catalog/database.h create mode 100644 src/mongo/db/catalog/database_holder.cpp create mode 100644 src/mongo/db/catalog/database_holder.h create mode 100644 src/mongo/db/catalog/index_catalog.cpp create mode 100644 src/mongo/db/catalog/index_catalog.h create mode 100644 src/mongo/db/catalog/index_catalog_entry.cpp create mode 100644 src/mongo/db/catalog/index_catalog_entry.h create mode 100644 src/mongo/db/catalog/index_create.cpp create mode 100644 src/mongo/db/catalog/index_create.h create mode 100644 src/mongo/db/catalog/index_key_validate.cpp create mode 100644 src/mongo/db/catalog/index_key_validate.h create mode 100644 src/mongo/db/catalog/index_pregen.cpp create mode 100644 src/mongo/db/catalog/index_pregen.h delete mode 100644 src/mongo/db/cc_by_loc.h create mode 100644 src/mongo/db/clientlistplugin.cpp delete mode 100644 src/mongo/db/cmdline.cpp delete mode 100644 src/mongo/db/cmdline.h delete mode 100644 src/mongo/db/cmdline_test.cpp delete mode 100644 src/mongo/db/collection.h create mode 100644 src/mongo/db/commands/apply_ops.cpp create mode 100644 src/mongo/db/commands/auth_schema_upgrade_d.cpp create mode 100644 src/mongo/db/commands/cleanup_orphaned_cmd.cpp create mode 100644 src/mongo/db/commands/collection_to_capped.cpp create mode 100644 src/mongo/db/commands/compact.cpp create mode 100644 src/mongo/db/commands/copydb.h create mode 100644 src/mongo/db/commands/copydb_common.cpp create mode 100644 src/mongo/db/commands/create_indexes.cpp create mode 100644 src/mongo/db/commands/dbhash.cpp create mode 100644 src/mongo/db/commands/dbhash.h create mode 100644 src/mongo/db/commands/drop_indexes.cpp create mode 100644 src/mongo/db/commands/find_cmd.cpp create mode 100644 src/mongo/db/commands/geonear.cpp create mode 100644 src/mongo/db/commands/get_last_error.cpp create mode 100644 src/mongo/db/commands/index_filter_commands.cpp create mode 100644 src/mongo/db/commands/index_filter_commands.h create mode 100644 src/mongo/db/commands/index_filter_commands_test.cpp create mode 100644 src/mongo/db/commands/merge_chunks_cmd.cpp create mode 100644 src/mongo/db/commands/oplog_note.cpp create mode 100644 src/mongo/db/commands/parallel_collection_scan.cpp create mode 100644 src/mongo/db/commands/plan_cache_commands.cpp create mode 100644 src/mongo/db/commands/plan_cache_commands.h create mode 100644 src/mongo/db/commands/plan_cache_commands_test.cpp create mode 100644 src/mongo/db/commands/rename_collection.cpp create mode 100644 src/mongo/db/commands/shutdown.cpp create mode 100644 src/mongo/db/commands/shutdown.h create mode 100644 src/mongo/db/commands/test_commands.cpp create mode 100644 src/mongo/db/commands/user_management_commands.cpp create mode 100644 src/mongo/db/commands/user_management_commands.h create mode 100644 src/mongo/db/commands/validate.cpp create mode 100644 src/mongo/db/commands/write_commands/batch_executor.cpp create mode 100644 src/mongo/db/commands/write_commands/batch_executor.h create mode 100644 src/mongo/db/commands/write_commands/write_commands.cpp create mode 100644 src/mongo/db/commands/write_commands/write_commands.h create mode 100644 src/mongo/db/commands/write_commands/write_commands_common.cpp create mode 100644 src/mongo/db/commands/write_commands/write_commands_common.h delete mode 100644 src/mongo/db/common.cpp delete mode 100644 src/mongo/db/compact.cpp create mode 100644 src/mongo/db/conn_pool_options.cpp create mode 100644 src/mongo/db/conn_pool_options.h delete mode 100644 src/mongo/db/connection_factory.cpp create mode 100644 src/mongo/db/curop_test.cpp delete mode 100644 src/mongo/db/cursor.cpp delete mode 100644 src/mongo/db/cursor.h delete mode 100644 src/mongo/db/database.cpp delete mode 100644 src/mongo/db/database.h delete mode 100644 src/mongo/db/databaseholder.h create mode 100644 src/mongo/db/dbmessage_test.cpp create mode 100644 src/mongo/db/exec/2d.cpp create mode 100644 src/mongo/db/exec/2d.h create mode 100644 src/mongo/db/exec/2dcommon.cpp create mode 100644 src/mongo/db/exec/2dcommon.h create mode 100644 src/mongo/db/exec/2dnear.cpp create mode 100644 src/mongo/db/exec/2dnear.h create mode 100644 src/mongo/db/exec/SConscript create mode 100644 src/mongo/db/exec/and_common-inl.h create mode 100644 src/mongo/db/exec/and_hash.cpp create mode 100644 src/mongo/db/exec/and_hash.h create mode 100644 src/mongo/db/exec/and_sorted.cpp create mode 100644 src/mongo/db/exec/and_sorted.h create mode 100644 src/mongo/db/exec/collection_scan.cpp create mode 100644 src/mongo/db/exec/collection_scan.h create mode 100644 src/mongo/db/exec/collection_scan_common.h create mode 100644 src/mongo/db/exec/count.cpp create mode 100644 src/mongo/db/exec/count.h create mode 100644 src/mongo/db/exec/distinct_scan.cpp create mode 100644 src/mongo/db/exec/distinct_scan.h create mode 100644 src/mongo/db/exec/fetch.cpp create mode 100644 src/mongo/db/exec/fetch.h create mode 100644 src/mongo/db/exec/filter.h create mode 100644 src/mongo/db/exec/index_scan.cpp create mode 100644 src/mongo/db/exec/index_scan.h create mode 100644 src/mongo/db/exec/keep_mutations.cpp create mode 100644 src/mongo/db/exec/keep_mutations.h create mode 100644 src/mongo/db/exec/limit.cpp create mode 100644 src/mongo/db/exec/limit.h create mode 100644 src/mongo/db/exec/merge_sort.cpp create mode 100644 src/mongo/db/exec/merge_sort.h create mode 100644 src/mongo/db/exec/mock_stage.cpp create mode 100644 src/mongo/db/exec/mock_stage.h create mode 100644 src/mongo/db/exec/oplogstart.cpp create mode 100644 src/mongo/db/exec/oplogstart.h create mode 100644 src/mongo/db/exec/or.cpp create mode 100644 src/mongo/db/exec/or.h create mode 100644 src/mongo/db/exec/plan_stage.h create mode 100644 src/mongo/db/exec/plan_stats.cpp create mode 100644 src/mongo/db/exec/plan_stats.h create mode 100644 src/mongo/db/exec/plan_stats_test.cpp create mode 100644 src/mongo/db/exec/projection.cpp create mode 100644 src/mongo/db/exec/projection.h create mode 100644 src/mongo/db/exec/projection_exec.cpp create mode 100644 src/mongo/db/exec/projection_exec.h create mode 100644 src/mongo/db/exec/projection_exec_test.cpp create mode 100644 src/mongo/db/exec/s2near.cpp create mode 100644 src/mongo/db/exec/s2near.h create mode 100644 src/mongo/db/exec/shard_filter.cpp create mode 100644 src/mongo/db/exec/shard_filter.h create mode 100644 src/mongo/db/exec/skip.cpp create mode 100644 src/mongo/db/exec/skip.h create mode 100644 src/mongo/db/exec/sort.cpp create mode 100644 src/mongo/db/exec/sort.h create mode 100644 src/mongo/db/exec/sort_test.cpp create mode 100644 src/mongo/db/exec/stagedebug_cmd.cpp create mode 100644 src/mongo/db/exec/text.cpp create mode 100644 src/mongo/db/exec/text.h create mode 100644 src/mongo/db/exec/working_set.cpp create mode 100644 src/mongo/db/exec/working_set.h create mode 100644 src/mongo/db/exec/working_set_common.cpp create mode 100644 src/mongo/db/exec/working_set_common.h create mode 100644 src/mongo/db/exec/working_set_computed_data.h create mode 100644 src/mongo/db/exec/working_set_test.cpp delete mode 100644 src/mongo/db/explain.cpp delete mode 100644 src/mongo/db/explain.h create mode 100644 src/mongo/db/field_parser-inl.h create mode 100644 src/mongo/db/field_parser.cpp create mode 100644 src/mongo/db/field_parser.h create mode 100644 src/mongo/db/field_parser_test.cpp create mode 100644 src/mongo/db/field_ref_set.cpp create mode 100644 src/mongo/db/field_ref_set.h create mode 100644 src/mongo/db/field_ref_set_test.cpp create mode 100644 src/mongo/db/fts/fts_element_iterator.cpp create mode 100644 src/mongo/db/fts/fts_element_iterator.h create mode 100644 src/mongo/db/fts/fts_element_iterator_test.cpp delete mode 100644 src/mongo/db/fts/fts_enabled.h delete mode 100644 src/mongo/db/fts/fts_index.cpp delete mode 100644 src/mongo/db/fts/fts_index.h create mode 100644 src/mongo/db/fts/fts_language.cpp create mode 100644 src/mongo/db/fts/fts_language.h create mode 100644 src/mongo/db/fts/fts_language_test.cpp delete mode 100644 src/mongo/db/fts/fts_search.cpp delete mode 100644 src/mongo/db/fts/fts_search.h create mode 100644 src/mongo/db/fts/fts_spec_legacy.cpp delete mode 100644 src/mongo/db/geo/2d.cpp delete mode 100644 src/mongo/db/geo/2d.h create mode 100644 src/mongo/db/geo/geoconstants.h delete mode 100644 src/mongo/db/geo/geonear.cpp delete mode 100644 src/mongo/db/geo/geonear.h create mode 100644 src/mongo/db/geo/s2.h delete mode 100644 src/mongo/db/geo/s2cursor.cpp delete mode 100644 src/mongo/db/geo/s2cursor.h delete mode 100644 src/mongo/db/geo/s2index.cpp delete mode 100644 src/mongo/db/geo/s2index.h delete mode 100644 src/mongo/db/geo/s2nearcursor.cpp delete mode 100644 src/mongo/db/geo/s2nearcursor.h create mode 100644 src/mongo/db/hasher_test.cpp delete mode 100644 src/mongo/db/hashindex.cpp delete mode 100644 src/mongo/db/hashindex.h delete mode 100644 src/mongo/db/index.cpp delete mode 100644 src/mongo/db/index.h create mode 100644 src/mongo/db/index/2d_access_method.cpp create mode 100644 src/mongo/db/index/2d_access_method.h create mode 100644 src/mongo/db/index/2d_common.h create mode 100644 src/mongo/db/index/2d_key_generator.cpp create mode 100644 src/mongo/db/index/2d_key_generator.h create mode 100644 src/mongo/db/index/SConscript create mode 100644 src/mongo/db/index/btree_access_method.cpp create mode 100644 src/mongo/db/index/btree_access_method.h create mode 100644 src/mongo/db/index/btree_based_access_method.cpp create mode 100644 src/mongo/db/index/btree_based_access_method.h create mode 100644 src/mongo/db/index/btree_index_cursor.cpp create mode 100644 src/mongo/db/index/btree_index_cursor.h create mode 100644 src/mongo/db/index/btree_interface.cpp create mode 100644 src/mongo/db/index/btree_interface.h create mode 100644 src/mongo/db/index/btree_key_generator.cpp create mode 100644 src/mongo/db/index/btree_key_generator.h create mode 100644 src/mongo/db/index/expression_index.h create mode 100644 src/mongo/db/index/expression_params.h create mode 100644 src/mongo/db/index/external_key_generator.cpp create mode 100644 src/mongo/db/index/external_key_generator.h create mode 100644 src/mongo/db/index/fts_access_method.cpp create mode 100644 src/mongo/db/index/fts_access_method.h create mode 100644 src/mongo/db/index/fts_key_generator.cpp create mode 100644 src/mongo/db/index/fts_key_generator.h create mode 100644 src/mongo/db/index/hash_access_method.cpp create mode 100644 src/mongo/db/index/hash_access_method.h create mode 100644 src/mongo/db/index/hash_key_generator.cpp create mode 100644 src/mongo/db/index/hash_key_generator.h create mode 100644 src/mongo/db/index/haystack_access_method.cpp create mode 100644 src/mongo/db/index/haystack_access_method.h create mode 100644 src/mongo/db/index/haystack_access_method_internal.h create mode 100644 src/mongo/db/index/haystack_key_generator.cpp create mode 100644 src/mongo/db/index/haystack_key_generator.h create mode 100644 src/mongo/db/index/index_access_method.h create mode 100644 src/mongo/db/index/index_cursor.h create mode 100644 src/mongo/db/index/index_descriptor.cpp create mode 100644 src/mongo/db/index/index_descriptor.h create mode 100644 src/mongo/db/index/key_generator.cpp create mode 100644 src/mongo/db/index/key_generator.h create mode 100644 src/mongo/db/index/s2_access_method.cpp create mode 100644 src/mongo/db/index/s2_access_method.h create mode 100644 src/mongo/db/index/s2_key_generator.cpp create mode 100644 src/mongo/db/index/s2_key_generator.h create mode 100644 src/mongo/db/index_builder.cpp create mode 100644 src/mongo/db/index_builder.h delete mode 100644 src/mongo/db/index_insertion_continuation.h create mode 100644 src/mongo/db/index_legacy.cpp create mode 100644 src/mongo/db/index_legacy.h create mode 100644 src/mongo/db/index_names.cpp create mode 100644 src/mongo/db/index_names.h delete mode 100644 src/mongo/db/index_set.cpp delete mode 100644 src/mongo/db/index_set.h delete mode 100644 src/mongo/db/index_set_test.cpp delete mode 100644 src/mongo/db/index_update.cpp delete mode 100644 src/mongo/db/index_update.h delete mode 100644 src/mongo/db/indexkey.cpp delete mode 100644 src/mongo/db/indexkey.h delete mode 100644 src/mongo/db/intervalbtreecursor.cpp delete mode 100644 src/mongo/db/intervalbtreecursor.h create mode 100644 src/mongo/db/invalidation_type.h delete mode 100644 src/mongo/db/key.cpp delete mode 100644 src/mongo/db/key.h create mode 100644 src/mongo/db/log_process_details.cpp create mode 100644 src/mongo/db/log_process_details.h delete mode 100644 src/mongo/db/matcher.cpp create mode 100644 src/mongo/db/matcher/expression.cpp create mode 100644 src/mongo/db/matcher/expression.h create mode 100644 src/mongo/db/matcher/expression_array.cpp create mode 100644 src/mongo/db/matcher/expression_array.h create mode 100644 src/mongo/db/matcher/expression_array_test.cpp create mode 100644 src/mongo/db/matcher/expression_geo.cpp create mode 100644 src/mongo/db/matcher/expression_geo.h create mode 100644 src/mongo/db/matcher/expression_geo_test.cpp create mode 100644 src/mongo/db/matcher/expression_leaf.cpp create mode 100644 src/mongo/db/matcher/expression_leaf.h create mode 100644 src/mongo/db/matcher/expression_leaf_test.cpp create mode 100644 src/mongo/db/matcher/expression_parser.cpp create mode 100644 src/mongo/db/matcher/expression_parser.h create mode 100644 src/mongo/db/matcher/expression_parser_array_test.cpp create mode 100644 src/mongo/db/matcher/expression_parser_geo.cpp create mode 100644 src/mongo/db/matcher/expression_parser_geo_test.cpp create mode 100644 src/mongo/db/matcher/expression_parser_leaf_test.cpp create mode 100644 src/mongo/db/matcher/expression_parser_test.cpp create mode 100644 src/mongo/db/matcher/expression_parser_text.cpp create mode 100644 src/mongo/db/matcher/expression_parser_text_test.cpp create mode 100644 src/mongo/db/matcher/expression_parser_tree.cpp create mode 100644 src/mongo/db/matcher/expression_parser_tree_test.cpp create mode 100644 src/mongo/db/matcher/expression_test.cpp create mode 100644 src/mongo/db/matcher/expression_text.cpp create mode 100644 src/mongo/db/matcher/expression_text.h create mode 100644 src/mongo/db/matcher/expression_tree.cpp create mode 100644 src/mongo/db/matcher/expression_tree.h create mode 100644 src/mongo/db/matcher/expression_tree_test.cpp create mode 100644 src/mongo/db/matcher/expression_where.cpp create mode 100644 src/mongo/db/matcher/match_details.cpp create mode 100644 src/mongo/db/matcher/match_details.h create mode 100644 src/mongo/db/matcher/matchable.cpp create mode 100644 src/mongo/db/matcher/matchable.h create mode 100644 src/mongo/db/matcher/matcher.cpp create mode 100644 src/mongo/db/matcher/matcher.h create mode 100644 src/mongo/db/matcher/path.cpp create mode 100644 src/mongo/db/matcher/path.h create mode 100644 src/mongo/db/matcher/path_internal.cpp create mode 100644 src/mongo/db/matcher/path_internal.h create mode 100644 src/mongo/db/matcher/path_test.cpp delete mode 100644 src/mongo/db/matcher_covered.cpp create mode 100644 src/mongo/db/max_time.h delete mode 100644 src/mongo/db/memconcept.cpp delete mode 100644 src/mongo/db/memconcept.h delete mode 100644 src/mongo/db/module.cpp delete mode 100644 src/mongo/db/module.h create mode 100644 src/mongo/db/mongod_options.cpp create mode 100644 src/mongo/db/mongod_options.h create mode 100644 src/mongo/db/mongod_options_init.cpp delete mode 100644 src/mongo/db/mongod_sm.sln delete mode 100644 src/mongo/db/mongod_sm.vcxproj delete mode 100644 src/mongo/db/mongod_sm.vcxproj.filters delete mode 100644 src/mongo/db/mongommf.cpp delete mode 100644 src/mongo/db/mongommf.h delete mode 100644 src/mongo/db/namespace-inl.h delete mode 100644 src/mongo/db/namespace.cpp delete mode 100644 src/mongo/db/namespace.h delete mode 100644 src/mongo/db/namespace_details-inl.h delete mode 100644 src/mongo/db/namespace_details.cpp delete mode 100644 src/mongo/db/namespace_details.h create mode 100644 src/mongo/db/namespace_string-inl.h create mode 100644 src/mongo/db/namespace_string.h create mode 100644 src/mongo/db/namespace_string_test.cpp delete mode 100644 src/mongo/db/namespacestring.h delete mode 100644 src/mongo/db/namespacestring_test.cpp delete mode 100644 src/mongo/db/oplog.cpp delete mode 100644 src/mongo/db/oplog.h delete mode 100644 src/mongo/db/oplogreader.h create mode 100644 src/mongo/db/ops/SConscript create mode 100644 src/mongo/db/ops/delete_executor.cpp create mode 100644 src/mongo/db/ops/delete_executor.h create mode 100644 src/mongo/db/ops/delete_request.h create mode 100644 src/mongo/db/ops/field_checker.cpp create mode 100644 src/mongo/db/ops/field_checker.h create mode 100644 src/mongo/db/ops/field_checker_test.cpp create mode 100644 src/mongo/db/ops/insert.cpp create mode 100644 src/mongo/db/ops/insert.h create mode 100644 src/mongo/db/ops/log_builder.cpp create mode 100644 src/mongo/db/ops/log_builder.h create mode 100644 src/mongo/db/ops/log_builder_test.cpp create mode 100644 src/mongo/db/ops/modifier_add_to_set.cpp create mode 100644 src/mongo/db/ops/modifier_add_to_set.h create mode 100644 src/mongo/db/ops/modifier_add_to_set_test.cpp delete mode 100644 src/mongo/db/ops/modifier_base.h create mode 100644 src/mongo/db/ops/modifier_bit.cpp create mode 100644 src/mongo/db/ops/modifier_bit.h create mode 100644 src/mongo/db/ops/modifier_bit_test.cpp create mode 100644 src/mongo/db/ops/modifier_compare.cpp create mode 100644 src/mongo/db/ops/modifier_compare.h create mode 100644 src/mongo/db/ops/modifier_compare_test.cpp create mode 100644 src/mongo/db/ops/modifier_current_date.cpp create mode 100644 src/mongo/db/ops/modifier_current_date.h create mode 100644 src/mongo/db/ops/modifier_current_date_test.cpp create mode 100644 src/mongo/db/ops/modifier_inc.cpp create mode 100644 src/mongo/db/ops/modifier_inc.h create mode 100644 src/mongo/db/ops/modifier_inc_test.cpp create mode 100644 src/mongo/db/ops/modifier_interface.h create mode 100644 src/mongo/db/ops/modifier_object_replace.cpp create mode 100644 src/mongo/db/ops/modifier_object_replace.h create mode 100644 src/mongo/db/ops/modifier_object_replace_test.cpp create mode 100644 src/mongo/db/ops/modifier_pop.cpp create mode 100644 src/mongo/db/ops/modifier_pop.h create mode 100644 src/mongo/db/ops/modifier_pop_test.cpp create mode 100644 src/mongo/db/ops/modifier_pull.cpp create mode 100644 src/mongo/db/ops/modifier_pull.h create mode 100644 src/mongo/db/ops/modifier_pull_all.cpp create mode 100644 src/mongo/db/ops/modifier_pull_all.h create mode 100644 src/mongo/db/ops/modifier_pull_all_test.cpp create mode 100644 src/mongo/db/ops/modifier_pull_test.cpp create mode 100644 src/mongo/db/ops/modifier_push.cpp create mode 100644 src/mongo/db/ops/modifier_push.h create mode 100644 src/mongo/db/ops/modifier_push_sorter.h create mode 100644 src/mongo/db/ops/modifier_push_sorter_test.cpp create mode 100644 src/mongo/db/ops/modifier_push_test.cpp create mode 100644 src/mongo/db/ops/modifier_rename.cpp create mode 100644 src/mongo/db/ops/modifier_rename.h create mode 100644 src/mongo/db/ops/modifier_rename_test.cpp create mode 100644 src/mongo/db/ops/modifier_set.cpp create mode 100644 src/mongo/db/ops/modifier_set.h create mode 100644 src/mongo/db/ops/modifier_set_test.cpp create mode 100644 src/mongo/db/ops/modifier_table.cpp create mode 100644 src/mongo/db/ops/modifier_table.h create mode 100644 src/mongo/db/ops/modifier_table_test.cpp create mode 100644 src/mongo/db/ops/modifier_unset.cpp create mode 100644 src/mongo/db/ops/modifier_unset.h create mode 100644 src/mongo/db/ops/modifier_unset_test.cpp create mode 100644 src/mongo/db/ops/path_support.cpp create mode 100644 src/mongo/db/ops/path_support.h create mode 100644 src/mongo/db/ops/path_support_test.cpp delete mode 100644 src/mongo/db/ops/query.cpp delete mode 100644 src/mongo/db/ops/query.h create mode 100644 src/mongo/db/ops/update_driver.cpp create mode 100644 src/mongo/db/ops/update_driver.h create mode 100644 src/mongo/db/ops/update_driver_test.cpp create mode 100644 src/mongo/db/ops/update_executor.cpp create mode 100644 src/mongo/db/ops/update_executor.h delete mode 100644 src/mongo/db/ops/update_internal.cpp delete mode 100644 src/mongo/db/ops/update_internal.h create mode 100644 src/mongo/db/ops/update_lifecycle.h create mode 100644 src/mongo/db/ops/update_lifecycle_impl.cpp create mode 100644 src/mongo/db/ops/update_lifecycle_impl.h create mode 100644 src/mongo/db/ops/update_request.h create mode 100644 src/mongo/db/ops/update_result.h delete mode 100644 src/mongo/db/pipeline/accumulator.cpp delete mode 100644 src/mongo/db/pipeline/accumulator_single_value.cpp delete mode 100644 src/mongo/db/pipeline/builder.cpp delete mode 100644 src/mongo/db/pipeline/builder.h create mode 100644 src/mongo/db/pipeline/dependencies.cpp create mode 100644 src/mongo/db/pipeline/dependencies.h delete mode 100644 src/mongo/db/pipeline/doc_mem_monitor.cpp delete mode 100644 src/mongo/db/pipeline/doc_mem_monitor.h delete mode 100644 src/mongo/db/pipeline/document_source_filter.cpp delete mode 100644 src/mongo/db/pipeline/document_source_filter_base.cpp create mode 100644 src/mongo/db/pipeline/document_source_merge_cursors.cpp create mode 100644 src/mongo/db/pipeline/document_source_redact.cpp delete mode 100644 src/mongo/db/pipeline/expression_context.cpp create mode 100644 src/mongo/db/pipeline/pipeline_optimizations.h create mode 100644 src/mongo/db/query/SConscript create mode 100644 src/mongo/db/query/cached_plan_runner.cpp create mode 100644 src/mongo/db/query/cached_plan_runner.h create mode 100644 src/mongo/db/query/canonical_query.cpp create mode 100644 src/mongo/db/query/canonical_query.h create mode 100644 src/mongo/db/query/canonical_query_test.cpp create mode 100644 src/mongo/db/query/eof_runner.cpp create mode 100644 src/mongo/db/query/eof_runner.h create mode 100644 src/mongo/db/query/explain_plan.cpp create mode 100644 src/mongo/db/query/explain_plan.h create mode 100644 src/mongo/db/query/find_constants.h create mode 100644 src/mongo/db/query/get_runner.cpp create mode 100644 src/mongo/db/query/get_runner.h create mode 100644 src/mongo/db/query/get_runner_test.cpp create mode 100644 src/mongo/db/query/idhack_runner.cpp create mode 100644 src/mongo/db/query/idhack_runner.h create mode 100644 src/mongo/db/query/index_bounds.cpp create mode 100644 src/mongo/db/query/index_bounds.h create mode 100644 src/mongo/db/query/index_bounds_builder.cpp create mode 100644 src/mongo/db/query/index_bounds_builder.h create mode 100644 src/mongo/db/query/index_bounds_builder_test.cpp create mode 100644 src/mongo/db/query/index_bounds_test.cpp create mode 100644 src/mongo/db/query/index_entry.h create mode 100644 src/mongo/db/query/index_tag.cpp create mode 100644 src/mongo/db/query/index_tag.h create mode 100644 src/mongo/db/query/indexability.h create mode 100644 src/mongo/db/query/internal_plans.h create mode 100644 src/mongo/db/query/internal_runner.cpp create mode 100644 src/mongo/db/query/internal_runner.h create mode 100644 src/mongo/db/query/interval.cpp create mode 100644 src/mongo/db/query/interval.h create mode 100644 src/mongo/db/query/interval_test.cpp create mode 100644 src/mongo/db/query/lite_parsed_query.cpp create mode 100644 src/mongo/db/query/lite_parsed_query.h create mode 100644 src/mongo/db/query/lite_parsed_query_test.cpp create mode 100644 src/mongo/db/query/lru_key_value.h create mode 100644 src/mongo/db/query/lru_key_value_test.cpp create mode 100644 src/mongo/db/query/multi_plan_runner.cpp create mode 100644 src/mongo/db/query/multi_plan_runner.h create mode 100644 src/mongo/db/query/new_find.cpp create mode 100644 src/mongo/db/query/new_find.h create mode 100644 src/mongo/db/query/parsed_projection.cpp create mode 100644 src/mongo/db/query/parsed_projection.h create mode 100644 src/mongo/db/query/parsed_projection_test.cpp create mode 100644 src/mongo/db/query/plan_cache.cpp create mode 100644 src/mongo/db/query/plan_cache.h create mode 100644 src/mongo/db/query/plan_cache_test.cpp create mode 100644 src/mongo/db/query/plan_enumerator.cpp create mode 100644 src/mongo/db/query/plan_enumerator.h create mode 100644 src/mongo/db/query/plan_executor.cpp create mode 100644 src/mongo/db/query/plan_executor.h create mode 100644 src/mongo/db/query/plan_ranker.cpp create mode 100644 src/mongo/db/query/plan_ranker.h create mode 100644 src/mongo/db/query/planner_access.cpp create mode 100644 src/mongo/db/query/planner_access.h create mode 100644 src/mongo/db/query/planner_analysis.cpp create mode 100644 src/mongo/db/query/planner_analysis.h create mode 100644 src/mongo/db/query/planner_analysis_test.cpp create mode 100644 src/mongo/db/query/planner_ixselect.cpp create mode 100644 src/mongo/db/query/planner_ixselect.h create mode 100644 src/mongo/db/query/planner_ixselect_test.cpp create mode 100644 src/mongo/db/query/qlog.cpp create mode 100644 src/mongo/db/query/qlog.h create mode 100644 src/mongo/db/query/query_knobs.cpp create mode 100644 src/mongo/db/query/query_knobs.h create mode 100644 src/mongo/db/query/query_planner.cpp create mode 100644 src/mongo/db/query/query_planner.h create mode 100644 src/mongo/db/query/query_planner_common.h create mode 100644 src/mongo/db/query/query_planner_params.h create mode 100644 src/mongo/db/query/query_planner_test.cpp create mode 100644 src/mongo/db/query/query_planner_test_lib.cpp create mode 100644 src/mongo/db/query/query_planner_test_lib.h create mode 100644 src/mongo/db/query/query_planner_text_test.cpp create mode 100644 src/mongo/db/query/query_settings.cpp create mode 100644 src/mongo/db/query/query_settings.h create mode 100644 src/mongo/db/query/query_solution.cpp create mode 100644 src/mongo/db/query/query_solution.h create mode 100644 src/mongo/db/query/runner.h create mode 100644 src/mongo/db/query/runner_yield_policy.h create mode 100644 src/mongo/db/query/single_solution_runner.cpp create mode 100644 src/mongo/db/query/single_solution_runner.h create mode 100644 src/mongo/db/query/stage_builder.cpp create mode 100644 src/mongo/db/query/stage_builder.h create mode 100644 src/mongo/db/query/stage_types.h create mode 100644 src/mongo/db/query/subplan_runner.cpp create mode 100644 src/mongo/db/query/subplan_runner.h create mode 100644 src/mongo/db/query/type_explain.cpp create mode 100644 src/mongo/db/query/type_explain.h delete mode 100644 src/mongo/db/queryoptimizer.cpp delete mode 100644 src/mongo/db/queryoptimizer.h delete mode 100644 src/mongo/db/queryoptimizercursor.h delete mode 100644 src/mongo/db/queryoptimizercursorimpl.cpp delete mode 100644 src/mongo/db/queryoptimizercursorimpl.h delete mode 100644 src/mongo/db/querypattern.cpp delete mode 100644 src/mongo/db/querypattern.h create mode 100644 src/mongo/db/range_deleter.cpp create mode 100644 src/mongo/db/range_deleter.h create mode 100644 src/mongo/db/range_deleter_db_env.cpp create mode 100644 src/mongo/db/range_deleter_db_env.h create mode 100644 src/mongo/db/range_deleter_mock_env.cpp create mode 100644 src/mongo/db/range_deleter_mock_env.h create mode 100644 src/mongo/db/range_deleter_service.cpp create mode 100644 src/mongo/db/range_deleter_service.h create mode 100644 src/mongo/db/range_deleter_stat_test.cpp create mode 100644 src/mongo/db/range_deleter_stats.cpp create mode 100644 src/mongo/db/range_deleter_stats.h create mode 100644 src/mongo/db/range_deleter_test.cpp create mode 100644 src/mongo/db/range_preserver.h delete mode 100644 src/mongo/db/record.cpp delete mode 100644 src/mongo/db/record.h create mode 100644 src/mongo/db/repair_database.cpp create mode 100644 src/mongo/db/repair_database.h delete mode 100644 src/mongo/db/repl.cpp delete mode 100644 src/mongo/db/repl.h create mode 100644 src/mongo/db/repl/is_master.h create mode 100644 src/mongo/db/repl/master_slave.cpp create mode 100644 src/mongo/db/repl/master_slave.h create mode 100644 src/mongo/db/repl/oplog.cpp create mode 100644 src/mongo/db/repl/oplog.h create mode 100644 src/mongo/db/repl/oplogreader.cpp create mode 100644 src/mongo/db/repl/oplogreader.h create mode 100644 src/mongo/db/repl/repl_reads_ok.cpp create mode 100644 src/mongo/db/repl/repl_reads_ok.h create mode 100644 src/mongo/db/repl/repl_start.cpp create mode 100644 src/mongo/db/repl/repl_start.h create mode 100644 src/mongo/db/repl/replication_server_status.cpp create mode 100644 src/mongo/db/repl/replication_server_status.h create mode 100644 src/mongo/db/repl/replset_web_handler.cpp create mode 100644 src/mongo/db/repl/resync.cpp delete mode 100644 src/mongo/db/repl/rs_optime.h create mode 100644 src/mongo/db/repl/sync.cpp create mode 100644 src/mongo/db/repl/sync.h create mode 100644 src/mongo/db/repl/sync_source_feedback.cpp create mode 100644 src/mongo/db/repl/sync_source_feedback.h create mode 100644 src/mongo/db/repl/write_concern.cpp create mode 100644 src/mongo/db/repl/write_concern.h delete mode 100644 src/mongo/db/repl_block.cpp delete mode 100644 src/mongo/db/repl_block.h delete mode 100644 src/mongo/db/replutil.h delete mode 100644 src/mongo/db/scanandorder.cpp delete mode 100644 src/mongo/db/scanandorder.h create mode 100644 src/mongo/db/server_options.cpp create mode 100644 src/mongo/db/server_options.h create mode 100644 src/mongo/db/server_options_helpers.cpp create mode 100644 src/mongo/db/server_options_helpers.h create mode 100644 src/mongo/db/server_options_test.cpp create mode 100644 src/mongo/db/sorter/SConscript create mode 100644 src/mongo/db/sorter/sorter.cpp create mode 100644 src/mongo/db/sorter/sorter.h create mode 100644 src/mongo/db/sorter/sorter_test.cpp create mode 100644 src/mongo/db/startup_warnings.cpp create mode 100644 src/mongo/db/startup_warnings.h create mode 100644 src/mongo/db/stats/snapshots_webplugins.cpp create mode 100644 src/mongo/db/storage/data_file.cpp create mode 100644 src/mongo/db/storage/data_file.h create mode 100644 src/mongo/db/storage/durable_mapped_file.cpp create mode 100644 src/mongo/db/storage/durable_mapped_file.h create mode 100644 src/mongo/db/storage/extent.cpp create mode 100644 src/mongo/db/storage/extent.h create mode 100644 src/mongo/db/storage/extent_manager.cpp create mode 100644 src/mongo/db/storage/extent_manager.h create mode 100644 src/mongo/db/storage/record.cpp create mode 100644 src/mongo/db/storage/record.h create mode 100644 src/mongo/db/storage/storage_engine_metadata.cpp create mode 100644 src/mongo/db/storage/storage_engine_metadata.h create mode 100644 src/mongo/db/storage/storage_engine_metadata_test.cpp create mode 100644 src/mongo/db/storage_options.cpp create mode 100644 src/mongo/db/storage_options.h create mode 100644 src/mongo/db/structure/btree/btree.cpp create mode 100644 src/mongo/db/structure/btree/btree.h create mode 100644 src/mongo/db/structure/btree/btree_stats.cpp create mode 100644 src/mongo/db/structure/btree/btree_stats.h create mode 100644 src/mongo/db/structure/btree/btreebuilder.cpp create mode 100644 src/mongo/db/structure/btree/btreebuilder.h create mode 100644 src/mongo/db/structure/btree/key.cpp create mode 100644 src/mongo/db/structure/btree/key.h create mode 100644 src/mongo/db/structure/catalog/cap.cpp create mode 100644 src/mongo/db/structure/catalog/hashtab.h create mode 100644 src/mongo/db/structure/catalog/index_details.cpp create mode 100644 src/mongo/db/structure/catalog/index_details.h create mode 100644 src/mongo/db/structure/catalog/namespace-inl.h create mode 100644 src/mongo/db/structure/catalog/namespace.cpp create mode 100644 src/mongo/db/structure/catalog/namespace.h create mode 100644 src/mongo/db/structure/catalog/namespace_details-inl.h create mode 100644 src/mongo/db/structure/catalog/namespace_details.cpp create mode 100644 src/mongo/db/structure/catalog/namespace_details.h create mode 100644 src/mongo/db/structure/catalog/namespace_index.cpp create mode 100644 src/mongo/db/structure/catalog/namespace_index.h create mode 100644 src/mongo/db/structure/catalog/namespace_test.cpp create mode 100644 src/mongo/db/structure/collection_compact.cpp create mode 100644 src/mongo/db/structure/collection_iterator.cpp create mode 100644 src/mongo/db/structure/collection_iterator.h create mode 100644 src/mongo/db/structure/record_store.cpp create mode 100644 src/mongo/db/structure/record_store.h create mode 100644 src/mongo/db/update_index_data.cpp create mode 100644 src/mongo/db/update_index_data.h create mode 100644 src/mongo/db/update_index_data_test.cpp create mode 100644 src/mongo/db/wire_version.h create mode 100644 src/mongo/db/write_concern.cpp create mode 100644 src/mongo/db/write_concern.h create mode 100644 src/mongo/db/write_concern_options.cpp create mode 100644 src/mongo/db/write_concern_options.h delete mode 100644 src/mongo/dbtests/background_job_test.cpp delete mode 100644 src/mongo/dbtests/btreepositiontests.cpp create mode 100644 src/mongo/dbtests/chunk_manager_targeter_test.cpp delete mode 100644 src/mongo/dbtests/cursortests.cpp delete mode 100644 src/mongo/dbtests/d_chunk_manager_tests.cpp create mode 100644 src/mongo/dbtests/dbclient_multi_command_test.cpp create mode 100644 src/mongo/dbtests/dbhelper_tests.cpp create mode 100644 src/mongo/dbtests/framework_options.cpp create mode 100644 src/mongo/dbtests/framework_options.h create mode 100644 src/mongo/dbtests/framework_options_init.cpp create mode 100644 src/mongo/dbtests/indexcatalogtests.cpp delete mode 100644 src/mongo/dbtests/intervalbtreecursortests.cpp delete mode 100644 src/mongo/dbtests/jsobjhashingtests.cpp create mode 100644 src/mongo/dbtests/merge_chunk_tests.cpp create mode 100644 src/mongo/dbtests/oplogstarttests.cpp create mode 100644 src/mongo/dbtests/plan_ranking.cpp create mode 100644 src/mongo/dbtests/query_multi_plan_runner.cpp create mode 100644 src/mongo/dbtests/query_single_solution_runner.cpp create mode 100644 src/mongo/dbtests/query_stage_and.cpp create mode 100644 src/mongo/dbtests/query_stage_collscan.cpp create mode 100644 src/mongo/dbtests/query_stage_count.cpp create mode 100644 src/mongo/dbtests/query_stage_distinct.cpp create mode 100644 src/mongo/dbtests/query_stage_fetch.cpp create mode 100644 src/mongo/dbtests/query_stage_ixscan.cpp create mode 100644 src/mongo/dbtests/query_stage_keep.cpp create mode 100644 src/mongo/dbtests/query_stage_limit_skip.cpp create mode 100644 src/mongo/dbtests/query_stage_merge_sort.cpp create mode 100644 src/mongo/dbtests/query_stage_sort.cpp create mode 100644 src/mongo/dbtests/query_stage_tests.cpp create mode 100644 src/mongo/dbtests/query_subplan_runner.cpp delete mode 100644 src/mongo/dbtests/queryoptimizercursortests.cpp delete mode 100644 src/mongo/dbtests/queryoptimizertests.cpp delete mode 100644 src/mongo/dbtests/queryoptimizertests2.cpp delete mode 100644 src/mongo/dbtests/removetests.cpp create mode 100644 src/mongo/dbtests/runner_registry.cpp delete mode 100644 src/mongo/dbtests/spin_lock_test.cpp delete mode 100644 src/mongo/dbtests/test_sm.sln delete mode 100644 src/mongo/dbtests/test_sm.vcxproj delete mode 100644 src/mongo/dbtests/test_sm.vcxproj.filters create mode 100644 src/mongo/installer/msi/SConscript create mode 100644 src/mongo/installer/msi/wxs/BinaryFragment.wxs create mode 100644 src/mongo/installer/msi/wxs/FeatureFragment.wxs create mode 100644 src/mongo/installer/msi/wxs/Installer.wxs create mode 100644 src/mongo/installer/msi/wxs/Installer_64.wxs create mode 100644 src/mongo/installer/msi/wxs/LicensingFragment.wxs create mode 100644 src/mongo/logger/SConscript create mode 100644 src/mongo/logger/appender.h create mode 100644 src/mongo/logger/console.cpp create mode 100644 src/mongo/logger/console.h create mode 100644 src/mongo/logger/console_appender.h create mode 100644 src/mongo/logger/console_test.cpp create mode 100644 src/mongo/logger/encoder.h create mode 100644 src/mongo/logger/labeled_level.h create mode 100644 src/mongo/logger/log_domain-impl.h create mode 100644 src/mongo/logger/log_domain.h create mode 100644 src/mongo/logger/log_manager.cpp create mode 100644 src/mongo/logger/log_manager.h create mode 100644 src/mongo/logger/log_severity-inl.h create mode 100644 src/mongo/logger/log_severity.cpp create mode 100644 src/mongo/logger/log_severity.h create mode 100644 src/mongo/logger/log_test.cpp create mode 100644 src/mongo/logger/logger.cpp create mode 100644 src/mongo/logger/logger.h create mode 100644 src/mongo/logger/logstream_builder.cpp create mode 100644 src/mongo/logger/logstream_builder.h create mode 100644 src/mongo/logger/message_event.h create mode 100644 src/mongo/logger/message_event_utf8_encoder.cpp create mode 100644 src/mongo/logger/message_event_utf8_encoder.h create mode 100644 src/mongo/logger/message_log_domain.cpp create mode 100644 src/mongo/logger/message_log_domain.h create mode 100644 src/mongo/logger/ramlog.cpp create mode 100644 src/mongo/logger/ramlog.h create mode 100644 src/mongo/logger/rotatable_file_appender.h create mode 100644 src/mongo/logger/rotatable_file_manager.cpp create mode 100644 src/mongo/logger/rotatable_file_manager.h create mode 100644 src/mongo/logger/rotatable_file_writer.cpp create mode 100644 src/mongo/logger/rotatable_file_writer.h create mode 100644 src/mongo/logger/rotatable_file_writer_test.cpp create mode 100644 src/mongo/logger/syslog_appender.h create mode 100644 src/mongo/logger/tee.h delete mode 100644 src/mongo/platform/atomic_intrinsics_gcc.h create mode 100644 src/mongo/platform/atomic_intrinsics_gcc_generic.h create mode 100644 src/mongo/platform/atomic_intrinsics_gcc_intel.h create mode 100644 src/mongo/platform/hash_namespace.h create mode 100644 src/mongo/platform/process_id.cpp create mode 100644 src/mongo/platform/process_id.h create mode 100644 src/mongo/platform/process_id_test.cpp create mode 100644 src/mongo/s/bson_serializable.h create mode 100644 src/mongo/s/chunk_diff-inl.cpp delete mode 100644 src/mongo/s/chunk_diff.hpp create mode 100644 src/mongo/s/chunk_manager_targeter.cpp create mode 100644 src/mongo/s/chunk_manager_targeter.h create mode 100644 src/mongo/s/cluster_write.cpp create mode 100644 src/mongo/s/cluster_write.h delete mode 100644 src/mongo/s/collection_manager.cpp delete mode 100644 src/mongo/s/collection_manager.h delete mode 100644 src/mongo/s/collection_manager_test.cpp create mode 100644 src/mongo/s/collection_metadata.cpp create mode 100644 src/mongo/s/collection_metadata.h create mode 100644 src/mongo/s/collection_metadata_test.cpp create mode 100644 src/mongo/s/commands/auth_schema_upgrade_s.cpp create mode 100644 src/mongo/s/commands/cluster_index_filter_cmd.cpp create mode 100644 src/mongo/s/commands/cluster_merge_chunks_cmd.cpp create mode 100644 src/mongo/s/commands/cluster_plan_cache_cmd.cpp create mode 100644 src/mongo/s/commands/cluster_write_cmd.cpp create mode 100644 src/mongo/s/config_server_tests.cpp delete mode 100644 src/mongo/s/config_upgrade_v0_to_v4.cpp create mode 100644 src/mongo/s/config_upgrade_v0_to_v5.cpp delete mode 100644 src/mongo/s/config_upgrade_v3_to_v4.cpp create mode 100644 src/mongo/s/config_upgrade_v4_to_v5.cpp delete mode 100644 src/mongo/s/d_chunk_manager.cpp delete mode 100644 src/mongo/s/d_chunk_manager.h create mode 100644 src/mongo/s/d_merge.cpp create mode 100644 src/mongo/s/d_merge.h create mode 100644 src/mongo/s/dbclient_multi_command.cpp create mode 100644 src/mongo/s/dbclient_multi_command.h create mode 100644 src/mongo/s/dbclient_shard_resolver.cpp create mode 100644 src/mongo/s/dbclient_shard_resolver.h create mode 100644 src/mongo/s/distlock.cpp create mode 100644 src/mongo/s/distlock.h create mode 100644 src/mongo/s/distlock_test.cpp delete mode 100644 src/mongo/s/field_parser-inl.h delete mode 100644 src/mongo/s/field_parser.cpp delete mode 100644 src/mongo/s/field_parser.h delete mode 100644 src/mongo/s/field_parser_test.cpp create mode 100644 src/mongo/s/mock_multi_write_command.h create mode 100644 src/mongo/s/mock_ns_targeter.h create mode 100644 src/mongo/s/mock_shard_resolver.h create mode 100644 src/mongo/s/mongos_options.cpp create mode 100644 src/mongo/s/mongos_options.h create mode 100644 src/mongo/s/mongos_options_init.cpp create mode 100644 src/mongo/s/mongos_persistence_stubs.cpp delete mode 100644 src/mongo/s/mongos_sm.sln delete mode 100644 src/mongo/s/mongos_sm.vcxproj delete mode 100644 src/mongo/s/mongos_sm.vcxproj.filters create mode 100644 src/mongo/s/multi_command_dispatch.h create mode 100644 src/mongo/s/multi_host_query.cpp create mode 100644 src/mongo/s/multi_host_query.h create mode 100644 src/mongo/s/multi_host_query_test.cpp create mode 100644 src/mongo/s/ns_targeter.h create mode 100644 src/mongo/s/range_arithmetic.cpp create mode 100644 src/mongo/s/range_arithmetic.h create mode 100644 src/mongo/s/range_arithmetic_test.cpp create mode 100644 src/mongo/s/scc_fast_query_handler.cpp create mode 100644 src/mongo/s/scc_fast_query_handler.h create mode 100644 src/mongo/s/shard_key_pattern.cpp create mode 100644 src/mongo/s/shard_key_pattern.h create mode 100644 src/mongo/s/shard_resolver.h delete mode 100644 src/mongo/s/strategy_shard.cpp delete mode 100644 src/mongo/s/strategy_single.cpp create mode 100644 src/mongo/s/version_mongos.cpp create mode 100644 src/mongo/s/version_mongos.h create mode 100644 src/mongo/s/write_ops/batch_downconvert.cpp create mode 100644 src/mongo/s/write_ops/batch_downconvert.h create mode 100644 src/mongo/s/write_ops/batch_downconvert_test.cpp create mode 100644 src/mongo/s/write_ops/batch_upconvert.cpp create mode 100644 src/mongo/s/write_ops/batch_upconvert.h create mode 100644 src/mongo/s/write_ops/batch_upconvert_test.cpp create mode 100644 src/mongo/s/write_ops/batch_write_exec.cpp create mode 100644 src/mongo/s/write_ops/batch_write_exec.h create mode 100644 src/mongo/s/write_ops/batch_write_exec_test.cpp create mode 100644 src/mongo/s/write_ops/batch_write_op.cpp create mode 100644 src/mongo/s/write_ops/batch_write_op.h create mode 100644 src/mongo/s/write_ops/batch_write_op_test.cpp create mode 100644 src/mongo/s/write_ops/batched_command_request.cpp create mode 100644 src/mongo/s/write_ops/batched_command_request.h create mode 100644 src/mongo/s/write_ops/batched_command_response.cpp create mode 100644 src/mongo/s/write_ops/batched_command_response.h create mode 100644 src/mongo/s/write_ops/batched_command_response_test.cpp create mode 100644 src/mongo/s/write_ops/batched_delete_document.cpp create mode 100644 src/mongo/s/write_ops/batched_delete_document.h create mode 100644 src/mongo/s/write_ops/batched_delete_request.cpp create mode 100644 src/mongo/s/write_ops/batched_delete_request.h create mode 100644 src/mongo/s/write_ops/batched_delete_request_test.cpp create mode 100644 src/mongo/s/write_ops/batched_insert_request.cpp create mode 100644 src/mongo/s/write_ops/batched_insert_request.h create mode 100644 src/mongo/s/write_ops/batched_insert_request_test.cpp create mode 100644 src/mongo/s/write_ops/batched_request_metadata.cpp create mode 100644 src/mongo/s/write_ops/batched_request_metadata.h create mode 100644 src/mongo/s/write_ops/batched_request_metadata_test.cpp create mode 100644 src/mongo/s/write_ops/batched_update_document.cpp create mode 100644 src/mongo/s/write_ops/batched_update_document.h create mode 100644 src/mongo/s/write_ops/batched_update_request.cpp create mode 100644 src/mongo/s/write_ops/batched_update_request.h create mode 100644 src/mongo/s/write_ops/batched_update_request_test.cpp create mode 100644 src/mongo/s/write_ops/batched_upsert_detail.cpp create mode 100644 src/mongo/s/write_ops/batched_upsert_detail.h create mode 100644 src/mongo/s/write_ops/config_coordinator.cpp create mode 100644 src/mongo/s/write_ops/config_coordinator.h create mode 100644 src/mongo/s/write_ops/config_coordinator_test.cpp create mode 100644 src/mongo/s/write_ops/dbclient_safe_writer.cpp create mode 100644 src/mongo/s/write_ops/dbclient_safe_writer.h create mode 100644 src/mongo/s/write_ops/wc_error_detail.cpp create mode 100644 src/mongo/s/write_ops/wc_error_detail.h create mode 100644 src/mongo/s/write_ops/write_error_detail.cpp create mode 100644 src/mongo/s/write_ops/write_error_detail.h create mode 100644 src/mongo/s/write_ops/write_op.cpp create mode 100644 src/mongo/s/write_ops/write_op.h create mode 100644 src/mongo/s/write_ops/write_op_test.cpp delete mode 100644 src/mongo/scripting/engine_spidermonkey.cpp delete mode 100644 src/mongo/scripting/engine_spidermonkey.h delete mode 100644 src/mongo/scripting/engine_spidermonkey_internal.h delete mode 100644 src/mongo/scripting/sm_db.cpp create mode 100644 src/mongo/shell/bulk_api.js delete mode 100644 src/mongo/shell/mongo_sm.sln delete mode 100644 src/mongo/shell/mongo_sm.vcxproj delete mode 100644 src/mongo/shell/mongo_sm.vcxproj.filters create mode 100644 src/mongo/shell/shell_options.cpp create mode 100644 src/mongo/shell/shell_options.h create mode 100644 src/mongo/shell/shell_options_init.cpp create mode 100644 src/mongo/shell/upgrade_check.js delete mode 100644 src/mongo/targetver.h create mode 100644 src/mongo/tools/bsondump_options.cpp create mode 100644 src/mongo/tools/bsondump_options.h create mode 100644 src/mongo/tools/bsondump_options_init.cpp delete mode 100644 src/mongo/tools/docgenerator.cpp delete mode 100644 src/mongo/tools/docgenerator.h delete mode 100644 src/mongo/tools/docgeneratormain.cpp delete mode 100644 src/mongo/tools/loadgenerator.cpp create mode 100644 src/mongo/tools/mongobridge_options.cpp create mode 100644 src/mongo/tools/mongobridge_options.h create mode 100644 src/mongo/tools/mongobridge_options_init.cpp create mode 100644 src/mongo/tools/mongodump_options.cpp create mode 100644 src/mongo/tools/mongodump_options.h create mode 100644 src/mongo/tools/mongodump_options_init.cpp create mode 100644 src/mongo/tools/mongoexport_options.cpp create mode 100644 src/mongo/tools/mongoexport_options.h create mode 100644 src/mongo/tools/mongoexport_options_init.cpp create mode 100644 src/mongo/tools/mongofiles_options.cpp create mode 100644 src/mongo/tools/mongofiles_options.h create mode 100644 src/mongo/tools/mongofiles_options_init.cpp create mode 100644 src/mongo/tools/mongoimport_options.cpp create mode 100644 src/mongo/tools/mongoimport_options.h create mode 100644 src/mongo/tools/mongoimport_options_init.cpp create mode 100644 src/mongo/tools/mongooplog_options.cpp create mode 100644 src/mongo/tools/mongooplog_options.h create mode 100644 src/mongo/tools/mongooplog_options_init.cpp create mode 100644 src/mongo/tools/mongorestore_options.cpp create mode 100644 src/mongo/tools/mongorestore_options.h create mode 100644 src/mongo/tools/mongorestore_options_init.cpp create mode 100644 src/mongo/tools/mongostat_options.cpp create mode 100644 src/mongo/tools/mongostat_options.h create mode 100644 src/mongo/tools/mongostat_options_init.cpp create mode 100644 src/mongo/tools/mongotop_options.cpp create mode 100644 src/mongo/tools/mongotop_options.h create mode 100644 src/mongo/tools/mongotop_options_init.cpp create mode 100644 src/mongo/tools/tool_logger.cpp create mode 100644 src/mongo/tools/tool_logger.h create mode 100644 src/mongo/tools/tool_options.cpp create mode 100644 src/mongo/tools/tool_options.h create mode 100644 src/mongo/unittest/temp_dir.cpp create mode 100644 src/mongo/unittest/temp_dir.h create mode 100644 src/mongo/unittest/temp_dir_test.cpp create mode 100644 src/mongo/util/background_job_test.cpp create mode 100644 src/mongo/util/cmdline_utils/SConscript create mode 100644 src/mongo/util/cmdline_utils/censor_cmdline.cpp create mode 100644 src/mongo/util/cmdline_utils/censor_cmdline.h create mode 100644 src/mongo/util/cmdline_utils/censor_cmdline_test.cpp create mode 100644 src/mongo/util/concurrency/SConscript create mode 100644 src/mongo/util/concurrency/spin_lock_test.cpp create mode 100644 src/mongo/util/concurrency/thread_name.cpp create mode 100644 src/mongo/util/concurrency/thread_name.h create mode 100644 src/mongo/util/gcov.h delete mode 100644 src/mongo/util/hashtab.h create mode 100644 src/mongo/util/hex.cpp delete mode 100644 src/mongo/util/lame_stacktrace_test.cpp create mode 100644 src/mongo/util/net/socket_poll.cpp create mode 100644 src/mongo/util/net/socket_poll.h create mode 100644 src/mongo/util/net/ssl_options.cpp create mode 100644 src/mongo/util/net/ssl_options.h delete mode 100644 src/mongo/util/optime.h create mode 100644 src/mongo/util/options_parser/SConscript create mode 100644 src/mongo/util/options_parser/constraints.cpp create mode 100644 src/mongo/util/options_parser/constraints.h create mode 100644 src/mongo/util/options_parser/environment.cpp create mode 100644 src/mongo/util/options_parser/environment.h create mode 100644 src/mongo/util/options_parser/environment_test.cpp create mode 100644 src/mongo/util/options_parser/option_description.cpp create mode 100644 src/mongo/util/options_parser/option_description.h create mode 100644 src/mongo/util/options_parser/option_section.cpp create mode 100644 src/mongo/util/options_parser/option_section.h create mode 100644 src/mongo/util/options_parser/options_parser.cpp create mode 100644 src/mongo/util/options_parser/options_parser.h create mode 100644 src/mongo/util/options_parser/options_parser_init.cpp create mode 100644 src/mongo/util/options_parser/options_parser_test.cpp create mode 100644 src/mongo/util/options_parser/startup_option_init.cpp create mode 100644 src/mongo/util/options_parser/startup_option_init.h create mode 100644 src/mongo/util/options_parser/startup_options.cpp create mode 100644 src/mongo/util/options_parser/startup_options.h create mode 100644 src/mongo/util/options_parser/test_config_files/empty.json create mode 100644 src/mongo/util/options_parser/test_config_files/good.conf create mode 100644 src/mongo/util/options_parser/test_config_files/good.json create mode 100644 src/mongo/util/options_parser/value.cpp create mode 100644 src/mongo/util/options_parser/value.h create mode 100644 src/mongo/util/password_digest.cpp create mode 100644 src/mongo/util/password_digest.h create mode 100644 src/mongo/util/paths.cpp delete mode 100644 src/mongo/util/ramlog.cpp create mode 100644 src/mongo/util/safe_num-inl.h create mode 100644 src/mongo/util/signal_win32.cpp create mode 100644 src/mongo/util/signal_win32.h delete mode 100644 src/mongo/util/string_map_internal.h delete mode 100755 src/mongo/util/string_writer.h delete mode 100755 src/mongo/util/systeminfo.h delete mode 100755 src/mongo/util/systeminfo_linux2.cpp delete mode 100755 src/mongo/util/systeminfo_none.cpp delete mode 100755 src/mongo/util/systeminfo_win32.cpp create mode 100644 src/mongo/util/tcmalloc_server_status_section.cpp create mode 100644 src/mongo/util/time_support_test.cpp create mode 100644 src/mongo/util/version_reporting.cpp create mode 100644 src/mongo/util/version_reporting.h create mode 100644 src/mongo/util/version_test.cpp delete mode 100644 src/mongodb.sln delete mode 100644 src/mongodb_sm.sln delete mode 100644 src/third_party/js-1.7/Makefile.in delete mode 100644 src/third_party/js-1.7/Makefile.ref delete mode 100644 src/third_party/js-1.7/README.html delete mode 100644 src/third_party/js-1.7/SConscript delete mode 100644 src/third_party/js-1.7/SpiderMonkey-prebuild.vcxproj delete mode 100644 src/third_party/js-1.7/SpiderMonkey-prebuild.vcxproj.filters delete mode 100644 src/third_party/js-1.7/Y.js delete mode 100644 src/third_party/js-1.7/config.mk delete mode 100644 src/third_party/js-1.7/config/AIX4.1.mk delete mode 100644 src/third_party/js-1.7/config/AIX4.2.mk delete mode 100644 src/third_party/js-1.7/config/AIX4.3.mk delete mode 100644 src/third_party/js-1.7/config/CVS/Entries delete mode 100644 src/third_party/js-1.7/config/CVS/Repository delete mode 100644 src/third_party/js-1.7/config/CVS/Root delete mode 100644 src/third_party/js-1.7/config/CVS/Tag delete mode 100644 src/third_party/js-1.7/config/Darwin.mk delete mode 100755 src/third_party/js-1.7/config/Darwin1.3.mk delete mode 100755 src/third_party/js-1.7/config/Darwin1.4.mk delete mode 100755 src/third_party/js-1.7/config/Darwin5.2.mk delete mode 100644 src/third_party/js-1.7/config/Darwin5.3.mk delete mode 100644 src/third_party/js-1.7/config/HP-UXB.10.10.mk delete mode 100644 src/third_party/js-1.7/config/HP-UXB.10.20.mk delete mode 100644 src/third_party/js-1.7/config/HP-UXB.11.00.mk delete mode 100644 src/third_party/js-1.7/config/IRIX.mk delete mode 100644 src/third_party/js-1.7/config/IRIX5.3.mk delete mode 100644 src/third_party/js-1.7/config/IRIX6.1.mk delete mode 100644 src/third_party/js-1.7/config/IRIX6.2.mk delete mode 100644 src/third_party/js-1.7/config/IRIX6.3.mk delete mode 100644 src/third_party/js-1.7/config/IRIX6.5.mk delete mode 100644 src/third_party/js-1.7/config/Linux_All.mk delete mode 100755 src/third_party/js-1.7/config/Mac_OS10.0.mk delete mode 100644 src/third_party/js-1.7/config/OSF1V4.0.mk delete mode 100644 src/third_party/js-1.7/config/OSF1V5.0.mk delete mode 100644 src/third_party/js-1.7/config/SunOS4.1.4.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.3.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.4.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.5.1.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.5.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.6.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.7.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.8.mk delete mode 100644 src/third_party/js-1.7/config/SunOS5.9.mk delete mode 100644 src/third_party/js-1.7/config/WINNT4.0.mk delete mode 100644 src/third_party/js-1.7/config/WINNT5.0.mk delete mode 100644 src/third_party/js-1.7/config/WINNT5.1.mk delete mode 100644 src/third_party/js-1.7/config/WINNT5.2.mk delete mode 100644 src/third_party/js-1.7/config/dgux.mk delete mode 100644 src/third_party/js-1.7/fdlibm/.cvsignore delete mode 100644 src/third_party/js-1.7/fdlibm/CVS/Entries delete mode 100644 src/third_party/js-1.7/fdlibm/CVS/Repository delete mode 100644 src/third_party/js-1.7/fdlibm/CVS/Root delete mode 100644 src/third_party/js-1.7/fdlibm/CVS/Tag delete mode 100644 src/third_party/js-1.7/fdlibm/Makefile.in delete mode 100644 src/third_party/js-1.7/fdlibm/Makefile.ref delete mode 100644 src/third_party/js-1.7/fdlibm/e_acos.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_acosh.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_asin.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_atan2.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_atanh.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_cosh.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_exp.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_fmod.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_gamma.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_gamma_r.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_hypot.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_j0.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_j1.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_jn.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_lgamma.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_lgamma_r.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_log.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_log10.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_pow.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_rem_pio2.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_remainder.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_scalb.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_sinh.c delete mode 100644 src/third_party/js-1.7/fdlibm/e_sqrt.c delete mode 100644 src/third_party/js-1.7/fdlibm/fdlibm.h delete mode 100644 src/third_party/js-1.7/fdlibm/fdlibm.mak delete mode 100644 src/third_party/js-1.7/fdlibm/fdlibm.mdp delete mode 100644 src/third_party/js-1.7/fdlibm/k_cos.c delete mode 100644 src/third_party/js-1.7/fdlibm/k_rem_pio2.c delete mode 100644 src/third_party/js-1.7/fdlibm/k_sin.c delete mode 100644 src/third_party/js-1.7/fdlibm/k_standard.c delete mode 100644 src/third_party/js-1.7/fdlibm/k_tan.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_asinh.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_atan.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_cbrt.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_ceil.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_copysign.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_cos.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_erf.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_expm1.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_fabs.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_finite.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_floor.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_frexp.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_ilogb.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_isnan.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_ldexp.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_lib_version.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_log1p.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_logb.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_matherr.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_modf.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_nextafter.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_rint.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_scalbn.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_signgam.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_significand.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_sin.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_tan.c delete mode 100644 src/third_party/js-1.7/fdlibm/s_tanh.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_acos.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_acosh.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_asin.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_atan2.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_atanh.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_cosh.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_exp.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_fmod.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_gamma.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_gamma_r.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_hypot.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_j0.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_j1.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_jn.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_lgamma.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_lgamma_r.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_log.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_log10.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_pow.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_remainder.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_scalb.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_sinh.c delete mode 100644 src/third_party/js-1.7/fdlibm/w_sqrt.c delete mode 100644 src/third_party/js-1.7/js.c delete mode 100644 src/third_party/js-1.7/js.mak delete mode 100644 src/third_party/js-1.7/js.mdp delete mode 100644 src/third_party/js-1.7/js.msg delete mode 100644 src/third_party/js-1.7/js.pkg delete mode 100644 src/third_party/js-1.7/js3240.rc delete mode 100644 src/third_party/js-1.7/jsOS240.def delete mode 100644 src/third_party/js-1.7/jsapi.c delete mode 100644 src/third_party/js-1.7/jsapi.h delete mode 100644 src/third_party/js-1.7/jsarena.c delete mode 100644 src/third_party/js-1.7/jsarena.h delete mode 100644 src/third_party/js-1.7/jsarray.c delete mode 100644 src/third_party/js-1.7/jsarray.h delete mode 100644 src/third_party/js-1.7/jsatom.c delete mode 100644 src/third_party/js-1.7/jsatom.h delete mode 100644 src/third_party/js-1.7/jsbit.h delete mode 100644 src/third_party/js-1.7/jsbool.c delete mode 100644 src/third_party/js-1.7/jsbool.h delete mode 100644 src/third_party/js-1.7/jsclist.h delete mode 100644 src/third_party/js-1.7/jscntxt.c delete mode 100644 src/third_party/js-1.7/jscntxt.h delete mode 100644 src/third_party/js-1.7/jscompat.h delete mode 100644 src/third_party/js-1.7/jsconfig.h delete mode 100644 src/third_party/js-1.7/jsconfig.mk delete mode 100644 src/third_party/js-1.7/jscpucfg.c delete mode 100644 src/third_party/js-1.7/jscpucfg.h delete mode 100644 src/third_party/js-1.7/jsdate.c delete mode 100644 src/third_party/js-1.7/jsdate.h delete mode 100644 src/third_party/js-1.7/jsdbgapi.c delete mode 100644 src/third_party/js-1.7/jsdbgapi.h delete mode 100644 src/third_party/js-1.7/jsdhash.c delete mode 100644 src/third_party/js-1.7/jsdhash.h delete mode 100644 src/third_party/js-1.7/jsdtoa.c delete mode 100644 src/third_party/js-1.7/jsdtoa.h delete mode 100644 src/third_party/js-1.7/jsemit.c delete mode 100644 src/third_party/js-1.7/jsemit.h delete mode 100644 src/third_party/js-1.7/jsexn.c delete mode 100644 src/third_party/js-1.7/jsexn.h delete mode 100644 src/third_party/js-1.7/jsfile.c delete mode 100644 src/third_party/js-1.7/jsfile.h delete mode 100644 src/third_party/js-1.7/jsfile.msg delete mode 100644 src/third_party/js-1.7/jsfun.c delete mode 100644 src/third_party/js-1.7/jsfun.h delete mode 100644 src/third_party/js-1.7/jsgc.c delete mode 100644 src/third_party/js-1.7/jsgc.h delete mode 100644 src/third_party/js-1.7/jshash.c delete mode 100644 src/third_party/js-1.7/jshash.h delete mode 100644 src/third_party/js-1.7/jsify.pl delete mode 100644 src/third_party/js-1.7/jsinterp.c delete mode 100644 src/third_party/js-1.7/jsinterp.h delete mode 100644 src/third_party/js-1.7/jsiter.c delete mode 100644 src/third_party/js-1.7/jsiter.h delete mode 100644 src/third_party/js-1.7/jskeyword.tbl delete mode 100644 src/third_party/js-1.7/jskwgen.c delete mode 100644 src/third_party/js-1.7/jslibmath.h delete mode 100644 src/third_party/js-1.7/jslock.c delete mode 100644 src/third_party/js-1.7/jslock.h delete mode 100644 src/third_party/js-1.7/jslocko.asm delete mode 100644 src/third_party/js-1.7/jslog2.c delete mode 100644 src/third_party/js-1.7/jslong.c delete mode 100644 src/third_party/js-1.7/jslong.h delete mode 100644 src/third_party/js-1.7/jsmath.c delete mode 100644 src/third_party/js-1.7/jsmath.h delete mode 100644 src/third_party/js-1.7/jsnum.c delete mode 100644 src/third_party/js-1.7/jsnum.h delete mode 100644 src/third_party/js-1.7/jsobj.c delete mode 100644 src/third_party/js-1.7/jsobj.h delete mode 100644 src/third_party/js-1.7/jsopcode.c delete mode 100644 src/third_party/js-1.7/jsopcode.h delete mode 100644 src/third_party/js-1.7/jsopcode.tbl delete mode 100644 src/third_party/js-1.7/jsosdep.h delete mode 100644 src/third_party/js-1.7/jsotypes.h delete mode 100644 src/third_party/js-1.7/jsparse.c delete mode 100644 src/third_party/js-1.7/jsparse.h delete mode 100644 src/third_party/js-1.7/jsprf.c delete mode 100644 src/third_party/js-1.7/jsprf.h delete mode 100644 src/third_party/js-1.7/jsproto.tbl delete mode 100644 src/third_party/js-1.7/jsprvtd.h delete mode 100644 src/third_party/js-1.7/jspubtd.h delete mode 100644 src/third_party/js-1.7/jsregexp.c delete mode 100644 src/third_party/js-1.7/jsregexp.h delete mode 100644 src/third_party/js-1.7/jsscan.c delete mode 100644 src/third_party/js-1.7/jsscan.h delete mode 100644 src/third_party/js-1.7/jsscope.c delete mode 100644 src/third_party/js-1.7/jsscope.h delete mode 100644 src/third_party/js-1.7/jsscript.c delete mode 100644 src/third_party/js-1.7/jsscript.h delete mode 100644 src/third_party/js-1.7/jsshell.msg delete mode 100644 src/third_party/js-1.7/jsstddef.h delete mode 100644 src/third_party/js-1.7/jsstr.c delete mode 100644 src/third_party/js-1.7/jsstr.h delete mode 100644 src/third_party/js-1.7/jstypes.h delete mode 100644 src/third_party/js-1.7/jsutil.c delete mode 100644 src/third_party/js-1.7/jsutil.h delete mode 100644 src/third_party/js-1.7/jsxdrapi.c delete mode 100644 src/third_party/js-1.7/jsxdrapi.h delete mode 100644 src/third_party/js-1.7/jsxml.c delete mode 100644 src/third_party/js-1.7/jsxml.h delete mode 100644 src/third_party/js-1.7/lock_SunOS.s delete mode 100644 src/third_party/js-1.7/perfect.js delete mode 100644 src/third_party/js-1.7/plify_jsdhash.sed delete mode 100644 src/third_party/js-1.7/prmjtime.c delete mode 100644 src/third_party/js-1.7/prmjtime.h delete mode 100644 src/third_party/js-1.7/resource.h delete mode 100644 src/third_party/js-1.7/rules.mk delete mode 100644 src/third_party/js-1.7/win32.order create mode 100755 src/third_party/pcre-8.36/132html create mode 100644 src/third_party/pcre-8.36/AUTHORS create mode 100644 src/third_party/pcre-8.36/CMakeLists.txt create mode 100644 src/third_party/pcre-8.36/COPYING create mode 100644 src/third_party/pcre-8.36/ChangeLog create mode 100755 src/third_party/pcre-8.36/CheckMan create mode 100755 src/third_party/pcre-8.36/CleanTxt create mode 100755 src/third_party/pcre-8.36/Detrail create mode 100644 src/third_party/pcre-8.36/HACKING create mode 100644 src/third_party/pcre-8.36/INSTALL create mode 100644 src/third_party/pcre-8.36/LICENCE create mode 100644 src/third_party/pcre-8.36/Makefile.am create mode 100644 src/third_party/pcre-8.36/Makefile.in create mode 100644 src/third_party/pcre-8.36/NEWS create mode 100644 src/third_party/pcre-8.36/NON-AUTOTOOLS-BUILD create mode 100644 src/third_party/pcre-8.36/NON-UNIX-USE create mode 100755 src/third_party/pcre-8.36/PrepareRelease create mode 100644 src/third_party/pcre-8.36/README create mode 100755 src/third_party/pcre-8.36/RunGrepTest create mode 100755 src/third_party/pcre-8.36/RunTest create mode 100644 src/third_party/pcre-8.36/RunTest.bat create mode 100644 src/third_party/pcre-8.36/SConscript create mode 100644 src/third_party/pcre-8.36/aclocal.m4 create mode 100755 src/third_party/pcre-8.36/ar-lib create mode 100644 src/third_party/pcre-8.36/build_posix/config.h create mode 100644 src/third_party/pcre-8.36/build_solaris/config.h create mode 100644 src/third_party/pcre-8.36/build_windows/config.h create mode 100755 src/third_party/pcre-8.36/compile create mode 100644 src/third_party/pcre-8.36/config-cmake.h.in create mode 100755 src/third_party/pcre-8.36/config.guess create mode 100644 src/third_party/pcre-8.36/config.h.generic create mode 100644 src/third_party/pcre-8.36/config.h.in create mode 100755 src/third_party/pcre-8.36/config.sub create mode 100755 src/third_party/pcre-8.36/configure create mode 100644 src/third_party/pcre-8.36/configure.ac create mode 100755 src/third_party/pcre-8.36/depcomp create mode 100644 src/third_party/pcre-8.36/dftables.c create mode 100755 src/third_party/pcre-8.36/install-sh create mode 100644 src/third_party/pcre-8.36/libpcre.pc.in create mode 100644 src/third_party/pcre-8.36/libpcre16.pc.in create mode 100644 src/third_party/pcre-8.36/libpcre32.pc.in create mode 100644 src/third_party/pcre-8.36/libpcrecpp.pc.in create mode 100644 src/third_party/pcre-8.36/libpcreposix.pc.in create mode 100644 src/third_party/pcre-8.36/ltmain.sh create mode 100644 src/third_party/pcre-8.36/makevp.bat create mode 100644 src/third_party/pcre-8.36/makevp_c.txt create mode 100644 src/third_party/pcre-8.36/makevp_l.txt create mode 100755 src/third_party/pcre-8.36/missing create mode 100644 src/third_party/pcre-8.36/pcre-config.in create mode 100644 src/third_party/pcre-8.36/pcre.h create mode 100644 src/third_party/pcre-8.36/pcre.h.generic create mode 100644 src/third_party/pcre-8.36/pcre.h.in create mode 100644 src/third_party/pcre-8.36/pcre16_byte_order.c create mode 100644 src/third_party/pcre-8.36/pcre16_chartables.c create mode 100644 src/third_party/pcre-8.36/pcre16_compile.c create mode 100644 src/third_party/pcre-8.36/pcre16_config.c create mode 100644 src/third_party/pcre-8.36/pcre16_dfa_exec.c create mode 100644 src/third_party/pcre-8.36/pcre16_exec.c create mode 100644 src/third_party/pcre-8.36/pcre16_fullinfo.c create mode 100644 src/third_party/pcre-8.36/pcre16_get.c create mode 100644 src/third_party/pcre-8.36/pcre16_globals.c create mode 100644 src/third_party/pcre-8.36/pcre16_jit_compile.c create mode 100644 src/third_party/pcre-8.36/pcre16_maketables.c create mode 100644 src/third_party/pcre-8.36/pcre16_newline.c create mode 100644 src/third_party/pcre-8.36/pcre16_ord2utf16.c create mode 100644 src/third_party/pcre-8.36/pcre16_printint.c create mode 100644 src/third_party/pcre-8.36/pcre16_refcount.c create mode 100644 src/third_party/pcre-8.36/pcre16_string_utils.c create mode 100644 src/third_party/pcre-8.36/pcre16_study.c create mode 100644 src/third_party/pcre-8.36/pcre16_tables.c create mode 100644 src/third_party/pcre-8.36/pcre16_ucd.c create mode 100644 src/third_party/pcre-8.36/pcre16_utf16_utils.c create mode 100644 src/third_party/pcre-8.36/pcre16_valid_utf16.c create mode 100644 src/third_party/pcre-8.36/pcre16_version.c create mode 100644 src/third_party/pcre-8.36/pcre16_xclass.c create mode 100644 src/third_party/pcre-8.36/pcre32_byte_order.c create mode 100644 src/third_party/pcre-8.36/pcre32_chartables.c create mode 100644 src/third_party/pcre-8.36/pcre32_compile.c create mode 100644 src/third_party/pcre-8.36/pcre32_config.c create mode 100644 src/third_party/pcre-8.36/pcre32_dfa_exec.c create mode 100644 src/third_party/pcre-8.36/pcre32_exec.c create mode 100644 src/third_party/pcre-8.36/pcre32_fullinfo.c create mode 100644 src/third_party/pcre-8.36/pcre32_get.c create mode 100644 src/third_party/pcre-8.36/pcre32_globals.c create mode 100644 src/third_party/pcre-8.36/pcre32_jit_compile.c create mode 100644 src/third_party/pcre-8.36/pcre32_maketables.c create mode 100644 src/third_party/pcre-8.36/pcre32_newline.c create mode 100644 src/third_party/pcre-8.36/pcre32_ord2utf32.c create mode 100644 src/third_party/pcre-8.36/pcre32_printint.c create mode 100644 src/third_party/pcre-8.36/pcre32_refcount.c create mode 100644 src/third_party/pcre-8.36/pcre32_string_utils.c create mode 100644 src/third_party/pcre-8.36/pcre32_study.c create mode 100644 src/third_party/pcre-8.36/pcre32_tables.c create mode 100644 src/third_party/pcre-8.36/pcre32_ucd.c create mode 100644 src/third_party/pcre-8.36/pcre32_utf32_utils.c create mode 100644 src/third_party/pcre-8.36/pcre32_valid_utf32.c create mode 100644 src/third_party/pcre-8.36/pcre32_version.c create mode 100644 src/third_party/pcre-8.36/pcre32_xclass.c create mode 100644 src/third_party/pcre-8.36/pcre_byte_order.c create mode 100644 src/third_party/pcre-8.36/pcre_chartables.c create mode 100644 src/third_party/pcre-8.36/pcre_chartables.c.dist create mode 100644 src/third_party/pcre-8.36/pcre_compile.c create mode 100644 src/third_party/pcre-8.36/pcre_config.c create mode 100644 src/third_party/pcre-8.36/pcre_dfa_exec.c create mode 100644 src/third_party/pcre-8.36/pcre_exec.c create mode 100644 src/third_party/pcre-8.36/pcre_fullinfo.c create mode 100644 src/third_party/pcre-8.36/pcre_get.c create mode 100644 src/third_party/pcre-8.36/pcre_globals.c create mode 100644 src/third_party/pcre-8.36/pcre_internal.h create mode 100644 src/third_party/pcre-8.36/pcre_jit_compile.c create mode 100644 src/third_party/pcre-8.36/pcre_jit_test.c create mode 100644 src/third_party/pcre-8.36/pcre_maketables.c create mode 100644 src/third_party/pcre-8.36/pcre_newline.c create mode 100644 src/third_party/pcre-8.36/pcre_ord2utf8.c create mode 100644 src/third_party/pcre-8.36/pcre_printint.c create mode 100644 src/third_party/pcre-8.36/pcre_refcount.c create mode 100644 src/third_party/pcre-8.36/pcre_scanner.cc create mode 100644 src/third_party/pcre-8.36/pcre_scanner.h create mode 100644 src/third_party/pcre-8.36/pcre_scanner_unittest.cc create mode 100644 src/third_party/pcre-8.36/pcre_string_utils.c create mode 100644 src/third_party/pcre-8.36/pcre_stringpiece.cc create mode 100644 src/third_party/pcre-8.36/pcre_stringpiece.h create mode 100644 src/third_party/pcre-8.36/pcre_stringpiece.h.in create mode 100644 src/third_party/pcre-8.36/pcre_stringpiece_unittest.cc create mode 100644 src/third_party/pcre-8.36/pcre_study.c create mode 100644 src/third_party/pcre-8.36/pcre_tables.c create mode 100644 src/third_party/pcre-8.36/pcre_ucd.c create mode 100644 src/third_party/pcre-8.36/pcre_valid_utf8.c create mode 100644 src/third_party/pcre-8.36/pcre_version.c create mode 100644 src/third_party/pcre-8.36/pcre_xclass.c create mode 100644 src/third_party/pcre-8.36/pcrecpp.cc create mode 100644 src/third_party/pcre-8.36/pcrecpp.h create mode 100644 src/third_party/pcre-8.36/pcrecpp_internal.h create mode 100644 src/third_party/pcre-8.36/pcrecpp_unittest.cc create mode 100644 src/third_party/pcre-8.36/pcrecpparg.h create mode 100644 src/third_party/pcre-8.36/pcrecpparg.h.in create mode 100644 src/third_party/pcre-8.36/pcredemo.c create mode 100644 src/third_party/pcre-8.36/pcregexp.pas create mode 100644 src/third_party/pcre-8.36/pcregrep.c create mode 100644 src/third_party/pcre-8.36/pcreposix.c create mode 100644 src/third_party/pcre-8.36/pcreposix.h create mode 100644 src/third_party/pcre-8.36/pcretest.c create mode 100755 src/third_party/pcre-8.36/perltest.pl create mode 100755 src/third_party/pcre-8.36/test-driver create mode 100644 src/third_party/pcre-8.36/ucp.h create mode 100755 src/third_party/pcre-8.37/132html create mode 100644 src/third_party/pcre-8.37/AUTHORS create mode 100644 src/third_party/pcre-8.37/CMakeLists.txt create mode 100644 src/third_party/pcre-8.37/COPYING create mode 100644 src/third_party/pcre-8.37/ChangeLog create mode 100755 src/third_party/pcre-8.37/CheckMan create mode 100755 src/third_party/pcre-8.37/CleanTxt create mode 100755 src/third_party/pcre-8.37/Detrail create mode 100644 src/third_party/pcre-8.37/HACKING create mode 100644 src/third_party/pcre-8.37/INSTALL create mode 100644 src/third_party/pcre-8.37/LICENCE create mode 100644 src/third_party/pcre-8.37/Makefile.am create mode 100644 src/third_party/pcre-8.37/Makefile.in create mode 100644 src/third_party/pcre-8.37/NEWS create mode 100644 src/third_party/pcre-8.37/NON-AUTOTOOLS-BUILD create mode 100644 src/third_party/pcre-8.37/NON-UNIX-USE create mode 100755 src/third_party/pcre-8.37/PrepareRelease create mode 100644 src/third_party/pcre-8.37/README create mode 100755 src/third_party/pcre-8.37/RunGrepTest create mode 100755 src/third_party/pcre-8.37/RunTest create mode 100644 src/third_party/pcre-8.37/RunTest.bat create mode 100644 src/third_party/pcre-8.37/SConscript create mode 100644 src/third_party/pcre-8.37/aclocal.m4 create mode 100755 src/third_party/pcre-8.37/ar-lib create mode 100644 src/third_party/pcre-8.37/build_posix/config.h create mode 100644 src/third_party/pcre-8.37/build_solaris/config.h create mode 100644 src/third_party/pcre-8.37/build_windows/config.h create mode 100755 src/third_party/pcre-8.37/compile create mode 100644 src/third_party/pcre-8.37/config-cmake.h.in create mode 100755 src/third_party/pcre-8.37/config.guess create mode 100644 src/third_party/pcre-8.37/config.h.generic create mode 100644 src/third_party/pcre-8.37/config.h.in create mode 100755 src/third_party/pcre-8.37/config.sub create mode 100755 src/third_party/pcre-8.37/configure create mode 100644 src/third_party/pcre-8.37/configure.ac create mode 100755 src/third_party/pcre-8.37/depcomp create mode 100644 src/third_party/pcre-8.37/dftables.c create mode 100755 src/third_party/pcre-8.37/install-sh create mode 100644 src/third_party/pcre-8.37/libpcre.pc.in create mode 100644 src/third_party/pcre-8.37/libpcre16.pc.in create mode 100644 src/third_party/pcre-8.37/libpcre32.pc.in create mode 100644 src/third_party/pcre-8.37/libpcrecpp.pc.in create mode 100644 src/third_party/pcre-8.37/libpcreposix.pc.in create mode 100644 src/third_party/pcre-8.37/ltmain.sh create mode 100644 src/third_party/pcre-8.37/makevp.bat create mode 100644 src/third_party/pcre-8.37/makevp_c.txt create mode 100644 src/third_party/pcre-8.37/makevp_l.txt create mode 100755 src/third_party/pcre-8.37/missing create mode 100644 src/third_party/pcre-8.37/pcre-config.in create mode 100644 src/third_party/pcre-8.37/pcre.h create mode 100644 src/third_party/pcre-8.37/pcre.h.generic create mode 100644 src/third_party/pcre-8.37/pcre.h.in create mode 100644 src/third_party/pcre-8.37/pcre16_byte_order.c create mode 100644 src/third_party/pcre-8.37/pcre16_chartables.c create mode 100644 src/third_party/pcre-8.37/pcre16_compile.c create mode 100644 src/third_party/pcre-8.37/pcre16_config.c create mode 100644 src/third_party/pcre-8.37/pcre16_dfa_exec.c create mode 100644 src/third_party/pcre-8.37/pcre16_exec.c create mode 100644 src/third_party/pcre-8.37/pcre16_fullinfo.c create mode 100644 src/third_party/pcre-8.37/pcre16_get.c create mode 100644 src/third_party/pcre-8.37/pcre16_globals.c create mode 100644 src/third_party/pcre-8.37/pcre16_jit_compile.c create mode 100644 src/third_party/pcre-8.37/pcre16_maketables.c create mode 100644 src/third_party/pcre-8.37/pcre16_newline.c create mode 100644 src/third_party/pcre-8.37/pcre16_ord2utf16.c create mode 100644 src/third_party/pcre-8.37/pcre16_printint.c create mode 100644 src/third_party/pcre-8.37/pcre16_refcount.c create mode 100644 src/third_party/pcre-8.37/pcre16_string_utils.c create mode 100644 src/third_party/pcre-8.37/pcre16_study.c create mode 100644 src/third_party/pcre-8.37/pcre16_tables.c create mode 100644 src/third_party/pcre-8.37/pcre16_ucd.c create mode 100644 src/third_party/pcre-8.37/pcre16_utf16_utils.c create mode 100644 src/third_party/pcre-8.37/pcre16_valid_utf16.c create mode 100644 src/third_party/pcre-8.37/pcre16_version.c create mode 100644 src/third_party/pcre-8.37/pcre16_xclass.c create mode 100644 src/third_party/pcre-8.37/pcre32_byte_order.c create mode 100644 src/third_party/pcre-8.37/pcre32_chartables.c create mode 100644 src/third_party/pcre-8.37/pcre32_compile.c create mode 100644 src/third_party/pcre-8.37/pcre32_config.c create mode 100644 src/third_party/pcre-8.37/pcre32_dfa_exec.c create mode 100644 src/third_party/pcre-8.37/pcre32_exec.c create mode 100644 src/third_party/pcre-8.37/pcre32_fullinfo.c create mode 100644 src/third_party/pcre-8.37/pcre32_get.c create mode 100644 src/third_party/pcre-8.37/pcre32_globals.c create mode 100644 src/third_party/pcre-8.37/pcre32_jit_compile.c create mode 100644 src/third_party/pcre-8.37/pcre32_maketables.c create mode 100644 src/third_party/pcre-8.37/pcre32_newline.c create mode 100644 src/third_party/pcre-8.37/pcre32_ord2utf32.c create mode 100644 src/third_party/pcre-8.37/pcre32_printint.c create mode 100644 src/third_party/pcre-8.37/pcre32_refcount.c create mode 100644 src/third_party/pcre-8.37/pcre32_string_utils.c create mode 100644 src/third_party/pcre-8.37/pcre32_study.c create mode 100644 src/third_party/pcre-8.37/pcre32_tables.c create mode 100644 src/third_party/pcre-8.37/pcre32_ucd.c create mode 100644 src/third_party/pcre-8.37/pcre32_utf32_utils.c create mode 100644 src/third_party/pcre-8.37/pcre32_valid_utf32.c create mode 100644 src/third_party/pcre-8.37/pcre32_version.c create mode 100644 src/third_party/pcre-8.37/pcre32_xclass.c create mode 100644 src/third_party/pcre-8.37/pcre_byte_order.c create mode 100644 src/third_party/pcre-8.37/pcre_chartables.c create mode 100644 src/third_party/pcre-8.37/pcre_chartables.c.dist create mode 100644 src/third_party/pcre-8.37/pcre_compile.c create mode 100644 src/third_party/pcre-8.37/pcre_config.c create mode 100644 src/third_party/pcre-8.37/pcre_dfa_exec.c create mode 100644 src/third_party/pcre-8.37/pcre_exec.c create mode 100644 src/third_party/pcre-8.37/pcre_fullinfo.c create mode 100644 src/third_party/pcre-8.37/pcre_get.c create mode 100644 src/third_party/pcre-8.37/pcre_globals.c create mode 100644 src/third_party/pcre-8.37/pcre_internal.h create mode 100644 src/third_party/pcre-8.37/pcre_jit_compile.c create mode 100644 src/third_party/pcre-8.37/pcre_jit_test.c create mode 100644 src/third_party/pcre-8.37/pcre_maketables.c create mode 100644 src/third_party/pcre-8.37/pcre_newline.c create mode 100644 src/third_party/pcre-8.37/pcre_ord2utf8.c create mode 100644 src/third_party/pcre-8.37/pcre_printint.c create mode 100644 src/third_party/pcre-8.37/pcre_refcount.c create mode 100644 src/third_party/pcre-8.37/pcre_scanner.cc create mode 100644 src/third_party/pcre-8.37/pcre_scanner.h create mode 100644 src/third_party/pcre-8.37/pcre_scanner_unittest.cc create mode 100644 src/third_party/pcre-8.37/pcre_string_utils.c create mode 100644 src/third_party/pcre-8.37/pcre_stringpiece.cc create mode 100644 src/third_party/pcre-8.37/pcre_stringpiece.h create mode 100644 src/third_party/pcre-8.37/pcre_stringpiece.h.in create mode 100644 src/third_party/pcre-8.37/pcre_stringpiece_unittest.cc create mode 100644 src/third_party/pcre-8.37/pcre_study.c create mode 100644 src/third_party/pcre-8.37/pcre_tables.c create mode 100644 src/third_party/pcre-8.37/pcre_ucd.c create mode 100644 src/third_party/pcre-8.37/pcre_valid_utf8.c create mode 100644 src/third_party/pcre-8.37/pcre_version.c create mode 100644 src/third_party/pcre-8.37/pcre_xclass.c create mode 100644 src/third_party/pcre-8.37/pcrecpp.cc create mode 100644 src/third_party/pcre-8.37/pcrecpp.h create mode 100644 src/third_party/pcre-8.37/pcrecpp_internal.h create mode 100644 src/third_party/pcre-8.37/pcrecpp_unittest.cc create mode 100644 src/third_party/pcre-8.37/pcrecpparg.h create mode 100644 src/third_party/pcre-8.37/pcrecpparg.h.in create mode 100644 src/third_party/pcre-8.37/pcredemo.c create mode 100644 src/third_party/pcre-8.37/pcregexp.pas create mode 100644 src/third_party/pcre-8.37/pcregrep.c create mode 100644 src/third_party/pcre-8.37/pcreposix.c create mode 100644 src/third_party/pcre-8.37/pcreposix.h create mode 100644 src/third_party/pcre-8.37/pcretest.c create mode 100755 src/third_party/pcre-8.37/perltest.pl create mode 100755 src/third_party/pcre-8.37/test-driver create mode 100644 src/third_party/pcre-8.37/ucp.h delete mode 100644 src/third_party/shim_spidermonkey.cpp create mode 100644 src/third_party/shim_stemmer.cpp create mode 100644 src/third_party/shim_tz.cpp create mode 100644 src/third_party/shim_yaml.cpp create mode 100644 src/third_party/tz/SConscript create mode 100644 src/third_party/tz/timegm.c create mode 100644 src/third_party/tz/timegm_private.h create mode 100644 src/third_party/tz/timegm_tzfile.h create mode 100644 src/third_party/yaml-cpp-0.5.1/SConscript create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/anchor.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/binary.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/contrib/anchordict.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/contrib/graphbuilder.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/dll.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/emitfromevents.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/emitter.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/emitterdef.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/emittermanip.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/eventhandler.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/exceptions.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/mark.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/convert.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/bool_type.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/impl.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/iterator.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/iterator_fwd.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/memory.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/node.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/node_data.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/node_iterator.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/detail/node_ref.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/emit.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/impl.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/iterator.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/node.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/parse.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/ptr.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/node/type.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/noncopyable.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/null.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/ostream_wrapper.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/parser.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/stlemitter.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/traits.h create mode 100644 src/third_party/yaml-cpp-0.5.1/include/yaml-cpp/yaml.h create mode 100644 src/third_party/yaml-cpp-0.5.1/license.txt create mode 100644 src/third_party/yaml-cpp-0.5.1/src/binary.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/collectionstack.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/convert.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/directives.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/directives.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/emit.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/emitfromevents.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/emitter.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/emitterstate.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/emitterstate.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/emitterutils.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/emitterutils.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/exp.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/exp.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/graphbuilder.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/graphbuilderadapter.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/graphbuilderadapter.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/indentation.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/memory.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/node.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/node_data.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/nodebuilder.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/nodebuilder.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/nodeevents.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/nodeevents.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/null.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/ostream_wrapper.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/parse.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/parser.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/ptr_stack.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/ptr_vector.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/regex.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/regex.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/regeximpl.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/scanner.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/scanner.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/scanscalar.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/scanscalar.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/scantag.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/scantag.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/scantoken.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/setting.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/simplekey.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/singledocparser.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/singledocparser.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/stream.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/stream.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/streamcharsource.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/stringsource.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/tag.cpp create mode 100644 src/third_party/yaml-cpp-0.5.1/src/tag.h create mode 100644 src/third_party/yaml-cpp-0.5.1/src/token.h create mode 100644 src/vs2010_mongodb.sln (limited to 'src') diff --git a/src/SConscript.client b/src/SConscript.client index ff959e95f27..c17009ae0b0 100644 --- a/src/SConscript.client +++ b/src/SConscript.client @@ -1,332 +1,52 @@ # -*- mode: python -*- -# This SConscript describes build and install rules for the Mongo C++ driver and associated exmaple -# programs. -import os - -Import('env has_option installSetup use_system_version_of_library') - -Import('nix linux darwin windows') - -buildShared = False -if has_option("sharedclient"): - buildShared = True - -env.Command(['mongo/base/error_codes.h', 'mongo/base/error_codes.cpp',], - ['mongo/base/generate_error_codes.py', 'mongo/base/error_codes.err'], - '$PYTHON $SOURCES $TARGETS') - -env.Command(['mongo/db/auth/action_type.h', 'mongo/db/auth/action_type.cpp'], - ['mongo/db/auth/generate_action_types.py', 'mongo/db/auth/action_types.txt'], - '$PYTHON $SOURCES $TARGETS') - -clientSourceBasic = [ - 'mongo/base/configuration_variable_manager.cpp', - 'mongo/base/error_codes.cpp', - 'mongo/base/global_initializer.cpp', - 'mongo/base/global_initializer_registerer.cpp', - 'mongo/base/init.cpp', - 'mongo/base/initializer.cpp', - 'mongo/base/initializer_context.cpp', - 'mongo/base/initializer_dependency_graph.cpp', - 'mongo/base/make_string_vector.cpp', - 'mongo/base/parse_number.cpp', - 'mongo/base/status.cpp', - 'mongo/base/string_data.cpp', - 'mongo/bson/bson_validate.cpp', - 'mongo/bson/oid.cpp', - 'mongo/bson/util/bson_extract.cpp', - 'mongo/buildinfo.cpp', - 'mongo/client/clientAndShell.cpp', - 'mongo/client/clientOnly.cpp', - 'mongo/client/connection_factory.cpp', - 'mongo/client/connpool.cpp', - 'mongo/client/dbclient.cpp', - 'mongo/client/dbclient_rs.cpp', - 'mongo/client/dbclientcursor.cpp', - 'mongo/client/gridfs.cpp', - 'mongo/client/model.cpp', - 'mongo/client/sasl_client_authenticate.cpp', - 'mongo/client/syncclusterconnection.cpp', - 'mongo/db/jsobj.cpp', - 'mongo/db/json.cpp', - 'mongo/db/lasterror.cpp', - 'mongo/db/namespace.cpp', - 'mongo/db/dbmessage.cpp', - 'mongo/pch.cpp', - 'mongo/platform/backtrace.cpp', - 'mongo/platform/posix_fadvise.cpp', - 'mongo/platform/random.cpp', - 'mongo/util/assert_util.cpp', - 'mongo/util/background.cpp', - 'mongo/util/base64.cpp', - 'mongo/util/concurrency/rwlockimpl.cpp', - 'mongo/util/concurrency/spin_lock.cpp', - 'mongo/util/concurrency/synchronization.cpp', - 'mongo/util/concurrency/task.cpp', - 'mongo/util/concurrency/thread_pool.cpp', - 'mongo/util/concurrency/mutexdebugger.cpp', - 'mongo/util/debug_util.cpp', - 'mongo/util/stacktrace.cpp', - 'mongo/util/file.cpp', - 'mongo/util/file_allocator.cpp', - 'mongo/util/fail_point.cpp', - 'mongo/util/fail_point_registry.cpp', - 'mongo/util/fail_point_service.cpp', - 'mongo/util/histogram.cpp', - 'mongo/util/intrusive_counter.cpp', - 'mongo/util/log.cpp', - 'mongo/util/md5.cpp', - 'mongo/util/md5main.cpp', - 'mongo/util/net/httpclient.cpp', - 'mongo/util/net/listen.cpp', - 'mongo/util/net/message.cpp', - 'mongo/util/net/message_port.cpp', - 'mongo/util/net/sock.cpp', - 'mongo/util/net/ssl_manager.cpp', - 'mongo/util/password.cpp', - 'mongo/util/processinfo.cpp', - 'mongo/util/ramlog.cpp', - 'mongo/util/signal_handlers.cpp', - 'mongo/util/stringutils.cpp', - 'mongo/util/text.cpp', - 'mongo/util/time_support.cpp', - 'mongo/util/timer.cpp', - 'mongo/util/trace.cpp', - 'mongo/util/util.cpp', - 'mongo/util/version.cpp', - 'third_party/murmurhash3/MurmurHash3.cpp', - ] - -clientSourceSasl = ['mongo/client/sasl_client_authenticate_impl.cpp', - 'mongo/client/sasl_client_session.cpp'] - -clientSourceProcessInfo = [ - 'mongo/util/processinfo_darwin.cpp', - 'mongo/util/processinfo_freebsd.cpp', - 'mongo/util/processinfo_linux2.cpp', - 'mongo/util/processinfo_none.cpp', - 'mongo/util/processinfo_sunos5.cpp', - 'mongo/util/processinfo_win32.cpp' -] - -clientSourceAll = clientSourceBasic + clientSourceSasl + clientSourceProcessInfo - -usingSasl = env['MONGO_BUILD_SASL_CLIENT'] - -clientSource = list(clientSourceBasic) -if usingSasl: - clientSource += clientSourceSasl +# This SConscript used to describe the build and install rules for the Mongo C++ driver. Now it +# causes the relevant targets to emit an error message informing users that the client driver +# is now maintained in a separate repo, and provides a link to the documentation on how to +# build the driver. -processInfoPlatformFile = env.File( "mongo/util/processinfo_${PYSYSPLATFORM}.cpp" ) -# NOTE: See comment about similar code in src/mongo/SConscript -if not os.path.exists( str( processInfoPlatformFile ) ): - processInfoPlatformFile = env.File( "mongo/util/processinfo_none.cpp" ) -clientSource += [processInfoPlatformFile] - -exampleSourceMap = [ - ('authTest', 'mongo/client/examples/authTest.cpp'), - ('clientTest', 'mongo/client/examples/clientTest.cpp'), - ('firstExample', 'mongo/client/examples/first.cpp'), - ('httpClientTest', 'mongo/client/examples/httpClientTest.cpp'), - ('rsExample', 'mongo/client/examples/rs.cpp'), - ('secondExample', 'mongo/client/examples/second.cpp'), - ('tutorial', 'mongo/client/examples/tutorial.cpp'), - ('whereExample', 'mongo/client/examples/whereExample.cpp'), - ] - -clientHeaderDirectories = [ - "base/", - "bson/", - "bson/util/", - "client/", - "db/", - "db/auth/", - "db/stats/", - "db/repl/", - "db/ops/", - "platform/", - "s/", - "scripting/", - "shell/", - "util/", - "util/concurrency/", - "util/mongoutils/", - "util/net/", - "" - ] - -clientHeaders = [] -for path in clientHeaderDirectories: - clientHeaders.extend(Glob('mongo/%s/*.h' % path)) - clientHeaders.extend(Glob('mongo/%s/*.hpp' % path)) - -# This relies on static and shared objects being the same, since we will link these object -# files twice: once into a .a, and another time into a .so -clientObjects = [env.Object(source) for source in clientSource] - -mongoClientLibs = [] -mongoClientLibDeps = [] -mongoClientSysLibDeps = [] - -if usingSasl: - mongoClientSysLibDeps += ["sasl2"] - -if not use_system_version_of_library("boost"): - mongoClientLibDeps.append(['$BUILD_DIR/third_party/shim_boost']) - -mongoClientInstalls = [] -mongoClientPrefixInstalls = [] - -if windows and buildShared: - print("Building the client driver as a DLL is not supported, see SERVER-5650") +import os +import textwrap + +Import('env') + +def failClient(target, source, env): + failClientMessage = ''' + PLEASE NOTE: The C++ client driver repository has been moved to + http://github.com/mongodb/mongo-cxx-driver + + As of MongoDB 2.6.0, the C++ client library has been separated from the core database + server repository into a separate git repository. The targets specific to the C++ driver + can no longer be built from the server codebase. + + In particular, the following targets are no longer availble in this repository: + mongoclient + install-mongoclient + check-install-mongoclient + clientTests + smokeClient + + Additionally, the following SCons options no longer have any effect in this repository, + though they are accepted without error for backward compatibility: + --sharedclient + --full + --disable-declspec-thread + + For detailed instructions on how to build the C++ driver, please + see http://dochub.mongodb.org/core/build-cpp-driver + ''' + print(textwrap.dedent(failClientMessage)) Exit(1) -staticLibEnv = env.Clone() -staticLibEnv.AppendUnique( - LIBDEPS=mongoClientLibDeps, - SYSLIBDEPS=mongoClientSysLibDeps) - -mongoClientStaticLib = staticLibEnv.StaticLibrary( - 'mongoclient', clientObjects), -mongoClientInstalls.append(staticLibEnv.Install('#/', mongoClientStaticLib)) - -if installSetup.libraries: - mongoClientPrefixInstalls.append(env.Install("$INSTALL_DIR/lib", mongoClientStaticLib)) - -mongoClientSharedLib = None - -if buildShared: - - # TODO: When we are ready to set a SONAME for mongoclient, set SHLIBVERSION=x.y.z in this - # environment to enable SCons versioned shared library support, and then change the two - # 'Install' calls in this block to 'InstallVersionedLibrary'. SHLIBVERSION and - # InstallVersionedLibrary support is only stable in SCons > 2.3.0, so if you add support - # here, be sure to add an EnsuredSconsVersion here as well. - sharedLibEnv = env.Clone() - sharedLibEnv.AppendUnique( - LIBS=mongoClientLibs + mongoClientSysLibDeps, - # TODO: This currently causes the files for the libdep to get dragged into dependents - # of this shared library, incorrectly. We need to patch up libdeps to treat shared - # libraries as dependency terminals. - LIBDEPS=mongoClientLibDeps) - - if linux: - sharedLibEnv.AppendUnique(SHLINKFLAGS=["-Wl,--as-needed", "-Wl,-zdefs"]) - - mongoClientSharedLib = sharedLibEnv.SharedLibrary('mongoclient', clientObjects) - - mongoClientSharedLibInstall = sharedLibEnv.Install( - '#/sharedclient', mongoClientSharedLib) - - if darwin: - # Set up the copy of the client library in #/sharedclient so that things that link - # against it record the local directory as the install_name. - sharedLibEnv.AddPostAction( - mongoClientSharedLibInstall, - "install_name_tool -id @executable_path/%s %s" % ( - mongoClientSharedLibInstall[0].name, - mongoClientSharedLibInstall[0] - )) - mongoClientInstalls.append(mongoClientSharedLibInstall) - - if installSetup.libraries: - mongoClientSharedLibPrefixInstall = sharedLibEnv.Install( - '$INSTALL_DIR/lib', mongoClientSharedLib) - if darwin: - sharedLibEnv.AddPostAction( - mongoClientSharedLibPrefixInstall, - "install_name_tool -id %s %s" % ( - mongoClientSharedLibPrefixInstall[0], - mongoClientSharedLibPrefixInstall[0] - )) - mongoClientPrefixInstalls.append(mongoClientSharedLibPrefixInstall) - -env.Alias('mongoclient', mongoClientInstalls) - -if installSetup.headers: - for x in clientHeaderDirectories: - inst = env.Install("$INSTALL_DIR/include/mongo/" + x, - [Glob('mongo/%s*.h' % x), Glob('mongo/%s*.hpp' % x)]) - env.AddPostAction(inst, Chmod('$TARGET', 0644)) - mongoClientPrefixInstalls.append(inst) - -if installSetup.headers or installSetup.libraries: - env.Alias('install-mongoclient', mongoClientPrefixInstalls) - -clientEnv = env.Clone() -clientEnv['CPPDEFINES'].remove('MONGO_EXPOSE_MACROS') - -# Compile the example files to .o so that we can link them twice: once statically, once shared. -exampleObjMap = [(target, clientEnv.Object(source)) for (target, source) in exampleSourceMap] - -# Create an environment for linking the examples to the static library. For out of tree builds, -# we need to use LIBS. For in-tree builds we need LIBDEPS. -staticClientEnv = clientEnv.Clone() -if '_LIBDEPS' in clientEnv: - staticClientEnv.PrependUnique(LIBDEPS=[mongoClientStaticLib]) -else: - # We need the mongo client library to preceed the boost libraries. - staticClientEnv.PrependUnique(LIBS=[mongoClientStaticLib]) - -# Build each statically linked client program -staticClientPrograms = [staticClientEnv.Program(target, obj) for (target, obj) in exampleObjMap] - -# Install them to the root, and append the install targets to the list of client tests -clientTests = staticClientEnv.Install("#/", staticClientPrograms) - -# Do the same for the shared library case, if we are doing that. -if buildShared: - sharedClientEnv = clientEnv.Clone() - - # Arrange for the tests to link against the mongoclient in the #/sharedclient directory. - sharedClientEnv.PrependUnique( - LIBS=['mongoclient'], - LIBPATH=["#/sharedclient"] - ) - - # Deal with the different lookup models between regular UNIX and Darwin. For regular unix, - # we set $ORIGIN to pull the copy we run against from the current directory - # (#/sharedclient). On Darwin, the staged copy of the mongoclient dylib in #sharedclient - # has @executable path set as its install_name, so we pick up the same behavior that way. - if nix and not darwin: - sharedClientEnv.PrependUnique( - LINKFLAGS="-Wl,-z,origin", - RPATH=[sharedClientEnv.Literal("\\$$ORIGIN")]) - - sharedClientPrograms = [ - sharedClientEnv.Program("sharedclient/" + target, obj) for (target, obj) in exampleObjMap] - env.Depends(sharedClientPrograms, mongoClientInstalls) - - sharedClientProgramInstalls = sharedClientEnv.Install("#/sharedclient", sharedClientPrograms) - clientTests.append(sharedClientProgramInstalls) - -clientTests.append( - clientEnv.Install('#/', clientEnv.Program('bsondemo', 'mongo/bson/bsondemo/bsondemo.cpp'))) - -clientEnv.Alias('clientTests', clientTests, []) +failClientAction = env.Action(failClient, cmdstr=None) +failClientTargets = [ + 'check-install-mongoclient', + 'clientTests', + 'install-mongoclient', + 'mongoclient', + 'smokeClient', +] -env.Install( - '#/', - env.Command('$CLIENT_ARCHIVE', - ['#buildscripts/make_archive.py', - '$CLIENT_SCONSTRUCT', - '$CLIENT_LICENSE', - 'SConscript.client', - 'mongo/base/generate_error_codes.py', - 'mongo/base/error_codes.err', - 'mongo/db/auth/generate_action_types.py', - 'mongo/db/auth/action_types.txt', - 'third_party/murmurhash3/MurmurHash3.h', - '#buildscripts/make_archive.py', - clientSourceAll, - clientHeaders, - [source for (target, source) in exampleSourceMap], - 'mongo/bson/bsondemo/bsondemo.cpp', - ], - '${PYTHON} ${SOURCES[0]} -o $TARGET ' - '--transform ${str(Dir(BUILD_DIR))}/client_build=$CLIENT_DIST_BASENAME/src ' - '--transform ${str(Dir(BUILD_DIR))}=$CLIENT_DIST_BASENAME/src ' - '--transform distsrc/client=$CLIENT_DIST_BASENAME ' - '--transform =$CLIENT_DIST_BASENAME/ ' - '${TEMPFILE(SOURCES[1:])}')) +env.AlwaysBuild( + [env.Alias(target, [], failClientAction) for target in failClientTargets] +) diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 816a6bfacf8..98e9b7ac434 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -10,49 +10,65 @@ Import("env") Import("shellEnv") Import("testEnv") Import("has_option") -Import("usesm usev8") -Import("installSetup") +Import("get_option") +Import("usev8") Import("enforce_glibc") Import("darwin windows solaris linux nix") env.SConscript(['base/SConscript', 'db/auth/SConscript', + 'db/exec/SConscript', 'db/fts/SConscript', + 'db/index/SConscript', + 'db/ops/SConscript', + 'db/query/SConscript', + 'db/sorter/SConscript', 'db/SConscript', + 'installer/msi/SConscript', + 'logger/SConscript', 'platform/SConscript', 's/SConscript', - 'unittest/SConscript']) + 'unittest/SConscript', + 'util/concurrency/SConscript', + 'util/options_parser/SConscript', + 'util/cmdline_utils/SConscript']) def add_exe( v ): return "${PROGPREFIX}%s${PROGSUFFIX}" % v # ------ SOURCE FILE SETUP ----------- -env.StaticLibrary('foundation', - [ 'util/assert_util.cpp', - 'util/concurrency/mutexdebugger.cpp', - 'util/debug_util.cpp', - 'util/exception_filter_win32.cpp', - 'util/file.cpp', - 'util/log.cpp', - 'util/platform_init.cpp', - 'util/signal_handlers.cpp', - 'util/text.cpp', - 'util/time_support.cpp', - 'util/timer.cpp' - ], - LIBDEPS=['stacktrace', - '$BUILD_DIR/mongo/base/base', - '$BUILD_DIR/mongo/platform/platform', - '$BUILD_DIR/third_party/shim_allocator', - '$BUILD_DIR/third_party/shim_boost']) +env.Library('foundation', + [ 'util/assert_util.cpp', + 'util/concurrency/mutexdebugger.cpp', + 'util/debug_util.cpp', + 'util/exception_filter_win32.cpp', + 'util/file.cpp', + 'util/log.cpp', + 'util/platform_init.cpp', + 'util/text.cpp', + 'util/time_support.cpp', + 'util/timer.cpp', + "util/util.cpp", + "util/startup_test.cpp", + ], + LIBDEPS=['stacktrace', + '$BUILD_DIR/mongo/base/base', + '$BUILD_DIR/mongo/logger/logger', + '$BUILD_DIR/mongo/platform/platform', + '$BUILD_DIR/mongo/util/concurrency/thread_name', + '$BUILD_DIR/third_party/shim_allocator', + '$BUILD_DIR/third_party/shim_boost', + '$BUILD_DIR/third_party/shim_tz']) env.CppUnitTest('text_test', 'util/text_test.cpp', LIBDEPS=['foundation']) +env.CppUnitTest('util/time_support_test', 'util/time_support_test.cpp', LIBDEPS=['foundation']) -env.StaticLibrary('stringutils', ['util/stringutils.cpp', 'util/base64.cpp',]) +env.Library('stringutils', ['util/stringutils.cpp', 'util/base64.cpp', 'util/hex.cpp']) -env.StaticLibrary('md5', [ - 'util/md5.cpp' +env.Library('md5', [ + 'util/md5.cpp', + 'util/password_digest.cpp', ]) env.CppUnitTest( "md5_test", ["util/md5_test.cpp", "util/md5main.cpp" ], @@ -61,15 +77,14 @@ env.CppUnitTest( "md5_test", ["util/md5_test.cpp", "util/md5main.cpp" ], env.CppUnitTest( "stringutils_test", [ "util/stringutils_test.cpp" ], LIBDEPS=["stringutils"] ) -env.StaticLibrary('bson', [ - 'bson/mutable/mutable_bson.cpp', - 'bson/mutable/mutable_bson_builder.cpp', - 'bson/mutable/mutable_bson_heap.cpp', - 'bson/mutable/mutable_bson_internal.cpp', +env.Library('bson', [ + 'bson/mutable/document.cpp', + 'bson/mutable/element.cpp', 'bson/util/bson_extract.cpp', 'util/safe_num.cpp', 'bson/bson_validate.cpp', 'bson/oid.cpp', + "bson/optime.cpp", 'db/jsobj.cpp', 'db/json.cpp' ], LIBDEPS=[ @@ -79,20 +94,18 @@ env.StaticLibrary('bson', [ '$BUILD_DIR/mongo/platform/platform', ]) +env.Library('mutable_bson_test_utils', [ + 'bson/mutable/mutable_bson_test_utils.cpp' + ], LIBDEPS=['bson']) + env.CppUnitTest('builder_test', ['bson/util/builder_test.cpp'], LIBDEPS=['bson']) env.CppUnitTest('mutable_bson_test', ['bson/mutable/mutable_bson_test.cpp'], - LIBDEPS=['bson']) - -env.CppUnitTest('mutable_bson_heap_test', ['bson/mutable/mutable_bson_heap_test.cpp'], - LIBDEPS=['bson']) - -env.CppUnitTest('mutable_bson_builder_test', ['bson/mutable/mutable_bson_builder_test.cpp'], - LIBDEPS=['bson']) + LIBDEPS=['bson', 'mutable_bson_test_utils']) env.CppUnitTest('mutable_bson_algo_test', ['bson/mutable/mutable_bson_algo_test.cpp'], - LIBDEPS=['bson']) + LIBDEPS=['bson', 'mutable_bson_test_utils']) env.CppUnitTest('safe_num_test', ['util/safe_num_test.cpp'], LIBDEPS=['bson']) @@ -104,14 +117,85 @@ env.CppUnitTest('string_map_test', ['util/string_map_test.cpp'], env.CppUnitTest('bson_field_test', ['bson/bson_field_test.cpp'], LIBDEPS=['bson']) +env.CppUnitTest('bson_obj_test', ['bson/bson_obj_test.cpp'], + LIBDEPS=['bson']) + env.CppUnitTest('bson_validate_test', ['bson/bson_validate_test.cpp'], LIBDEPS=['bson']) -env.CppUnitTest('namespacestring_test', ['db/namespacestring_test.cpp'], +env.CppUnitTest('bsonobjbuilder_test', ['bson/bsonobjbuilder_test.cpp'], LIBDEPS=['bson']) -env.CppUnitTest('index_set_test', ['db/index_set_test.cpp'], - LIBDEPS=['bson','index_set']) +env.CppUnitTest('namespacestring_test', ['db/namespace_string_test.cpp'], + LIBDEPS=['bson']) + +env.CppUnitTest('namespace_test', ['db/structure/catalog/namespace_test.cpp'], + LIBDEPS=['foundation']) + +env.CppUnitTest('update_index_data_test', ['db/update_index_data_test.cpp'], + LIBDEPS=['bson','update_index_data','db/common']) + +env.Library('path', + ['db/matcher/path.cpp', + 'db/matcher/path_internal.cpp'], + LIBDEPS=['bson', + '$BUILD_DIR/mongo/db/common']) + +env.CppUnitTest('path_test', ['db/matcher/path_test.cpp'], + LIBDEPS=['path']) + + +env.Library('expressions', + ['db/matcher/expression.cpp', + 'db/matcher/expression_array.cpp', + 'db/matcher/expression_leaf.cpp', + 'db/matcher/expression_tree.cpp', + 'db/matcher/expression_parser.cpp', + 'db/matcher/expression_parser_tree.cpp', + 'db/matcher/matchable.cpp', + 'db/matcher/match_details.cpp'], + LIBDEPS=['bson', + 'path', + '$BUILD_DIR/mongo/db/common', + '$BUILD_DIR/third_party/shim_pcrecpp' + ] ) + +env.Library('expressions_geo', + ['db/matcher/expression_geo.cpp', + 'db/matcher/expression_parser_geo.cpp'], + LIBDEPS=['expressions','geoquery','geoparser'] ) + +env.Library('expressions_text', + ['db/matcher/expression_text.cpp', + 'db/matcher/expression_parser_text.cpp'], + LIBDEPS=['expressions','db/fts/base'] ) + +env.Library('expressions_where', + ['db/matcher/expression_where.cpp'], + LIBDEPS=['expressions'] ) + +env.CppUnitTest('expression_test', + ['db/matcher/expression_test.cpp', + 'db/matcher/expression_leaf_test.cpp', + 'db/matcher/expression_tree_test.cpp', + 'db/matcher/expression_array_test.cpp'], + LIBDEPS=['expressions'] ) + +env.CppUnitTest('expression_geo_test', + ['db/matcher/expression_geo_test.cpp', + 'db/matcher/expression_parser_geo_test.cpp'], + LIBDEPS=['expressions_geo'] ) + +env.CppUnitTest('expression_text_test', + ['db/matcher/expression_parser_text_test.cpp'], + LIBDEPS=['expressions_text'] ) + +env.CppUnitTest('expression_parser_test', + ['db/matcher/expression_parser_test.cpp', + 'db/matcher/expression_parser_array_test.cpp', + 'db/matcher/expression_parser_tree_test.cpp', + 'db/matcher/expression_parser_leaf_test.cpp'], + LIBDEPS=['expressions'] ) env.CppUnitTest('bson_extract_test', ['bson/util/bson_extract_test.cpp'], LIBDEPS=['bson']) @@ -121,58 +205,119 @@ env.CppUnitTest('descriptive_stats_test', LIBDEPS=['foundation', 'bson']); env.CppUnitTest('sock_test', ['util/net/sock_test.cpp'], - LIBDEPS=['mongocommon', 'notmongodormongos'], - NO_CRUTCH=True) - -env.StaticLibrary( 'mongohasher', [ "db/hasher.cpp" ] ) + LIBDEPS=['network', + 'synchronization', + ]) +env.CppUnitTest('curop_test', + ['db/curop_test.cpp'], + LIBDEPS=['serveronly', 'coredb', 'coreserver'], + NO_CRUTCH=True) +env.Library('index_names',["db/index_names.cpp"]) + +env.Library( 'mongohasher', [ "db/hasher.cpp" ] ) + +env.Library('synchronization', [ 'util/concurrency/synchronization.cpp' ]) + +env.Library('auth_helpers', ['client/auth_helpers.cpp']) + +env.Library('spin_lock', ["util/concurrency/spin_lock.cpp"]) +env.CppUnitTest('spin_lock_test', ['util/concurrency/spin_lock_test.cpp'], + LIBDEPS=['spin_lock', '$BUILD_DIR/third_party/shim_boost']) + +env.Library('network', [ + "util/net/sock.cpp", + "util/net/socket_poll.cpp", + "util/net/ssl_manager.cpp", + "util/net/ssl_options.cpp", + "util/net/httpclient.cpp", + "util/net/message.cpp", + "util/net/message_port.cpp", + "util/net/listen.cpp" ], + LIBDEPS=['$BUILD_DIR/mongo/util/options_parser/options_parser', + 'background_job', + 'fail_point', + 'foundation', + 'server_options_core', + ]) + +env.Library( + target='index_key_validate', + source=[ + "db/catalog/index_key_validate.cpp", + ], + LIBDEPS=[ + 'bson', + 'db/common', + 'index_names', + ]) + +env.Library('clientdriver', [ + "client/connpool.cpp", + "client/dbclient.cpp", + "client/dbclient_rs.cpp", + "client/dbclientcursor.cpp", + "client/replica_set_monitor.cpp", + 'client/sasl_client_authenticate.cpp', + "client/syncclusterconnection.cpp", + "db/dbmessage.cpp" + ], + LIBDEPS=['auth_helpers', + 'network', + ]) + +env.CppUnitTest("replica_set_monitor_test", + ["client/replica_set_monitor_test.cpp"], + LIBDEPS=["clientdriver"]) + +env.Library('lasterror', [ + "db/lasterror.cpp", + ], + LIBDEPS=['network', + 'foundation', + ]) + +env.Library('version', + [ + 'buildinfo.cpp', + 'util/version.cpp' + ], + LIBDEPS=[ + 'bson', + '$BUILD_DIR/mongo/base/base' + ]) commonFiles = [ "pch.cpp", - "buildinfo.cpp", - "db/lasterror.cpp", - "db/namespace.cpp", + "db/structure/catalog/namespace.cpp", "shell/mongo.cpp", - "util/background.cpp", "util/intrusive_counter.cpp", - "util/util.cpp", "util/file_allocator.cpp", "util/trace.cpp", - "util/ramlog.cpp", + "util/paths.cpp", "util/progress_meter.cpp", "util/concurrency/task.cpp", "util/concurrency/thread_pool.cpp", "util/password.cpp", "util/concurrency/rwlockimpl.cpp", "util/histogram.cpp", - "util/concurrency/spin_lock.cpp", "util/text_startuptest.cpp", + 'util/signal_win32.cpp', "util/stack_introspect.cpp", - "util/concurrency/synchronization.cpp", - "util/net/sock.cpp", - "util/net/ssl_manager.cpp", - "util/net/httpclient.cpp", - "util/net/message.cpp", - "util/net/message_port.cpp", - "util/net/listen.cpp", - "util/startup_test.cpp", - "util/version.cpp", - "client/connpool.cpp", - "client/dbclient.cpp", - "client/dbclient_rs.cpp", - "client/dbclientcursor.cpp", - "client/model.cpp", - 'client/sasl_client_authenticate.cpp', - "client/syncclusterconnection.cpp", - "db/dbmessage.cpp" + "util/version_reporting.cpp", ] extraCommonLibdeps = [] if env['MONGO_BUILD_SASL_CLIENT']: - env.StaticLibrary('sasl_client_session', - ['client/sasl_client_session.cpp'], - LIBDEPS=['foundation'], - SYSLIBDEPS=['sasl2']) + saslLibs = ['sasl2'] + if env['PYSYSPLATFORM'] == "win32": + saslLibs.extend(["secur32"]) + + env.Library('sasl_client_session', + ['client/sasl_client_session.cpp', + 'client/sasl_sspi.cpp'], + LIBDEPS=['foundation'], + SYSLIBDEPS=saslLibs) commonFiles.extend(['client/sasl_client_authenticate_impl.cpp']) extraCommonLibdeps.append('sasl_client_session') @@ -187,48 +332,86 @@ if not os.path.exists( str( processInfoPlatformFile ) ): processInfoFiles.append( processInfoPlatformFile ) -env.StaticLibrary("processinfo", - processInfoFiles, - LIBDEPS=["foundation", "bson"]) +env.Library("processinfo", + processInfoFiles, + LIBDEPS=["foundation", "bson"]) env.CppUnitTest("processinfo_test", ["util/processinfo_test.cpp"], LIBDEPS=["processinfo"]) -env.StaticLibrary("server_parameters", - ["db/server_parameters.cpp"], - LIBDEPS=["foundation","bson"]) +env.Library("server_parameters", + ["db/server_parameters.cpp"], + LIBDEPS=["foundation","bson"]) env.CppUnitTest("server_parameters_test", [ "db/server_parameters_test.cpp" ], LIBDEPS=["server_parameters"] ) -env.StaticLibrary("fail_point", - ["util/fail_point.cpp", - "util/fail_point_registry.cpp", - "util/fail_point_service.cpp"], - LIBDEPS=["foundation", "bson"]) - -env.StaticLibrary('mongocommon', commonFiles, - LIBDEPS=['bson', - 'foundation', - 'mongohasher', - 'md5', - 'processinfo', - 'stacktrace', - 'stringutils', - 'fail_point', - '$BUILD_DIR/third_party/pcrecpp', - '$BUILD_DIR/third_party/murmurhash3/murmurhash3', - '$BUILD_DIR/third_party/shim_boost'] + - extraCommonLibdeps) - -env.StaticLibrary("coredb", [ +env.Library("fail_point", + ["util/fail_point.cpp", + "util/fail_point_registry.cpp", + "util/fail_point_service.cpp"], + LIBDEPS=["foundation", "bson"]) + +env.Library('mongocommon', commonFiles, + LIBDEPS=['auth_helpers', + 'bson', + 'background_job', + 'clientdriver', + 'fail_point', + 'foundation', + 'lasterror', + 'md5', + 'mongohasher', + 'network', + 'processinfo', + 'spin_lock', + 'stacktrace', + 'stringutils', + 'synchronization', + 'util/concurrency/thread_name', + 'version', + '$BUILD_DIR/third_party/shim_pcrecpp', + '$BUILD_DIR/third_party/murmurhash3/murmurhash3', + '$BUILD_DIR/third_party/shim_boost', + '$BUILD_DIR/mongo/util/options_parser/options_parser', + ] + + extraCommonLibdeps) + +env.CppUnitTest( + target="util/version_test", + source=["util/version_test.cpp"], + LIBDEPS=["mongocommon"] +) + +env.Library('background_job', ["util/background.cpp"], + LIBDEPS=['spin_lock']) + +env.CppUnitTest( + target="background_job_test", + source=[ + "util/background_job_test.cpp", + ], + LIBDEPS=[ + "background_job", + "network", # Temporary crutch since the ssl cleanup is hard coded in background.cpp + "synchronization", + ] +) + +tcmallocServerStatus = [] +if get_option('allocator') == 'tcmalloc': + tcmallocServerStatus.append("util/tcmalloc_server_status_section.cpp") + +env.Library("coredb", [ "client/parallel.cpp", + "db/audit.cpp", "db/commands.cpp", "db/commands/authentication_commands.cpp", "db/commands/connection_status.cpp", + "db/commands/copydb_common.cpp", "db/commands/fail_point_cmd.cpp", "db/commands/find_and_modify_common.cpp", "db/commands/hashcmd.cpp", @@ -236,65 +419,77 @@ env.StaticLibrary("coredb", [ "db/commands/mr_common.cpp", "db/commands/rename_collection_common.cpp", "db/commands/server_status.cpp", + "db/commands/shutdown.cpp", "db/commands/parameters.cpp", + "db/commands/user_management_commands.cpp", + "db/commands/write_commands/write_commands_common.cpp", "db/pipeline/pipeline.cpp", "db/dbcommands_generic.cpp", - "db/dbwebserver.cpp", "db/keypattern.cpp", - "db/matcher.cpp", - "db/pipeline/accumulator.cpp", + "db/matcher/matcher.cpp", "db/pipeline/accumulator_add_to_set.cpp", "db/pipeline/accumulator_avg.cpp", "db/pipeline/accumulator_first.cpp", "db/pipeline/accumulator_last.cpp", "db/pipeline/accumulator_min_max.cpp", "db/pipeline/accumulator_push.cpp", - "db/pipeline/accumulator_single_value.cpp", "db/pipeline/accumulator_sum.cpp", - "db/pipeline/builder.cpp", - "db/pipeline/doc_mem_monitor.cpp", + "db/pipeline/dependencies.cpp", "db/pipeline/document.cpp", "db/pipeline/document_source.cpp", "db/pipeline/document_source_bson_array.cpp", "db/pipeline/document_source_command_shards.cpp", - "db/pipeline/document_source_filter.cpp", - "db/pipeline/document_source_filter_base.cpp", "db/pipeline/document_source_geo_near.cpp", "db/pipeline/document_source_group.cpp", "db/pipeline/document_source_limit.cpp", "db/pipeline/document_source_match.cpp", + "db/pipeline/document_source_merge_cursors.cpp", "db/pipeline/document_source_out.cpp", "db/pipeline/document_source_project.cpp", + "db/pipeline/document_source_redact.cpp", "db/pipeline/document_source_skip.cpp", "db/pipeline/document_source_sort.cpp", "db/pipeline/document_source_unwind.cpp", "db/pipeline/expression.cpp", - "db/pipeline/expression_context.cpp", "db/pipeline/field_path.cpp", "db/pipeline/value.cpp", "db/projection.cpp", - "db/querypattern.cpp", "db/queryutil.cpp", "db/stats/timer_stats.cpp", - "db/stats/top.cpp", "s/shardconnection.cpp", - ], + ] + + tcmallocServerStatus + , LIBDEPS=['db/auth/serverauth', 'db/common', 'server_parameters', 'geoparser', 'geoquery', - '$BUILD_DIR/mongo/foundation']) + 'expressions', + 'expressions_geo', + 'expressions_where', + 'expressions_text', + 'index_names', + 'db/exec/working_set', + 'db/index/key_generator', + '$BUILD_DIR/mongo/foundation', + '$BUILD_DIR/third_party/shim_snappy', + 'server_options', + '$BUILD_DIR/mongo/util/cmdline_utils/cmdline_utils', + 'clientdriver', + ]) coreServerFiles = [ "db/client_basic.cpp", - "db/common.cpp", "util/net/miniwebserver.cpp", - "db/indexkey.cpp", "db/stats/counters.cpp", "db/stats/service_stats.cpp", + "db/log_process_details.cpp", + "db/conn_pool_options.cpp" ] -env.StaticLibrary('ntservice', ['util/ntservice.cpp'], LIBDEPS=['foundation']) +env.Library('ntservice', ['util/ntservice.cpp'], + LIBDEPS=['foundation', + '$BUILD_DIR/mongo/util/options_parser/options_parser']) if windows: env.CppUnitTest('ntservice_test', 'util/ntservice_test.cpp', LIBDEPS=['ntservice'], @@ -305,24 +500,20 @@ scripting_common_files = [ "scripting/engine.cpp", "scripting/bench.cpp", ] -env.StaticLibrary('bson_template_evaluator', ["scripting/bson_template_evaluator.cpp"], - LIBDEPS=['bson']) +env.Library('bson_template_evaluator', ["scripting/bson_template_evaluator.cpp"], + LIBDEPS=['bson']) env.CppUnitTest('bson_template_evaluator_test', ['scripting/bson_template_evaluator_test.cpp'], LIBDEPS=['bson_template_evaluator']) -if usesm: - env.StaticLibrary('scripting', scripting_common_files + ['scripting/engine_spidermonkey.cpp', - 'scripting/sm_db.cpp'], - LIBDEPS=['$BUILD_DIR/third_party/js-1.7/js', 'bson_template_evaluator']) -elif usev8: - env.StaticLibrary('scripting', scripting_common_files + ['scripting/engine_v8.cpp', - 'scripting/v8_db.cpp', - 'scripting/v8_utils.cpp', - 'scripting/v8_profiler.cpp'], - LIBDEPS=['bson_template_evaluator', '$BUILD_DIR/third_party/shim_v8']) +if usev8: + env.Library('scripting', scripting_common_files + ['scripting/engine_v8.cpp', + 'scripting/v8_db.cpp', + 'scripting/v8_utils.cpp', + 'scripting/v8_profiler.cpp'], + LIBDEPS=['bson_template_evaluator', '$BUILD_DIR/third_party/shim_v8']) else: - env.StaticLibrary('scripting', scripting_common_files + ['scripting/engine_none.cpp'], - LIBDEPS=['bson_template_evaluator']) + env.Library('scripting', scripting_common_files + ['scripting/engine_none.cpp'], + LIBDEPS=['bson_template_evaluator']) mmapFiles = [ "util/mmap.cpp" ] @@ -331,24 +522,15 @@ if has_option( "mm" ): else: mmapFiles += [ "util/mmap_${OS_FAMILY}.cpp" ] -# handle systeminfo* -systemInfoPlatformFile = env.File( "util/systeminfo_${PYSYSPLATFORM}.cpp" ) -# NOTE( schwerin ): This is a very un-scons-y way to make this decision, and prevents one from using -# code generation to produce util/systeminfo_$PYSYSPLATFORM.cpp. -if not os.path.exists( str( systemInfoPlatformFile ) ): - systemInfoPlatformFile = env.File( "util/systeminfo_none.cpp" ) - -coreServerFiles.append( systemInfoPlatformFile ) - if has_option( "asio" ): coreServerFiles += [ "util/net/message_server_asio.cpp" ] -env.StaticLibrary('index_set', [ 'db/index_set.cpp' ] ) +env.Library('update_index_data', [ 'db/update_index_data.cpp' ], LIPDEPS=[ 'db/common' ]) -# mongod files - also files used in tools. present in dbtests, but not in mongos and not in client libs. +# mongod files - also files used in tools. present in dbtests, but not in mongos and not in client +# libs. serverOnlyFiles = [ "db/curop.cpp", "db/kill_current_op.cpp", - "db/memconcept.cpp", "db/interrupt_status_mongod.cpp", "db/d_globals.cpp", "db/pagefault.cpp", @@ -357,13 +539,13 @@ serverOnlyFiles = [ "db/curop.cpp", "db/d_concurrency.cpp", "db/lockstat.cpp", "db/lockstate.cpp", - "db/key.cpp", - "db/btreebuilder.cpp", + "db/structure/btree/key.cpp", + "db/structure/btree/btreebuilder.cpp", "util/logfile.cpp", "util/alignedbuilder.cpp", "util/elapsed_tracker.cpp", "util/touch_pages.cpp", - "db/mongommf.cpp", + "db/storage/durable_mapped_file.cpp", "db/dur.cpp", "db/durop.cpp", "db/dur_writetodatafiles.cpp", @@ -372,11 +554,13 @@ serverOnlyFiles = [ "db/curop.cpp", "db/dur_recover.cpp", "db/dur_journal.cpp", "db/introspect.cpp", - "db/btree.cpp", - "db/btree_stats.cpp", + "db/structure/btree/btree.cpp", + "db/structure/btree/btree_stats.cpp", "db/clientcursor.cpp", "db/tests.cpp", - "db/repl.cpp", + "db/range_deleter_db_env.cpp", + "db/range_deleter_service.cpp", + "db/repl/repl_start.cpp", "db/repl/rs.cpp", "db/repl/consensus.cpp", "db/repl/rs_initiate.cpp", @@ -389,64 +573,128 @@ serverOnlyFiles = [ "db/curop.cpp", "db/repl/rs_sync.cpp", "db/repl/rs_initialsync.cpp", "db/repl/bgsync.cpp", - "db/oplog.cpp", + "db/repl/master_slave.cpp", + "db/repl/sync.cpp", + "db/repl/sync_source_feedback.cpp", + "db/repl/oplogreader.cpp", + "db/repl/replication_server_status.cpp", + "db/repl/repl_reads_ok.cpp", + "db/repl/resync.cpp", + "db/repl/oplog.cpp", "db/prefetch.cpp", - "db/repl_block.cpp", - "db/btreecursor.cpp", - "db/intervalbtreecursor.cpp", - "db/btreeposition.cpp", + "db/repl/write_concern.cpp", + "db/index_legacy.cpp", + "db/index/2d_access_method.cpp", + "db/index/btree_access_method.cpp", + "db/index/btree_based_access_method.cpp", + "db/index/btree_index_cursor.cpp", + "db/index/btree_interface.cpp", + "db/index/index_descriptor.cpp", + "db/index/fts_access_method.cpp", + "db/index/hash_access_method.cpp", + "db/index/haystack_access_method.cpp", + "db/index/s2_access_method.cpp", "db/cloner.cpp", - "db/namespace_details.cpp", - "db/cap.cpp", - "db/matcher_covered.cpp", + "db/structure/catalog/namespace_details.cpp", + "db/structure/catalog/namespace_index.cpp", + "db/structure/catalog/cap.cpp", "db/dbeval.cpp", - "db/restapi.cpp", "db/dbhelpers.cpp", "db/instance.cpp", "db/client.cpp", - "db/database.cpp", + "db/catalog/database.cpp", + "db/catalog/index_catalog.cpp", + "db/catalog/index_catalog_entry.cpp", + "db/catalog/index_create.cpp", + "db/catalog/index_pregen.cpp", + "db/catalog/collection.cpp", + "db/structure/collection_compact.cpp", + "db/catalog/collection_cursor_cache.cpp", + "db/catalog/collection_info_cache.cpp", + "db/structure/collection_iterator.cpp", + "db/catalog/database_holder.cpp", + "db/background.cpp", "db/pdfile.cpp", - "db/record.cpp", - "db/cursor.cpp", - "db/queryoptimizer.cpp", - "db/queryoptimizercursorimpl.cpp", + "db/repair_database.cpp", + "db/storage/data_file.cpp", + "db/storage/extent.cpp", + "db/storage/extent_manager.cpp", + "db/structure/catalog/index_details.cpp", + "db/structure/record_store.cpp", "db/extsort.cpp", - "db/index.cpp", - "db/index_update.cpp", + "db/index_builder.cpp", "db/index_rebuilder.cpp", - "db/scanandorder.cpp", - "db/explain.cpp", - "db/geo/2d.cpp", - "db/geo/geonear.cpp", + "db/storage/record.cpp", + "db/commands/geonear.cpp", "db/geo/haystack.cpp", "db/geo/s2common.cpp", - "db/geo/s2cursor.cpp", - "db/geo/s2index.cpp", - "db/geo/s2nearcursor.cpp", - "db/hashindex.cpp", "db/ops/count.cpp", "db/ops/delete.cpp", - "db/ops/query.cpp", + "db/ops/delete_executor.cpp", + "db/ops/insert.cpp", "db/ops/update.cpp", - "db/ops/update_internal.cpp", + "db/ops/update_executor.cpp", "db/dbcommands.cpp", "db/dbcommands_admin.cpp", + "db/write_concern.cpp", + "db/startup_warnings.cpp", + "db/storage_options.cpp", + "db/ops/update_lifecycle_impl.cpp", # most commands are only for mongod + "db/stats/top.cpp", + "db/commands/apply_ops.cpp", + "db/commands/compact.cpp", + "db/commands/auth_schema_upgrade_d.cpp", + "db/commands/create_indexes.cpp", + "db/commands/dbhash.cpp", + "db/commands/merge_chunks_cmd.cpp", + "db/commands/cleanup_orphaned_cmd.cpp", + "db/commands/collection_to_capped.cpp", + "db/commands/drop_indexes.cpp", "db/commands/fsync.cpp", + "db/commands/get_last_error.cpp", + "db/commands/write_commands/write_commands.cpp", + "db/commands/write_commands/batch_executor.cpp", "db/commands/distinct.cpp", "db/commands/find_and_modify.cpp", "db/commands/group.cpp", + "db/commands/index_filter_commands.cpp", "db/commands/index_stats.cpp", "db/commands/mr.cpp", + "db/commands/oplog_note.cpp", "db/commands/pipeline_command.cpp", + "db/commands/parallel_collection_scan.cpp", + "db/commands/plan_cache_commands.cpp", + "db/commands/rename_collection.cpp", "db/commands/storage_details.cpp", + "db/commands/test_commands.cpp", + "db/commands/validate.cpp", "db/pipeline/pipeline_d.cpp", "db/pipeline/document_source_cursor.cpp", "db/driverHelpers.cpp" ] -env.Library( "dbcmdline", "db/cmdline.cpp", LIBDEPS=['bson', 'server_parameters'] ) -env.CppUnitTest('cmdline_test', 'db/cmdline_test.cpp', LIBDEPS=['dbcmdline']) +# This library exists because some libraries, such as our networking library, need access to server +# options, but not to the helpers to set them from the command line. libserver_options_core.a just +# has the structure for storing the server options, while libserver_options.a has the code to set +# them via the command line. +env.Library("server_options_core", ["db/server_options.cpp"], + LIBDEPS=['bson']) + +env.Library("server_options", [ + "db/server_options_helpers.cpp" + ], + LIBDEPS=['bson', + 'network', # temporary crutch that should go away once the networking + # library has separate options + 'server_options_core', + 'server_parameters', + '$BUILD_DIR/mongo/util/cmdline_utils/cmdline_utils', + '$BUILD_DIR/mongo/util/options_parser/options_parser', + ]) + +env.CppUnitTest('server_options_test', 'db/server_options_test.cpp', + LIBDEPS=['server_options']) env.CppUnitTest('diskloc_test', 'db/diskloc_test.cpp', LIBDEPS=[]) @@ -454,70 +702,72 @@ env.CppUnitTest('v8_deadline_monitor_test', 'scripting/v8_deadline_monitor_test. env.Library('stacktrace', 'util/stacktrace.cpp') -env.Program('lame_stacktrace_test', 'util/lame_stacktrace_test.cpp', - LIBDEPS=['stacktrace', - '$BUILD_DIR/mongo/base/base']) - serverOnlyFiles += mmapFiles serverOnlyFiles += [ "db/stats/snapshots.cpp" ] -env.Library('coreshard', ['client/distlock.cpp', +env.Library('coreshard', [# This is only here temporarily for auto-split logic in chunk.cpp. + 's/balancer_policy.cpp', + 's/distlock.cpp', 's/config.cpp', 's/grid.cpp', 's/chunk.cpp', + # No good reason to be here other than chunk.cpp needs this. + 's/config_server_checker_service.cpp', 's/shard.cpp', - 's/shardkey.cpp'], - LIBDEPS=['s/base']); + 's/shardkey.cpp', + 's/shard_key_pattern.cpp'], + LIBDEPS=['s/base', + 's/cluster_ops_impl']); mongosLibraryFiles = [ "s/interrupt_status_mongos.cpp", "s/strategy.cpp", - "s/strategy_shard.cpp", - "s/strategy_single.cpp", "s/commands_admin.cpp", "s/commands_public.cpp", + "s/commands/auth_schema_upgrade_s.cpp", + "s/commands/cluster_index_filter_cmd.cpp", + "s/commands/cluster_merge_chunks_cmd.cpp", + "s/commands/cluster_plan_cache_cmd.cpp", + "s/commands/cluster_write_cmd.cpp", "s/request.cpp", "s/client_info.cpp", - "s/config_server_checker_service.cpp", "s/cursors.cpp", "s/s_only.cpp", "s/balance.cpp", - "s/balancer_policy.cpp", "s/writeback_listener.cpp", "s/version_manager.cpp", + "s/version_mongos.cpp", + "s/mongos_persistence_stubs.cpp", ] env.Library( "mongoscore", mongosLibraryFiles, LIBDEPS=['db/auth/authmongos', - 'db/fts/ftsmongos' - ] ) + 'db/fts/ftsmongos', + 'db/query/lite_parsed_query', + 's/cluster_ops', + 's/cluster_write_op_conversion', + 's/upgrade', + ] ) env.CppUnitTest( "balancer_policy_test" , [ "s/balancer_policy_tests.cpp" ] , - LIBDEPS=["mongoscore", "coreshard", "mongocommon","coreserver","coredb","dbcmdline","mongodandmongos"] , - NO_CRUTCH=True) + LIBDEPS=["mongoscore", + "coreshard", + "mongocommon", + "coreserver", + "coredb", + "message_server_port"]) env.CppUnitTest("dbclient_rs_test", [ "client/dbclient_rs_test.cpp" ], - LIBDEPS=[ - "coredb", - "coreserver", - "coreshard", - "dbcmdline", - "mocklib", - "mongocommon", - "mongodandmongos", - "mongoscore"], - NO_CRUTCH=True) - + LIBDEPS=['clientdriver', 'mocklib']) env.CppUnitTest("scoped_db_conn_test", [ "client/scoped_db_conn_test.cpp" ], LIBDEPS=[ "coredb", "coreserver", "coreshard", - "dbcmdline", "mongocommon", - "mongodandmongos", + "message_server_port", "mongoscore"], NO_CRUTCH=True) @@ -528,10 +778,9 @@ env.CppUnitTest("shard_conn_test", [ "s/shard_conn_test.cpp" ], "mongocommon", "coreserver", "coredb", - "dbcmdline", - "mongodandmongos", - "mocklib"], - NO_CRUTCH=True) + "message_server_port", + "mocklib", + "$BUILD_DIR/mongo/db/auth/authmocks"]) env.CppUnitTest("shard_test", [ "s/shard_test.cpp" ], LIBDEPS=[ "mongoscore", @@ -539,10 +788,17 @@ env.CppUnitTest("shard_test", [ "s/shard_test.cpp" ], "mongocommon", "coreserver", "coredb", - "dbcmdline", - "mongodandmongos", - "mocklib"], - NO_CRUTCH=True) + "message_server_port", + "mocklib"]) + +env.CppUnitTest('config_server_tests', [ 's/config_server_tests.cpp' ], + LIBDEPS=[ "mongoscore", + "coreshard", + "mongocommon", + "coreserver", + "coredb", + "message_server_port", + "mocklib"]) # Should only need stuff from util, unittest and platform @@ -554,63 +810,151 @@ serverOnlyFiles += [ "s/d_logic.cpp", "s/d_migrate.cpp", "s/d_state.cpp", "s/d_split.cpp", - "client/distlock_test.cpp", - "s/d_chunk_manager.cpp", - "db/module.cpp" ] - -env.StaticLibrary("defaultversion", "s/default_version.cpp") - -env.StaticLibrary("geometry", [ "db/geo/hash.cpp", "db/geo/shapes.cpp", ], LIBDEPS = [ "bson" ]) -env.StaticLibrary("geoparser", [ "db/geo/geoparser.cpp", ], - LIBDEPS = [ "bson", - "geometry", - '$BUILD_DIR/third_party/s2/s2' ]) -env.StaticLibrary("geoquery", [ "db/geo/geoquery.cpp", ], - LIBDEPS = [ "bson", - "geometry", - '$BUILD_DIR/third_party/s2/s2' ]) + "s/d_merge.cpp", + "s/distlock_test.cpp" ] + +env.Library("defaultversion", "s/default_version.cpp") + +# Geo +env.Library("geometry", [ "db/geo/hash.cpp", "db/geo/shapes.cpp", ], LIBDEPS = [ "bson" ]) +env.Library("geoparser", [ "db/geo/geoparser.cpp", ], + LIBDEPS = [ "bson", + "geometry", + '$BUILD_DIR/third_party/s2/s2' ]) +env.Library("geoquery", [ "db/geo/geoquery.cpp", ], + LIBDEPS = [ "bson", + "geometry", + '$BUILD_DIR/third_party/s2/s2' ]) env.CppUnitTest("hash_test", [ "db/geo/hash_test.cpp" ], LIBDEPS = ["geometry" ]) env.CppUnitTest("geoparser_test", [ "db/geo/geoparser_test.cpp" ], LIBDEPS = ["geoparser"]) -env.StaticLibrary("serveronly", serverOnlyFiles, - LIBDEPS=["coreshard", - "db/auth/authmongod", - "db/fts/ftsmongod", - "db/common", - "dbcmdline", - "defaultversion", - "geoparser", - "geoquery", - "index_set", - '$BUILD_DIR/third_party/shim_snappy']) + +env.CppUnitTest( + target="index_filter_commands_test", + source=[ + "db/commands/index_filter_commands_test.cpp", + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/serveronly", + "$BUILD_DIR/mongo/coreserver", + "$BUILD_DIR/mongo/coredb", + ], + NO_CRUTCH = True, +) + +env.CppUnitTest( + target="plan_cache_commands_test", + source=[ + "db/commands/plan_cache_commands_test.cpp", + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/serveronly", + "$BUILD_DIR/mongo/coreserver", + "$BUILD_DIR/mongo/coredb", + ], + NO_CRUTCH = True, +) + +env.Library('range_deleter', + [ 'db/range_deleter.cpp', + 'db/range_deleter_mock_env.cpp', + 'db/range_deleter_stats.cpp' + ], + LIBDEPS = [ + '$BUILD_DIR/mongo/s/base', # range_arithmetic.cpp + 'base/base', + 'bson', + 'synchronization' + ]) + +env.CppUnitTest('range_deleter_test', + [ 'db/range_deleter_test.cpp' ], + LIBDEPS = [ 'range_deleter', 'db/common' ]); + +env.CppUnitTest('range_deleter_stat_test', + [ 'db/range_deleter_stat_test.cpp' ], + LIBDEPS = [ 'range_deleter', 'db/common' ]); + +env.Library( + target='storage_engine_metadata', + source=[ + 'db/storage/storage_engine_metadata.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/bson', + ] +) + +env.CppUnitTest( + target= 'storage_engine_metadata_test', + source = 'db/storage/storage_engine_metadata_test.cpp', + LIBDEPS=[ + 'storage_engine_metadata', + ], +) + +env.Library("serveronly", serverOnlyFiles, + LIBDEPS=["coreshard", + "db/auth/authmongod", + "db/fts/ftsmongod", + "db/common", + "db/ops/update_driver", + "defaultversion", + "geoparser", + "geoquery", + "index_key_validate", + 'range_deleter', + "update_index_data", + 's/metadata', + 's/batch_write_types', + "db/exec/working_set", + "db/exec/exec", + "db/query/query", + '$BUILD_DIR/third_party/shim_snappy']) + + +env.Library("message_server_port", "util/net/message_server_port.cpp") # These files go into mongos and mongod only, not into the shell or any tools. mongodAndMongosFiles = [ - "db/connection_factory.cpp", "db/initialize_server_global_state.cpp", "db/server_extra_log_context.cpp", - "util/net/message_server_port.cpp", + "db/dbwebserver.cpp", + "util/signal_handlers.cpp", ] -env.StaticLibrary("mongodandmongos", mongodAndMongosFiles) +env.Library("mongodandmongos", mongodAndMongosFiles, + LIBDEPS=["message_server_port"]) -# These files go into the tools, tests, shell, etc. but not mongod or mongos -everythingButMongodAndMongosFiles = [ - "client/connection_factory.cpp", - ] -env.StaticLibrary("notmongodormongos", everythingButMongodAndMongosFiles) +env.Library("mongodwebserver", + [ + "db/clientlistplugin.cpp", + "db/repl/replset_web_handler.cpp", + "db/restapi.cpp", + "db/stats/snapshots_webplugins.cpp", + ], + LIBDEPS=["coredb", "mongodandmongos"]) -mongodOnlyFiles = [ "db/db.cpp", "db/compact.cpp", "db/commands/touch.cpp" ] +mongodOnlyFiles = [ "db/db.cpp", "db/commands/touch.cpp", "db/mongod_options_init.cpp" ] # ----- TARGETS ------ -env.StaticLibrary("gridfs", "client/gridfs.cpp") +env.Library("gridfs", "client/gridfs.cpp") if has_option( 'use-cpu-profiler' ): coreServerFiles.append( 'db/commands/cpuprofile.cpp' ) env.Append(LIBS=['unwind']) -env.StaticLibrary("coreserver", coreServerFiles, LIBDEPS=["mongocommon", "scripting"]) +env.Library("coreserver", coreServerFiles, LIBDEPS=["mongocommon", "scripting"]) + +# mongod options +env.Library("mongod_options", ["db/mongod_options.cpp"], + LIBDEPS=['server_options', + 'mongocommon', + 'serveronly', + 'coreserver', + 'coredb', + '$BUILD_DIR/mongo/util/options_parser/options_parser_init']) # main db target mongod = env.Install( @@ -618,79 +962,130 @@ mongod = env.Install( LIBDEPS=["coredb", "coreserver", "mongodandmongos", + "mongodwebserver", "ntservice", "serveronly", - ] + env['MODULE_LIBDEPS_MONGOD'] ) ) + "mongod_options", + "storage_engine_metadata", + ] ) ) Default( mongod ) # tools -allToolFiles = [ "tools/tool.cpp", "tools/stat_util.cpp" ] -env.StaticLibrary("alltools", allToolFiles, LIBDEPS=["serveronly", "coreserver", "coredb", - "notmongodormongos"]) +allToolFiles = ["tools/tool.cpp", + "tools/stat_util.cpp", + "tools/tool_logger.cpp"] +env.Library("tool_options", "tools/tool_options.cpp", + LIBDEPS=["server_options", + "$BUILD_DIR/mongo/util/options_parser/options_parser_init", + "serveronly", + "coreserver", + "coredb", + ]) +env.Library("alltools", + allToolFiles, + LIBDEPS=["serveronly", + "coreserver", + "coredb", + "$BUILD_DIR/mongo/util/options_parser/options_parser", + "$BUILD_DIR/mongo/db/auth/authmocks", + "$BUILD_DIR/mongo/db/auth/authmongod"]) normalTools = [ "dump", "restore", "export", "import", "stat", "top", "oplog" ] -env.Alias( "tools", [ "#/${PROGPREFIX}mongo" + x + "${PROGSUFFIX}" for x in normalTools ] ) -for x in normalTools: - env.Install( '#/', env.Program( "mongo" + x, [ "tools/" + x + ".cpp" ], - LIBDEPS=["alltools"]) ) - -env.StaticLibrary("docgenerator", "tools/docgenerator.cpp") +env.Alias( "tools", [ "#/${PROGPREFIX}mongo" + name + "${PROGSUFFIX}" for name in normalTools ] ) +for name in normalTools: + env.Install( '#/', env.Program("mongo" + name, + ["tools/" + name + ".cpp", "tools/mongo" + name + "_options_init.cpp"], + LIBDEPS=["alltools", "mongo" + name + "_options"]) ) + env.Library("mongo" + name + "_options", ["tools/mongo" + name + "_options.cpp"], + LIBDEPS=['tool_options']) #some special tools +env.Library("mongofiles_options", ["tools/mongofiles_options.cpp"], + LIBDEPS=['tool_options']) +env.Library("bsondump_options", ["tools/bsondump_options.cpp"], + LIBDEPS=['tool_options']) +env.Library("mongobridge_options", ["tools/mongobridge_options.cpp"], + LIBDEPS=['tool_options']) + env.Install( '#/', [ - env.Program( "mongofiles", "tools/files.cpp", LIBDEPS=["alltools", "gridfs"] ), - env.Program( "docgen", "tools/docgeneratormain.cpp", LIBDEPS=["alltools", "docgenerator"] ), - env.Program( "loadgen", "tools/loadgenerator.cpp", LIBDEPS=["alltools", "docgenerator"] ), - env.Program( "bsondump", "tools/bsondump.cpp", LIBDEPS=["alltools"]), - env.Program( "mongobridge", "tools/bridge.cpp", LIBDEPS=["alltools"]), - env.Program( "mongoperf", "client/examples/mongoperf.cpp", LIBDEPS=["alltools"] ), + env.Program( "mongofiles", ["tools/files.cpp", "tools/mongofiles_options_init.cpp"], + LIBDEPS=["alltools", "gridfs", "mongofiles_options"] ), + env.Program( "bsondump", ["tools/bsondump.cpp", "tools/bsondump_options_init.cpp"], + LIBDEPS=["alltools", "bsondump_options"]), + env.Program( "mongobridge", ["tools/bridge.cpp", "tools/mongobridge_options_init.cpp"], + LIBDEPS=["serveronly", "coreserver", "coredb", "mongobridge_options"] ), + env.Program( "mongoperf", "client/examples/mongoperf.cpp", + LIBDEPS=["serveronly", "coreserver", "coredb"] ), ] ) +# mongos options +env.Library("mongos_options", ["s/mongos_options.cpp"], + LIBDEPS=['mongoscore', + 'coreshard', + 'mongocommon', + 'coreserver', + 'coredb', + '$BUILD_DIR/mongo/util/options_parser/options_parser_init']) + # mongos mongos = env.Program( - "mongos", [ "s/server.cpp"] , - LIBDEPS=["mongoscore", "coreserver", "coredb", "mongocommon", "coreshard", "dbcmdline", "ntservice", - "mongodandmongos", "s/upgrade"] + env['MODULE_LIBDEPS_MONGOS']) + "mongos", [ "s/server.cpp", "s/mongos_options_init.cpp" ] , + LIBDEPS=["mongoscore", "coreserver", "coredb", "mongocommon", "coreshard", "ntservice", + "mongodandmongos", "s/upgrade", "mongos_options" ]) env.Install( '#/', mongos ) env.Library("clientandshell", ["client/clientAndShell.cpp"], LIBDEPS=["mongocommon", "defaultversion", - "gridfs", - "notmongodormongos"]) + "gridfs"]) env.Library("allclient", "client/clientOnly.cpp", LIBDEPS=["clientandshell"]) +# dbtests test binary options +env.Library("framework_options", ["dbtests/framework_options.cpp"], + LIBDEPS=['$BUILD_DIR/mongo/util/options_parser/options_parser_init']) + # dbtests test binary -env.StaticLibrary('testframework', ['dbtests/framework.cpp'], LIBDEPS=['unittest/unittest']) +env.Library('testframework', ['dbtests/framework.cpp', 'dbtests/framework_options_init.cpp'], + LIBDEPS=['unittest/unittest', + 'framework_options', + ]) -env.StaticLibrary('mocklib', [ +env.Library('mocklib', [ 'dbtests/mock/mock_conn_registry.cpp', 'dbtests/mock/mock_dbclient_connection.cpp', 'dbtests/mock/mock_dbclient_cursor.cpp', 'dbtests/mock/mock_remote_db_server.cpp', 'dbtests/mock/mock_replica_set.cpp' ], - LIBDEPS=['mongocommon']) + LIBDEPS=['clientdriver']) test = testEnv.Install( '#/', testEnv.Program("test", - [ f for f in Glob("dbtests/*.cpp") if not str(f).endswith('framework.cpp') ], + [ f for f in Glob("dbtests/*.cpp") + if not str(f).endswith('framework.cpp') and + not str(f).endswith('framework_options.cpp') and + not str(f).endswith('framework_options_init.cpp') ], LIBDEPS = [ + "mutable_bson_test_utils", "mongocommon", "serveronly", "coreserver", "coredb", "testframework", "gridfs", - "notmongodormongos", "s/upgrade", - "mocklib"])) + "s/cluster_ops", + "s/cluster_ops_impl", + "mocklib", + "db/exec/mock_stage", + "$BUILD_DIR/mongo/db/auth/authmocks", + "$BUILD_DIR/mongo/db/query/query"])) if len(testEnv.subst('$PROGSUFFIX')): testEnv.Alias( "test", "#/${PROGPREFIX}test${PROGSUFFIX}" ) -env.Install( '#/', testEnv.Program( "perftest", [ "dbtests/perf/perftest.cpp" ], LIBDEPS=["serveronly", "coreserver", "coredb", "testframework", "notmongodormongos" ] ) ) +env.Install( '#/', testEnv.Program( "perftest", [ "dbtests/perf/perftest.cpp" ], LIBDEPS=["serveronly", "coreserver", "coredb", "testframework" ] ) ) # --- sniffer --- mongosniff_built = False @@ -705,36 +1100,58 @@ if darwin or env["_HAVEPCAP"]: sniffEnv.Append( LIBS=[ "wpcap" ] ) sniffEnv.Install( '#/', sniffEnv.Program( "mongosniff", "tools/sniffer.cpp", - LIBDEPS=["gridfs", "serveronly", "coreserver", "coredb", "notmongodormongos"])) + LIBDEPS=["gridfs", "serveronly", "coreserver", "coredb"])) # --- shell --- # if you add a file here, you need to add it in scripting/engine.cpp and shell/createCPPfromJavaScriptFiles.js as well env.JSHeader("shell/mongo.cpp", - ["shell/assert.js", "shell/types.js", "shell/utils.js", "shell/utils_sh.js", - "shell/db.js", "shell/mongo.js", "shell/mr.js", "shell/query.js", - "shell/collection.js"]) + [ + "shell/assert.js", + "shell/bulk_api.js", + "shell/collection.js", + "shell/db.js", + "shell/mongo.js", + "shell/mr.js", + "shell/query.js", + "shell/types.js", + "shell/upgrade_check.js", + "shell/utils.js", + "shell/utils_sh.js", + ]) # if you add a file here, you need to add it in shell/shell_utils.cpp and shell/createCPPfromJavaScriptFiles.js as well env.JSHeader("shell/mongo-server.cpp", ["shell/servers.js", "shell/shardingtest.js", "shell/servers_misc.js", "shell/replsettest.js", "shell/replsetbridge.js"]) -coreShellFiles = [ "shell/dbshell.cpp", - "shell/shell_utils.cpp", +coreShellFiles = [ "shell/shell_utils.cpp", "shell/shell_utils_extended.cpp", "shell/shell_utils_launcher.cpp", "shell/mongo-server.cpp", "shell/linenoise.cpp", "shell/linenoise_utf8.cpp", - "shell/mk_wcwidth.cpp"] + "shell/mk_wcwidth.cpp", + "shell/shell_options_init.cpp" ] if shellEnv is not None: + env.Library("shell_core", coreShellFiles, + LIBDEPS=['clientandshell', + 'db/index/external_key_generator', + 'index_key_validate', + 'scripting', + 'mongocommon']) + # mongo shell options + env.Library("shell_options", ["shell/shell_options.cpp"], + LIBDEPS=['$BUILD_DIR/mongo/util/options_parser/options_parser_init']) + mongo_shell = shellEnv.Program( "mongo", - coreShellFiles, - LIBDEPS=[ "clientandshell", "mongocommon", "scripting", - "$BUILD_DIR/third_party/pcrecpp"] + env['MODULE_LIBDEPS_MONGOSHELL'] ) + "shell/dbshell.cpp", + LIBDEPS=["$BUILD_DIR/third_party/shim_pcrecpp", + "shell_options", + "shell_core", + ]) shellEnv.Install( '#/', mongo_shell ) @@ -771,10 +1188,6 @@ def installBinary( e, name ): e.AddPostAction(name, 'strip $TARGET') distBinaries.append(name) - - if not installSetup.binaries: - return - inst = e.Install( "$INSTALL_DIR/bin", name ) if nix: @@ -822,20 +1235,21 @@ if not len(module_banner_filenames) == len(env['MODULE_BANNERS']): module_banner_dirs = set([Dir('#').rel_path(f.get_dir()) for f in env['MODULE_BANNERS']]) module_banner_transforms = ["--transform %s=$SERVER_DIST_BASENAME" % d for d in module_banner_dirs] +# Allow modules to map original file name directories to subdirectories +# within the archive (e.g. { "src/mongo/db/modules/enterprise/docs": "snmp"}) +archive_addition_transforms = [] +for full_dir, archive_dir in env["ARCHIVE_ADDITION_DIR_MAP"].items(): + archive_addition_transforms.append("--transform \"%s=$SERVER_DIST_BASENAME/%s\"" % + (full_dir, archive_dir)) + env.Command( '#/${SERVER_ARCHIVE}', - ['#buildscripts/make_archive.py'] + env["MODULE_BANNERS"] + distBinaries, - ' '.join(['$PYTHON ${SOURCES[0]} -o $TARGET'] + module_banner_transforms + [ + ['#buildscripts/make_archive.py'] + env["MODULE_BANNERS"] + env["ARCHIVE_ADDITIONS"] + + distBinaries, ' '.join(['$PYTHON ${SOURCES[0]} -o $TARGET'] + archive_addition_transforms + + module_banner_transforms + [ '--transform ${str(Dir(BUILD_DIR))}/mongo/stripped=$SERVER_DIST_BASENAME/bin', '--transform ${str(Dir(BUILD_DIR))}/mongo=$SERVER_DIST_BASENAME/bin', '${TEMPFILE(SOURCES[1:])}'])) #final alias env.Alias( "install", "$INSTALL_DIR" ) - -env.Alias("clientBuild", ['#buildscripts/build_and_test_client.py', - '#/${PROGPREFIX}mongod${PROGSUFFIX}', - '#$CLIENT_ARCHIVE'], - '$PYTHON ${SOURCES[0]} ${SOURCES[2]} ${EXTRAPATH and "--extrapath"} $EXTRAPATH' - ) -env.AlwaysBuild("clientBuild") diff --git a/src/mongo/base/SConscript b/src/mongo/base/SConscript index 725a0f6286f..5b712bba073 100644 --- a/src/mongo/base/SConscript +++ b/src/mongo/base/SConscript @@ -5,26 +5,22 @@ Import("env") env.Command(['error_codes.h', 'error_codes.cpp'], ['generate_error_codes.py', 'error_codes.err'], '$PYTHON $SOURCES $TARGETS') -env.StaticLibrary('base', ['configuration_variable_manager.cpp', - 'error_codes.cpp', - 'global_initializer.cpp', - 'global_initializer_registerer.cpp', - 'init.cpp', - 'initializer.cpp', - 'initializer_context.cpp', - 'initializer_dependency_graph.cpp', - 'make_string_vector.cpp', - 'parse_number.cpp', - 'status.cpp', - 'string_data.cpp', - ], - LIBDEPS=[ - '$BUILD_DIR/third_party/murmurhash3/murmurhash3', - ]) - -env.CppUnitTest('configuration_variable_manager_test', - ['configuration_variable_manager_test.cpp'], - LIBDEPS=['base']) +env.Library('base', ['error_codes.cpp', + 'global_initializer.cpp', + 'global_initializer_registerer.cpp', + 'init.cpp', + 'initializer.cpp', + 'initializer_context.cpp', + 'initializer_dependency_graph.cpp', + 'validate_locale.cpp', + 'make_string_vector.cpp', + 'parse_number.cpp', + 'status.cpp', + 'string_data.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/third_party/murmurhash3/murmurhash3', + ]) env.CppUnitTest('initializer_dependency_graph_test', ['initializer_dependency_graph_test.cpp'], diff --git a/src/mongo/base/configuration_variable_manager.cpp b/src/mongo/base/configuration_variable_manager.cpp deleted file mode 100644 index db99cc5fc2e..00000000000 --- a/src/mongo/base/configuration_variable_manager.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/base/configuration_variable_manager.h" - -#include - -#include "mongo/base/parse_number.h" - -namespace mongo { - - ConfigurationVariableManager::ConfigurationVariableManager() {} - - ConfigurationVariableManager::~ConfigurationVariableManager() {} - - Status ConfigurationVariableManager::registerVariableFn(const std::string& name, - const SetFromStringFn setter) { - if (!setter) - return Status(ErrorCodes::BadValue, "setter function invalid"); - - SetFromStringFn& existingSetter = _variables[name]; - if (existingSetter) - return Status(ErrorCodes::DuplicateKey, name); - - existingSetter = setter; - return Status::OK(); - } - - Status ConfigurationVariableManager::setVariable(const std::string& name, - const std::string& value) const { - const VariableMap::const_iterator iter = _variables.find(name); - if (_variables.end() == iter) - return Status(ErrorCodes::NoSuchKey, name); - - return iter->second(value); - } - - template <> - Status ConfigurationVariableManager::SetFromStringImpl::operator()( - const std::string& stringValue) const { - *_storage = stringValue; - return Status::OK(); - } - - template - Status ConfigurationVariableManager::SetFromStringImpl::operator()( - const std::string& stringValue) const { - - return parseNumberFromString(stringValue, _storage); - } - - template class ConfigurationVariableManager::SetFromStringImpl; - template class ConfigurationVariableManager::SetFromStringImpl; - template class ConfigurationVariableManager::SetFromStringImpl; - template class ConfigurationVariableManager::SetFromStringImpl; - template class ConfigurationVariableManager::SetFromStringImpl; - template class ConfigurationVariableManager::SetFromStringImpl; - - template <> - Status ConfigurationVariableManager::SetFromStringImpl::operator()( - const std::string& stringValue) const { - if (stringValue == "true") { - *_storage = true; - return Status::OK(); - } - if (stringValue == "false") { - *_storage = false; - return Status::OK(); - } - return Status(ErrorCodes::FailedToParse, - "Could not parse boolean value out of \"" + stringValue + "\""); - } - -} // namespace mongo diff --git a/src/mongo/base/configuration_variable_manager.h b/src/mongo/base/configuration_variable_manager.h deleted file mode 100644 index 26a05a9f71d..00000000000 --- a/src/mongo/base/configuration_variable_manager.h +++ /dev/null @@ -1,121 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/status.h" -#include "mongo/platform/unordered_map.h" - -namespace mongo { - - /** - * Utility class for setting and getting the values of configuration variables. - * - * A common kind of global variable is a configuration setting for a module or set of modules. - * The primary purpose of this class is to generically expose these settings so that they may be - * configured at program startup (say by inspecting argv, the environment or config files). The - * secondary purpose is to provide a facility for introspection on these settings, for use by - * reporting utilities built into a mongo application. - * - * By providing modules with access to an instance of this variable during a registration phase - * of the application, those modules may register their global variables with unique names, - * using "registerVariable()". Then, during a configuration phase, the application may set - * values for configurables to non-default values using "setVariable()". - * - * After the configuration phase, modules may directly access their registered variables via the - * storage they provided at registration time; there is no need to maintain access to the - * configuration manager at all. - * - * Instances of this class provide no facilities for coordinating activity among threads, so in - * a multi-threaded scenario, it is the responsibility of the programmer to ensure that modules - * do not access configuration variables from separate threads before the configuration phase - * completes. The easiest way to accomplish this is to perform configuration in a - * single-threaded context. - * - * TODO: Support introspection of configuration variables, for use in reporting tools, etc. - */ - class ConfigurationVariableManager { - MONGO_DISALLOW_COPYING(ConfigurationVariableManager); - - public: - typedef boost::function SetFromStringFn; - - ConfigurationVariableManager(); - ~ConfigurationVariableManager(); - - /** - * Register a variable named "name" whose value may be set from a string - * using the function-like object "setter". - * - * Returns ErrorCodes::DuplicateKey if another module has already registered "name", - * or ErrorCodes::BadValue if "setter" is an invalid function object. - */ - Status registerVariableFn(const std::string& name, const SetFromStringFn setter); - - /** - * Register a variable of type "T", named "name", stored at "storage". - * - * Returns ErrorCodes::DuplicateKey if another module has already registered "name", - * or ErrorCodes::BadValue if "storage" is NULL. - * - * Uses a generic SetFromString function, based on the type T. - */ - template - Status registerVariable(const std::string& name, T* storage) { - if (!storage) - return Status(ErrorCodes::BadValue, "Storage is null"); - return registerVariableFn(name, SetFromStringImpl(storage)); - } - - /** - * Set the variable named "name" to the value "value". - * - * If "name" was not previously added, returns ErrorCodes::NoSuchKey. - * - * Returns Status::OK() and sets the named variable to the parsed value of "value" on - * success, and returns the error from the associated SetFromStringFn on failure. - */ - Status setVariable(const std::string& name, const std::string& value) const; - - private: - - typedef unordered_map VariableMap; - - /** - * Template type of generic set-from-string function objects, used in the - * registerVariable method, above. Available implementations can - * be seen in configuration_variable_manager.cpp. - */ - template - class SetFromStringImpl { - public: - explicit SetFromStringImpl(T* storage) : _storage(storage) {} - Status operator()(const std::string& stringValue) const; - - private: - T* _storage; - }; - - - /// Map from names of registered variables to their storage location and type information. - VariableMap _variables; - }; - -} // namespace mongo diff --git a/src/mongo/base/configuration_variable_manager_test.cpp b/src/mongo/base/configuration_variable_manager_test.cpp deleted file mode 100644 index 1802f4845fa..00000000000 --- a/src/mongo/base/configuration_variable_manager_test.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Unit tests of the ConfigurationVariableManager type. - */ - -#include "mongo/base/configuration_variable_manager.h" -#include "mongo/base/status.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -namespace { - - TEST(ConfigurationVariableManagerTest, CorrectRegisterAndSet) { - ConfigurationVariableManager cvars; - - int firstInt = 1; - int secondInt = 2; - std::string aString = "Hello"; - - ASSERT_OK(cvars.registerVariable("firstInt", &firstInt)); - ASSERT_OK(cvars.registerVariable("secondInt", &secondInt)); - ASSERT_OK(cvars.registerVariable("aString", &aString)); - - // Registering doesn't change values. - ASSERT_EQUALS(1, firstInt); - ASSERT_EQUALS(2, secondInt); - ASSERT_EQUALS("Hello", aString); - - ASSERT_OK(cvars.setVariable("firstInt", "7")); - ASSERT_EQUALS(7, firstInt); - ASSERT_OK(cvars.setVariable("firstInt", "8")); - ASSERT_EQUALS(8, firstInt); - ASSERT_OK(cvars.setVariable("secondInt", "9")); - ASSERT_OK(cvars.setVariable("aString", "Goodbye")); - } - - TEST(ConfigurationVariableManager, ParseFancyNumbers) { - ConfigurationVariableManager cvars; - int v = 0; - ASSERT_OK(cvars.registerVariable("v", &v)); - ASSERT_OK(cvars.setVariable("v", "0xf")); - ASSERT_EQUALS(0xf, v); - ASSERT_OK(cvars.setVariable("v", "010")); - ASSERT_EQUALS(010, v); - } - - TEST(ConfigurationVariableManagerTest, DoubleRegisterFails) { - ConfigurationVariableManager cvars; - int a1, a2; - ASSERT_OK(cvars.registerVariable("a", &a1)); - ASSERT_EQUALS(ErrorCodes::DuplicateKey, cvars.registerVariable("a", &a2)); - } - - TEST(ConfigurationVariableManagerTest, RegisterNullFails) { - ConfigurationVariableManager cvars; - ASSERT_EQUALS(ErrorCodes::BadValue, cvars.registerVariable("a", NULL)); - } - - TEST(ConfigurationVariableManagerTest, IncompatibleSetFails) { - ConfigurationVariableManager cvars; - unsigned int v = 12; - ASSERT_OK(cvars.registerVariable("v", &v)); - ASSERT_OK(cvars.setVariable("v", "15")); - ASSERT_EQUALS(15U, v); - ASSERT_EQUALS(ErrorCodes::FailedToParse, cvars.setVariable("v", "fifteen")); - ASSERT_EQUALS(15U, v); - ASSERT_EQUALS(ErrorCodes::FailedToParse, cvars.setVariable("v", "-15")); - ASSERT_EQUALS(15U, v); - } - - TEST(ConfigurationVariableManagerTest, StringsWithSpacesSettable) { - ConfigurationVariableManager cvars; - std::string v; - ASSERT_OK(cvars.registerVariable("v", &v)); - ASSERT_OK(cvars.setVariable("v", "new value")); - ASSERT_EQUALS("new value", v); - } - - TEST(ConfigurationVariableManagerTest, SettingUnregisteredVariableFails) { - ConfigurationVariableManager cvars; - ASSERT_EQUALS(ErrorCodes::NoSuchKey, cvars.setVariable("v", "15")); - } - - TEST(ConfigurationVariableManagerTest, ParseBool) { - ConfigurationVariableManager cvars; - bool a; - ASSERT_OK(cvars.registerVariable("a", &a)); - ASSERT_OK(cvars.setVariable("a", "false")); - ASSERT_FALSE(a); - ASSERT_OK(cvars.setVariable("a", "true")); - ASSERT_TRUE(a); - ASSERT_OK(cvars.setVariable("a", "false")); - ASSERT_FALSE(a); - ASSERT_EQUALS(ErrorCodes::FailedToParse, cvars.setVariable("a", "False")); - } - -} // namespace -} // namespace mongo diff --git a/src/mongo/base/counter.h b/src/mongo/base/counter.h index 7dcc760a8a9..07ec888475d 100644 --- a/src/mongo/base/counter.h +++ b/src/mongo/base/counter.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ #pragma once diff --git a/src/mongo/base/counter_test.cpp b/src/mongo/base/counter_test.cpp index 0966cbddd78..94998192eb1 100644 --- a/src/mongo/base/counter_test.cpp +++ b/src/mongo/base/counter_test.cpp @@ -13,14 +13,14 @@ * limitations under the License. */ -#include "mongo/platform/basic.h" -#include +#include "mongo/platform/basic.h" #include #include +#include -#include "counter.h" +#include "mongo/base/counter.h" #include "mongo/unittest/unittest.h" namespace mongo { diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 1dfa1702b64..ac900d23d27 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -1,9 +1,9 @@ -# -*- python-mode -*- +# -*- mode: python -*- error_code("OK", 0) error_code("InternalError", 1) error_code("BadValue", 2) -error_code("DuplicateKey", 3) +error_code("OBSOLETE_DuplicateKey", 3) error_code("NoSuchKey", 4) error_code("GraphContainsCycle", 5) error_code("HostUnreachable", 6) @@ -26,5 +26,79 @@ error_code("InvalidBSON", 22) error_code("AlreadyInitialized", 23) error_code("LockTimeout", 24) error_code("RemoteValidationError", 25) +error_code("NamespaceNotFound", 26) +error_code("IndexNotFound", 27) +error_code("PathNotViable", 28) +error_code("NonExistentPath", 29) +error_code("InvalidPath", 30) +error_code("RoleNotFound", 31) +error_code("RolesNotRelated", 32) +error_code("PrivilegeNotFound", 33) +error_code("CannotBackfillArray", 34) +error_code("UserModificationFailed", 35) +error_code("RemoteChangeDetected", 36) +error_code("FileRenameFailed", 37) +error_code("FileNotOpen", 38) +error_code("FileStreamFailed", 39) +error_code("ConflictingUpdateOperators", 40) +error_code("FileAlreadyOpen", 41) +error_code("LogWriteFailed", 42) +error_code("CursorNotFound", 43) +error_code("UserDataInconsistent", 45) +error_code("LockBusy", 46) +error_code("NoMatchingDocument", 47) +error_code("NamespaceExists", 48) +error_code("InvalidRoleModification", 49) +error_code("ExceededTimeLimit", 50) +error_code("ManualInterventionRequired", 51) +error_code("DollarPrefixedFieldName", 52) +error_code("InvalidIdField", 53) +error_code("NotSingleValueField", 54) +error_code("InvalidDBRef", 55) +error_code("EmptyFieldName", 56) +error_code("DottedFieldName", 57) +error_code("RoleModificationFailed", 58) +error_code("CommandNotFound", 59) +error_code("DatabaseNotFound", 60) +error_code("ShardKeyNotFound", 61) +error_code("OplogOperationUnsupported", 62) +error_code("StaleShardVersion", 63) +error_code("WriteConcernFailed", 64) +error_code("MultipleErrorsOccurred", 65) +error_code("ImmutableField", 66) +error_code("CannotCreateIndex", 67 ) +error_code("IndexAlreadyExists", 68 ) +error_code("AuthSchemaIncompatible", 69) +error_code("ShardNotFound", 70) +error_code("ReplicaSetNotFound", 71) +error_code("InvalidOptions", 72) +error_code("InvalidNamespace", 73) +error_code("NodeNotFound", 74) +error_code("WriteConcernLegacyOK", 75) +error_code("NoReplicationEnabled", 76) +error_code("OperationIncomplete", 77) +error_code("CommandResultSchemaViolation", 78) +error_code("UnknownReplWriteConcern", 79) +error_code("RoleDataInconsistent", 80) +error_code("NoClientContext", 81) +error_code("NoProgressMade", 82) +error_code("RemoteResultsUnavailable", 83) +error_code("IndexOptionsConflict", 85 ) +error_code("IndexKeySpecsConflict", 86 ) +error_code("OutdatedClient", 101) +error_code("IncompatibleAuditMetadata", 102) -error_class("NetworkError", ["HostUnreachable", "HostNotFound"]) +# Non-sequential error codes (for compatibility only) +error_code("NetworkTimeout", 89) +error_code("NotMaster", 10107) #this comes from assert_util.h +error_code("DuplicateKey", 11000) +error_code("InterruptedAtShutdown", 11600) +error_code("Interrupted", 11601) +error_code("OutOfDiskSpace", 14031 ) +error_code("BackgroundOperationInProgressForDatabase", 12586); +error_code("BackgroundOperationInProgressForNamespace", 12587); + +error_class("NetworkError", ["HostUnreachable", "HostNotFound", "NetworkTimeout"]) +error_class("Interruption", ["Interrupted", "InterruptedAtShutdown", "ExceededTimeLimit"]) +error_class("IndexCreationError", ["CannotCreateIndex", "IndexOptionsConflict", + "IndexKeySpecsConflict", "IndexAlreadyExists"]) diff --git a/src/mongo/base/generate_error_codes.py b/src/mongo/base/generate_error_codes.py index 86f0f62532c..eee41afce2a 100644 --- a/src/mongo/base/generate_error_codes.py +++ b/src/mongo/base/generate_error_codes.py @@ -158,7 +158,10 @@ header_template = '''// AUTO-GENERATED FILE DO NOT EDIT #pragma once +#include + #include "mongo/base/string_data.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -169,26 +172,26 @@ namespace mongo { * Do not update this file directly. Update src/mongo/base/error_codes.err instead. */ - class ErrorCodes { + class MONGO_CLIENT_API ErrorCodes { public: enum Error { %(error_code_enum_declarations)s, MaxError }; - static const char* errorString(Error err); + static std::string errorString(Error err); /** - * Parse an Error from its "name". Returns UnknownError if "name" is unrecognized. + * Parses an Error from its "name". Returns UnknownError if "name" is unrecognized. * * NOTE: Also returns UnknownError for the string "UnknownError". */ static Error fromString(const StringData& name); /** - * Parse an Error from its "code". Returns UnknownError if "code" is unrecognized. - * - * NOTE: Also returns UnknownError for the integer code for UnknownError. + * Casts an integer "code" to an Error. Unrecognized codes are preserved, meaning + * that the result of a call to fromInt() may not be one of the values in the + * Error enumeration. */ static Error fromInt(int code); @@ -217,13 +220,16 @@ source_template = '''// AUTO-GENERATED FILE DO NOT EDIT #include "mongo/base/error_codes.h" -#include +#include + +#include "mongo/util/mongoutils/str.h" namespace mongo { - const char* ErrorCodes::errorString(Error err) { + + std::string ErrorCodes::errorString(Error err) { switch (err) { %(symbol_to_string_cases)s; - default: return "Unknown error code"; + default: return mongoutils::str::stream() << "Location" << err; } } @@ -233,14 +239,14 @@ namespace mongo { } ErrorCodes::Error ErrorCodes::fromInt(int code) { - switch (code) { - %(int_to_symbol_cases)s; - default: - return UnknownError; - } + return static_cast(code); } %(error_code_class_predicate_definitions)s + +namespace { + BOOST_STATIC_ASSERT(sizeof(ErrorCodes::Error) == sizeof(int)); +} // namespace } // namespace mongo ''' diff --git a/src/mongo/base/init.cpp b/src/mongo/base/init.cpp index c03e63dd786..e88d0f96b4e 100644 --- a/src/mongo/base/init.cpp +++ b/src/mongo/base/init.cpp @@ -16,8 +16,3 @@ #include "mongo/base/init.h" MONGO_INITIALIZER_GROUP(default, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS) - -MONGO_INITIALIZER_GROUP(globalVariableConfigurationStarted, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS) -MONGO_INITIALIZER_GROUP(globalVariablesDeclared, ("globalVariableConfigurationStarted"), MONGO_NO_DEPENDENTS) -MONGO_INITIALIZER_GROUP(globalVariablesSet, ("globalVariablesDeclared"), MONGO_NO_DEPENDENTS) -MONGO_INITIALIZER_GROUP(globalVariablesConfigured, ("globalVariablesDeclared"), ("default")) diff --git a/src/mongo/base/init.h b/src/mongo/base/init.h index 9fec726ce9b..7ba6c2cc0a5 100644 --- a/src/mongo/base/init.h +++ b/src/mongo/base/init.h @@ -14,7 +14,7 @@ */ /** - * Utility macros for declaring global initializers and configurable variables. + * Utility macros for declaring global initializers * * Should NOT be included by other header files. Include only in source files. * @@ -24,33 +24,10 @@ * Initializer functions take a parameter of type ::mongo::InitializerContext*, and return * a Status. Any status other than Status::OK() is considered a failure that will stop further * intializer processing. - * - * Global configuration variables are declared and set using initializers and a few groups. All - * global configuration declarations have "globalVariableConfigurationStarted" as a prerequisite, - * and "globalVariablesDeclared" as a dependent. The easiest way for programs to then configure - * those values to non-default settings is to use the MONGO_CONFIG_VARIABLE_SETTER macro - * to declare exactly one function that has "globalVariablesDeclared" as a prerequisite and - * "globalVariablesSet" as a dependent. - * - * Initializers that wish to use configurable global variables must have "globalVariablesConfigured" - * as a direct or indirect prerequisite. The "default" prerequisite depends on - * "globalVariablesConfigured", so most initializer functions can safely use global configurable - * variables. - * - * Programmers may validate global variables after they are set using an initializer declared as - * MONGO_CONFIG_VARIABLE_VALIDATOR, which has "globalVariablesSet" as prerequisite and - * "globalVariablesConfigured" as dependent. - * - * In summary, the following partial order is provided: - * All MONGO_CONFIG_VARIABLE_REGISTER()s are evaluated before - * The MONGO_CONFIG_VARIABLE_SETTER is evaluated before - * All MONGO_CONFIG_VARIABLE_VALIDATORs are evaluated before - * Things dependent on "default" are evaluated. */ #pragma once -#include "mongo/base/configuration_variable_manager.h" #include "mongo/base/initializer.h" #include "mongo/base/initializer_context.h" #include "mongo/base/initializer_function.h" @@ -62,12 +39,12 @@ /** * Convenience parameter representing an empty set of prerequisites for an initializer function. */ -#define MONGO_NO_PREREQUISITES () +#define MONGO_NO_PREREQUISITES (NULL) /** * Convenience parameter representing an empty set of dependents of an initializer function. */ -#define MONGO_NO_DEPENDENTS () +#define MONGO_NO_DEPENDENTS (NULL) /** * Convenience parameter representing the default set of dependents for initializer functions. @@ -148,43 +125,6 @@ MONGO_INITIALIZER_GENERAL(NAME, PREREQUISITES, DEPENDENTS)( \ ::mongo::InitializerContext*) { return ::mongo::Status::OK(); } - -/** - * Macro to register a configurable global variable. - * - * "NAME" is the string name through which the variable's storage may be accessed - * in the ConfigurationVariableManager supplied as part of the InitializerContext - * to global initializer functions. "STORAGE" is a pointer to the location in - * memory where the variable is stored, and "DEFAULT_VALUE" is the value to be - * assigned as the default, at registration time (once main has started). This - * allows DEFAULT_VALUE to be constructed after main() begins, so some options - * that are not available to static initializers may be available here. - */ -#define MONGO_CONFIG_VARIABLE_REGISTER(NAME, STORAGE, DEFAULT_VALUE) \ - MONGO_INITIALIZER_GENERAL(cvr_##NAME, \ - ("globalVariableConfigurationStarted"), \ - ("globalVariablesDeclared"))( \ - ::mongo::InitializerContext* context) { \ - *(STORAGE) = (DEFAULT_VALUE); \ - return ::mongo::getGlobalInitializer().getConfigurationVariableManager().registerVariable( \ - #NAME, (STORAGE)); \ - } - -/** - * Convenience macro for functions that validate already-set values of global - * variables. Run after the MONGO_CONFIG_VARIABLE_SETTER completes. - */ -#define MONGO_CONFIG_VARIABLE_VALIDATOR(NAME) \ - MONGO_INITIALIZER_GENERAL(NAME, ("globalVariablesConfigured"), ("default")) - -/** - * Convenience macro for declaring the global variable setting function. - */ -#define MONGO_CONFIG_VARIABLE_SETTER \ - MONGO_INITIALIZER_GENERAL(globalVariableSetter, \ - ("globalVariablesDeclared"), \ - ("globalVariablesSet")) - /** * Macro to produce a name for a mongo initializer function for an initializer operation * named "NAME". diff --git a/src/mongo/base/initializer.cpp b/src/mongo/base/initializer.cpp index 8792ec2de97..1bb77c967ca 100644 --- a/src/mongo/base/initializer.cpp +++ b/src/mongo/base/initializer.cpp @@ -16,6 +16,7 @@ #include "mongo/base/initializer.h" #include +#include "mongo/util/assert_util.h" #include "mongo/base/global_initializer.h" namespace mongo { @@ -31,7 +32,7 @@ namespace mongo { if (Status::OK() != status) return status; - InitializerContext context(args, env, &_configVariables); + InitializerContext context(args, env); for (size_t i = 0; i < sortedNodes.size(); ++i) { InitializerFunction fn = _graph.getInitializerFunction(sortedNodes[i]); @@ -40,7 +41,12 @@ namespace mongo { "topSort returned a node that has no associated function: \"" + sortedNodes[i] + '"'); } - status = fn(&context); + try { + status = fn(&context); + } catch( const DBException& xcp ) { + return xcp.toStatus(); + } + if (Status::OK() != status) return status; } @@ -49,21 +55,10 @@ namespace mongo { Status runGlobalInitializers(const InitializerContext::ArgumentVector& args, const InitializerContext::EnvironmentMap& env) { - return getGlobalInitializer().execute(args, env); } - void runGlobalInitializersOrDie(const InitializerContext::ArgumentVector& args, - const InitializerContext::EnvironmentMap& env) { - - Status status = runGlobalInitializers(args, env); - if (Status::OK() != status) { - std::cerr << "Failed global initialization: " << status << std::endl; - ::_exit(1); - } - } - - void runGlobalInitializersOrDie(int argc, const char* const* argv, const char* const* envp) { + Status runGlobalInitializers(int argc, const char* const* argv, const char* const* envp) { InitializerContext::ArgumentVector args(argc); std::copy(argv, argv + argc, args.begin()); @@ -73,14 +68,21 @@ namespace mongo { for(; *envp; ++envp) { const char* firstEqualSign = strchr(*envp, '='); if (!firstEqualSign) { - std::cerr << "Failed global initialization: malformed environment block\n"; - ::_exit(1); + return Status(ErrorCodes::BadValue, "malformed environment block"); } env[std::string(*envp, firstEqualSign)] = std::string(firstEqualSign + 1); } } - runGlobalInitializersOrDie(args, env); + return runGlobalInitializers(args, env); + } + + void runGlobalInitializersOrDie(int argc, const char* const* argv, const char* const* envp) { + Status status = runGlobalInitializers(argc, argv, envp); + if (!status.isOK()) { + std::cerr << "Failed global initialization: " << status << std::endl; + ::_exit(1); + } } } // namespace mongo diff --git a/src/mongo/base/initializer.h b/src/mongo/base/initializer.h index e54560eb72b..d81645302ea 100644 --- a/src/mongo/base/initializer.h +++ b/src/mongo/base/initializer.h @@ -18,11 +18,11 @@ #include #include -#include "mongo/base/configuration_variable_manager.h" #include "mongo/base/disallow_copying.h" #include "mongo/base/initializer_context.h" #include "mongo/base/initializer_dependency_graph.h" #include "mongo/base/status.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -30,13 +30,11 @@ namespace mongo { * Class representing an initialization process. * * Such a process is described by a directed acyclic graph of initialization operations, the - * InitializerDependencyGraph, and a collection of mutable global state, the - * ConfigurationVariableManager. One constructs an initialization process by adding nodes and - * edges to the graph, and variable mappings in the variable manager. Then, one executes the - * process, causing each initialization operation to execute in an order that respects the - * programmer-established prerequistes. + * InitializerDependencyGraph. One constructs an initialization process by adding nodes and + * edges to the graph. Then, one executes the process, causing each initialization operation to + * execute in an order that respects the programmer-established prerequistes. */ - class Initializer { + class MONGO_CLIENT_API Initializer { MONGO_DISALLOW_COPYING(Initializer); public: Initializer(); @@ -47,12 +45,6 @@ namespace mongo { */ InitializerDependencyGraph& getInitializerDependencyGraph() { return _graph; } - /** - * Get the configuration variable manager, for the purpose of describing more configurable - * variables. - */ - ConfigurationVariableManager& getConfigurationVariableManager() { return _configVariables; } - /** * Execute the initializer process, using the given argv and environment data as input. * @@ -65,7 +57,6 @@ namespace mongo { private: InitializerDependencyGraph _graph; - ConfigurationVariableManager _configVariables; }; /** @@ -77,16 +68,17 @@ namespace mongo { * This means that the few initializers that might want to terminate the program by failing * should probably arrange to terminate the process themselves. */ - Status runGlobalInitializers(const InitializerContext::ArgumentVector& args, - const InitializerContext::EnvironmentMap& env); + MONGO_CLIENT_API Status runGlobalInitializers(const InitializerContext::ArgumentVector& args, + const InitializerContext::EnvironmentMap& env); + + MONGO_CLIENT_API Status runGlobalInitializers( + int argc, const char* const* argv, const char* const* envp); /** * Same as runGlobalInitializers(), except prints a brief message to std::cerr * and terminates the process on failure. */ - void runGlobalInitializersOrDie(const InitializerContext::ArgumentVector& args, - const InitializerContext::EnvironmentMap& env); - - void runGlobalInitializersOrDie(int argc, const char* const* argv, const char* const* envp); + MONGO_CLIENT_API void runGlobalInitializersOrDie( + int argc, const char* const* argv, const char* const* envp); } // namespace mongo diff --git a/src/mongo/base/initializer_context.cpp b/src/mongo/base/initializer_context.cpp index 9c2e54caf3f..f944d577480 100644 --- a/src/mongo/base/initializer_context.cpp +++ b/src/mongo/base/initializer_context.cpp @@ -18,8 +18,7 @@ namespace mongo { InitializerContext::InitializerContext(const ArgumentVector& args, - const EnvironmentMap& env, - const ConfigurationVariableManager* configVariables) - : _args(args), _env(env), _configVariables(configVariables) {} + const EnvironmentMap& env) + : _args(args), _env(env) {} } // namespace mongo diff --git a/src/mongo/base/initializer_context.h b/src/mongo/base/initializer_context.h index 915e18f600e..60acb97eaa7 100644 --- a/src/mongo/base/initializer_context.h +++ b/src/mongo/base/initializer_context.h @@ -19,7 +19,6 @@ #include #include -#include "mongo/base/configuration_variable_manager.h" #include "mongo/base/disallow_copying.h" namespace mongo { @@ -37,17 +36,14 @@ namespace mongo { typedef std::map EnvironmentMap; InitializerContext(const ArgumentVector& args, - const EnvironmentMap& env, - const ConfigurationVariableManager* configVariables); + const EnvironmentMap& env); const ArgumentVector& args() const { return _args; } const EnvironmentMap& env() const { return _env; } - const ConfigurationVariableManager* configurationVariables() { return _configVariables; } private: ArgumentVector _args; EnvironmentMap _env; - const ConfigurationVariableManager* _configVariables; }; } // namespace mongo diff --git a/src/mongo/base/initializer_dependency_graph_test.cpp b/src/mongo/base/initializer_dependency_graph_test.cpp index 3b35dfff419..7ea03524c37 100644 --- a/src/mongo/base/initializer_dependency_graph_test.cpp +++ b/src/mongo/base/initializer_dependency_graph_test.cpp @@ -17,6 +17,7 @@ * Unit tests of the InitializerDependencyGraph type. */ +#include "mongo/base/init.h" #include "mongo/base/initializer_dependency_graph.h" #include "mongo/base/make_string_vector.h" #include "mongo/unittest/unittest.h" @@ -47,13 +48,17 @@ namespace { TEST(InitializerDependencyGraphTest, InsertNullFunctionFails) { InitializerDependencyGraph graph; - ASSERT_EQUALS(ErrorCodes::BadValue, ADD_INITIALIZER(graph, "A", InitializerFunction(), (), ())); + ASSERT_EQUALS(ErrorCodes::BadValue, ADD_INITIALIZER( + graph, "A", InitializerFunction(), + MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS)); } TEST(InitializerDependencyGraphTest, InsertSameNameTwiceFails) { InitializerDependencyGraph graph; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ()); - ASSERT_EQUALS(ErrorCodes::DuplicateKey, ADD_INITIALIZER(graph, "A", doNothing, (), ())); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_EQUALS(ErrorCodes::DuplicateKey, ADD_INITIALIZER( + graph, "A", doNothing, + MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS)); } TEST(InitializerDependencyGraphTest, TopSortEmptyGraph) { @@ -66,9 +71,9 @@ namespace { TEST(InitializerDependencyGraphTest, TopSortGraphNoDeps) { InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ()); - ASSERT_ADD_INITIALIZER(graph, "B", doNothing, (), ()); - ASSERT_ADD_INITIALIZER(graph, "C", doNothing, (), ()); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "B", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "C", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); ASSERT_EQUALS(Status::OK(), graph.topSort(&nodeNames)); ASSERT_EQUALS(3U, nodeNames.size()); ASSERT_EXACTLY_ONE_IN_CONTAINER(nodeNames, "A"); @@ -90,10 +95,10 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ()); - ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("B", "C"), ()); - ASSERT_ADD_INITIALIZER(graph, "B", doNothing, ("A"), ()); - ASSERT_ADD_INITIALIZER(graph, "C", doNothing, ("A"), ()); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("B", "C"), MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "B", doNothing, ("A"), MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "C", doNothing, ("A"), MONGO_NO_DEPENDENTS); ASSERT_EQUALS(Status::OK(), graph.topSort(&nodeNames)); ASSERT_EQUALS(4U, nodeNames.size()); ASSERT_EXACTLY_ONE_IN_CONTAINER(nodeNames, "A"); @@ -118,10 +123,10 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ("B", "C")); - ASSERT_ADD_INITIALIZER(graph, "D", doNothing, (), ()); - ASSERT_ADD_INITIALIZER(graph, "B", doNothing, (), ("D")); - ASSERT_ADD_INITIALIZER(graph, "C", doNothing, (), ("D")); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, ("B", "C")); + ASSERT_ADD_INITIALIZER(graph, "D", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "B", doNothing, MONGO_NO_PREREQUISITES, ("D")); + ASSERT_ADD_INITIALIZER(graph, "C", doNothing, MONGO_NO_PREREQUISITES, ("D")); ASSERT_EQUALS(Status::OK(), graph.topSort(&nodeNames)); ASSERT_EQUALS(4U, nodeNames.size()); ASSERT_EXACTLY_ONE_IN_CONTAINER(nodeNames, "A"); @@ -147,8 +152,8 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ()); - ASSERT_ADD_INITIALIZER(graph, "D", doNothing, (), ()); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "D", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); ASSERT_ADD_INITIALIZER(graph, "B", doNothing, ("A"), ("D")); ASSERT_ADD_INITIALIZER(graph, "C", doNothing, ("A"), ("D")); ASSERT_EQUALS(Status::OK(), graph.topSort(&nodeNames)); @@ -176,10 +181,10 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ("B", "C")); - ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("C", "B"), ()); - ASSERT_ADD_INITIALIZER(graph, "B", doNothing, (), ()); - ASSERT_ADD_INITIALIZER(graph, "C", doNothing, (), ()); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, ("B", "C")); + ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("C", "B"), MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "B", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "C", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); ASSERT_EQUALS(Status::OK(), graph.topSort(&nodeNames)); ASSERT_EQUALS(4U, nodeNames.size()); ASSERT_EXACTLY_ONE_IN_CONTAINER(nodeNames, "A"); @@ -205,8 +210,8 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ("B", "C")); - ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("C", "B"), ()); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, ("B", "C")); + ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("C", "B"), MONGO_NO_DEPENDENTS); ASSERT_ADD_INITIALIZER(graph, "B", doNothing, ("A"), ("D")); ASSERT_ADD_INITIALIZER(graph, "C", doNothing, ("A"), ("D")); ASSERT_EQUALS(Status::OK(), graph.topSort(&nodeNames)); @@ -233,10 +238,10 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ("B", "C")); - ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("C", "B"), ()); - ASSERT_ADD_INITIALIZER(graph, "B", doNothing, (), ()); - ASSERT_ADD_INITIALIZER(graph, "C", doNothing, (), ()); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, ("B", "C")); + ASSERT_ADD_INITIALIZER(graph, "D", doNothing, ("C", "B"), MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "B", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); + ASSERT_ADD_INITIALIZER(graph, "C", doNothing, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); ASSERT_ADD_INITIALIZER(graph, "E", doNothing, ("D"), ("B")); ASSERT_EQUALS(ErrorCodes::GraphContainsCycle, graph.topSort(&nodeNames)); ASSERT_EQUALS(4U, nodeNames.size()); @@ -254,7 +259,7 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "B", doNothing, ("A"), ()); + ASSERT_ADD_INITIALIZER(graph, "B", doNothing, ("A"), MONGO_NO_DEPENDENTS); ASSERT_EQUALS(ErrorCodes::BadValue, graph.topSort(&nodeNames)); } @@ -264,7 +269,7 @@ namespace { */ InitializerDependencyGraph graph; std::vector nodeNames; - ASSERT_ADD_INITIALIZER(graph, "A", doNothing, (), ("B")); + ASSERT_ADD_INITIALIZER(graph, "A", doNothing, MONGO_NO_PREREQUISITES, ("B")); ASSERT_EQUALS(ErrorCodes::BadValue, graph.topSort(&nodeNames)); } diff --git a/src/mongo/base/initializer_test.cpp b/src/mongo/base/initializer_test.cpp index ae3fb8d56e6..0635a5c2a2a 100644 --- a/src/mongo/base/initializer_test.cpp +++ b/src/mongo/base/initializer_test.cpp @@ -17,6 +17,7 @@ * Unit tests of the Initializer type. */ +#include "mongo/base/init.h" #include "mongo/base/initializer.h" #include "mongo/base/initializer_dependency_graph.h" #include "mongo/base/make_string_vector.h" @@ -50,15 +51,15 @@ #define CONSTRUCT_DEPENDENCY_GRAPH(GRAPH, FN0, FN1, FN2, FN3, FN4, FN5, FN6, FN7, FN8) \ do { \ InitializerDependencyGraph& _graph_ = (GRAPH); \ - ASSERT_ADD_INITIALIZER(_graph_, "n0", FN0, (), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n1", FN1, (), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n2", FN2, ("n0", "n1"), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n3", FN3, ("n0", "n2"), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n4", FN4, ("n2", "n1"), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n5", FN5, ("n3", "n4"), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n6", FN6, ("n4"), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n7", FN7, ("n3"), ()); \ - ASSERT_ADD_INITIALIZER(_graph_, "n8", FN8, ("n5", "n6", "n7"), ()); \ + ASSERT_ADD_INITIALIZER(_graph_, "n0", FN0, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n1", FN1, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n2", FN2, ("n0", "n1"), MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n3", FN3, ("n0", "n2"), MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n4", FN4, ("n2", "n1"), MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n5", FN5, ("n3", "n4"), MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n6", FN6, ("n4"), MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n7", FN7, ("n3"), MONGO_NO_DEPENDENTS); \ + ASSERT_ADD_INITIALIZER(_graph_, "n8", FN8, ("n5", "n6", "n7"), MONGO_NO_DEPENDENTS); \ } while (false) namespace mongo { diff --git a/src/mongo/base/make_string_vector.cpp b/src/mongo/base/make_string_vector.cpp index 1865f19b859..1818bb9ff53 100644 --- a/src/mongo/base/make_string_vector.cpp +++ b/src/mongo/base/make_string_vector.cpp @@ -24,12 +24,8 @@ namespace mongo { std::vector _makeStringVector(int ignored, ...) { va_list ap; va_start(ap, ignored); - const char* arg = va_arg(ap, const char *); - if (arg) { - std::cerr << "Internal error!\n"; - std::abort(); - } std::vector result; + const char* arg = NULL; while ((arg = va_arg(ap, const char *))) result.push_back(arg); va_end(ap); diff --git a/src/mongo/base/make_string_vector.h b/src/mongo/base/make_string_vector.h index 39c5167641e..6dd4dfad391 100644 --- a/src/mongo/base/make_string_vector.h +++ b/src/mongo/base/make_string_vector.h @@ -25,7 +25,7 @@ * Usage: MONGO_MAKE_STRING_VECTOR("a", "b", "c") returns a vector containing * std::strings "a", "b", "c", in that order. */ -#define MONGO_MAKE_STRING_VECTOR(...) ::mongo::_makeStringVector(0, NULL, ##__VA_ARGS__, NULL) +#define MONGO_MAKE_STRING_VECTOR(...) ::mongo::_makeStringVector(0, __VA_ARGS__, NULL) namespace mongo { @@ -35,9 +35,10 @@ namespace mongo { * WARNING: Only intended for use by MONGO_MAKE_STRING_VECTOR macro, defined above. Aborts * ungracefully if you misuse it, so stick to the macro. * - * The first parameter is ignored in all circumstances. The second parameter must be NULL, as - * must be the last parameter. The third through penultimate parameters should be const char* - * C-style strings. + * The first parameter is ignored in all circumstances. The subsequent parameters must be + * const char* C-style strings, or NULL. Of these parameters, at least one must be + * NULL. Parameters at and beyond the NULL are not inserted. Typically, the NULL will be + * the last parameter. The MONGO_MAKE_STRING_VECTOR macro enforces this. * * Returns a vector of std::strings. */ diff --git a/src/mongo/base/owned_pointer_map.h b/src/mongo/base/owned_pointer_map.h index 4f2f3d5dd45..15f2cbe9e01 100644 --- a/src/mongo/base/owned_pointer_map.h +++ b/src/mongo/base/owned_pointer_map.h @@ -24,38 +24,40 @@ namespace mongo { /** * An std::map wrapper that deletes pointers within a vector on destruction. The objects * referenced by the vector's pointers are 'owned' by an object of this class. - * NOTE that an OwnedPointerMap wraps an std::map. + * NOTE that an OwnedPointerMap wraps an std::map. */ - template + template > class OwnedPointerMap { MONGO_DISALLOW_COPYING(OwnedPointerMap); public: + typedef typename std::map MapType; + OwnedPointerMap(); ~OwnedPointerMap(); /** Access the map. */ - const std::map& map() { return _map; } - std::map& mutableMap() { return _map; } + const MapType& map() const { return _map; } + MapType& mutableMap() { return _map; } void clear(); private: - std::map _map; + MapType _map; }; - template - OwnedPointerMap::OwnedPointerMap() { + template + OwnedPointerMap::OwnedPointerMap() { } - template - OwnedPointerMap::~OwnedPointerMap() { + template + OwnedPointerMap::~OwnedPointerMap() { clear(); } - template - void OwnedPointerMap::clear() { - for( typename std::map::iterator i = _map.begin(); i != _map.end(); ++i ) { + template + void OwnedPointerMap::clear() { + for( typename MapType::iterator i = _map.begin(); i != _map.end(); ++i ) { delete i->second; } _map.clear(); diff --git a/src/mongo/base/owned_pointer_map_test.cpp b/src/mongo/base/owned_pointer_map_test.cpp index f813da4915d..32831286cdd 100644 --- a/src/mongo/base/owned_pointer_map_test.cpp +++ b/src/mongo/base/owned_pointer_map_test.cpp @@ -37,6 +37,8 @@ namespace { static std::vector& destroyedNames() { return _destroyedNames; } + string getName() { return _name; } + private: string _name; static std::vector _destroyedNames; @@ -79,5 +81,32 @@ namespace { ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[ 1 ] ); } + TEST(OwnedPointerMapTest, OwnedPointersWithCompare) { + DestructionLogger::destroyedNames().clear(); + { + OwnedPointerMap > owned; + owned.mutableMap().insert( make_pair( 0, new DestructionLogger( "0" ) ) ); + owned.mutableMap().insert( make_pair( 1, new DestructionLogger( "1" ) ) ); + + // use std::greater rather than the default std::less + std::map >::iterator it = owned.mutableMap().begin(); + + ASSERT( owned.mutableMap().end() != it); + // "1" should be sorted to be the first item. + ASSERT_EQUALS( "1", it->second->getName() ); + + it++; + ASSERT( owned.mutableMap().end() != it); + ASSERT_EQUALS( "0", it->second->getName() ); + + // owned destroyed + } + // destroyed in descending order + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "1", DestructionLogger::destroyedNames()[ 0 ] ); + ASSERT_EQUALS( "0", DestructionLogger::destroyedNames()[ 1 ] ); + } + + } // namespace } // namespace mongo diff --git a/src/mongo/base/owned_pointer_vector.h b/src/mongo/base/owned_pointer_vector.h index 4c3f83ae9d5..0f44c448b9b 100644 --- a/src/mongo/base/owned_pointer_vector.h +++ b/src/mongo/base/owned_pointer_vector.h @@ -15,6 +15,7 @@ #pragma once +#include #include #include "mongo/base/disallow_copying.h" @@ -31,30 +32,103 @@ namespace mongo { MONGO_DISALLOW_COPYING(OwnedPointerVector); public: - OwnedPointerVector(); - ~OwnedPointerVector(); + OwnedPointerVector() {} + ~OwnedPointerVector() { clear(); } + + /** + * Takes ownership of all pointers contained in 'other'. + * NOTE: argument is intentionally taken by value. + */ + OwnedPointerVector(std::vector other) { _vector.swap(other); } + + /** + * Takes ownership of all pointers contained in 'other'. + * NOTE: argument is intentionally taken by value. + */ + OwnedPointerVector& operator=(std::vector other) { + clear(); + _vector.swap(other); + return *this; + } + + typedef typename std::vector::const_iterator const_iterator; + typedef typename std::vector::const_reverse_iterator const_reverse_iterator; /** Access the vector. */ - const std::vector& vector() { return _vector; } + const std::vector& vector() const { return _vector; } std::vector& mutableVector() { return _vector; } + std::size_t size() const { return _vector.size(); } + bool empty() const { return _vector.empty(); } + + const_iterator begin() const { return _vector.begin(); } + const_reverse_iterator rbegin() const { return _vector.rbegin(); } + const_iterator end() const { return _vector.end(); } + const_reverse_iterator rend() const { return _vector.rend(); } + + T* operator[] (size_t i) const { return _vector[i]; } + T* back() const { return _vector.back(); } + T* front() const { return _vector.front(); } + + void push_back(T* ptr) { _vector.push_back(ptr); } + + /** + * Deletes all pointers in the vector, then sets its size to 0. + */ void clear(); + /** + * Deletes the pointer at 'it', then erases it from the vector. + */ + void erase(const_iterator it) { + delete *it; + // vector::erase(const_iterator) is new in c++11, so converting to non-const iterator. + _vector.erase(_vector.begin() + (it - begin())); + } + + // + // extensions + // + + /** + * Releases the entire vector to allow you to transfer ownership. + * + * Leaves the OwnedPointerVector empty. + * Named after the similar method and pattern in std::auto_ptr. + */ + std::vector release() { + std::vector out; + out.swap(_vector); + return out; + } + + /** + * Releases ownership of a single element. + * + * Sets that element to NULL and does not change size(). + */ + T* releaseAt(size_t i) { + T* out = _vector[i]; + _vector[i] = NULL; + return out; + } + + T* popAndReleaseBack() { + T* out = _vector.back(); + _vector.pop_back(); + return out; + } + + void popAndDeleteBack() { + delete popAndReleaseBack(); + } + private: std::vector _vector; }; template - OwnedPointerVector::OwnedPointerVector() { - } - - template - OwnedPointerVector::~OwnedPointerVector() { - clear(); - } - - template - void OwnedPointerVector::clear() { + inline void OwnedPointerVector::clear() { for( typename std::vector::iterator i = _vector.begin(); i != _vector.end(); ++i ) { delete *i; } diff --git a/src/mongo/base/owned_pointer_vector_test.cpp b/src/mongo/base/owned_pointer_vector_test.cpp index 95ec5b5e379..ac8683a96ad 100644 --- a/src/mongo/base/owned_pointer_vector_test.cpp +++ b/src/mongo/base/owned_pointer_vector_test.cpp @@ -57,7 +57,7 @@ namespace { DestructionLogger::destroyedNames().clear(); { OwnedPointerVector owned; - owned.mutableVector().push_back( new DestructionLogger( "foo" ) ); + owned.push_back( new DestructionLogger( "foo" ) ); // owned destroyed } ASSERT_EQUALS( 1U, DestructionLogger::destroyedNames().size() ); @@ -68,14 +68,276 @@ namespace { DestructionLogger::destroyedNames().clear(); { OwnedPointerVector owned; - owned.mutableVector().push_back( new DestructionLogger( "first" ) ); - owned.mutableVector().push_back( new DestructionLogger( "second" ) ); + owned.push_back( new DestructionLogger( "first" ) ); + owned.push_back( new DestructionLogger( "second" ) ); + // owned destroyed + } + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames()[ 0 ] ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[ 1 ] ); + } + + TEST(OwnedPointerVectorTest, ClearDestroyedInOrder) { + DestructionLogger::destroyedNames().clear(); + { + OwnedPointerVector owned; + owned.push_back( new DestructionLogger( "first" ) ); + owned.push_back( new DestructionLogger( "second" ) ); + + owned.clear(); + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames()[ 0 ] ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[ 1 ] ); + ASSERT_EQUALS( 0U, owned.size() ); + // owned destroyed + } + // no additional deletion should have occured when owned was destroyed + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + } + + TEST(OwnedPointerVectorTest, EraseDestroysAsCalled) { + DestructionLogger::destroyedNames().clear(); + { + // names are order of erasure + OwnedPointerVector owned; + owned.push_back( new DestructionLogger( "third" ) ); + owned.push_back( new DestructionLogger( "first" ) ); + owned.push_back( new DestructionLogger( "second" ) ); + owned.push_back( new DestructionLogger( "fourth" ) ); + + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + + // erase "first", sliding "second" down to index 1 + owned.erase(owned.begin() + 1); + ASSERT_EQUALS( 1U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames().back() ); + ASSERT_EQUALS( 3U, owned.size() ); + + // erase "second" + owned.erase(owned.begin() + 1); + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames().back() ); + ASSERT_EQUALS( 2U, owned.size() ); + + // erase "third" + owned.erase(owned.begin() + 0); + ASSERT_EQUALS( 3U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "third", DestructionLogger::destroyedNames().back() ); + ASSERT_EQUALS( 1U, owned.size() ); + // owned destroyed } + + // only "four" should have been deleted when owned was destroyed + ASSERT_EQUALS( 4U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "fourth", DestructionLogger::destroyedNames().back() ); + } + + TEST(OwnedPointerVectorTest, Accessors) { + OwnedPointerVector owned; + ASSERT_TRUE( owned.empty() ); + ASSERT_EQUALS( 0U, owned.size() ); + + owned.push_back( new int(0) ); + owned.push_back( new int(1) ); + owned.push_back( new int(2) ); + + ASSERT_FALSE( owned.empty() ); + ASSERT_EQUALS( 3U, owned.size() ); + + ASSERT_EQUALS( 0, *owned[0] ); + ASSERT_EQUALS( 1, *owned[1] ); + ASSERT_EQUALS( 2, *owned[2] ); + + ASSERT_EQUALS( 0, *owned.front() ); + ASSERT_EQUALS( 2, *owned.back() ); + } + + TEST(OwnedPointerVectorTest, TransferConstructor) { + DestructionLogger::destroyedNames().clear(); + { + OwnedPointerVector source; + source.push_back( new DestructionLogger( "first" ) ); + source.push_back( new DestructionLogger( "second" ) ); + + { + OwnedPointerVector dest(source.release()); + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 0U, source.size() ); + ASSERT_EQUALS( 2U, dest.size() ); + // dest destroyed + } + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames()[ 0 ] ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[ 1 ] ); + + // source destroyed + } + // no additional deletions + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + } + + TEST(OwnedPointerVectorTest, TransferConstructorDoesntModifyArgument) { + DestructionLogger::destroyedNames().clear(); + { + std::vector source; + source.push_back( new DestructionLogger( "first" ) ); + source.push_back( new DestructionLogger( "second" ) ); + + { + OwnedPointerVector dest(source); + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 2U, source.size() ); + ASSERT_EQUALS( 2U, dest.size() ); + ASSERT( source == dest.vector() ); // can't use ASSERT_EQUALS + // dest destroyed + } + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames()[ 0 ] ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[ 1 ] ); + + ASSERT_EQUALS( 2U, source.size() ); + // source destroyed + } + // no additional deletions + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + } + + TEST(OwnedPointerVectorTest, TransferAssignment) { + DestructionLogger::destroyedNames().clear(); + { + OwnedPointerVector dest; + { + OwnedPointerVector source; + source.push_back( new DestructionLogger( "first" ) ); + source.push_back( new DestructionLogger( "second" ) ); + + dest = source.release(); + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 0U, source.size() ); + ASSERT_EQUALS( 2U, dest.size() ); + // source destroyed + } + + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 2U, dest.size() ); + // dest destroyed + } ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); ASSERT_EQUALS( "first", DestructionLogger::destroyedNames()[ 0 ] ); ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[ 1 ] ); } + + TEST(OwnedPointerVectorTest, TransferAssignmentDoesntModifyArgument) { + DestructionLogger::destroyedNames().clear(); + { + OwnedPointerVector dest; + { + std::vector source; + source.push_back( new DestructionLogger( "first" ) ); + source.push_back( new DestructionLogger( "second" ) ); + + dest = source; + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 2U, source.size() ); + ASSERT_EQUALS( 2U, dest.size() ); + ASSERT( source == dest.vector() ); // can't use ASSERT_EQUALS + // source destroyed + } + + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 2U, dest.size() ); + // dest destroyed + } + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames()[ 0 ] ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[ 1 ] ); + } + + TEST(OwnedPointerVectorTest, ReleaseAt) { + DestructionLogger::destroyedNames().clear(); + + boost::scoped_ptr holder; + { + // names are order of deletion + OwnedPointerVector owned; + owned.push_back( new DestructionLogger( "first" ) ); + owned.push_back( new DestructionLogger( "third" ) ); + owned.push_back( new DestructionLogger( "second" ) ); + + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + + // transfer ownership of "third" to holder + holder.reset(owned.releaseAt(1)); + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 3U, owned.size() ); + ASSERT_EQUALS( static_cast(NULL), owned[1] ); + + // owned destroyed + } + // owned deleted "first" and "second", but not "third" + ASSERT_EQUALS( 2U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames()[0] ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[1] ); + + // delete "third" + holder.reset(); + ASSERT_EQUALS( 3U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "third", DestructionLogger::destroyedNames().back() ); + } + + TEST(OwnedPointerVectorTest, PopAndReleaseBack) { + DestructionLogger::destroyedNames().clear(); + + { + // names are order of deletion + OwnedPointerVector owned; + owned.push_back( new DestructionLogger( "second" ) ); + owned.push_back( new DestructionLogger( "third" ) ); + owned.push_back( new DestructionLogger( "first" ) ); + + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + + { + // transfer ownership of "third" to holder + boost::scoped_ptr holder(owned.popAndReleaseBack()); + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( 2U, owned.size() ); + // holder destroyed + } + ASSERT_EQUALS( 1U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames().back() ); + // owned destroyed + } + // owned destructor deleted "second" and "third", but not "first" + ASSERT_EQUALS( 3U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[1] ); + ASSERT_EQUALS( "third", DestructionLogger::destroyedNames()[2] ); + } + + TEST(OwnedPointerVectorTest, PopAndDeleteBack) { + DestructionLogger::destroyedNames().clear(); + + { + // names are order of deletion + OwnedPointerVector owned; + owned.push_back( new DestructionLogger( "second" ) ); + owned.push_back( new DestructionLogger( "third" ) ); + owned.push_back( new DestructionLogger( "first" ) ); + + ASSERT_EQUALS( 0U, DestructionLogger::destroyedNames().size() ); + + owned.popAndDeleteBack(); + ASSERT_EQUALS( 2U, owned.size() ); + ASSERT_EQUALS( 1U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "first", DestructionLogger::destroyedNames().back() ); + // owned destroyed + } + // owned destructor deleted "second" and "third", but not "first" + ASSERT_EQUALS( 3U, DestructionLogger::destroyedNames().size() ); + ASSERT_EQUALS( "second", DestructionLogger::destroyedNames()[1] ); + ASSERT_EQUALS( "third", DestructionLogger::destroyedNames()[2] ); + } } // namespace } // namespace mongo diff --git a/src/mongo/base/parse_number.cpp b/src/mongo/base/parse_number.cpp index 775727d7337..2e75c6ef7cd 100644 --- a/src/mongo/base/parse_number.cpp +++ b/src/mongo/base/parse_number.cpp @@ -13,9 +13,15 @@ * limitations under the License. */ +#include "mongo/platform/basic.h" + #include "mongo/base/parse_number.h" +#include +#include +#include #include +#include #include "mongo/platform/cstdint.h" @@ -169,7 +175,7 @@ namespace mongo { // Definition of the various supported implementations of parseNumberFromStringWithBase. #define DEFINE_PARSE_NUMBER_FROM_STRING_WITH_BASE(NUMBER_TYPE) \ - template Status parseNumberFromStringWithBase(const StringData&, int, NUMBER_TYPE*); + template MONGO_COMPILER_API_EXPORT Status parseNumberFromStringWithBase(const StringData&, int, NUMBER_TYPE*); DEFINE_PARSE_NUMBER_FROM_STRING_WITH_BASE(long) DEFINE_PARSE_NUMBER_FROM_STRING_WITH_BASE(long long) @@ -181,5 +187,72 @@ namespace mongo { DEFINE_PARSE_NUMBER_FROM_STRING_WITH_BASE(unsigned int) DEFINE_PARSE_NUMBER_FROM_STRING_WITH_BASE(int8_t); DEFINE_PARSE_NUMBER_FROM_STRING_WITH_BASE(uint8_t); +#undef DEFINE_PARSE_NUMBER_FROM_STRING_WITH_BASE + +#ifdef _WIN32 + +namespace { + + /** + * Converts ascii c-locale uppercase characters to lower case, leaves other char values + * unchanged. + */ + char toLowerAscii(char c) { + if (isascii(c) && isupper(c)) + return _tolower(c); + return c; + } + +} // namespace + +#endif // defined(_WIN32) + + template <> + Status parseNumberFromStringWithBase(const StringData& stringValue, + int base, + double* result) { + if (base != 0) { + return Status(ErrorCodes::BadValue, + "Must pass 0 as base to parseNumberFromStringWithBase."); + } + if (stringValue.empty()) + return Status(ErrorCodes::FailedToParse, "Empty string"); + + if (isspace(stringValue[0])) + return Status(ErrorCodes::FailedToParse, "Leading whitespace"); + + std::string str = stringValue.toString(); + const char* cStr = str.c_str(); + char* endp; + errno = 0; + double d = strtod(cStr, &endp); + int actualErrno = errno; + if (endp != stringValue.size() + cStr) { +#ifdef _WIN32 + // The Windows libc implementation of strtod cannot parse +/-infinity or nan, + // so handle that here. + std::transform(str.begin(), str.end(), str.begin(), toLowerAscii); + if (str == StringData("nan", StringData::LiteralTag())) { + *result = std::numeric_limits::quiet_NaN(); + return Status::OK(); + } + else if (str == StringData("+infinity", StringData::LiteralTag()) || + str == StringData("infinity", StringData::LiteralTag())) { + *result = std::numeric_limits::infinity(); + return Status::OK(); + } + else if (str == StringData("-infinity", StringData::LiteralTag())) { + *result = -std::numeric_limits::infinity(); + return Status::OK(); + } +#endif // defined(_WIN32) + + return Status(ErrorCodes::FailedToParse, "Did not consume whole number."); + } + if (actualErrno == ERANGE) + return Status(ErrorCodes::FailedToParse, "Out of range"); + *result = d; + return Status::OK(); + } } // namespace mongo diff --git a/src/mongo/base/parse_number.h b/src/mongo/base/parse_number.h index 9a5d252f8d6..efe7247785f 100644 --- a/src/mongo/base/parse_number.h +++ b/src/mongo/base/parse_number.h @@ -21,6 +21,7 @@ #include "mongo/base/status.h" #include "mongo/base/string_data.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -42,7 +43,7 @@ namespace mongo { * See parse_number.cpp for the available instantiations, and add any new instantiations there. */ template - Status parseNumberFromStringWithBase(const StringData& stringValue, int base, NumberType* result); + MONGO_CLIENT_API Status parseNumberFromStringWithBase(const StringData& stringValue, int base, NumberType* result); template static Status parseNumberFromString(const StringData& stringValue, NumberType* result) { diff --git a/src/mongo/base/parse_number_test.cpp b/src/mongo/base/parse_number_test.cpp index e18066b742d..585b6f33377 100644 --- a/src/mongo/base/parse_number_test.cpp +++ b/src/mongo/base/parse_number_test.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ #include "mongo/pch.h" @@ -21,6 +33,7 @@ #include "mongo/base/parse_number.h" #include "mongo/base/status.h" #include "mongo/platform/cstdint.h" +#include "mongo/platform/float_utils.h" #include "mongo/util/mongoutils/str.h" // for str::stream()! #include "mongo/unittest/unittest.h" @@ -222,5 +235,77 @@ namespace { ASSERT_PARSES(uint8_t, std::string(mongoutils::str::stream() << i), i); } + TEST(Double, TestRejectingBadBases) { + double ignored; + + // Only supported base for parseNumberFromStringWithBase is 0. + ASSERT_EQUALS(ErrorCodes::BadValue, parseNumberFromStringWithBase("0", -1, &ignored)); + ASSERT_EQUALS(ErrorCodes::BadValue, parseNumberFromStringWithBase("0", 1, &ignored)); + ASSERT_EQUALS(ErrorCodes::BadValue, parseNumberFromStringWithBase("0", 8, &ignored)); + ASSERT_EQUALS(ErrorCodes::BadValue, parseNumberFromStringWithBase("0", 10, &ignored)); + ASSERT_EQUALS(ErrorCodes::BadValue, parseNumberFromStringWithBase("0", 16, &ignored)); + } + + TEST(Double, TestParsingGarbage) { + double d; + CommonNumberParsingTests::TestParsingGarbage(); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("1.0.1", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("1.0-1", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString(" 1.0", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("1.0P4", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("1e6 ", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString(" 1e6", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("1e6 ", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString(" 1e6", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("0xabcab.defPa", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, + parseNumberFromString(StringData("1.0\0garbage", + StringData::LiteralTag()), + &d)); + } + + TEST(Double, TestParsingOverflow) { + double d; + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("1e309", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("-1e309", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("1e-400", &d)); + ASSERT_EQUALS(ErrorCodes::FailedToParse, parseNumberFromString("-1e-400", &d)); + } + + TEST(Double, TestParsingNan) { + double d = 0; + ASSERT_OK(parseNumberFromString("NaN", &d)); + ASSERT_TRUE(isNaN(d)); + } + + TEST(Double, TestParsingInfinity) { + double d = 0; + ASSERT_OK(parseNumberFromString("infinity", &d)); + ASSERT_TRUE(isInf(d)); + d = 0; + ASSERT_OK(parseNumberFromString("-Infinity", &d)); + ASSERT_TRUE(isInf(d)); + } + + TEST(Double, TestParsingNormal) { + ASSERT_PARSES(double, "10", 10); + ASSERT_PARSES(double, "0", 0); + ASSERT_PARSES(double, "1", 1); + ASSERT_PARSES(double, "-10", -10); + ASSERT_PARSES(double, "1e8", 1e8); + ASSERT_PARSES(double, "1e-8", 1e-8); + ASSERT_PARSES(double, "12e-8", 12e-8); + ASSERT_PARSES(double, "-485.381e-8", -485.381e-8); + +#if !(defined(_WIN32) || defined(__sunos__)) + // Parse hexadecimal representations of a double. Hex literals not supported by MSVC, and + // not parseable by the Windows SDK libc or the Solaris libc in the mode we build. + + ASSERT_PARSES(double, "0xff", 0xff); + ASSERT_PARSES(double, "-0xff", -0xff); + ASSERT_PARSES(double, "0xabcab.defdefP-10", 0xabcab.defdefP-10); +#endif + } + } // namespace } // namespace mongo diff --git a/src/mongo/base/status-inl.h b/src/mongo/base/status-inl.h new file mode 100644 index 00000000000..add962e2369 --- /dev/null +++ b/src/mongo/base/status-inl.h @@ -0,0 +1,86 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { + + inline Status Status::OK() { + return Status(); + } + + inline Status::Status(const Status& other) + : _error(other._error) { + ref(_error); + } + + inline Status& Status::operator=(const Status& other) { + ref(other._error); + unref(_error); + _error = other._error; + return *this; + } + + inline Status::~Status() { + unref(_error); + } + + inline bool Status::isOK() const { + return code() == ErrorCodes::OK; + } + + inline ErrorCodes::Error Status::code() const { + return _error ? _error->code : ErrorCodes::OK; + } + + inline std::string Status::codeString() const { + return ErrorCodes::errorString(code()); + } + + inline std::string Status::reason() const { + return _error ? _error->reason : std::string(); + } + + inline int Status::location() const { + return _error ? _error->location : 0; + } + + inline AtomicUInt32::WordType Status::refCount() const { + return _error ? _error->refs.load() : 0; + } + + inline Status::Status() + : _error(NULL) { + } + + inline void Status::ref(ErrorInfo* error) { + if (error) + error->refs.fetchAndAdd(1); + } + + inline void Status::unref(ErrorInfo* error) { + if (error && (error->refs.subtractAndFetch(1) == 0)) + delete error; + } + + inline bool operator==(const ErrorCodes::Error lhs, const Status& rhs) { + return rhs == lhs; + } + + inline bool operator!=(const ErrorCodes::Error lhs, const Status& rhs) { + return rhs != lhs; + } + +} // namespace mongo diff --git a/src/mongo/base/status.cpp b/src/mongo/base/status.cpp index e32f880ffa2..731dd44cdff 100644 --- a/src/mongo/base/status.cpp +++ b/src/mongo/base/status.cpp @@ -20,48 +20,31 @@ namespace mongo { - Status::ErrorInfo* Status::getOKInfo() { - static ErrorInfo* okInfo = new ErrorInfo(ErrorCodes::OK, "", 0); - return okInfo; + Status::ErrorInfo::ErrorInfo(ErrorCodes::Error aCode, const StringData& aReason, int aLocation) + : code(aCode), reason(aReason.toString()), location(aLocation) { } - Status::ErrorInfo::ErrorInfo(ErrorCodes::Error aCode, const std::string& aReason, int aLocation) - : code(aCode), reason(aReason), location(aLocation) {} - - Status::Status(ErrorCodes::Error code, const char* reason, int location) { - _error = new ErrorInfo(code, std::string(reason), location); - ref(_error); + Status::ErrorInfo* Status::ErrorInfo::create(ErrorCodes::Error c, const StringData& r, int l) { + const bool needRep = ((c != ErrorCodes::OK) || + !r.empty() || + (l != 0)); + return needRep ? new ErrorInfo(c, r, l) : NULL; } - Status::Status(ErrorCodes::Error code, const std::string& reason, int location) { - _error = new ErrorInfo(code, reason, location); + Status::Status(ErrorCodes::Error code, const std::string& reason, int location) + : _error(ErrorInfo::create(code, reason, location)) { ref(_error); } - Status::Status(ErrorInfo* info) { - _error = info; - ref(info); - } - - Status::Status(const Status& other) { - ref(other._error); - _error = other._error; - } - - Status& Status::operator=(const Status& other) { - ref(other._error); - unref(_error); - _error = other._error; - return *this; - } - - Status::~Status() { - unref(_error); + Status::Status(ErrorCodes::Error code, const char* reason, int location) + : _error(ErrorInfo::create(code, reason, location)) { + ref(_error); } bool Status::compare(const Status& other) const { - return _error->code == other._error->code && - _error->location == other._error->location; + return + code() == other.code() && + location() == other.location(); } bool Status::operator==(const Status& other) const { @@ -73,7 +56,7 @@ namespace mongo { } bool Status::compareCode(const ErrorCodes::Error other) const { - return _error->code == other; + return code() == other; } bool Status::operator==(const ErrorCodes::Error other) const { @@ -84,25 +67,6 @@ namespace mongo { return ! compareCode(other); } - void Status::ref(ErrorInfo* error) { - // okInfo is never deallocated, so no need to bump ref here. - if (error == getOKInfo()) { - return; - } - error->refs.fetchAndAdd(1); - } - - void Status::unref(ErrorInfo* error) { - // okInfo is never deallocated. - if (error == getOKInfo()) { - return; - } - - if (error->refs.subtractAndFetch(1) == 0) { - delete error; - } - } - std::ostream& operator<<(std::ostream& os, const Status& status) { return os << status.codeString() << " " << status.reason(); } @@ -121,10 +85,4 @@ namespace mongo { return ss.str(); } -namespace { - /// Ensure that Status::OK() is called at least once in single threaded context, - /// by creating a global variable whose static initializer calls it. - Status okStatusInstance = Status::OK(); -} // namespace - } // namespace mongo diff --git a/src/mongo/base/status.h b/src/mongo/base/status.h index cdd3980bae2..ad301f8b008 100644 --- a/src/mongo/base/status.h +++ b/src/mongo/base/status.h @@ -19,6 +19,7 @@ #include #include "mongo/base/error_codes.h" +#include "mongo/client/export_macros.h" #include "mongo/platform/atomic_word.h" namespace mongo { @@ -46,10 +47,10 @@ namespace mongo { * TODO: generate base/error_codes.h out of a description file * TODO: check 'location' duplicates against assert numbers */ - class Status { + class MONGO_CLIENT_API Status { public: // Short-hand for returning an OK status. - static Status OK() { return Status(getOKInfo()); } + static inline Status OK(); /** * Builds an error status given the error code, a textual description of what @@ -58,9 +59,11 @@ namespace mongo { */ Status(ErrorCodes::Error code, const std::string& reason, int location = 0); Status(ErrorCodes::Error code, const char* reason, int location = 0); - Status(const Status& other); - Status& operator=(const Status& other); - ~Status(); + + inline Status(const Status& other); + inline Status& operator=(const Status& other); + + inline ~Status(); /** * Returns true if 'other's error code and location are equal/different to this @@ -82,11 +85,15 @@ namespace mongo { // accessors // - bool isOK() const { return code() == ErrorCodes::OK; } - ErrorCodes::Error code() const { return _error->code; } - const char* codeString() const { return ErrorCodes::errorString(_error->code); } - const std::string& reason() const { return _error->reason; } - int location() const { return _error->location; } + inline bool isOK() const; + + inline ErrorCodes::Error code() const; + + inline std::string codeString() const; + + inline std::string reason() const; + + inline int location() const; std::string toString() const; @@ -94,21 +101,22 @@ namespace mongo { // Below interface used for testing code only. // - int refCount() const { return _error->refs.load(); } + inline AtomicUInt32::WordType refCount() const; private: - struct ErrorInfo { - AtomicUInt32 refs; // reference counter - ErrorCodes::Error code; // error code - std::string reason; // description of error cause - int location; // unique location of the triggering line in the code + inline Status(); - ErrorInfo(ErrorCodes::Error aCode, const std::string& aReason, int aLocation); - }; + struct ErrorInfo { + AtomicUInt32 refs; // reference counter + const ErrorCodes::Error code; // error code + const std::string reason; // description of error cause + const int location; // unique location of the triggering line in the code - static ErrorInfo *getOKInfo(); + static ErrorInfo* create(ErrorCodes::Error code, + const StringData& reason, int location); - explicit Status(ErrorInfo *info); + ErrorInfo(ErrorCodes::Error code, const StringData& reason, int location); + }; ErrorInfo* _error; @@ -117,23 +125,21 @@ namespace mongo { * * @param error ErrorInfo to be incremented */ - static void ref(ErrorInfo* error); - static void unref(ErrorInfo* error); + static inline void ref(ErrorInfo* error); + static inline void unref(ErrorInfo* error); }; - static inline bool operator==(const ErrorCodes::Error lhs, const Status& rhs) { - return rhs == lhs; - } + MONGO_CLIENT_API inline bool operator==(const ErrorCodes::Error lhs, const Status& rhs); - static inline bool operator!=(const ErrorCodes::Error lhs, const Status& rhs) { - return rhs != lhs; - } + MONGO_CLIENT_API inline bool operator!=(const ErrorCodes::Error lhs, const Status& rhs); // // Convenience method for unittest code. Please use accessors otherwise. // - std::ostream& operator<<(std::ostream& os, const Status& status); - std::ostream& operator<<(std::ostream& os, ErrorCodes::Error); + MONGO_CLIENT_API std::ostream& operator<<(std::ostream& os, const Status& status); + MONGO_CLIENT_API std::ostream& operator<<(std::ostream& os, ErrorCodes::Error); } // namespace mongo + +#include "mongo/base/status-inl.h" diff --git a/src/mongo/base/status_test.cpp b/src/mongo/base/status_test.cpp index 00f40a42962..abda898c7f5 100644 --- a/src/mongo/base/status_test.cpp +++ b/src/mongo/base/status_test.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ #include "mongo/base/status.h" @@ -45,36 +57,29 @@ namespace { TEST(Cloning, Copy) { Status orig(ErrorCodes::MaxError, "error"); - ASSERT_EQUALS(orig.refCount(), 1); + ASSERT_EQUALS(orig.refCount(), 1U); Status dest(orig); ASSERT_EQUALS(dest.code(), ErrorCodes::MaxError); ASSERT_EQUALS(dest.reason(), "error"); - ASSERT_EQUALS(dest.refCount(), 2); - ASSERT_EQUALS(orig.refCount(), 2); + ASSERT_EQUALS(dest.refCount(), 2U); + ASSERT_EQUALS(orig.refCount(), 2U); } TEST(Cloning, OKIsNotRefCounted) { - ASSERT_EQUALS(Status::OK().refCount(), 0); + ASSERT_EQUALS(Status::OK().refCount(), 0U); Status myOk = Status::OK(); - ASSERT_EQUALS(myOk.refCount(), 0); - ASSERT_EQUALS(Status::OK().refCount(), 0); + ASSERT_EQUALS(myOk.refCount(), 0U); + ASSERT_EQUALS(Status::OK().refCount(), 0U); } TEST(Parsing, CodeToEnum) { ASSERT_EQUALS(ErrorCodes::TypeMismatch, ErrorCodes::fromInt(ErrorCodes::TypeMismatch)); ASSERT_EQUALS(ErrorCodes::UnknownError, ErrorCodes::fromInt(ErrorCodes::UnknownError)); - ASSERT_EQUALS(ErrorCodes::UnknownError, ErrorCodes::fromInt(ErrorCodes::MaxError)); + ASSERT_EQUALS(ErrorCodes::MaxError, ErrorCodes::fromInt(ErrorCodes::MaxError)); ASSERT_EQUALS(ErrorCodes::OK, ErrorCodes::fromInt(0)); } - TEST(Parsing, StringToEnum) { - ASSERT_EQUALS(ErrorCodes::TypeMismatch, ErrorCodes::fromString("TypeMismatch")); - ASSERT_EQUALS(ErrorCodes::UnknownError, ErrorCodes::fromString("UnknownError")); - ASSERT_EQUALS(ErrorCodes::UnknownError, ErrorCodes::fromString("Garbage")); - ASSERT_EQUALS(ErrorCodes::OK, ErrorCodes::fromString("OK")); - } - } // unnamed namespace diff --git a/src/mongo/base/status_with.h b/src/mongo/base/status_with.h new file mode 100644 index 00000000000..384246d5bf9 --- /dev/null +++ b/src/mongo/base/status_with.h @@ -0,0 +1,78 @@ +// status_with.h + +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongo/base/status.h" + +namespace mongo { + + + /* + * StatusWith is used to return an error or a value + * this is designed to make exception code free cleaner by not needing as many out paramters + * example: + StatusWith fib( int n ) { + if ( n < 0 ) return StatusWith( ErrorCodes::BadValue, "paramter to fib has to be >= 0" ); + if ( n <= 1 ) return StatusWith( 1 ); + StatusWith a = fib( n - 1 ); + StatusWith b = fib( n - 2 ); + if ( !a.isOK() ) return a; + if ( !b.isOK() ) return b; + return StatusWith( a.getValue() + b.getValue() ); + } + + * Note: the value is copied in the current implementation, so should be small (int, int*) + * not a vector + */ + template + class StatusWith { + public: + /** + * for the error case + */ + StatusWith( ErrorCodes::Error code, const std::string& reason, int location = 0 ) + : _status( Status( code, reason, location ) ) { + } + + /** + * for the error case + */ + explicit StatusWith( const Status& status ) + : _status( status ) { + // verify(( !status.isOK() ); // TODO + } + + /** + * for the OK case + */ + explicit StatusWith( const T& t ) + : _status( Status::OK() ), _t( t ) { + } + + const T& getValue() const { /* verify( isOK() ); */ return _t; } // TODO + const Status& getStatus() const { return _status;} + + bool isOK() const { return _status.isOK(); } + + string toString() const { return _status.toString(); } + private: + Status _status; + T _t; + }; + +} diff --git a/src/mongo/base/string_data.cpp b/src/mongo/base/string_data.cpp index 31ab21f1144..51f05fb6839 100644 --- a/src/mongo/base/string_data.cpp +++ b/src/mongo/base/string_data.cpp @@ -20,14 +20,33 @@ namespace mongo { + namespace { + + template + size_t murmur3(const StringData& str); + + template <> + size_t murmur3<4>(const StringData& str) { + uint32_t hash; + MurmurHash3_x86_32(str.rawData(), str.size(), 0, &hash); + return hash; + } + + template <> + size_t murmur3<8>(const StringData& str) { + uint64_t hash[2]; + MurmurHash3_x64_128(str.rawData(), str.size(), 0, hash); + return static_cast(hash[0]); + } + + } // namespace + std::ostream& operator<<(std::ostream& stream, const StringData& value) { return stream.write(value.rawData(), value.size()); } size_t StringData::Hasher::operator() (const StringData& str) const { - unsigned out; - MurmurHash3_x86_32(str.rawData(), str.size(), 0, &out); - return out; + return murmur3(str); } } // namespace mongo diff --git a/src/mongo/base/string_data.h b/src/mongo/base/string_data.h index 3ffbba834e8..2c4864943a0 100644 --- a/src/mongo/base/string_data.h +++ b/src/mongo/base/string_data.h @@ -135,6 +135,15 @@ namespace mongo { size_t operator() (const StringData& str) const; }; + // + // iterators + // + + typedef const char* const_iterator; + + const_iterator begin() const { return rawData(); } + const_iterator end() const { return rawData() + size(); } + private: const char* _data; // is not guaranted to be null terminated (see "notes" above) mutable size_t _size; // 'size' does not include the null terminator @@ -174,4 +183,4 @@ namespace mongo { } // namespace mongo -#include "string_data-inl.h" +#include "mongo/base/string_data-inl.h" diff --git a/src/mongo/base/string_data_test.cpp b/src/mongo/base/string_data_test.cpp index bbd40011002..dca13e44dba 100644 --- a/src/mongo/base/string_data_test.cpp +++ b/src/mongo/base/string_data_test.cpp @@ -12,9 +12,23 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ +#include #include +#include #include "mongo/base/string_data.h" #include "mongo/unittest/unittest.h" @@ -123,6 +137,42 @@ namespace { ASSERT_EQUALS( string("foo").find( "" ), StringData("foo").find( "" ) ); } + // Helper function for Test(Hasher, Str1) + template + void SDHasher_check(void); + + template <> + void SDHasher_check<4>(void) { + ASSERT_EQUALS(StringData::Hasher()(""), + static_cast(0)); + ASSERT_EQUALS(StringData::Hasher()("foo"), + static_cast(4138058784ULL)); + ASSERT_EQUALS(StringData::Hasher()("pizza"), + static_cast(3587803311ULL)); + ASSERT_EQUALS(StringData::Hasher()("mongo"), + static_cast(3724335885ULL)); + ASSERT_EQUALS(StringData::Hasher()("murmur"), + static_cast(1945310157ULL)); + } + + template <> + void SDHasher_check<8>(void) { + ASSERT_EQUALS(StringData::Hasher()(""), + static_cast(0)); + ASSERT_EQUALS(StringData::Hasher()("foo"), + static_cast(16316970633193145697ULL)); + ASSERT_EQUALS(StringData::Hasher()("pizza"), + static_cast(12165495155477134356ULL)); + ASSERT_EQUALS(StringData::Hasher()("mongo"), + static_cast(2861051452199491487ULL)); + ASSERT_EQUALS(StringData::Hasher()("murmur"), + static_cast(18237957392784716687ULL)); + } + + TEST(Hasher, Str1) { + SDHasher_check(); + } + TEST(Rfind, Char1) { ASSERT_EQUALS( string::npos, StringData( "foo" ).rfind( 'a' ) ); @@ -224,4 +274,47 @@ namespace { ASSERT(!StringData("abcde").substr(0, 3).endsWith("cde")); } + TEST(ConstIterator, StdCopy) { + std::vector chars; + const char rawData[] = "This is some raw data."; + StringData data(rawData, StringData::LiteralTag()); + + chars.resize(data.size()); + std::copy(data.begin(), data.end(), chars.begin()); + + for (size_t i = 0; i < data.size(); ++i) { + ASSERT_EQUALS(data[i], chars[i]); + } + } + + TEST(ConstIterator, StdReverseCopy) { + std::vector chars; + const char rawData[] = "This is some raw data."; + StringData data(rawData, StringData::LiteralTag()); + + chars.resize(data.size()); + std::reverse_copy(data.begin(), data.end(), chars.begin()); + + const char rawDataExpected[] = ".atad war emos si sihT"; + + for (size_t i = 0; i < data.size(); ++i) { + ASSERT_EQUALS(rawDataExpected[i], chars[i]); + } + } + + TEST(ConstIterator, StdReplaceCopy) { + std::vector chars; + const char rawData[] = "This is some raw data."; + StringData data(rawData, StringData::LiteralTag()); + + chars.resize(data.size()); + std::replace_copy(data.begin(), data.end(), chars.begin(), ' ', '_'); + + const char rawDataExpected[] = "This_is_some_raw_data."; + + for (size_t i = 0; i < data.size(); ++i) { + ASSERT_EQUALS(rawDataExpected[i], chars[i]); + } + } + } // unnamed namespace diff --git a/src/mongo/base/validate_locale.cpp b/src/mongo/base/validate_locale.cpp new file mode 100644 index 00000000000..54d9d1a0c22 --- /dev/null +++ b/src/mongo/base/validate_locale.cpp @@ -0,0 +1,54 @@ +/* Copyright 2014 MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include + +#include + +#include "mongo/base/init.h" + +namespace mongo { + +MONGO_INITIALIZER_GENERAL(ValidateLocale, + MONGO_NO_PREREQUISITES, + MONGO_DEFAULT_PREREQUISITES) + (InitializerContext*) { + try { + // Validate that boost can correctly load the user's locale + boost::filesystem::path("/").has_root_directory(); + } + catch (const std::runtime_error&) { + return Status(ErrorCodes::BadValue, "Invalid or no user locale set." +#ifndef _WIN32 + " Please ensure LANG and/or LC_* environment variables are set correctly." +#endif + ); + } + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/bson/bson-inl.h b/src/mongo/bson/bson-inl.h index dfe756860c3..e70dba1b47e 100644 --- a/src/mongo/bson/bson-inl.h +++ b/src/mongo/bson/bson-inl.h @@ -208,7 +208,7 @@ dodouble: inline NOINLINE_DECL void BSONObj::_assertInvalid() const { StringBuilder ss; int os = objsize(); - ss << "BSONObj size: " << os << " (0x" << toHex( &os, 4 ) << ") is invalid. " + ss << "BSONObj size: " << os << " (0x" << integerToHex( os ) << ") is invalid. " << "Size must be between 0 and " << BSONObjMaxInternalSize << "(" << ( BSONObjMaxInternalSize/(1024*1024) ) << "MB)"; try { @@ -272,29 +272,27 @@ dodouble: return BSONElement(); } - inline int BSONObj::getIntField(const char *name) const { + inline int BSONObj::getIntField(const StringData& name) const { BSONElement e = getField(name); return e.isNumber() ? (int) e.number() : std::numeric_limits< int >::min(); } - inline bool BSONObj::getBoolField(const char *name) const { + inline bool BSONObj::getBoolField(const StringData& name) const { BSONElement e = getField(name); return e.type() == Bool ? e.boolean() : false; } - inline const char * BSONObj::getStringField(const char *name) const { + inline const char * BSONObj::getStringField(const StringData& name) const { BSONElement e = getField(name); return e.type() == String ? e.valuestr() : ""; } /* add all the fields from the object specified to this object */ inline BSONObjBuilder& BSONObjBuilder::appendElements(BSONObj x) { - BSONObjIterator it(x); - while ( it.moreWithEOO() ) { - BSONElement e = it.next(); - if ( e.eoo() ) break; - append(e); - } + if (!x.isEmpty()) + _b.appendBuf( + x.objdata() + 4, // skip over leading length + x.objsize() - 5); // ignore leading length and trailing \0 return *this; } @@ -369,8 +367,8 @@ dodouble: if ( haveSubobj() ) { verify( _fieldName.rawData() ); _builder->append( _fieldName, subobj()->done() ); + _subobj.reset(); } - _subobj.reset(); _fieldName = nextFieldName; } @@ -454,14 +452,14 @@ dodouble: } inline std::string BSONObj::toString( bool isArray, bool full ) const { - if ( isEmpty() ) return "{}"; + if ( isEmpty() ) return (isArray ? "[]" : "{}"); StringBuilder s; toString(s, isArray, full); return s.str(); } inline void BSONObj::toString( StringBuilder& s, bool isArray, bool full, int depth ) const { if ( isEmpty() ) { - s << "{}"; + s << (isArray ? "[]" : "{}"); return; } @@ -476,7 +474,6 @@ dodouble: int offset = (int) (e.rawdata() - this->objdata()); massert( 10330 , "Element extends past end of object", e.size() + offset <= this->objsize() ); - e.validate(); bool end = ( e.size() + offset == this->objsize() ); if ( e.eoo() ) { massert( 10331 , "EOO Before end of object", end ); @@ -491,45 +488,6 @@ dodouble: s << ( isArray ? " ]" : " }" ); } - inline void BSONElement::validate() const { - const BSONType t = type(); - - switch( t ) { - case DBRef: - case Code: - case Symbol: - case mongo::String: { - unsigned x = (unsigned) valuestrsize(); - bool lenOk = x > 0 && x < (unsigned) BSONObjMaxInternalSize; - if( lenOk && valuestr()[x-1] == 0 ) - return; - StringBuilder buf; - buf << "Invalid dbref/code/string/symbol size: " << x; - if( lenOk ) - buf << " strnlen:" << mongo::strnlen( valuestr() , x ); - msgasserted( 10321 , buf.str() ); - break; - } - case CodeWScope: { - int totalSize = *( int * )( value() ); - massert( 10322 , "Invalid CodeWScope size", totalSize >= 8 ); - int strSizeWNull = *( int * )( value() + 4 ); - massert( 10323 , "Invalid CodeWScope string size", totalSize >= strSizeWNull + 4 + 4 ); - massert( 10324 , "Invalid CodeWScope string size", - strSizeWNull > 0 && - (strSizeWNull - 1) == mongo::strnlen( codeWScopeCode(), strSizeWNull ) ); - massert( 10325 , "Invalid CodeWScope size", totalSize >= strSizeWNull + 4 + 4 + 4 ); - int objSize = *( int * )( value() + 4 + 4 + strSizeWNull ); - massert( 10326 , "Invalid CodeWScope object size", totalSize == 4 + 4 + strSizeWNull + objSize ); - // Subobject validation handled elsewhere. - } - case Object: - // We expect Object size validation to be handled elsewhere. - default: - break; - } - } - inline int BSONElement::size( int maxLen ) const { if ( totalSize >= 0 ) return totalSize; @@ -779,11 +737,16 @@ dodouble: s << __oid() << "')"; break; case BinData: - s << "BinData"; - if (full) { + s << "BinData(" << binDataType() << ", "; + { int len; - const char* data = binDataClean(len); - s << '(' << binDataType() << ", " << toHex(data, len) << ')'; + const char *data = binDataClean(len); + if ( !full && len > 80 ) { + s << toHex(data, 70) << "...)"; + } + else { + s << toHex(data, len) << ")"; + } } break; case Timestamp: @@ -798,21 +761,22 @@ dodouble: /* return has eoo() true if no match supports "." notation to reach into embedded objects */ - inline BSONElement BSONObj::getFieldDotted(const char *name) const { - BSONElement e = getField( name ); - if ( e.eoo() ) { - const char *p = strchr(name, '.'); - if ( p ) { - std::string left(name, p-name); - BSONObj sub = getObjectField(left.c_str()); - return sub.isEmpty() ? BSONElement() : sub.getFieldDotted(p+1); + inline BSONElement BSONObj::getFieldDotted(const StringData& name) const { + BSONElement e = getField(name); + if (e.eoo()) { + size_t dot_offset = name.find('.'); + if (dot_offset != string::npos) { + StringData left = name.substr(0, dot_offset); + StringData right = name.substr(dot_offset + 1); + BSONObj sub = getObjectField(left); + return sub.isEmpty() ? BSONElement() : sub.getFieldDotted(right); } } return e; } - inline BSONObj BSONObj::getObjectField(const char *name) const { + inline BSONObj BSONObj::getObjectField(const StringData& name) const { BSONElement e = getField(name); BSONType t = e.type(); return t == Object || t == Array ? e.embeddedObject() : BSONObj(); @@ -1039,6 +1003,13 @@ dodouble: return true; } + template<> inline bool BSONElement::coerce( long long* out ) const { + if ( !isNumber() ) + return false; + *out = numberLong(); + return true; + } + template<> inline bool BSONElement::coerce( double* out ) const { if ( !isNumber() ) return false; diff --git a/src/mongo/bson/bson.h b/src/mongo/bson/bson.h index 2c8d61bd3da..3ccbaec5a21 100644 --- a/src/mongo/bson/bson.h +++ b/src/mongo/bson/bson.h @@ -1,6 +1,6 @@ /** @file bson.h - Main bson include file for mongodb c++ clients. MongoDB includes ../db/jsobj.h instead. + Main bson include file for mongodb c++ clients. MongoDB includes ../db/jsobj.h instead. This file, however, pulls in much less code / dependencies. @see bsondemo @@ -23,7 +23,8 @@ */ /** - Main include file for C++ BSON module when using standalone (sans MongoDB client). + Main include file for C++ BSON. This pulls in fewer dependencies than + mongo/client/dbclient.h, but still requires libmongoclient to link. "BSON" stands for "binary JSON" -- ie a binary way to represent objects that would be represented in JSON (plus a few extensions useful for databases & other languages). @@ -33,75 +34,15 @@ #pragma once -#if defined(MONGO_EXPOSE_MACROS) -#error this header is for client programs, not the mongo database itself. include jsobj.h instead. -/* because we define simplistic assert helpers here that don't pull in a bunch of util -- so that - BSON can be used header only. - */ +#ifdef MONGO_EXPOSE_MACROS +#error bson.h is for C++ driver consumer use only #endif -#include -#include -#include -#include +#define LIBMONGOCLIENT_CONSUMER -#include "mongo/platform/compiler.h" +#include "mongo/client/redef_macros.h" -namespace bson { - - using std::string; - using std::stringstream; - - class assertion : public std::exception { - public: - assertion( unsigned u , const std::string& s ) - : id( u ) , msg( s ) { - std::stringstream ss; - ss << "BsonAssertion id: " << u << " " << s; - full = ss.str(); - } - - virtual ~assertion() throw() {} - - virtual const char* what() const throw() { return full.c_str(); } - - unsigned id; - std::string msg; - std::string full; - }; -} - -namespace mongo { -#if !defined(verify) - inline void verify(bool expr) { - if(!expr) { - throw bson::assertion( 0 , "assertion failure in bson library" ); - } - } -#endif -#if !defined(uassert) - MONGO_COMPILER_NORETURN inline void uasserted(int msgid, const std::string &s) { - throw bson::assertion( msgid , s ); - } - - inline void uassert(unsigned msgid, const std::string& msg, bool expr) { - if( !expr ) - uasserted( msgid , msg ); - } - MONGO_COMPILER_NORETURN inline void msgasserted(int msgid, const char *msg) { - throw bson::assertion( msgid , msg ); - } - MONGO_COMPILER_NORETURN inline void msgasserted(int msgid, const std::string &msg) { - msgasserted(msgid, msg.c_str()); - } - inline void massert(int msgid, const std::string& msg, bool expr) { - if(!expr) { - std::cout << "assertion failure in bson library: " << msgid << ' ' << msg << std::endl; - throw bson::assertion( msgid , msg ); - } - } -#endif -} +#include "mongo/pch.h" #include "mongo/bson/bsonelement.h" #include "mongo/bson/bsonobj.h" @@ -111,3 +52,5 @@ namespace mongo { #include "mongo/bson/bson-inl.h" #include "mongo/bson/oid.h" #include "mongo/bson/util/builder.h" + +#include "mongo/client/undef_macros.h" diff --git a/src/mongo/bson/bson_db.h b/src/mongo/bson/bson_db.h index c421816e50a..31cadb76ff2 100644 --- a/src/mongo/bson/bson_db.h +++ b/src/mongo/bson/bson_db.h @@ -26,8 +26,8 @@ #pragma once -#include "../util/optime.h" -#include "../util/time_support.h" +#include "mongo/bson/optime.h" +#include "mongo/util/time_support.h" namespace mongo { diff --git a/src/mongo/bson/bson_field.h b/src/mongo/bson/bson_field.h index c8fcfb1fe2f..7e1bb94a13e 100644 --- a/src/mongo/bson/bson_field.h +++ b/src/mongo/bson/bson_field.h @@ -78,7 +78,7 @@ namespace mongo { : _name(name), _defaultSet(false) {} BSONField(const std::string& name, const T& defaultVal) - : _name(name), _default(defaultVal), _defaultSet(true) {} + : _name(name), _default(defaultVal) , _defaultSet(true) {} BSONFieldValue make(const T& t) const { return BSONFieldValue(_name, t); diff --git a/src/mongo/bson/bson_obj_test.cpp b/src/mongo/bson/bson_obj_test.cpp new file mode 100644 index 00000000000..6e4d857ee44 --- /dev/null +++ b/src/mongo/bson/bson_obj_test.cpp @@ -0,0 +1,30 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" + +#include "mongo/unittest/unittest.h" + +namespace { + + TEST(ToString, EmptyArray) { + const char text[] = "{ x: [] }"; + mongo::BSONObj o1 = mongo::fromjson(text); + const std::string o1_str = o1.toString(); + ASSERT_EQUALS(text, o1_str); + } + +} // unnamed namespace diff --git a/src/mongo/bson/bson_validate.cpp b/src/mongo/bson/bson_validate.cpp index 9ac0bcbbbfb..bc27ccae854 100644 --- a/src/mongo/bson/bson_validate.cpp +++ b/src/mongo/bson/bson_validate.cpp @@ -15,15 +15,32 @@ * limitations under the License. */ +#include #include +#include #include "mongo/bson/bson_validate.h" #include "mongo/bson/oid.h" +#include "mongo/db/jsobj.h" namespace mongo { namespace { + /** + * Creates a status with InvalidBSON code and adds information about _id if available. + * WARNING: only pass in a non-EOO idElem if it has been fully validated already! + */ + Status makeError(std::string baseMsg, BSONElement idElem) { + if (idElem.eoo()) { + baseMsg += " in object with unknown _id"; + } + else { + baseMsg += " in object with " + idElem.toString(/*field name=*/true, /*full=*/true); + } + return Status(ErrorCodes::InvalidBSON, baseMsg); + } + class Buffer { public: Buffer( const char* buffer, uint64_t maxLength ) @@ -45,7 +62,7 @@ namespace mongo { Status readCString( StringData* out ) { const void* x = memchr( _buffer + _position, 0, _maxLength - _position ); if ( !x ) - return Status( ErrorCodes::InvalidBSON, "no end of c-string" ); + return makeError("no end of c-string", _idElem); uint64_t len = static_cast( static_cast(x) - ( _buffer + _position ) ); StringData data( _buffer + _position, len ); @@ -60,21 +77,26 @@ namespace mongo { Status readUTF8String( StringData* out ) { int sz; if ( !readNumber( &sz ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", _idElem); + + if ( sz <= 0 ) { + // must have NULL at the very least + return makeError("invalid bson", _idElem); + } if ( out ) { *out = StringData( _buffer + _position, sz ); } if ( !skip( sz - 1 ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", _idElem); char c; if ( !readNumber( &c ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", _idElem); if ( c != 0 ) - return Status( ErrorCodes::InvalidBSON, "not null terminate string" ); + return makeError("not null terminated string", _idElem); return Status::OK(); } @@ -88,10 +110,22 @@ namespace mongo { return _position; } + const char* getBasePtr() const { + return _buffer; + } + + /** + * WARNING: only pass in a non-EOO idElem if it has been fully validated already! + */ + void setIdElem(BSONElement idElem) { + _idElem = idElem; + } + private: const char* _buffer; uint64_t _position; uint64_t _maxLength; + BSONElement _idElem; }; struct ValidationState { @@ -127,12 +161,17 @@ namespace mongo { int _startPosition; }; - Status validateElementInfo(Buffer* buffer, ValidationState::State* nextState) { + /** + * WARNING: only pass in a non-EOO idElem if it has been fully validated already! + */ + Status validateElementInfo(Buffer* buffer, + ValidationState::State* nextState, + BSONElement idElem) { Status status = Status::OK(); - char type; - if ( !buffer->readNumber(&type) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + signed char type; + if ( !buffer->readNumber(&type) ) + return makeError("invalid bson", idElem); if ( type == EOO ) { *nextState = ValidationState::EndObj; @@ -153,17 +192,17 @@ namespace mongo { case jstOID: if ( !buffer->skip( sizeof(OID) ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", idElem); return Status::OK(); case NumberInt: if ( !buffer->skip( sizeof(int32_t) ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", idElem); return Status::OK(); case Bool: if ( !buffer->skip( sizeof(int8_t) ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", idElem); return Status::OK(); @@ -172,7 +211,7 @@ namespace mongo { case Timestamp: case Date: if ( !buffer->skip( sizeof(int64_t) ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", idElem); return Status::OK(); case DBRef: @@ -203,9 +242,11 @@ namespace mongo { case BinData: { int sz; if ( !buffer->readNumber( &sz ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", idElem); + if ( sz < 0 || sz == std::numeric_limits::max() ) + return makeError("invalid size in bson", idElem); if ( !buffer->skip( 1 + sz ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson" ); + return makeError("invalid bson", idElem); return Status::OK(); } case CodeWScope: @@ -217,7 +258,7 @@ namespace mongo { return Status::OK(); default: - return Status( ErrorCodes::InvalidBSON, "invalid bson type" ); + return makeError("invalid bson type", idElem); } } @@ -226,6 +267,9 @@ namespace mongo { ValidationObjectFrame* curr = NULL; ValidationState::State state = ValidationState::BeginObj; + uint64_t idElemStartPos = 0; // will become idElem once validated + BSONElement idElem; + while (state != ValidationState::Done) { switch (state) { case ValidationState::BeginObj: @@ -234,22 +278,40 @@ namespace mongo { curr->setStartPosition(buffer->position()); curr->setIsCodeWithScope(false); if (!buffer->readNumber(&curr->expectedSize)) { - return Status(ErrorCodes::InvalidBSON, - "bson size is larger than buffer size"); + return makeError("bson size is larger than buffer size", idElem); } state = ValidationState::WithinObj; // fall through case ValidationState::WithinObj: { - Status status = validateElementInfo(buffer, &state); + const bool atTopLevel = frames.size() == 1; + // check if we've finished validating idElem and are at start of next element. + if (atTopLevel && idElemStartPos) { + idElem = BSONElement(buffer->getBasePtr() + idElemStartPos); + buffer->setIdElem(idElem); + idElemStartPos = 0; + } + + const uint64_t elemStartPos = buffer->position(); + ValidationState::State nextState = state; + Status status = validateElementInfo(buffer, &nextState, idElem); if (!status.isOK()) return status; + + // we've already validated that fieldname is safe to access as long as we aren't + // at the end of the object, since EOO doesn't have a fieldname. + if (nextState != ValidationState::EndObj && idElem.eoo() && atTopLevel) { + if (strcmp(buffer->getBasePtr() + elemStartPos + 1/*type*/, "_id") == 0) { + idElemStartPos = elemStartPos; + } + } + + state = nextState; break; } case ValidationState::EndObj: { int actualLength = buffer->position() - curr->startPosition(); if ( actualLength != curr->expectedSize ) { - return Status( ErrorCodes::InvalidBSON, - "bson length doesn't match what we found" ); + return makeError("bson length doesn't match what we found", idElem); } frames.pop_back(); if (frames.empty()) { @@ -270,7 +332,7 @@ namespace mongo { curr->setStartPosition(buffer->position()); curr->setIsCodeWithScope(true); if ( !buffer->readNumber( &curr->expectedSize ) ) - return Status( ErrorCodes::InvalidBSON, "invalid bson CodeWScope size" ); + return makeError("invalid bson CodeWScope size", idElem); Status status = buffer->readUTF8String( NULL ); if ( !status.isOK() ) return status; @@ -280,12 +342,12 @@ namespace mongo { case ValidationState::EndCodeWScope: { int actualLength = buffer->position() - curr->startPosition(); if ( actualLength != curr->expectedSize ) { - return Status( ErrorCodes::InvalidBSON, - "bson length for CodeWScope doesn't match what we found" ); + return makeError("bson length for CodeWScope doesn't match what we found", + idElem); } frames.pop_back(); if (frames.empty()) - return Status(ErrorCodes::InvalidBSON, "unnested CodeWScope"); + return makeError("unnested CodeWScope", idElem); curr = &frames.back(); state = ValidationState::WithinObj; break; diff --git a/src/mongo/bson/bson_validate_test.cpp b/src/mongo/bson/bson_validate_test.cpp index 61e67f33c46..83f1739dda4 100644 --- a/src/mongo/bson/bson_validate_test.cpp +++ b/src/mongo/bson/bson_validate_test.cpp @@ -22,6 +22,14 @@ namespace { using namespace mongo; + void appendInvalidStringElement(const char* fieldName, BufBuilder* bb) { + // like a BSONObj string, but without a NUL terminator. + bb->appendChar(String); + bb->appendStr(fieldName, /*withNUL*/true); + bb->appendNum(4); + bb->appendStr("asdf", /*withNUL*/false); + } + TEST(BSONValidate, Basic) { BSONObj x; ASSERT_TRUE( x.valid() ); @@ -213,4 +221,72 @@ namespace { ASSERT_NOT_OK(validateBSON(x.objdata(), x.objsize() / 2)); } + TEST(BSONValidateFast, ErrorWithId) { + BufBuilder bb; + BSONObjBuilder ob(bb); + ob.append("_id", 1); + appendInvalidStringElement("not_id", &bb); + const BSONObj x = ob.done(); + const Status status = validateBSON(x.objdata(), x.objsize()); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(status.reason(), "not null terminated string in object with _id: 1"); + } + + TEST(BSONValidateFast, ErrorBeforeId) { + BufBuilder bb; + BSONObjBuilder ob(bb); + appendInvalidStringElement("not_id", &bb); + ob.append("_id", 1); + const BSONObj x = ob.done(); + const Status status = validateBSON(x.objdata(), x.objsize()); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id"); + } + + TEST(BSONValidateFast, ErrorNoId) { + BufBuilder bb; + BSONObjBuilder ob(bb); + appendInvalidStringElement("not_id", &bb); + const BSONObj x = ob.done(); + const Status status = validateBSON(x.objdata(), x.objsize()); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id"); + } + + TEST(BSONValidateFast, ErrorIsInId) { + BufBuilder bb; + BSONObjBuilder ob(bb); + appendInvalidStringElement("_id", &bb); + const BSONObj x = ob.done(); + const Status status = validateBSON(x.objdata(), x.objsize()); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id"); + } + + TEST(BSONValidateFast, NonTopLevelId) { + BufBuilder bb; + BSONObjBuilder ob(bb); + ob.append("not_id1", BSON("_id" << "not the real _id")); + appendInvalidStringElement("not_id2", &bb); + const BSONObj x = ob.done(); + const Status status = validateBSON(x.objdata(), x.objsize()); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id"); + } + + TEST(BSONValidateFast, StringHasSomething) { + BufBuilder bb; + BSONObjBuilder ob(bb); + bb.appendChar(String); + bb.appendStr("x", /*withNUL*/true); + bb.appendNum(0); + const BSONObj x = ob.done(); + ASSERT_EQUALS(5 // overhead + + 1 // type + + 2 // name + + 4 // size + , x.objsize()); + ASSERT_NOT_OK(validateBSON(x.objdata(), x.objsize())); + } + } diff --git a/src/mongo/bson/bsondemo/bsondemo.cpp b/src/mongo/bson/bsondemo/bsondemo.cpp index 5106595054a..d0f69bbcee5 100644 --- a/src/mongo/bson/bsondemo/bsondemo.cpp +++ b/src/mongo/bson/bsondemo/bsondemo.cpp @@ -30,7 +30,8 @@ Windows: project files are available in this directory for bsondemo.cpp for use with Visual Studio. */ -#include "../bson.h" +#include "mongo/client/dbclient.h" + #include #include diff --git a/src/mongo/bson/bsonelement.h b/src/mongo/bson/bsonelement.h index f094ab91994..7a150992936 100644 --- a/src/mongo/bson/bsonelement.h +++ b/src/mongo/bson/bsonelement.h @@ -23,6 +23,7 @@ #include "mongo/bson/bsontypes.h" #include "mongo/bson/oid.h" +#include "mongo/client/export_macros.h" #include "mongo/platform/cstdint.h" #include "mongo/platform/float_utils.h" @@ -58,14 +59,14 @@ namespace mongo { value() type() */ - class BSONElement { + class MONGO_CLIENT_API BSONElement { public: - /** These functions, which start with a capital letter, throw a UserException if the + /** These functions, which start with a capital letter, throw a MsgAssertionException if the element is not of the required type. Example: std::string foo = obj["foo"].String(); // std::exception if not a std::string type or DNE */ - std::string String() const { return chk(mongo::String).valuestr(); } + std::string String() const { return chk(mongo::String).str(); } Date_t Date() const { return chk(mongo::Date).date(); } double Number() const { return chk(isNumber()).number(); } double Double() const { return chk(NumberDouble)._numberDouble(); } @@ -74,12 +75,12 @@ namespace mongo { bool Bool() const { return chk(mongo::Bool).boolean(); } std::vector Array() const; // see implementation for detailed comments mongo::OID OID() const { return chk(jstOID).__oid(); } - void Null() const { chk(isNull()); } // throw UserException if not null - void OK() const { chk(ok()); } // throw UserException if element DNE + void Null() const { chk(isNull()); } // throw MsgAssertionException if not null + void OK() const { chk(ok()); } // throw MsgAssertionException if element DNE /** @return the embedded object associated with this field. - Note the returned object is a reference to within the parent bson object. If that - object is out of scope, this pointer will no longer be valid. Call getOwned() on the + Note the returned object is a reference to within the parent bson object. If that + object is out of scope, this pointer will no longer be valid. Call getOwned() on the returned BSONObj if you need your own copy. throws UserException if the element is not of type object. */ @@ -144,13 +145,19 @@ namespace mongo { return data + 1; } - + /** + * NOTE: size includes the NULL terminator. + */ int fieldNameSize() const { if ( fieldNameSize_ == -1 ) fieldNameSize_ = (int)strlen( fieldName() ) + 1; return fieldNameSize_; } + const StringData fieldNameStringData() const { + return StringData(fieldName(), fieldNameSize() - 1); + } + /** raw data of the element's value (so be careful). */ const char * value() const { return (data + fieldNameSize() + 1); @@ -230,7 +237,7 @@ namespace mongo { } /** Size (length) of a string element. - You must assure of type String first. + You must assure of type String first. @return string size including terminating null */ int valuestrsize() const { @@ -369,9 +376,6 @@ namespace mongo { /** Constructs an empty element */ BSONElement(); - /** Check that data is internally consistent. */ - void validate() const; - /** True if this element may contain subobjects. */ bool mayEncapsulate() const { switch ( type() ) { @@ -403,6 +407,10 @@ namespace mongo { return ((unsigned int*)(value() ))[0]; } + unsigned long long timestampValue() const { + return reinterpret_cast( value() )[0]; + } + const char * dbrefNS() const { uassert( 10063 , "not a dbref" , type() == DBRef ); return value() + 4; @@ -449,6 +457,19 @@ namespace mongo { } } + struct FieldNameSizeTag {}; // For disambiguation with ctor taking 'maxLen' above. + + /** Construct a BSONElement where you already know the length of the name. The value + * passed here includes the null terminator. The data pointed to by 'd' must not + * represent an EOO. You may pass -1 to indicate that you don't actually know the + * size. + */ + BSONElement(const char* d, int fieldNameSize, FieldNameSizeTag) + : data(d) + , fieldNameSize_(fieldNameSize) // internal size includes null terminator + , totalSize(-1) { + } + std::string _asCode() const; OpTime _opTime() const; diff --git a/src/mongo/bson/bsonmisc.h b/src/mongo/bson/bsonmisc.h index 2d7357a8d1f..147f1bc07db 100644 --- a/src/mongo/bson/bsonmisc.h +++ b/src/mongo/bson/bsonmisc.h @@ -20,6 +20,7 @@ #include #include "mongo/bson/bsonelement.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -69,45 +70,51 @@ namespace mongo { BSON( "a" << GT << 23.4 << NE << 30 << "b" << 2 ) produces the object { a: { \$gt: 23.4, \$ne: 30 }, b: 2 }. */ -#define BSON(x) (( mongo::BSONObjBuilder(64) << x ).obj()) +#define BSON(x) (( ::mongo::BSONObjBuilder(64) << x ).obj()) /** Use BSON_ARRAY macro like BSON macro, but without keys BSONArray arr = BSON_ARRAY( "hello" << 1 << BSON( "foo" << BSON_ARRAY( "bar" << "baz" << "qux" ) ) ); */ -#define BSON_ARRAY(x) (( mongo::BSONArrayBuilder() << x ).arr()) +#define BSON_ARRAY(x) (( ::mongo::BSONArrayBuilder() << x ).arr()) /* Utility class to auto assign object IDs. Example: std::cout << BSON( GENOID << "z" << 3 ); // { _id : ..., z : 3 } */ - extern struct GENOIDLabeler { } GENOID; + struct MONGO_CLIENT_API GENOIDLabeler { }; + extern MONGO_CLIENT_API GENOIDLabeler GENOID; /* Utility class to add a Date element with the current time Example: std::cout << BSON( "created" << DATENOW ); // { created : "2009-10-09 11:41:42" } */ - extern struct DateNowLabeler { } DATENOW; + struct MONGO_CLIENT_API DateNowLabeler { }; + extern MONGO_CLIENT_API DateNowLabeler DATENOW; /* Utility class to assign a NULL value to a given attribute Example: std::cout << BSON( "a" << BSONNULL ); // { a : null } */ - extern struct NullLabeler { } BSONNULL; + struct MONGO_CLIENT_API NullLabeler { }; + extern MONGO_CLIENT_API NullLabeler BSONNULL; /* Utility class to assign an Undefined value to a given attribute Example: std::cout << BSON( "a" << BSONUndefined ); // { a : undefined } */ - extern struct UndefinedLabeler { } BSONUndefined; + struct MONGO_CLIENT_API UndefinedLabeler { }; + extern MONGO_CLIENT_API UndefinedLabeler BSONUndefined; /* Utility class to add the minKey (minus infinity) to a given attribute Example: std::cout << BSON( "a" << MINKEY ); // { "a" : { "$minKey" : 1 } } */ - extern struct MinKeyLabeler { } MINKEY; - extern struct MaxKeyLabeler { } MAXKEY; + struct MONGO_CLIENT_API MinKeyLabeler { }; + extern MONGO_CLIENT_API MinKeyLabeler MINKEY; + struct MONGO_CLIENT_API MaxKeyLabeler { }; + extern MONGO_CLIENT_API MaxKeyLabeler MAXKEY; // Utility class to implement GT, GTE, etc as described above. class Labeler { @@ -172,13 +179,13 @@ namespace mongo { OID oid; }; - extern Labeler::Label GT; - extern Labeler::Label GTE; - extern Labeler::Label LT; - extern Labeler::Label LTE; - extern Labeler::Label NE; - extern Labeler::Label NIN; - extern Labeler::Label BSIZE; + extern MONGO_CLIENT_API Labeler::Label GT; + extern MONGO_CLIENT_API Labeler::Label GTE; + extern MONGO_CLIENT_API Labeler::Label LT; + extern MONGO_CLIENT_API Labeler::Label LTE; + extern MONGO_CLIENT_API Labeler::Label NE; + extern MONGO_CLIENT_API Labeler::Label NIN; + extern MONGO_CLIENT_API Labeler::Label BSIZE; // $or helper: OR(BSON("x" << GT << 7), BSON("y" << LT << 6)); @@ -191,7 +198,7 @@ namespace mongo { // definitions in bsonobjbuilder.h b/c of incomplete types // Utility class to implement BSON( key << val ) as described above. - class BSONObjBuilderValueStream : public boost::noncopyable { + class MONGO_CLIENT_API BSONObjBuilderValueStream : public boost::noncopyable { public: friend class Labeler; BSONObjBuilderValueStream( BSONObjBuilder * builder ); diff --git a/src/mongo/bson/bsonobj.h b/src/mongo/bson/bsonobj.h index ecf172af516..ba2821b2be1 100644 --- a/src/mongo/bson/bsonobj.h +++ b/src/mongo/bson/bsonobj.h @@ -28,6 +28,8 @@ #include "mongo/base/string_data.h" #include "mongo/bson/util/atomic_int.h" #include "mongo/bson/util/builder.h" +#include "mongo/client/export_macros.h" +#include "mongo/util/bufreader.h" namespace mongo { @@ -72,9 +74,9 @@ namespace mongo { Code With Scope: \endcode */ - class BSONObj { + class MONGO_CLIENT_API BSONObj { public: - + /** Construct a BSONObj from data in the proper format. * Use this constructor when something else owns msgdata's buffer */ @@ -96,7 +98,7 @@ namespace mongo { static BSONObj make( const Record* r ); - ~BSONObj() { + ~BSONObj() { _objdata = 0; // defensive } @@ -130,7 +132,7 @@ namespace mongo { */ bool isOwned() const { return _holder.get() != 0; } - /** assure the data buffer is under the control of this BSONObj and not a remote buffer + /** assure the data buffer is under the control of this BSONObj and not a remote buffer @see isOwned() */ BSONObj getOwned() const; @@ -171,14 +173,7 @@ namespace mongo { @param name field to find. supports dot (".") notation to reach into embedded objects. for example "x.y" means "in the nested object in field x, retrieve field y" */ - BSONElement getFieldDotted(const char *name) const; - /** @return the specified element. element.eoo() will be true if not found. - @param name field to find. supports dot (".") notation to reach into embedded objects. - for example "x.y" means "in the nested object in field x, retrieve field y" - */ - BSONElement getFieldDotted(const std::string& name) const { - return getFieldDotted( name.c_str() ); - } + BSONElement getFieldDotted(const StringData &name) const; /** Like getFieldDotted(), but expands arrays and returns all matching objects. * Turning off expandLastArray allows you to retrieve nested array objects instead of @@ -197,7 +192,7 @@ namespace mongo { */ BSONElement getField(const StringData& name) const; - /** Get several fields at once. This is faster than separate getField() calls as the size of + /** Get several fields at once. This is faster than separate getField() calls as the size of elements iterated can then be calculated only once each. @param n number of fieldNames, and number of elements in the fields array @param fields if a field is found its element is stored in its corresponding position in this array. @@ -208,11 +203,7 @@ namespace mongo { /** Get the field of the specified name. eoo() is true on the returned element if not found. */ - BSONElement operator[] (const char *field) const { - return getField(field); - } - - BSONElement operator[] (const std::string& field) const { + BSONElement operator[] (const StringData& field) const { return getField(field); } @@ -229,18 +220,18 @@ namespace mongo { bool hasElement(const StringData& name) const { return hasField(name); } /** @return "" if DNE or wrong type */ - const char * getStringField(const char *name) const; + const char * getStringField(const StringData& name) const; /** @return subobject of the given name */ - BSONObj getObjectField(const char *name) const; + BSONObj getObjectField(const StringData& name) const; /** @return INT_MIN if not present - does some type conversions */ - int getIntField(const char *name) const; + int getIntField(const StringData& name) const; - /** @return false if not present + /** @return false if not present @see BSONElement::trueValue() */ - bool getBoolField(const char *name) const; + bool getBoolField(const StringData& name) const; /** @param pattern a BSON obj indicating a set of (un-dotted) field * names. Element values are ignored. @@ -265,7 +256,8 @@ namespace mongo { BSONObj filterFieldsUndotted(const BSONObj &filter, bool inFilter) const; - BSONElement getFieldUsingIndexNames(const char *fieldName, const BSONObj &indexKey) const; + BSONElement getFieldUsingIndexNames(const StringData& fieldName, + const BSONObj &indexKey) const; /** arrays are bson objects with numeric and increasing field names @return true if field names are numeric and increasing @@ -282,10 +274,49 @@ namespace mongo { /** performs a cursory check on the object's size only. */ bool isValid() const; - /** @return if the user is a valid user doc - criter: isValid() no . or $ field names + /** @return ok if it can be stored as a valid embedded doc. + * Not valid if any field name: + * - contains a "." + * - starts with "$" + * -- unless it is a dbref ($ref/$id/[$db]/...) + */ + inline bool okForStorage() const { + return _okForStorage(false, true).isOK(); + } + + /** Same as above with the following extra restrictions + * Not valid if: + * - "_id" field is a + * -- Regex + * -- Array + */ + inline bool okForStorageAsRoot() const { + return _okForStorage(true, true).isOK(); + } + + /** + * Validates that this can be stored as an embedded document + * See details above in okForStorage + * + * If 'deep' is true then validation is done to children + * + * If not valid a user readable status message is returned. + */ + inline Status storageValidEmbedded(const bool deep = true) const { + return _okForStorage(false, deep); + } + + /** + * Validates that this can be stored as a document (in a collection) + * See details above in okForStorageAsRoot + * + * If 'deep' is true then validation is done to children + * + * If not valid a user readable status message is returned. */ - bool okForStorage() const; + inline Status storageValid(const bool deep = true) const { + return _okForStorage(true, deep); + } /** @return true if object is empty -- i.e., {} */ bool isEmpty() const { return objsize() <= 5; } @@ -353,15 +384,15 @@ namespace mongo { /** @return first field of the object */ BSONElement firstElement() const { return BSONElement(objdata() + 4); } - /** faster than firstElement().fieldName() - for the first element we can easily find the fieldname without + /** faster than firstElement().fieldName() - for the first element we can easily find the fieldname without computing the element size. */ - const char * firstElementFieldName() const { + const char * firstElementFieldName() const { const char *p = objdata() + 4; return *p == EOO ? "" : p+1; } - BSONType firstElementType() const { + BSONType firstElementType() const { const char *p = objdata() + 4; return (BSONType) *p; } @@ -508,6 +539,19 @@ namespace mongo { return *this; } + /// members for Sorter + struct SorterDeserializeSettings {}; // unused + void serializeForSorter(BufBuilder& buf) const { buf.appendBuf(objdata(), objsize()); } + static BSONObj deserializeForSorter(BufReader& buf, const SorterDeserializeSettings&) { + const int size = buf.peek(); + const void* ptr = buf.skip(size); + return BSONObj(static_cast(ptr)); + } + int memUsageForSorter() const { + // TODO consider ownedness? + return sizeof(BSONObj) + objsize(); + } + private: const char *_objdata; boost::intrusive_ptr< Holder > _holder; @@ -523,6 +567,14 @@ namespace mongo { if ( !isValid() ) _assertInvalid(); } + + /** + * Validate if the element is okay to be stored in a collection, maybe as the root element + * + * If 'root' is true then checks against _id are made. + * If 'deep' is false then do not traverse through children + */ + Status _okForStorage(bool root, bool deep) const; }; std::ostream& operator<<( std::ostream &s, const BSONObj &o ); diff --git a/src/mongo/bson/bsonobjbuilder.h b/src/mongo/bson/bsonobjbuilder.h index c0c3d0ba657..9d900a8ca60 100644 --- a/src/mongo/bson/bsonobjbuilder.h +++ b/src/mongo/bson/bsonobjbuilder.h @@ -33,6 +33,7 @@ #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bson_builder_base.h" #include "mongo/bson/bson_field.h" +#include "mongo/client/export_macros.h" #if defined(_DEBUG) && defined(MONGO_EXPOSE_MACROS) #include "mongo/util/log.h" @@ -48,12 +49,15 @@ namespace mongo { /** Utility for creating a BSONObj. See also the BSON() and BSON_ARRAY() macros. */ - class BSONObjBuilder : public BSONBuilderBase, private boost::noncopyable { + class MONGO_CLIENT_API BSONObjBuilder : public BSONBuilderBase, private boost::noncopyable { public: /** @param initsize this is just a hint as to the final size of the object */ BSONObjBuilder(int initsize=512) : _b(_buf), _buf(initsize + sizeof(unsigned)), _offset( sizeof(unsigned) ), _s( this ) , _tracker(0) , _doneCalled(false) { _b.appendNum((unsigned)0); // ref-count _b.skip(4); /*leave room for size field and ref-count*/ + + // Reserve space for the EOO byte. This means _done() can't fail. + _b.reserveBytes(1); } /** @param baseBuilder construct a BSONObjBuilder using an existing BufBuilder @@ -61,11 +65,17 @@ namespace mongo { */ BSONObjBuilder( BufBuilder &baseBuilder ) : _b( baseBuilder ), _buf( 0 ), _offset( baseBuilder.len() ), _s( this ) , _tracker(0) , _doneCalled(false) { _b.skip( 4 ); + + // Reserve space for the EOO byte. This means _done() can't fail. + _b.reserveBytes(1); } BSONObjBuilder( const BSONSizeTracker & tracker ) : _b(_buf) , _buf(tracker.getSize() + sizeof(unsigned) ), _offset( sizeof(unsigned) ), _s( this ) , _tracker( (BSONSizeTracker*)(&tracker) ) , _doneCalled(false) { _b.appendNum((unsigned)0); // ref-count _b.skip(4); + + // Reserve space for the EOO byte. This means _done() can't fail. + _b.reserveBytes(1); } ~BSONObjBuilder() { @@ -196,13 +206,15 @@ namespace mongo { /** appends a number. if n < max(int)/2 then uses int, otherwise long long */ BSONObjBuilder& appendIntOrLL( const StringData& fieldName , long long n ) { - long long x = n; - if ( x < 0 ) - x = x * -1; - if ( x < ( (std::numeric_limits::max)() / 2 ) ) // extra () to avoid max macro on windows - append( fieldName , (int)n ); - else + // extra () to avoid max macro on windows + static const long long maxInt = (std::numeric_limits::max)() / 2; + static const long long minInt = -maxInt; + if ( minInt < n && n < maxInt ) { + append( fieldName , static_cast( n ) ); + } + else { append( fieldName , n ); + } return *this; } @@ -230,15 +242,20 @@ namespace mongo { BSONObjBuilder& appendNumber( const StringData& fieldName, long long llNumber ) { static const long long maxInt = ( 1LL << 30 ); + static const long long minInt = -maxInt; static const long long maxDouble = ( 1LL << 40 ); + static const long long minDouble = -maxDouble; - long long nonNegative = llNumber >= 0 ? llNumber : -llNumber; - if ( nonNegative < maxInt ) + if ( minInt < llNumber && llNumber < maxInt ) { append( fieldName, static_cast( llNumber ) ); - else if ( nonNegative < maxDouble ) + } + else if ( minDouble < llNumber && llNumber < maxDouble ) { append( fieldName, static_cast( llNumber ) ); - else + } + else { append( fieldName, llNumber ); + } + return *this; } @@ -355,7 +372,7 @@ namespace mongo { return appendCode(fieldName, code.code); } - /** Append a string element. + /** Append a string element. @param sz size includes terminating null character */ BSONObjBuilder& append(const StringData& fieldName, const char *str, int sz) { _b.appendNum((char) String); @@ -592,10 +609,22 @@ namespace mongo { BSONObj asTempObj() { BSONObj temp(_done()); _b.setlen(_b.len()-1); //next append should overwrite the EOO + _b.reserveBytes(1); // Rereserve room for the real EOO _doneCalled = false; return temp; } + /** Make it look as if "done" has been called, so that our destructor is a no-op. Do + * this if you know that you don't care about the contents of the builder you are + * destroying. + * + * Note that it is invalid to call any method other than the destructor after invoking + * this method. + */ + void abandon() { + _doneCalled = true; + } + void decouple() { _b.decouple(); // post done() call version. be sure jsobj frees... } @@ -662,8 +691,14 @@ namespace mongo { return _b.buf() + _offset; _doneCalled = true; + + // TODO remove this or find some way to prevent it from failing. Since this is intended + // for use with BSON() literal queries, it is less likely to result in oversized BSON. _s.endField(); + + _b.claimReservedBytes(1); // Prevents adding EOO from failing. _b.appendNum((char) EOO); + char *data = _b.buf() + _offset; int size = _b.len() - _offset; *((int*)data) = size; @@ -774,7 +809,7 @@ namespace mongo { return _b.subarrayStart( num() ); } - // These should only be used where you really need interface compatability with BSONObjBuilder + // These should only be used where you really need interface compatibility with BSONObjBuilder // Currently they are only used by update.cpp and it should probably stay that way BufBuilder &subobjStart( const StringData& name ) { fill( name ); @@ -803,6 +838,11 @@ namespace mongo { return *this; } + BSONArrayBuilder& appendTimestamp(unsigned long long ts) { + _b.appendTimestamp(num(), ts); + return *this; + } + BSONArrayBuilder& append(const StringData& s) { _b.append(num(), s); return *this; @@ -815,6 +855,8 @@ namespace mongo { int len() const { return _b.len(); } int arrSize() const { return _i; } + BufBuilder& bb() { return _b.bb(); } + private: // These two are undefined privates to prevent their accidental // use as we don't support unsigned ints in BSON diff --git a/src/mongo/bson/bsonobjbuilder_test.cpp b/src/mongo/bson/bsonobjbuilder_test.cpp new file mode 100644 index 00000000000..f2b44c552f8 --- /dev/null +++ b/src/mongo/bson/bsonobjbuilder_test.cpp @@ -0,0 +1,264 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * tests for BSONObjBuilder + */ + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" + +#include +#include "mongo/unittest/unittest.h" + +namespace { + + using std::string; + using std::stringstream; + using mongo::BSONElement; + using mongo::BSONObj; + using mongo::BSONObjBuilder; + using mongo::BSONType; + using mongo::OpTime; + + const long long maxEncodableInt = (1 << 30) - 1; + const long long minEncodableInt = -maxEncodableInt; + + const long long maxInt = (std::numeric_limits::max)(); + const long long minInt = (std::numeric_limits::min)(); + + const long long maxEncodableDouble = (1LL << 40) - 1; + const long long minEncodableDouble = -maxEncodableDouble; + + const long long maxDouble = (1LL << std::numeric_limits::digits) - 1; + const long long minDouble = -maxDouble; + + const long long maxLongLong = (std::numeric_limits::max)(); + const long long minLongLong = (std::numeric_limits::min)(); + + template void assertBSONTypeEquals(BSONType actual, BSONType expected, T value, int i) { + if (expected != actual) { + stringstream ss; + ss << "incorrect type in bson object for " << (i+1) << "-th test value " << value + << ". actual: " << mongo::typeName(actual) + << "; expected: " << mongo::typeName(expected); + const string msg = ss.str(); + FAIL(msg); + } + } + + /** + * current conversion ranges in append(unsigned n) + * dbl/int max/min in comments refer to max/min encodable constants + * 0 <= n <= uint_max -----> int + */ + + TEST(BSONObjBuilderTest, AppendUnsignedInt) { + struct { unsigned int v; BSONType t; } data[] = { + { 0, mongo::NumberInt }, + { 100, mongo::NumberInt }, + { maxEncodableInt, mongo::NumberInt }, + { maxEncodableInt + 1, mongo::NumberInt }, + { static_cast(maxInt), mongo::NumberInt }, + { static_cast(maxInt) + 1U, mongo::NumberInt }, + { (std::numeric_limits::max)(), mongo::NumberInt }, + { 0, mongo::Undefined } + }; + for (int i=0; data[i].t != mongo::Undefined; i++) { + unsigned int v = data[i].v; + BSONObjBuilder b; + b.append("a", v); + BSONObj o = b.obj(); + ASSERT_EQUALS(o.nFields(), 1); + BSONElement e = o.getField("a"); + unsigned int n = e.numberLong(); + ASSERT_EQUALS(n, v); + assertBSONTypeEquals(e.type(), data[i].t, v, i); + } + } + + /** + * current conversion ranges in appendIntOrLL(long long n) + * dbl/int max/min in comments refer to max/min encodable constants + * n < dbl_min -----> long long + * dbl_min <= n < int_min -----> double + * int_min <= n <= int_max -----> int + * int_max < n <= dbl_max -----> double + * dbl_max < n -----> long long + */ + + TEST(BSONObjBuilderTest, AppendIntOrLL) { + struct { long long v; BSONType t; } data[] = { + { 0, mongo::NumberInt }, + { -100, mongo::NumberInt }, + { 100, mongo::NumberInt }, + { -(maxInt / 2 - 1), mongo::NumberInt }, + { maxInt / 2 - 1, mongo::NumberInt }, + { -(maxInt / 2), mongo::NumberLong }, + { maxInt / 2, mongo::NumberLong }, + { minEncodableInt, mongo::NumberLong }, + { maxEncodableInt, mongo::NumberLong }, + { minEncodableInt - 1, mongo::NumberLong }, + { maxEncodableInt + 1, mongo::NumberLong }, + { minInt, mongo::NumberLong }, + { maxInt, mongo::NumberLong }, + { minInt - 1, mongo::NumberLong }, + { maxInt + 1, mongo::NumberLong }, + { minLongLong, mongo::NumberLong }, + { maxLongLong, mongo::NumberLong }, + { 0, mongo::Undefined } + }; + for (int i=0; data[i].t != mongo::Undefined; i++) { + long long v = data[i].v; + BSONObjBuilder b; + b.appendIntOrLL("a", v); + BSONObj o = b.obj(); + ASSERT_EQUALS(o.nFields(), 1); + BSONElement e = o.getField("a"); + long long n = e.numberLong(); + ASSERT_EQUALS(n, v); + assertBSONTypeEquals(e.type(), data[i].t, v, i); + } + } + + /** + * current conversion ranges in appendNumber(size_t n) + * dbl/int max/min in comments refer to max/min encodable constants + * 0 <= n <= int_max -----> int + * int_max < n -----> long long + */ + + TEST(BSONObjBuilderTest, AppendNumberSizeT) { + struct { size_t v; BSONType t; } data[] = { + { 0, mongo::NumberInt }, + { 100, mongo::NumberInt }, + { maxEncodableInt, mongo::NumberInt }, + { maxEncodableInt + 1, mongo::NumberLong }, + { size_t(maxInt), mongo::NumberLong }, + { size_t(maxInt) + 1U, mongo::NumberLong }, + { (std::numeric_limits::max)(), mongo::NumberLong }, + { 0, mongo::Undefined } + }; + for (int i=0; data[i].t != mongo::Undefined; i++) { + size_t v = data[i].v; + BSONObjBuilder b; + b.appendNumber("a", v); + BSONObj o = b.obj(); + ASSERT_EQUALS(o.nFields(), 1); + BSONElement e = o.getField("a"); + size_t n = e.numberLong(); + ASSERT_EQUALS(n, v); + assertBSONTypeEquals(e.type(), data[i].t, v, i); + } + } + + /** + * current conversion ranges in appendNumber(long long n) + * dbl/int max/min in comments refer to max/min encodable constants + * n < dbl_min -----> long long + * dbl_min <= n < int_min -----> double + * int_min <= n <= int_max -----> int + * int_max < n <= dbl_max -----> double + * dbl_max < n -----> long long + */ + + TEST(BSONObjBuilderTest, AppendNumberLongLong) { + struct { long long v; BSONType t; } data[] = { + { 0, mongo::NumberInt }, + { -100, mongo::NumberInt }, + { 100, mongo::NumberInt }, + { minEncodableInt, mongo::NumberInt }, + { maxEncodableInt, mongo::NumberInt }, + { minEncodableInt - 1, mongo::NumberDouble }, + { maxEncodableInt + 1, mongo::NumberDouble }, + { minInt, mongo::NumberDouble }, + { maxInt, mongo::NumberDouble }, + { minInt - 1, mongo::NumberDouble }, + { maxInt + 1, mongo::NumberDouble }, + { minEncodableDouble, mongo::NumberDouble }, + { maxEncodableDouble, mongo::NumberDouble }, + { minEncodableDouble - 1, mongo::NumberLong }, + { maxEncodableDouble + 1, mongo::NumberLong }, + { minDouble, mongo::NumberLong }, + { maxDouble, mongo::NumberLong }, + { minDouble - 1, mongo::NumberLong }, + { maxDouble + 1, mongo::NumberLong }, + { minLongLong, mongo::NumberLong }, + { maxLongLong, mongo::NumberLong }, + { 0, mongo::Undefined } + }; + for (int i=0; data[i].t != mongo::Undefined; i++) { + long long v = data[i].v; + BSONObjBuilder b; + b.appendNumber("a", v); + BSONObj o = b.obj(); + ASSERT_EQUALS(o.nFields(), 1); + BSONElement e = o.getField("a"); + if (data[i].t != mongo::NumberDouble) { + long long n = e.numberLong(); + ASSERT_EQUALS(n, v); + } + else { + double n = e.numberDouble(); + ASSERT_APPROX_EQUAL(n, static_cast(v), 0.001); + } + assertBSONTypeEquals(e.type(), data[i].t, v, i); + } + } + + TEST(BSONObjBuilderTest, StreamLongLongMin) { + BSONObj o = BSON("a" << std::numeric_limits::min()); + ASSERT_EQUALS(o.nFields(), 1); + BSONElement e = o.getField("a"); + long long n = e.numberLong(); + ASSERT_EQUALS(n, std::numeric_limits::min()); + } + + TEST(BSONObjBuilderTest, AppendNumberLongLongMinCompareObject) { + BSONObjBuilder b; + b.appendNumber("a", std::numeric_limits::min()); + BSONObj o1 = b.obj(); + + BSONObj o2 = BSON("a" << std::numeric_limits::min()); + + ASSERT_EQUALS(o1, o2); + } + + TEST(BSONObjBuilderTest, AppendMaxTimestampOpTimeConversion) { + BSONObjBuilder b; + b.appendMaxForType("a", mongo::Timestamp); + BSONObj o1 = b.obj(); + + BSONElement e = o1.getField("a"); + ASSERT_FALSE(e.eoo()); + + OpTime opTime = e._opTime(); + ASSERT_FALSE(opTime.isNull()); + } + +} // unnamed namespace diff --git a/src/mongo/bson/bsonobjiterator.h b/src/mongo/bson/bsonobjiterator.h index e210ca22e13..87f53c99e5e 100644 --- a/src/mongo/bson/bsonobjiterator.h +++ b/src/mongo/bson/bsonobjiterator.h @@ -20,6 +20,7 @@ #include // like the ## operator but works with __LINE__ #include "mongo/bson/bsonobj.h" +#include "mongo/base/disallow_copying.h" namespace mongo { @@ -96,6 +97,7 @@ namespace mongo { /** Base class implementing ordered iteration through BSONElements. */ class BSONIteratorSorted { + MONGO_DISALLOW_COPYING(BSONIteratorSorted); public: ~BSONIteratorSorted() { verify( _fields ); diff --git a/src/mongo/bson/mutable/algorithm.h b/src/mongo/bson/mutable/algorithm.h new file mode 100644 index 00000000000..1edee2e0472 --- /dev/null +++ b/src/mongo/bson/mutable/algorithm.h @@ -0,0 +1,289 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/bson/mutable/const_element.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { +namespace mutablebson { + + /** For an overview of mutable BSON, please see the file document.h in this directory. + * + * This file defines, in analogy with , a collection of useful algorithms for + * use with mutable BSON classes. In particular, algorithms for searching, sorting, + * indexed access, and counting are included. + */ + + /** 'findElement' searches rightward among the sibiling Elements of 'first', returning an + * Element representing the first item matching the predicate 'predicate'. If no Element + * matches, then the 'ok' method on the returned Element will return false. + */ + template + inline ElementType findElement(ElementType first, Predicate predicate) { + while (first.ok() && !predicate(first)) + first = first.rightSibling(); + return first; + } + + /** A predicate for findElement that matches on the field name of Elements. */ + struct FieldNameEquals { + // The lifetime of this object must be a subset of the lifetime of 'fieldName'. + explicit FieldNameEquals(const StringData& fieldName) + : fieldName(fieldName) {} + + bool operator()(const ConstElement& element) const { + return (fieldName == element.getFieldName()); + } + + const StringData& fieldName; + }; + + /** An overload of findElement that delegates to the special implementation + * Element::findElementNamed to reduce traffic across the Element API. + */ + template + inline ElementType findElement(ElementType first, FieldNameEquals predicate) { + return first.ok() ? first.findElementNamed(predicate.fieldName) : first; + } + + /** A convenience wrapper around findElement. */ + template + inline ElementType findElementNamed(ElementType first, const StringData& fieldName) { + return findElement(first, FieldNameEquals(fieldName)); + } + + /** Finds the first child under 'parent' that matches the given predicate. If no such child + * Element is found, the returned Element's 'ok' method will return false. + */ + template + inline ElementType findFirstChild(ElementType parent, Predicate predicate) { + return findElement(parent.leftchild(), predicate); + } + + /** An overload of findFirstChild that delegates to the special implementation + * Element::findFirstChildNamed to reduce traffic across the Element API. + */ + template + inline ElementType findFirstChild(ElementType parent, FieldNameEquals predicate) { + return parent.ok() ? parent.findFirstChildNamed(predicate.fieldName) : parent; + } + + /** Finds the first child under 'parent' that matches the given field name. If no such child + * Element is found, the returned Element's 'ok' method will return false. + */ + template + inline ElementType findFirstChildNamed(ElementType parent, const StringData& fieldName) { + return findFirstChild(parent, FieldNameEquals(fieldName)); + } + + /** A less-than ordering for Elements that compares based on the Element field names. */ + class FieldNameLessThan { + // TODO: This should possibly derive from std::binary_function. + public: + inline bool operator()(const ConstElement& left, const ConstElement& right) const { + return left.getFieldName() < right.getFieldName(); + } + }; + + /** Sort any children of Element 'parent' by way of Comparator 'comp', which should provide + * an operator() that takes two const Element&'s and implements a strict weak ordering. + */ + template + void sortChildren(Element parent, Comparator comp) { + // TODO: The following works, but obviously is not ideal. + + // First, build a vector of the children. + std::vector children; + Element current = parent.leftChild(); + while (current.ok()) { + children.push_back(current); + current = current.rightSibling(); + } + + // Then, sort the child vector with our comparator. + std::sort(children.begin(), children.end(), comp); + + // Finally, reorder the children of parent according to the ordering established in + // 'children'. + std::vector::iterator where = children.begin(); + const std::vector::iterator end = children.end(); + for( ; where != end; ++where ) { + // Detach from its current location. + where->remove(); + // Make it the new rightmost element. + parent.pushBack(*where); + } + } + + /** Remove any consecutive children that compare as identical according to 'comp'. The + * children must be sorted (see sortChildren, above), and the equality comparator here + * must be compatible with the comparator used for the sort. + */ + template + void deduplicateChildren(Element parent, EqualityComparator equal) { + Element current = parent.leftChild(); + while (current.ok()) { + Element next = current.rightSibling(); + if (next.ok() && equal(current, next)) { + next.remove(); + } else { + current = next; + } + } + } + + /** A less-than ordering for Elements that compares based on woCompare */ + class woLess { + // TODO: This should possibly derive from std::binary_function. + public: + woLess(bool considerFieldName = true) + : _considerFieldName(considerFieldName) { + } + + inline bool operator()(const ConstElement& left, const ConstElement& right) const { + return left.compareWithElement(right, _considerFieldName) < 0; + } + private: + const bool _considerFieldName; + }; + + /** A greater-than ordering for Elements that compares based on woCompare */ + class woGreater { + // TODO: This should possibly derive from std::binary_function. + public: + woGreater(bool considerFieldName = true) + : _considerFieldName(considerFieldName) { + } + + inline bool operator()(const ConstElement& left, const ConstElement& right) const { + return left.compareWithElement(right, _considerFieldName) > 0; + } + private: + const bool _considerFieldName; + }; + + /** An equality predicate for elements that compares based on woCompare */ + class woEqual { + // TODO: This should possibly derive from std::binary_function. + public: + woEqual(bool considerFieldName = true) + : _considerFieldName(considerFieldName) { + } + + inline bool operator()(const ConstElement& left, const ConstElement& right) const { + return left.compareWithElement(right, _considerFieldName) == 0; + } + private: + const bool _considerFieldName; + }; + + /** An equality predicate for elements that compares based on woCompare */ + class woEqualTo { + // TODO: This should possibly derive from std::binary_function. + public: + woEqualTo(const ConstElement& value, bool considerFieldName = true) + : _value(value) + , _considerFieldName(considerFieldName) { + } + + inline bool operator()(const ConstElement& elt) const { + return _value.compareWithElement(elt, _considerFieldName) == 0; + } + private: + const ConstElement& _value; + const bool _considerFieldName; + }; + + // NOTE: Originally, these truly were algorithms, in that they executed the loop over a + // generic ElementType. However, these operations were later made intrinsic to + // Element/Document for performance reasons. These functions hare here for backward + // compatibility, and just delegate to the appropriate Element or ConstElement method of + // the same name. + + /** Return the element that is 'n' Elements to the left in the sibling chain of 'element'. */ + template + ElementType getNthLeftSibling(ElementType element, std::size_t n) { + return element.leftSibling(n); + } + + /** Return the element that is 'n' Elements to the right in the sibling chain of 'element'. */ + template + ElementType getNthRightSibling(ElementType element, std::size_t n) { + return element.rightSibling(n); + } + + /** Move 'n' Elements left or right in the sibling chain of 'element' */ + template + ElementType getNthSibling(ElementType element, int n) { + return (n < 0) ? + getNthLeftSibling(element, -n) : + getNthRightSibling(element, n); + } + + /** Get the child that is 'n' Elements to the right of 'element's left child. */ + template + ElementType getNthChild(ElementType element, std::size_t n) { + return element.findNthChild(n); + } + + /** Returns the number of valid siblings to the left of 'element'. */ + template + std::size_t countSiblingsLeft(ElementType element) { + return element.countSiblingsLeft(); + } + + /** Returns the number of valid siblings to the right of 'element'. */ + template + std::size_t countSiblingsRight(ElementType element) { + return element.countSiblingsRight(); + } + + /** Return the number of children of 'element'. */ + template + std::size_t countChildren(ElementType element) { + return element.countChildren(); + } + + /** Return the full (path) name of this element separating each name with the delim string. */ + template + std::string getFullName(ElementType element, char delim = '.') { + std::vector names; + ElementType curr = element; + while(curr.ok() && curr.parent().ok()) { + names.push_back(curr.getFieldName()); + curr = curr.parent(); + } + + mongoutils::str::stream name; + bool first = true; + for(std::vector::reverse_iterator it = names.rbegin(); + it != names.rend(); + ++it) { + if (!first) + name << delim; + name << *it; + first = false; + } + return name; + } +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/const_element-inl.h b/src/mongo/bson/mutable/const_element-inl.h new file mode 100644 index 00000000000..2e2c14516fe --- /dev/null +++ b/src/mongo/bson/mutable/const_element-inl.h @@ -0,0 +1,229 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { +namespace mutablebson { + + inline ConstElement::ConstElement(const Element& basis) + : _basis(basis) {} + + inline ConstElement ConstElement::leftChild() const { + return _basis.leftChild(); + } + + inline ConstElement ConstElement::rightChild() const { + return _basis.rightChild(); + } + + inline bool ConstElement::hasChildren() const { + return _basis.hasChildren(); + } + + inline ConstElement ConstElement::leftSibling(size_t distance) const { + return _basis.leftSibling(distance); + } + + inline ConstElement ConstElement::rightSibling(size_t distance) const { + return _basis.rightSibling(distance); + } + + inline ConstElement ConstElement::parent() const { + return _basis.parent(); + } + + inline ConstElement ConstElement::findNthChild(size_t n) const { + return _basis.findNthChild(n); + } + + inline ConstElement ConstElement::operator[](size_t n) const { + return _basis[n]; + } + + inline ConstElement ConstElement::findFirstChildNamed(const StringData& name) const { + return _basis.findFirstChildNamed(name); + } + + inline ConstElement ConstElement::operator[](const StringData& name) const { + return _basis[name]; + } + + inline ConstElement ConstElement::findElementNamed(const StringData& name) const { + return _basis.findElementNamed(name); + } + + inline size_t ConstElement::countSiblingsLeft() const { + return _basis.countSiblingsLeft(); + } + + inline size_t ConstElement::countSiblingsRight() const { + return _basis.countSiblingsRight(); + } + + inline size_t ConstElement::countChildren() const { + return _basis.countChildren(); + } + + inline bool ConstElement::hasValue() const { + return _basis.hasValue(); + } + + inline const BSONElement ConstElement::getValue() const { + return _basis.getValue(); + } + + inline double ConstElement::getValueDouble() const { + return _basis.getValueDouble(); + } + + inline StringData ConstElement::getValueString() const { + return _basis.getValueString(); + } + + inline BSONObj ConstElement::getValueObject() const { + return _basis.getValueObject(); + } + + inline BSONArray ConstElement::getValueArray() const { + return _basis.getValueArray(); + } + + inline bool ConstElement::isValueUndefined() const { + return _basis.isValueUndefined(); + } + + inline OID ConstElement::getValueOID() const { + return _basis.getValueOID(); + } + + inline bool ConstElement::getValueBool() const { + return _basis.getValueBool(); + } + + inline Date_t ConstElement::getValueDate() const { + return _basis.getValueDate(); + } + + inline bool ConstElement::isValueNull() const { + return _basis.isValueNull(); + } + + inline StringData ConstElement::getValueSymbol() const { + return _basis.getValueSymbol(); + } + + inline int32_t ConstElement::getValueInt() const { + return _basis.getValueInt(); + } + + inline OpTime ConstElement::getValueTimestamp() const { + return _basis.getValueTimestamp(); + } + + inline int64_t ConstElement::getValueLong() const { + return _basis.getValueLong(); + } + + inline bool ConstElement::isValueMinKey() const { + return _basis.isValueMinKey(); + } + + inline bool ConstElement::isValueMaxKey() const { + return _basis.isValueMaxKey(); + } + + inline SafeNum ConstElement::getValueSafeNum() const { + return _basis.getValueSafeNum(); + } + + inline int ConstElement::compareWithElement(const ConstElement& other, + bool considerFieldName) const { + return _basis.compareWithElement(other, considerFieldName); + } + + inline int ConstElement::compareWithBSONElement(const BSONElement& other, + bool considerFieldName) const { + return _basis.compareWithBSONElement(other, considerFieldName); + } + + inline int ConstElement::compareWithBSONObj(const BSONObj& other, + bool considerFieldName) const { + return _basis.compareWithBSONObj(other, considerFieldName); + } + + inline void ConstElement::writeTo(BSONObjBuilder* builder) const { + return _basis.writeTo(builder); + } + + inline void ConstElement::writeArrayTo(BSONArrayBuilder* builder) const { + return _basis.writeArrayTo(builder); + } + + inline bool ConstElement::ok() const { + return _basis.ok(); + } + + inline const Document& ConstElement::getDocument() const { + return _basis.getDocument(); + } + + inline BSONType ConstElement::getType() const { + return _basis.getType(); + } + + inline bool ConstElement::isType(BSONType type) const { + return _basis.isType(type); + } + + inline StringData ConstElement::getFieldName() const { + return _basis.getFieldName(); + } + + inline Element::RepIdx ConstElement::getIdx() const { + return _basis.getIdx(); + } + + inline std::string ConstElement::toString() const { + return _basis.toString(); + } + + inline bool operator==(const ConstElement& l, const ConstElement& r) { + return l._basis == r._basis; + } + + inline bool operator!=(const ConstElement& l, const ConstElement& r) { + return !(l == r); + } + + inline bool operator==(const Element& l, const ConstElement& r) { + return ConstElement(l) == r; + } + + inline bool operator!=(const Element& l, const ConstElement& r) { + return !(l == r); + } + + inline bool operator==(const ConstElement& l, const Element& r) { + return l == ConstElement(r); + } + + inline bool operator!=(const ConstElement& l, const Element& r) { + return !(l == r); + } + + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/const_element.h b/src/mongo/bson/mutable/const_element.h new file mode 100644 index 00000000000..171c25b28ce --- /dev/null +++ b/src/mongo/bson/mutable/const_element.h @@ -0,0 +1,125 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongo/bson/mutable/element.h" + +namespace mongo { +namespace mutablebson { + + /** For an overview of mutable BSON, please see the file document.h in this directory. */ + + /** ConstElement recapitulates all of the const methods of Element, but cannot be converted + * to an Element. This makes it safe to return as a value from a constant Document, since + * none of Element's non-const methods may be called on ConstElement or any values it + * yields. If you think of Element like an STL 'iterator', then ConstElement is a + * 'const_iterator'. + * + * For details on the API methods of ConstElement, please see the method comments for the + * analagous Element methods in the file element.h (in this same directory). + * + * All calls on ConstElement are simply forwarded to the underlying Element. + */ + class ConstElement { + public: + + // This one argument constructor is intentionally not explicit, since we want to be + // able to pass Elements to functions taking ConstElements without complaint. + inline ConstElement(const Element& basis); + + inline ConstElement leftChild() const; + inline ConstElement rightChild() const; + inline bool hasChildren() const; + inline ConstElement leftSibling(size_t distance = 1) const; + inline ConstElement rightSibling(size_t distance = 1) const; + inline ConstElement parent() const; + inline ConstElement findNthChild(size_t n) const; + inline ConstElement operator[](size_t n) const; + inline ConstElement findFirstChildNamed(const StringData& name) const; + inline ConstElement operator[](const StringData& n) const; + inline ConstElement findElementNamed(const StringData& name) const; + + inline size_t countSiblingsLeft() const; + inline size_t countSiblingsRight() const; + inline size_t countChildren() const; + + inline bool hasValue() const; + inline const BSONElement getValue() const; + + inline double getValueDouble() const; + inline StringData getValueString() const; + inline BSONObj getValueObject() const; + inline BSONArray getValueArray() const; + inline bool isValueUndefined() const; + inline OID getValueOID() const; + inline bool getValueBool() const; + inline Date_t getValueDate() const; + inline bool isValueNull() const; + inline StringData getValueSymbol() const; + inline int32_t getValueInt() const; + inline OpTime getValueTimestamp() const; + inline int64_t getValueLong() const; + inline bool isValueMinKey() const; + inline bool isValueMaxKey() const; + inline SafeNum getValueSafeNum() const; + + inline int compareWithElement(const ConstElement& other, + bool considerFieldName = true) const; + + inline int compareWithBSONElement(const BSONElement& other, + bool considerFieldName = true) const; + + inline int compareWithBSONObj(const BSONObj& other, + bool considerFieldName = true) const; + + inline void writeTo(BSONObjBuilder* builder) const; + inline void writeArrayTo(BSONArrayBuilder* builder) const; + + inline bool ok() const; + inline const Document& getDocument() const; + inline BSONType getType() const; + inline bool isType(BSONType type) const; + inline StringData getFieldName() const; + inline Element::RepIdx getIdx() const; + + inline std::string toString() const; + + friend bool operator==(const ConstElement&, const ConstElement&); + + private: + friend class Document; + + template + inline void writeElement(Builder* builder, const StringData* fieldName = NULL) const; + + Element _basis; + }; + + /** See notes for operator==(const Element&, const Element&). The multiple variants listed + * here enable cross type comparisons between Elements and ConstElements. + */ + inline bool operator==(const ConstElement& l, const ConstElement& r); + inline bool operator!=(const ConstElement& l, const ConstElement& r); + inline bool operator==(const Element& l, const ConstElement& r); + inline bool operator!=(const Element& l, const ConstElement& r); + inline bool operator==(const ConstElement& l, const Element& r); + inline bool operator!=(const ConstElement& l, const Element& r); + +} // namespace mutablebson +} // namespace mongo + +#include "mongo/bson/mutable/const_element-inl.h" + diff --git a/src/mongo/bson/mutable/damage_vector.h b/src/mongo/bson/mutable/damage_vector.h new file mode 100644 index 00000000000..130500f5700 --- /dev/null +++ b/src/mongo/bson/mutable/damage_vector.h @@ -0,0 +1,43 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace mongo { +namespace mutablebson { + + // A damage event represents a change of size 'size' byte at starting at offset + // 'target_offset' in some target buffer, with the replacement data being 'size' bytes of + // data from the 'source' offset. The base addresses against which these offsets are to be + // applied are not captured here. + struct DamageEvent { + typedef uint32_t OffsetSizeType; + + // Offset of source data (in some buffer held elsewhere). + OffsetSizeType sourceOffset; + + // Offset of target data (in some buffer held elsewhere). + OffsetSizeType targetOffset; + + // Size of the damage region. + size_t size; + }; + + typedef std::vector DamageVector; + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/document-inl.h b/src/mongo/bson/mutable/document-inl.h new file mode 100644 index 00000000000..5a6a2ce2cbb --- /dev/null +++ b/src/mongo/bson/mutable/document-inl.h @@ -0,0 +1,67 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { +namespace mutablebson { + + inline int Document::compareWith(const Document& other, bool considerFieldName) const { + // We cheat and use Element::compareWithElement since we know that 'other' is a + // Document and has a 'hidden' fieldname that is always indentical across all Document + // instances. + return root().compareWithElement(other.root(), considerFieldName); + } + + inline int Document::compareWithBSONObj(const BSONObj& other, bool considerFieldName) const { + return root().compareWithBSONObj(other, considerFieldName); + } + + inline void Document::writeTo(BSONObjBuilder* builder) const { + return root().writeTo(builder); + } + + inline BSONObj Document::getObject() const { + BSONObjBuilder builder; + writeTo(&builder); + return builder.obj(); + } + + inline Element Document::root() { + return _root; + } + + inline ConstElement Document::root() const { + return _root; + } + + inline Element Document::end() { + return Element(this, Element::kInvalidRepIdx); + } + + inline ConstElement Document::end() const { + return const_cast(this)->end(); + } + + inline std::string Document::toString() const { + return getObject().toString(); + } + + inline bool Document::isInPlaceModeEnabled() const { + return getCurrentInPlaceMode() == kInPlaceEnabled; + } + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/document.cpp b/src/mongo/bson/mutable/document.cpp new file mode 100644 index 00000000000..6ef13e1de43 --- /dev/null +++ b/src/mongo/bson/mutable/document.cpp @@ -0,0 +1,2639 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/mutable/document.h" + +#include +#include +#include +#include +#include +#include + +#include "mongo/bson/inline_decls.h" + +#include "mongo/bson/mutable/damage_vector.h" + +namespace mongo { +namespace mutablebson { + + /** Mutable BSON Implementation Overview + * + * If you haven't read it already, please read the 'Mutable BSON Overview' comment in + * document.h before reading further. + * + * In the following discussion, the capitalized terms 'Element' and 'Document' refer to + * the classes of the same name. At times, it is also necessary to refer to abstract + * 'elements' or 'documents', in the sense of bsonspec.org. These latter uses are + * non-capitalized. In the BSON specification, there are two 'classes' of + * elements. 'Primitive' or 'leaf' elements are those elements which do not contain other + * elements. In practice, all BSON types except 'Array' and 'Object' are primitives. The + * CodeWScope type is an exception, but one that we sidestep by considering its BSONObj + * payload to be opaque. + * + * A mutable BSON Document and its component Elements are implemented in terms of four + * data structures. These structures are owned by a Document::Impl object. Each Document + * owns a unique Document::Impl, which owns the relevant data structures and provides + * accessors, mutators, and helper methods related to those data structures. Understanding + * these data structures is critical for understanding how the system as a whole operates. + * + * - The 'Elements Vector': This is a std::vector, where 'ElementRep' is a + * structure type defined below that contains the detailed information about an entity + * in the Document (e.g. an Object, or an Array, or a NumberLong, etc.). The 'Element' + * and 'ConstElement' objects contain a pointer to a Document (which allows us to reach + * the Document::Impl for the Document), and an index into the Elements Vector in the + * Document::Impl. These two pieces of information make it possible for us to obtain the + * ElementRep associated with a given Element. Note that the Elements Vector is append + * only: ElementReps are never removed from it, even if the cooresponding Element is + * removed from the Document. By never removing ElementReps, and by using indexes into + * the Elements Vector, we can ensure that Elements are never invalidated. Note that + * every Document comes with an automatically provided 'root' element of mongo::Object + * type. The ElementRep for the root is always in the first slot (index zero) of the + * Elements Vector. + * + * - The 'Leaf Builder': This is a standard BSONObjBuilder. When a request is made to the + * Document to add new data to the Document via one of the Document::makeElement[TYPE] + * calls, the element is constructed by invoking the appropriate method on the Leaf + * Builder, forwarding the arguments provided to the call on Document. This results in a + * contiguous region of memory which encodes this element, capturing its field name, its + * type, and the bytes that encode its value, in the same way it normally does when + * using BSONObjBuilder. We then build an ElementRep that indexes into the BufBuilder + * behind the BSONObjBuilder (more on how this happens below, in the section on the + * 'Objects Vector'), then insert that new ElementRep into the ElementsVector, and + * finally return an Element that dereferences to the new ElementRep. Subsequently, + * requests for the type, fieldname or value bytes via the Element are satisfied by + * obtaining the contiguous memory region for the element, which may be used to + * construct a BSONElement over that memory region. + * + * - The 'Objects Vector': This is a std::vector. Any BSONObj object that + * provides values for parts of the Document is stored in the Objects Vector. For + * instance, in 'Example 2' from document.h, the Document we construct wraps an existing + * BSONObj, which is passed in to the Document constructor. That BSONObj would be stored + * in the Objects Vector. The data content of the BSONObj is not copied, but the BSONObj + * is copied, so the if the BSONObj is counted, we will up its refcount. In any event + * the lifetime of the BSONObj must exceed our lifetime by some mechanism. ElementReps + * that represent the component elements of the BSONObj store the index of their + * supporting BSONObj into the 'objIdx' field of ElementRep. Later, when Elements + * referring to those ElementReps are asked for properties like the field name or type + * of the Element, the underlying memory region in the appropriate BSONObj may be + * examined to provide the relevant data. + * + * - The 'Field Name Heap': For some elements, particularly those in the Leaf Builder or + * those embedded in a BSONObj in the Objects Vector, we can easily obtain the field + * name by reading it from the encoded BSON. However, some elements are not so + * fortunate. Newly created elements of mongo::Array or mongo::Object type, for + * instance, don't have a memory region that provides values. In such cases, the field + * name is stored in the field name heap, which is simply std::vector, where the + * field names are null-byte-delimited. ElementsReps for such elements store an offset + * into the Field Name Heap, and when asked for their field name simply return a pointer + * to the string data the offset identifies. This exploits the fact that in BSON, valid + * field names are null terinated and do not contain embedded null bytes. + * + * - The 'root' Element. Each Document contains a well known Element, which always refers + * to a pre-constructed ElementRep at offset zero in the Elements Vector. This is an + * Object element, and it is considered as the root of the document tree. It is possible + * for ElementReps to exist in the Document data structures, but not be in a child + * relationship to the root Element. Newly created Elements, for instance, are in this + * sort of 'detached' state until they are attched to another element. Only Element's + * that are children of the root element are traversed when calling top level + * serialization or comparision operations on Document. + * + * When you construct a Document that obtains its values from an underlying BSONObj, the + * entire BSONObj is not 'unpacked' into ElementReps at Document construction + * time. Instead, as you ask for Elements with the Element navigation API, the Elements + * for children and siblings are created on demand. Subobjects which are never visited + * will never have ElementReps constructed for them. Similarly, when writing a Document + * back out to a builder, regions of memory that provide values for the Document and which + * have not been modified will be block copied, instead of being recursively explored and + * written. + * + * To see how these data structures interoperate, we will walk through an example. You may + * want to read the comments for ElementRep before tackling the example, since we will + * refer to the internal state of ElementRep here. The example code used here exists as a + * unit test in mutable_bson_test.cpp as (Documentation, Example3). + * + * + * Legend: + * oi : objIdx + * +/- : bitfield state (s: serialized, a: array) + * x : invalid/empty rep idx + * ? : opaque rep idx + * ls/rs: left/right sibling + * lc/rc: left/right child + * p : parent + + static const char inJson[] = + "{" + " 'xs': { 'x' : 'x', 'X' : 'X' }," + " 'ys': { 'y' : 'y' }" + "}"; + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | ? | ? | x | + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | {} | + * +-----------------------------------------------------------------------------+ + + + mmb::Element root = doc.root(); + mmb::Element xs = root.leftChild(); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | ? | x | * + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | ? | ? | ? | 0 | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | {} | + * +-----------------------------------------------------------------------------+ + + + mmb::Element ys = xs.rightSibling(); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | ? | x | + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | 2 | ? | ? | 0 | * + * 2 | 1 | s:+ | ... | off of 'ys' into _objects[1] | 1 | ? | ? | ? | 0 | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | {} | + * +-----------------------------------------------------------------------------+ + + + mmb::Element dne = ys.rightSibling(); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | 2 | x | * + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | 2 | ? | ? | 0 | + * 2 | 1 | s:+ | ... | off of 'ys' into _objects[1] | 1 | x | ? | ? | 0 | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | {} | + * +-----------------------------------------------------------------------------+ + + + mmb::Element ycaps = doc.makeElementString("Y", "Y"); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | 2 | x | + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | 2 | ? | ? | 0 | + * 2 | 1 | s:+ | ... | off of 'ys' into _objects[1] | 1 | x | ? | ? | 0 | + * 3 | 0 | s:+ | ... | off of 'Y' into _objects[0] | x | x | x | x | x | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | { "Y" : "Y" } | * + * +-----------------------------------------------------------------------------+ + + + ys.pushBack(ycaps); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | 2 | x | + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | 2 | ? | ? | 0 | + * 2 | 1 | s:- | ... | off of 'ys' into _objects[1] | 1 | x | 4 | 3 | 0 | * + * 3 | 0 | s:+ | ... | off of 'Y' into _objects[0] | 4 | x | x | x | 2 | * + * 4 | 1 | s:+ | ... | off of 'ys.y' into _objects[1] | x | 3 | x | x | 2 | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | { "Y" : "Y" } | + * +-----------------------------------------------------------------------------+ + + + mmb::Element pun = doc.makeElementArray("why"); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | 2 | x | + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | 2 | ? | ? | 0 | + * 2 | 1 | s:- | ... | off of 'ys' into _objects[1] | 1 | x | 4 | 3 | 0 | + * 3 | 0 | s:+ | ... | off of 'Y' into _objects[0] | 4 | x | x | x | 2 | + * 4 | 1 | s:+ | ... | off of 'ys.y' into _objects[1] | x | 3 | x | x | 2 | + * 5 | -1 | s:- | a:+ | ... | off of 'why' into _fieldNames | x | x | x | x | x | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0why\0 | * + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | { "Y" : "Y" } | + * +-----------------------------------------------------------------------------+ + + + ys.pushBack(pun); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | 2 | x | + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | 2 | ? | ? | 0 | + * 2 | 1 | s:- | ... | off of 'ys' into _objects[1] | 1 | x | 4 | 5 | 0 | * + * 3 | 0 | s:+ | ... | off of 'Y' into _objects[0] | 4 | 5 | x | x | 2 | * + * 4 | 1 | s:+ | ... | off of 'ys.y' into _objects[1] | x | 3 | x | x | 2 | + * 5 | -1 | s:- | a:+ | ... | off of 'why' into _fieldNames | 3 | x | x | x | 2 | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0why\0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | { "Y" : "Y" } | + * +-----------------------------------------------------------------------------+ + + + pun.appendString("na", "not"); + + * _elements + * oi flags offset ls rs lc rc p + * +-----------------------------------------------------------------------------+ + * 0 | 1 | s:- | ... | off 0 into _fieldNames | x | x | 1 | 2 | x | + * 1 | 1 | s:+ | ... | off of 'xs' into _objects[1] | x | 2 | ? | ? | 0 | + * 2 | 1 | s:- | ... | off of 'ys' into _objects[1] | 1 | x | 4 | 5 | 0 | + * 3 | 0 | s:+ | ... | off of 'Y' into _objects[0] | 4 | 5 | x | x | 2 | + * 4 | 1 | s:+ | ... | off of 'ys.y' into _objects[1] | x | 3 | x | x | 2 | + * 5 | -1 | s:- | a:+ | ... | off of 'why' into _fieldNames | 3 | x | 6 | 6 | 2 | * + * 6 | 0 | s:+ | ... | off of 'na' into _objects[0] | x | x | x | x | 5 | * + * +-----------------------------------------------------------------------------+ + * + * _objects + * +-----------------------------------------------------------------------------+ + * | BSONObj for _leafBuilder | BSONObj for inObj | | + * +-----------------------------------------------------------------------------+ + * + * _fieldNames + * +-----------------------------------------------------------------------------+ + * | \0why\0 | + * +-----------------------------------------------------------------------------+ + * + * _leafBuf + * +-----------------------------------------------------------------------------+ + * | { "Y" : "Y", "na" : "not" } | * + * +-----------------------------------------------------------------------------+ + * + */ + +// Work around http://gcc.gnu.org/bugzilla/show_bug.cgi?id=29365. Note that the selection of +// minor version 4 is somewhat arbitrary. It does appear that the fix for this was backported +// to earlier versions. This is a conservative choice that we can revisit later. We need the +// __clang__ here because Clang claims to be gcc of some version. +#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4) + namespace { +#endif + + // The designated field name for the root element. + const char kRootFieldName[] = ""; + + // How many reps do we cache before we spill to heap. Use a power of two. + const size_t kFastReps = 128; + + // An ElementRep contains the information necessary to locate the data for an Element, + // and the topology information for how the Element is related to other Elements in the + // document. +#pragma pack(push, 1) + struct ElementRep { + + // The index of the BSONObj that provides the value for this Element. For nodes + // where serialized is 'false', this value may be kInvalidObjIdx to indicate that + // the Element does not have a supporting BSONObj. + typedef uint16_t ObjIdx; + ObjIdx objIdx; + + // This bit is true if this ElementRep identifies a completely serialized + // BSONElement (i.e. a region of memory with a bson type byte, a fieldname, and an + // encoded value). Changes to children of a serialized element will cause it to be + // marked as unserialized. + uint16_t serialized: 1; + + // For object like Elements where we cannot determine the type of the object by + // looking a region of memory, the 'array' bit allows us to determine whether we + // are an object or an array. + uint16_t array: 1; + + // Reserved for future use. + uint16_t reserved: 14; + + // This word either gives the offset into the BSONObj associated with this + // ElementRep where this serialized BSON element may be located, or the offset into + // the _fieldNames member of the Document where the field name for this BSON + // element may be located. + uint32_t offset; + + // The indexes of our left and right siblings in the Document. + struct { + Element::RepIdx left; + Element::RepIdx right; + } sibling; + + // The indexes of our left and right chidren in the Document. + struct { + Element::RepIdx left; + Element::RepIdx right; + } child; + + // The index of our parent in the Document. + Element::RepIdx parent; + + // The cached field name size of this element, or -1 if unknown. + int32_t fieldNameSize; + }; +#pragma pack(pop) + + BOOST_STATIC_ASSERT(sizeof(ElementRep) == 32); + + // We want ElementRep to be a POD so Document::Impl can grow the std::vector with + // memmove. + // + // TODO: C++11 static_assert(std::is_pod::value); + + // The ElementRep for the root element is always zero. + const Element::RepIdx kRootRepIdx = Element::RepIdx(0); + + // This is the object index for elements in the leaf heap. + const ElementRep::ObjIdx kLeafObjIdx = ElementRep::ObjIdx(0); + + // This is the sentinel value to indicate that we have no supporting BSONObj. + const ElementRep::ObjIdx kInvalidObjIdx = ElementRep::ObjIdx(-1); + + // This is the highest valid object index that does not overlap sentinel values. + const ElementRep::ObjIdx kMaxObjIdx = ElementRep::ObjIdx(-2); + + // Returns the offset of 'elt' within 'object' as a uint32_t. The element must be part + // of the object or the behavior is undefined. + uint32_t getElementOffset(const BSONObj& object, const BSONElement& elt) { + dassert(!elt.eoo()); + const char* const objRaw = object.objdata(); + const char* const eltRaw = elt.rawdata(); + dassert(objRaw < eltRaw); + dassert(eltRaw < objRaw + object.objsize()); + dassert(eltRaw + elt.size() <= objRaw + object.objsize()); + const ptrdiff_t offset = eltRaw - objRaw; + // BSON documents express their size as an int32_t so we should always be able to + // express the offset as a uint32_t. + verify(offset > 0); + verify(offset <= std::numeric_limits::max()); + return offset; + } + + // Returns true if this ElementRep is 'detached' from all other elements and can be + // added as a child, which helps ensure that we maintain a tree rather than a graph + // when adding new elements to the tree. The root element is never considered to be + // attachable. + bool canAttach(const Element::RepIdx id, const ElementRep& rep) { + return + (id != kRootRepIdx) && + (rep.sibling.left == Element::kInvalidRepIdx) && + (rep.sibling.right == Element::kInvalidRepIdx) && + (rep.parent == Element::kInvalidRepIdx); + } + + // Returns a Status describing why 'canAttach' returned false. This function should not + // be inlined since it just makes the callers larger for no real gain. + NOINLINE_DECL Status getAttachmentError(const ElementRep& rep); + Status getAttachmentError(const ElementRep& rep) { + if (rep.sibling.left != Element::kInvalidRepIdx) + return Status(ErrorCodes::IllegalOperation, "dangling left sibling"); + if (rep.sibling.right != Element::kInvalidRepIdx) + return Status(ErrorCodes::IllegalOperation, "dangling right sibling"); + if (rep.parent != Element::kInvalidRepIdx) + return Status(ErrorCodes::IllegalOperation, "dangling parent"); + return Status(ErrorCodes::IllegalOperation, "cannot add the root as a child"); + } + + + // Enable paranoid mode to force a reallocation on mutation of the princple data + // structures in Document::Impl. This is really slow, but can be very helpful if you + // suspect an invalidation logic error and want to find it with valgrind. Paranoid mode + // only works in debug mode; it is ignored in release builds. + const bool paranoid = false; + +#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4) + } // namespace +#endif + + /** Document::Impl holds the Document state. Please see the file comment above for details + * on the fields of Impl and how they are used to realize the implementation of mutable + * BSON. Impl provides various utility methods to insert, lookup, and interrogate the + * Elements, BSONObj objects, field names, and builders associated with the Document. + * + * TODO: At some point, we could remove the firewall and inline the members of Impl into + * Document. + */ + class Document::Impl { + MONGO_DISALLOW_COPYING(Impl); + + public: + Impl(Document::InPlaceMode inPlaceMode) + : _numElements(0) + , _slowElements() + , _objects() + , _fieldNames() + , _leafBuf() + , _leafBuilder(_leafBuf) + , _fieldNameScratch() + , _damages() + , _inPlaceMode(inPlaceMode) { + + // We always have a BSONObj for the leaves, and we often have + // one for our base document, so reserve 2. + _objects.reserve(2); + + // We always have at least one byte for the root field name, and we would like + // to be able to hold a few short field names without reallocation. + _fieldNames.reserve(8); + + // We need an object at _objects[0] so that we can access leaf elements we + // construct with the leaf builder in the same way we access elements serialized in + // other BSONObjs. So we call asTempObj on the builder and store the result in slot + // 0. + dassert(_objects.size() == kLeafObjIdx); + _objects.push_back(_leafBuilder.asTempObj()); + dassert(_leafBuf.len() != 0); + } + + ~Impl() { + _leafBuilder.abandon(); + } + + void reset(Document::InPlaceMode inPlaceMode) { + // Clear out the state in the vectors. + _slowElements.clear(); + _numElements = 0; + + _objects.clear(); + _fieldNames.clear(); + + // There is no way to reset the state of a BSONObjBuilder, so we need to call its + // dtor, reset the underlying buf, and re-invoke the constructor in-place. + _leafBuilder.abandon(); + _leafBuilder.~BSONObjBuilder(); + _leafBuf.reset(); + new (&_leafBuilder) BSONObjBuilder(_leafBuf); + + _fieldNameScratch.clear(); + _damages.clear(); + _inPlaceMode = inPlaceMode; + + // Ensure that we start in the same state as the ctor would leave us in. + _objects.push_back(_leafBuilder.asTempObj()); + } + + // Obtain the ElementRep for the given rep id. + ElementRep& getElementRep(Element::RepIdx id) { + return const_cast(const_cast(this)->getElementRep(id)); + } + + // Obtain the ElementRep for the given rep id. + const ElementRep& getElementRep(Element::RepIdx id) const { + dassert(id < _numElements); + if (id < kFastReps) + return _fastElements[id]; + else + return _slowElements[id - kFastReps]; + } + + // Construct and return a new default initialized ElementRep. The RepIdx identifying + // the new rep is returned in the out parameter. + ElementRep& makeNewRep(Element::RepIdx* newIdx) { + + const ElementRep defaultRep = { + kInvalidObjIdx, + false, false, 0, + 0, + { Element::kInvalidRepIdx, Element::kInvalidRepIdx }, + { Element::kInvalidRepIdx, Element::kInvalidRepIdx }, + Element::kInvalidRepIdx, + -1 + }; + + const Element::RepIdx id = *newIdx = _numElements++; + + if (id < kFastReps) { + return _fastElements[id] = defaultRep; + } + else { + verify(id <= Element::kMaxRepIdx); + + if (debug && paranoid) { + // Force all reps to new addresses to help catch invalid rep usage. + std::vector newSlowElements(_slowElements); + _slowElements.swap(newSlowElements); + } + + return *_slowElements.insert(_slowElements.end(), defaultRep); + } + } + + // Insert a new ElementRep for a leaf element at the given offset and return its ID. + Element::RepIdx insertLeafElement(int offset, int fieldNameSize = -1) { + // BufBuilder hands back sizes in 'int's. + Element::RepIdx inserted; + ElementRep& rep = makeNewRep(&inserted); + + rep.fieldNameSize = fieldNameSize; + rep.objIdx = kLeafObjIdx; + rep.serialized = true; + dassert(offset >= 0); + // TODO: Is this a legitimate possibility? + dassert(static_cast(offset) < std::numeric_limits::max()); + rep.offset = offset; + _objects[kLeafObjIdx] = _leafBuilder.asTempObj(); + return inserted; + } + + // Obtain the object builder for the leaves. + BSONObjBuilder& leafBuilder() { + return _leafBuilder; + } + + // Obtain the BSONObj for the given object id. + BSONObj& getObject(ElementRep::ObjIdx objIdx) { + dassert(objIdx < _objects.size()); + return _objects[objIdx]; + } + + // Obtain the BSONObj for the given object id. + const BSONObj& getObject(ElementRep::ObjIdx objIdx) const { + dassert(objIdx < _objects.size()); + return _objects[objIdx]; + } + + // Insert the given BSONObj and return an ID for it. + ElementRep::ObjIdx insertObject(const BSONObj& newObj) { + const size_t objIdx = _objects.size(); + verify(objIdx <= kMaxObjIdx); + _objects.push_back(newObj); + if (debug && paranoid) { + // Force reallocation to catch use after invalidation. + std::vector new_objects(_objects); + _objects.swap(new_objects); + } + return objIdx; + } + + // Given a RepIdx, return the BSONElement that it represents. + BSONElement getSerializedElement(const ElementRep& rep) const { + const BSONObj& object = getObject(rep.objIdx); + return BSONElement( + object.objdata() + rep.offset, + rep.fieldNameSize, + BSONElement::FieldNameSizeTag()); + } + + // A helper method that either inserts the field name into the field name heap and + // updates element. + void insertFieldName(ElementRep& rep, const StringData& fieldName) { + dassert(!rep.serialized); + rep.offset = insertFieldName(fieldName); + } + + // Retrieve the fieldName, given a rep. + StringData getFieldName(const ElementRep& rep) const { + // The root element has no field name. + if (&rep == &getElementRep(kRootRepIdx)) + return StringData(); + + if (rep.serialized || (rep.objIdx != kInvalidObjIdx)) + return getSerializedElement(rep).fieldNameStringData(); + + return getFieldName(rep.offset); + } + + StringData getFieldNameForNewElement(const ElementRep& rep) { + StringData result = getFieldName(rep); + if (rep.objIdx == kLeafObjIdx) { + _fieldNameScratch.assign(result.rawData(), result.size()); + result = StringData(_fieldNameScratch); + } + return result; + } + + // Retrieve the type, given a rep. + BSONType getType(const ElementRep& rep) const { + // The root element is always an Object. + if (&rep == &getElementRep(kRootRepIdx)) + return mongo::Object; + + if (rep.serialized || (rep.objIdx != kInvalidObjIdx)) + return getSerializedElement(rep).type(); + + return rep.array ? mongo::Array : mongo::Object; + } + + static bool isLeafType(BSONType type) { + return ((type != mongo::Object) && (type != mongo::Array)); + } + + // Returns true if rep is not an object or array. + bool isLeaf(const ElementRep& rep) const { + return isLeafType(getType(rep)); + } + + bool isLeaf(const BSONElement& elt) const { + return isLeafType(elt.type()); + } + + // Returns true if rep's value can be provided as a BSONElement. + bool hasValue(const ElementRep& rep) const { + // The root element may be marked serialized, but it doesn't have a BSONElement + // representation. + if (&rep == &getElementRep(kRootRepIdx)) + return false; + + return rep.serialized; + } + + // Return the index of the left child of the Element with index 'index', resolving the + // left child to a realized Element if it is currently opaque. This may also cause the + // parent elements child.right entry to be updated. + Element::RepIdx resolveLeftChild(Element::RepIdx index) { + dassert(index != Element::kInvalidRepIdx); + dassert(index != Element::kOpaqueRepIdx); + + // If the left child is anything other than opaque, then we are done here. + ElementRep* rep = &getElementRep(index); + if (rep->child.left != Element::kOpaqueRepIdx) + return rep->child.left; + + // It should be impossible to have an opaque left child and be non-serialized, + dassert(rep->serialized); + BSONElement childElt = ( + hasValue(*rep) ? + getSerializedElement(*rep).embeddedObject() : + getObject(rep->objIdx)).firstElement(); + + if (!childElt.eoo()) { + + // Do this now before other writes so compiler can exploit knowing + // that we are not eoo. + const int32_t fieldNameSize = childElt.fieldNameSize(); + + Element::RepIdx inserted; + ElementRep& newRep = makeNewRep(&inserted); + // Calling makeNewRep invalidates rep since it may cause a reallocation of + // the element vector. After calling insertElement, we reacquire rep. + rep = &getElementRep(index); + + newRep.serialized = true; + newRep.objIdx = rep->objIdx; + newRep.offset = + getElementOffset(getObject(rep->objIdx), childElt); + newRep.parent = index; + newRep.sibling.right = Element::kOpaqueRepIdx; + // If this new object has possible substructure, mark its children as opaque. + if (!isLeaf(childElt)) { + newRep.child.left = Element::kOpaqueRepIdx; + newRep.child.right = Element::kOpaqueRepIdx; + } + newRep.fieldNameSize = fieldNameSize; + rep->child.left = inserted; + } else { + rep->child.left = Element::kInvalidRepIdx; + rep->child.right = Element::kInvalidRepIdx; + } + + dassert(rep->child.left != Element::kOpaqueRepIdx); + return rep->child.left; + } + + // Return the index of the right child of the Element with index 'index', resolving any + // opaque nodes. Note that this may require resolving all of the right siblings of the + // left child. + Element::RepIdx resolveRightChild(Element::RepIdx index) { + dassert(index != Element::kInvalidRepIdx); + dassert(index != Element::kOpaqueRepIdx); + + Element::RepIdx current = getElementRep(index).child.right; + if (current == Element::kOpaqueRepIdx) { + current = resolveLeftChild(index); + while (current != Element::kInvalidRepIdx) { + Element::RepIdx next = resolveRightSibling(current); + if (next == Element::kInvalidRepIdx) + break; + current = next; + } + + // The resolveRightSibling calls should have eventually updated this nodes right + // child pointer to point to the node we are about to return. + dassert(getElementRep(index).child.right == current); + } + + return current; + } + + // Return the index of the right sibling of the Element with index 'index', resolving + // the right sibling to a realized Element if it is currently opaque. + Element::RepIdx resolveRightSibling(Element::RepIdx index) { + dassert(index != Element::kInvalidRepIdx); + dassert(index != Element::kOpaqueRepIdx); + + // If the right sibling is anything other than opaque, then we are done here. + ElementRep* rep = &getElementRep(index); + if (rep->sibling.right != Element::kOpaqueRepIdx) + return rep->sibling.right; + + BSONElement elt = getSerializedElement(*rep); + BSONElement rightElt(elt.rawdata() + elt.size()); + + if (!rightElt.eoo()) { + + // Do this now before other writes so compiler can exploit knowing + // that we are not eoo. + const int32_t fieldNameSize = rightElt.fieldNameSize(); + + Element::RepIdx inserted; + ElementRep& newRep = makeNewRep(&inserted); + // Calling makeNewRep invalidates rep since it may cause a reallocation of + // the element vector. After calling insertElement, we reacquire rep. + rep = &getElementRep(index); + + newRep.serialized = true; + newRep.objIdx = rep->objIdx; + newRep.offset = + getElementOffset(getObject(rep->objIdx), rightElt); + newRep.parent = rep->parent; + newRep.sibling.left = index; + newRep.sibling.right = Element::kOpaqueRepIdx; + // If this new object has possible substructure, mark its children as opaque. + if (!isLeaf(rightElt)) { + newRep.child.left = Element::kOpaqueRepIdx; + newRep.child.right = Element::kOpaqueRepIdx; + } + newRep.fieldNameSize = fieldNameSize; + rep->sibling.right = inserted; + } else { + rep->sibling.right = Element::kInvalidRepIdx; + // If we have found the end of this object, then our (necessarily existing) + // parent's necessarily opaque right child is now determined to be us. + dassert(rep->parent <= Element::kMaxRepIdx); + ElementRep& parentRep = getElementRep(rep->parent); + dassert(parentRep.child.right == Element::kOpaqueRepIdx); + parentRep.child.right = index; + } + + dassert(rep->sibling.right != Element::kOpaqueRepIdx); + return rep->sibling.right; + } + + // Find the ElementRep at index 'index', and mark it and all of its currently + // serialized parents as non-serialized. + void deserialize(Element::RepIdx index) { + while (index != Element::kInvalidRepIdx) { + ElementRep& rep = getElementRep(index); + // It does not make sense for leaf Elements to become deserialized, and + // requests to do so indicate a bug in the implementation of the library. + dassert(!isLeaf(rep)); + if (!rep.serialized) + break; + rep.serialized = false; + index = rep.parent; + } + } + + inline bool doesNotAlias(const StringData& s) const { + // StringData may come from either the field name heap or the leaf builder. + return doesNotAliasLeafBuilder(s) && !inFieldNameHeap(s.rawData()); + } + + inline bool doesNotAliasLeafBuilder(const StringData& s) const { + return !inLeafBuilder(s.rawData()); + } + + inline bool doesNotAlias(const BSONElement& e) const { + // A BSONElement could alias the leaf builder. + return !inLeafBuilder(e.rawdata()); + } + + inline bool doesNotAlias(const BSONObj& o) const { + // A BSONObj could alias the leaf buildr. + return !inLeafBuilder(o.objdata()); + } + + // Returns true if 'data' points within the leaf BufBuilder. + inline bool inLeafBuilder(const char* data) const { + // TODO: Write up something documenting that the following is technically UB due + // to illegality of comparing pointers to different aggregates for ordering. Also, + // do we need to do anything to prevent the optimizer from compiling this out on + // that basis? I've seen clang do that. We may need to declare these volatile. On + // the other hand, these should only be being called under a dassert, so the + // optimizer is maybe not in play, and the UB is unlikely to be a problem in + // practice. + const char* const start = _leafBuf.buf(); + const char* const end = start + _leafBuf.len(); + return (data >= start) && (data < end); + } + + // Returns true if 'data' points within the field name heap. + inline bool inFieldNameHeap(const char* data) const { + if (_fieldNames.empty()) + return false; + const char* const start = &_fieldNames.front(); + const char* const end = &_fieldNames.back(); + return (data >= start) && (data < end); + } + + void reserveDamageEvents(size_t expectedEvents) { + _damages.reserve(expectedEvents); + } + + bool getInPlaceUpdates(DamageVector* damages, const char** source, size_t* size) { + + // If some operations were not in-place, set source to NULL and return false to + // inform upstream that we are not returning in-place result data. + if (_inPlaceMode == Document::kInPlaceDisabled) { + damages->clear(); + *source = NULL; + if (size) + *size = 0; + return false; + } + + // Set up the source and source size out parameters. + *source = _objects[0].objdata(); + if (size) + *size = _objects[0].objsize(); + + // Swap our damage event queue with upstream, and reset ours to an empty vector. In + // princple, we can do another round of in-place updates. + damages->swap(_damages); + _damages.clear(); + + return true; + } + + void disableInPlaceUpdates() { + _inPlaceMode = Document::kInPlaceDisabled; + } + + Document::InPlaceMode getCurrentInPlaceMode() const { + return _inPlaceMode; + } + + bool isInPlaceModeEnabled() const { + return getCurrentInPlaceMode() == Document::kInPlaceEnabled; + } + + void recordDamageEvent(DamageEvent::OffsetSizeType targetOffset, + DamageEvent::OffsetSizeType sourceOffset, + size_t size) { + _damages.push_back(DamageEvent()); + _damages.back().targetOffset = targetOffset; + _damages.back().sourceOffset = sourceOffset; + _damages.back().size = size; + if (debug && paranoid) { + // Force damage events to new addresses to catch invalidation errors. + DamageVector new_damages(_damages); + _damages.swap(new_damages); + } + } + + // Not all types are currently permitted to be updated in-place. + bool canUpdateInPlace(const ElementRep& rep) { + const BSONType type = getType(rep); + switch(type) { + case mongo::NumberDouble: + case mongo::String: + case mongo::BinData: + case mongo::jstOID: + case mongo::Bool: + case mongo::Date: + case mongo::NumberInt: + case mongo::Timestamp: + case mongo::NumberLong: + return true; + default: + return false; + } + } + + template + void writeElement(Element::RepIdx repIdx, Builder* builder, + const StringData* fieldName = NULL) const; + + template + void writeChildren(Element::RepIdx repIdx, Builder* builder) const; + + private: + + // Insert the given field name into the field name heap, and return an ID for this + // field name. + int32_t insertFieldName(const StringData& fieldName) { + const uint32_t id = _fieldNames.size(); + if (!fieldName.empty()) + _fieldNames.insert( + _fieldNames.end(), + fieldName.rawData(), + fieldName.rawData() + fieldName.size()); + _fieldNames.push_back('\0'); + if (debug && paranoid) { + // Force names to new addresses to catch invalidation errors. + std::vector new_fieldNames(_fieldNames); + _fieldNames.swap(new_fieldNames); + } + return id; + } + + // Retrieve the field name with the given id. + StringData getFieldName(uint32_t fieldNameId) const { + dassert(fieldNameId < _fieldNames.size()); + return &_fieldNames[fieldNameId]; + } + + size_t _numElements; + ElementRep _fastElements[kFastReps]; + std::vector _slowElements; + + std::vector _objects; + std::vector _fieldNames; + + // We own a BufBuilder to avoid BSONObjBuilder's ref-count mechanism which would throw + // off our offset calculations. + BufBuilder _leafBuf; + BSONObjBuilder _leafBuilder; + + // Sometimes, we need a temporary storage area for a fieldName, because the source of + // the fieldName is in the same buffer that we want to write to, potentially + // reallocating it. In such cases, we temporarily store the value here, rather than + // creating and destroying a string and its buffer each time. + std::string _fieldNameScratch; + + // Queue of damage events and status bit for whether in-place updates are possible. + DamageVector _damages; + Document::InPlaceMode _inPlaceMode; + }; + + Status Element::addSiblingLeft(Element e) { + verify(ok()); + verify(e.ok()); + verify(_doc == e._doc); + + Document::Impl& impl = getDocument().getImpl(); + ElementRep& newRep = impl.getElementRep(e._repIdx); + + // check that new element roots a clean subtree. + if (!canAttach(e._repIdx, newRep)) + return getAttachmentError(newRep); + + ElementRep& thisRep = impl.getElementRep(_repIdx); + + dassert(thisRep.parent != kOpaqueRepIdx); + if (thisRep.parent == kInvalidRepIdx) + return Status( + ErrorCodes::IllegalOperation, + "Attempt to add a sibling to an element without a parent"); + + ElementRep& parentRep = impl.getElementRep(thisRep.parent); + dassert(!impl.isLeaf(parentRep)); + + impl.disableInPlaceUpdates(); + + // The new element shares our parent. + newRep.parent = thisRep.parent; + + // We are the new element's right sibling. + newRep.sibling.right = _repIdx; + + // The new element's left sibling is our left sibling. + newRep.sibling.left = thisRep.sibling.left; + + // If the new element has a left sibling after the adjustments above, then that left + // sibling must be updated to have the new element as its right sibling. + if (newRep.sibling.left != kInvalidRepIdx) + impl.getElementRep(thisRep.sibling.left).sibling.right = e._repIdx; + + // The new element becomes our left sibling. + thisRep.sibling.left = e._repIdx; + + // If we were our parent's left child, then we no longer are. Make the new right + // sibling the right child. + if (parentRep.child.left == _repIdx) + parentRep.child.left = e._repIdx; + + impl.deserialize(thisRep.parent); + + return Status::OK(); + } + + Status Element::addSiblingRight(Element e) { + verify(ok()); + verify(e.ok()); + verify(_doc == e._doc); + + Document::Impl& impl = getDocument().getImpl(); + ElementRep* newRep = &impl.getElementRep(e._repIdx); + + // check that new element roots a clean subtree. + if (!canAttach(e._repIdx, *newRep)) + return getAttachmentError(*newRep); + + ElementRep* thisRep = &impl.getElementRep(_repIdx); + + dassert(thisRep->parent != kOpaqueRepIdx); + if (thisRep->parent == kInvalidRepIdx) + return Status( + ErrorCodes::IllegalOperation, + "Attempt to add a sibling to an element without a parent"); + + ElementRep* parentRep = &impl.getElementRep(thisRep->parent); + dassert(!impl.isLeaf(*parentRep)); + + impl.disableInPlaceUpdates(); + + // If our current right sibling is opaque it needs to be resolved. This will invalidate + // our reps so we need to reacquire them. + Element::RepIdx rightSiblingIdx = thisRep->sibling.right; + if (rightSiblingIdx == kOpaqueRepIdx) { + rightSiblingIdx = impl.resolveRightSibling(_repIdx); + dassert(rightSiblingIdx != kOpaqueRepIdx); + newRep = &impl.getElementRep(e._repIdx); + thisRep = &impl.getElementRep(_repIdx); + parentRep = &impl.getElementRep(thisRep->parent); + } + + // The new element shares our parent. + newRep->parent = thisRep->parent; + + // We are the new element's left sibling. + newRep->sibling.left = _repIdx; + + // The new element right sibling is our right sibling. + newRep->sibling.right = rightSiblingIdx; + + // The new element becomes our right sibling. + thisRep->sibling.right = e._repIdx; + + // If the new element has a right sibling after the adjustments above, then that right + // sibling must be updated to have the new element as its left sibling. + if (newRep->sibling.right != kInvalidRepIdx) + impl.getElementRep(rightSiblingIdx).sibling.left = e._repIdx; + + // If we were our parent's right child, then we no longer are. Make the new right + // sibling the right child. + if (parentRep->child.right == _repIdx) + parentRep->child.right = e._repIdx; + + impl.deserialize(thisRep->parent); + + return Status::OK(); + } + + Status Element::remove() { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + // We need to realize any opaque right sibling, because we are going to need to set its + // left sibling. Do this before acquiring thisRep since otherwise we would potentially + // invalidate it. + impl.resolveRightSibling(_repIdx); + + ElementRep& thisRep = impl.getElementRep(_repIdx); + + if (thisRep.parent == kInvalidRepIdx) + return Status(ErrorCodes::IllegalOperation, "trying to remove a parentless element"); + impl.disableInPlaceUpdates(); + + // If our right sibling is not the end of the object, then set its left sibling to be + // our left sibling. + if (thisRep.sibling.right != kInvalidRepIdx) + impl.getElementRep(thisRep.sibling.right).sibling.left = thisRep.sibling.left; + + // Similarly, if our left sibling is not the beginning of the obejct, then set its + // right sibling to be our right sibling. + if (thisRep.sibling.left != kInvalidRepIdx) { + ElementRep& leftRep = impl.getElementRep(thisRep.sibling.left); + leftRep.sibling.right = thisRep.sibling.right; + } + + // If this element was our parent's right child, then our left sibling is the new right + // child. + ElementRep& parentRep = impl.getElementRep(thisRep.parent); + if (parentRep.child.right == _repIdx) + parentRep.child.right = thisRep.sibling.left; + + // Similarly, if this element was our parent's left child, then our right sibling is + // the new left child. + if (parentRep.child.left == _repIdx) + parentRep.child.left = thisRep.sibling.right; + + impl.deserialize(thisRep.parent); + + // The Element becomes detached. + thisRep.parent = kInvalidRepIdx; + thisRep.sibling.left = kInvalidRepIdx; + thisRep.sibling.right = kInvalidRepIdx; + + return Status::OK(); + } + + Status Element::rename(const StringData& newName) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + if (_repIdx == kRootRepIdx) + return Status(ErrorCodes::IllegalOperation, + "Invalid attempt to rename the root element of a document"); + + dassert(impl.doesNotAlias(newName)); + + // TODO: Some rename operations may be possible to do in-place. + impl.disableInPlaceUpdates(); + + // Operations below may invalidate thisRep, so we may need to reacquire it. + ElementRep* thisRep = &impl.getElementRep(_repIdx); + + // For non-leaf serialized elements, we can realize any opaque relatives and then + // convert ourselves to deserialized. + if (thisRep->objIdx != kInvalidObjIdx && !impl.isLeaf(*thisRep)) { + + const bool array = (impl.getType(*thisRep) == mongo::Array); + + // Realize any opaque right sibling or left child now, since otherwise we will lose + // the ability to do so. + impl.resolveLeftChild(_repIdx); + impl.resolveRightSibling(_repIdx); + + // The resolve calls above may have invalidated thisRep, we need to reacquire it. + thisRep = &impl.getElementRep(_repIdx); + + // Set this up as a non-supported deserialized element. We will set the fieldName + // in the else clause in the block below. + impl.deserialize(_repIdx); + + thisRep->array = array; + + // TODO: If we ever want to be able to add to the left or right of an opaque object + // without expanding, this may need to change. + thisRep->objIdx = kInvalidObjIdx; + } + + if (impl.hasValue(*thisRep)) { + // For leaf elements we just create a new Element with the current value and + // replace. Note that the 'setValue' call below will invalidate thisRep. + Element replacement = _doc->makeElementWithNewFieldName(newName, *this); + setValue(replacement._repIdx); + } else { + // The easy case: just update what our field name offset refers to. + impl.insertFieldName(*thisRep, newName); + } + + return Status::OK(); + } + + Element Element::leftChild() const { + verify(ok()); + + // Capturing Document::Impl by non-const ref exploits the constness loophole + // created by our Impl so that we can let leftChild be lazily evaluated, even for a + // const Element. + Document::Impl& impl = _doc->getImpl(); + const Element::RepIdx leftChildIdx = impl.resolveLeftChild(_repIdx); + dassert(leftChildIdx != kOpaqueRepIdx); + return Element(_doc, leftChildIdx); + } + + Element Element::rightChild() const { + verify(ok()); + + // Capturing Document::Impl by non-const ref exploits the constness loophole + // created by our Impl so that we can let leftChild be lazily evaluated, even for a + // const Element. + Document::Impl& impl = _doc->getImpl(); + const Element::RepIdx rightChildIdx = impl.resolveRightChild(_repIdx); + dassert(rightChildIdx != kOpaqueRepIdx); + return Element(_doc, rightChildIdx); + } + + bool Element::hasChildren() const { + verify(ok()); + // Capturing Document::Impl by non-const ref exploits the constness loophole + // created by our Impl so that we can let leftChild be lazily evaluated, even for a + // const Element. + Document::Impl& impl = _doc->getImpl(); + return impl.resolveLeftChild(_repIdx) != kInvalidRepIdx; + } + + Element Element::leftSibling(size_t distance) const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + Element::RepIdx current = _repIdx; + while ((current != kInvalidRepIdx) && (distance-- != 0)) { + // We are (currently) never left opaque, so don't need to resolve. + current = impl.getElementRep(current).sibling.left; + } + return Element(_doc, current); + } + + Element Element::rightSibling(size_t distance) const { + verify(ok()); + + // Capturing Document::Impl by non-const ref exploits the constness loophole + // created by our Impl so that we can let rightSibling be lazily evaluated, even for a + // const Element. + Document::Impl& impl = _doc->getImpl(); + Element::RepIdx current = _repIdx; + while ((current != kInvalidRepIdx) && (distance-- != 0)) + current = impl.resolveRightSibling(current); + return Element(_doc, current); + } + + Element Element::parent() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const Element::RepIdx parentIdx = impl.getElementRep(_repIdx).parent; + dassert(parentIdx != kOpaqueRepIdx); + return Element(_doc, parentIdx); + } + + Element Element::findNthChild(size_t n) const { + verify(ok()); + Document::Impl& impl = _doc->getImpl(); + Element::RepIdx current = _repIdx; + current = impl.resolveLeftChild(current); + while ((current != kInvalidRepIdx) && (n-- != 0)) + current = impl.resolveRightSibling(current); + return Element(_doc, current); + } + + Element Element::findFirstChildNamed(const StringData& name) const { + verify(ok()); + Document::Impl& impl = _doc->getImpl(); + Element::RepIdx current = _repIdx; + current = impl.resolveLeftChild(current); + // TODO: Could DRY this loop with the identical logic in findElementNamed. + while ((current != kInvalidRepIdx) && + (impl.getFieldName(impl.getElementRep(current)) != name)) + current = impl.resolveRightSibling(current); + return Element(_doc, current); + } + + Element Element::findElementNamed(const StringData& name) const { + verify(ok()); + Document::Impl& impl = _doc->getImpl(); + Element::RepIdx current = _repIdx; + while ((current != kInvalidRepIdx) && + (impl.getFieldName(impl.getElementRep(current)) != name)) + current = impl.resolveRightSibling(current); + return Element(_doc, current); + } + + size_t Element::countSiblingsLeft() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + Element::RepIdx current = _repIdx; + size_t result = 0; + while (true) { + // We are (currently) never left opaque, so don't need to resolve. + current = impl.getElementRep(current).sibling.left; + if (current == kInvalidRepIdx) + break; + ++result; + } + return result; + } + + size_t Element::countSiblingsRight() const { + verify(ok()); + Document::Impl& impl = _doc->getImpl(); + Element::RepIdx current = _repIdx; + size_t result = 0; + while (true) { + current = impl.resolveRightSibling(current); + if (current == kInvalidRepIdx) + break; + ++result; + } + return result; + } + + size_t Element::countChildren() const { + verify(ok()); + Document::Impl& impl = _doc->getImpl(); + Element::RepIdx current = _repIdx; + current = impl.resolveLeftChild(current); + size_t result = 0; + while (current != kInvalidRepIdx) { + ++result; + current = impl.resolveRightSibling(current); + } + return result; + } + + bool Element::hasValue() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + return impl.hasValue(thisRep); + } + + bool Element::isNumeric() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const BSONType type = impl.getType(thisRep); + return ((type == mongo::NumberLong) || + (type == mongo::NumberInt) || + (type == mongo::NumberDouble)); + } + + bool Element::isIntegral() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const BSONType type = impl.getType(thisRep); + return ((type == mongo::NumberLong) || + (type == mongo::NumberInt)); + } + + const BSONElement Element::getValue() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + if (impl.hasValue(thisRep)) + return impl.getSerializedElement(thisRep); + return BSONElement(); + } + + SafeNum Element::getValueSafeNum() const { + switch (getType()) { + case mongo::NumberInt: + return static_cast(getValueInt()); + case mongo::NumberLong: + return static_cast(getValueLong()); + case mongo::NumberDouble: + return getValueDouble(); + default: + return SafeNum(); + } + } + + int Element::compareWithElement(const ConstElement& other, bool considerFieldName) const { + verify(ok()); + verify(other.ok()); + + // Short circuit a tautological compare. + if ((_repIdx == other.getIdx()) && (_doc == &other.getDocument())) + return 0; + + // If either Element can represent its current value as a BSONElement, then we can + // obtain its value and use compareWithBSONElement. If both Elements have a + // representation as a BSONElement, compareWithBSONElement will notice that the first + // argument has a value and delegate to BSONElement::woCompare. + + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + + // Subtle: we must negate the comparison result here because we are reversing the + // argument order in this call. + // + // TODO: Andy has suggested that this may not be legal since woCompare is not reflexive + // in all cases. + if (impl.hasValue(thisRep)) + return -other.compareWithBSONElement( + impl.getSerializedElement(thisRep), considerFieldName); + + const Document::Impl& oimpl = other.getDocument().getImpl(); + const ElementRep& otherRep = oimpl.getElementRep(other.getIdx()); + + if (oimpl.hasValue(otherRep)) + return compareWithBSONElement( + oimpl.getSerializedElement(otherRep), considerFieldName); + + // Leaf elements should always have a value, so we should only be dealing with Objects + // or Arrays here. + dassert(!impl.isLeaf(thisRep)); + dassert(!oimpl.isLeaf(otherRep)); + + // Obtain the canonical types for this Element and the BSONElement, if they are + // different use the difference as the result. Please see BSONElement::woCompare for + // details. We know that thisRep is not a number, so we don't need to check that + // particular case. + const int leftCanonType = canonicalizeBSONType(impl.getType(thisRep)); + const int rightCanonType = canonicalizeBSONType(oimpl.getType(otherRep)); + const int diffCanon = leftCanonType - rightCanonType; + if (diffCanon != 0) + return diffCanon; + + // If we are considering field names, and the field names do not compare as equal, + // return the field name ordering as the element ordering. + if (considerFieldName) { + const int fnamesComp = impl.getFieldName(thisRep).compare(oimpl.getFieldName(otherRep)); + if (fnamesComp != 0) + return fnamesComp; + } + + const bool considerChildFieldNames = + (impl.getType(thisRep) != mongo::Array) && + (oimpl.getType(otherRep) != mongo::Array); + + // We are dealing with either two objects, or two arrays. We need to consider the child + // elements individually. We walk two iterators forward over the children and compare + // them. Length mismatches are handled by checking early for reaching the end of the + // children. + ConstElement thisIter = leftChild(); + ConstElement otherIter = other.leftChild(); + + while (true) { + if (!thisIter.ok()) + return !otherIter.ok() ? 0 : -1; + if (!otherIter.ok()) + return 1; + + const int result = thisIter.compareWithElement(otherIter, considerChildFieldNames); + if (result != 0) + return result; + + thisIter = thisIter.rightSibling(); + otherIter = otherIter.rightSibling(); + } + } + + int Element::compareWithBSONElement(const BSONElement& other, bool considerFieldName) const { + verify(ok()); + + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + + // If we have a representation as a BSONElement, we can just use BSONElement::woCompare + // to do the entire comparison. + if (impl.hasValue(thisRep)) + return impl.getSerializedElement(thisRep).woCompare(other, considerFieldName); + + // Leaf elements should always have a value, so we should only be dealing with Objects + // or Arrays here. + dassert(!impl.isLeaf(thisRep)); + + // Obtain the canonical types for this Element and the BSONElement, if they are + // different use the difference as the result. Please see BSONElement::woCompare for + // details. We know that thisRep is not a number, so we don't need to check that + // particular case. + const int leftCanonType = canonicalizeBSONType(impl.getType(thisRep)); + const int rightCanonType = canonicalizeBSONType(other.type()); + const int diffCanon = leftCanonType - rightCanonType; + if (diffCanon != 0) + return diffCanon; + + // If we are considering field names, and the field names do not compare as equal, + // return the field name ordering as the element ordering. + if (considerFieldName) { + const int fnamesComp = impl.getFieldName(thisRep).compare(other.fieldNameStringData()); + if (fnamesComp != 0) + return fnamesComp; + } + + const bool considerChildFieldNames = + (impl.getType(thisRep) != mongo::Array) && + (other.type() != mongo::Array); + + return compareWithBSONObj(other.Obj(), considerChildFieldNames); + } + + int Element::compareWithBSONObj(const BSONObj& other, bool considerFieldName) const { + verify(ok()); + + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + verify(!impl.isLeaf(thisRep)); + + // We are dealing with either two objects, or two arrays. We need to consider the child + // elements individually. We walk two iterators forward over the children and compare + // them. Length mismatches are handled by checking early for reaching the end of the + // children. + ConstElement thisIter = leftChild(); + BSONObjIterator otherIter(other); + + while (true) { + const BSONElement otherVal = otherIter.next(); + + if (!thisIter.ok()) + return otherVal.eoo() ? 0 : -1; + if (otherVal.eoo()) + return 1; + + const int result = thisIter.compareWithBSONElement(otherVal, considerFieldName); + if (result != 0) + return result; + + thisIter = thisIter.rightSibling(); + } + } + + void Element::writeTo(BSONObjBuilder* const builder) const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + verify(impl.getType(thisRep) == mongo::Object); + if (thisRep.parent == kInvalidRepIdx && _repIdx == kRootRepIdx) { + // If this is the root element, then we need to handle it differently, since it + // doesn't have a field name and should embed directly, rather than as an object. + impl.writeChildren(_repIdx, builder); + } else { + impl.writeElement(_repIdx, builder); + } + } + + void Element::writeArrayTo(BSONArrayBuilder* const builder) const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + verify(impl.getType(thisRep) == mongo::Array); + return impl.writeChildren(_repIdx, builder); + } + + Status Element::setValueDouble(const double value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + ElementRep thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementDouble(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueString(const StringData& value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(value)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementString(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueObject(const BSONObj& value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(value)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementObject(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueArray(const BSONObj& value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(value)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementArray(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueBinary(const uint32_t len, mongo::BinDataType binType, + const void* const data) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + // TODO: Alias check for binary data? + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementBinary( + fieldName, len, binType, data); + return setValue(newValue._repIdx); + } + + Status Element::setValueUndefined() { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementUndefined(fieldName); + return setValue(newValue._repIdx); + } + + Status Element::setValueOID(const OID value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementOID(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueBool(const bool value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + ElementRep thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementBool(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueDate(const Date_t value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementDate(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueNull() { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementNull(fieldName); + return setValue(newValue._repIdx); + } + + Status Element::setValueRegex(const StringData& re, const StringData& flags) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(re)); + dassert(impl.doesNotAlias(flags)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementRegex(fieldName, re, flags); + return setValue(newValue._repIdx); + } + + Status Element::setValueDBRef(const StringData& ns, const OID oid) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(ns)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementDBRef(fieldName, ns, oid); + return setValue(newValue._repIdx); + } + + Status Element::setValueCode(const StringData& value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(value)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementCode(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueSymbol(const StringData& value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(value)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementSymbol(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueCodeWithScope(const StringData& code, const BSONObj& scope) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(code)); + dassert(impl.doesNotAlias(scope)); + + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementCodeWithScope( + fieldName, code, scope); + return setValue(newValue._repIdx); + } + + Status Element::setValueInt(const int32_t value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + ElementRep thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementInt(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueTimestamp(const OpTime value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementTimestamp(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueLong(const int64_t value) { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + ElementRep thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementLong(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueMinKey() { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementMinKey(fieldName); + return setValue(newValue._repIdx); + } + + Status Element::setValueMaxKey() { + verify(ok()); + Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementMaxKey(fieldName); + return setValue(newValue._repIdx); + } + + Status Element::setValueBSONElement(const BSONElement& value) { + verify(ok()); + + if (value.type() == mongo::EOO) + return Status(ErrorCodes::IllegalOperation, "Can't set Element value to EOO"); + + Document::Impl& impl = getDocument().getImpl(); + + dassert(impl.doesNotAlias(value)); + + ElementRep thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementWithNewFieldName(fieldName, value); + return setValue(newValue._repIdx); + } + + Status Element::setValueSafeNum(const SafeNum value) { + verify(ok()); + switch (value.type()) { + case mongo::NumberInt: + return setValueInt(value._value.int32Val); + case mongo::NumberLong: + return setValueLong(value._value.int64Val); + case mongo::NumberDouble: + return setValueDouble(value._value.doubleVal); + default: + return Status( + ErrorCodes::UnsupportedFormat, + "Don't know how to handle unexpected SafeNum type"); + } + } + + BSONType Element::getType() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + return impl.getType(thisRep); + } + + StringData Element::getFieldName() const { + verify(ok()); + const Document::Impl& impl = getDocument().getImpl(); + const ElementRep& thisRep = impl.getElementRep(_repIdx); + return impl.getFieldName(thisRep); + } + + Status Element::addChild(Element e, bool front) { + // No need to verify(ok()) since we are only called from methods that have done so. + dassert(ok()); + + verify(e.ok()); + verify(_doc == e._doc); + + Document::Impl& impl = getDocument().getImpl(); + ElementRep& newRep = impl.getElementRep(e._repIdx); + + // check that new element roots a clean subtree. + if (!canAttach(e._repIdx, newRep)) + return getAttachmentError(newRep); + + // Check that this element is eligible for children. + ElementRep& thisRep = impl.getElementRep(_repIdx); + if (impl.isLeaf(thisRep)) + return Status( + ErrorCodes::IllegalOperation, + "Attempt to add a child element to a non-object element"); + + impl.disableInPlaceUpdates(); + + // TODO: In both of the following cases, we call two public API methods each. We can + // probably do better by writing this explicitly here and drying it with the public + // addSiblingLeft and addSiblingRight implementations. + if (front) { + // TODO: It is cheap to get the left child. However, it still means creating a rep + // for it. Can we do better? + Element lc = leftChild(); + if (lc.ok()) + return lc.addSiblingLeft(e); + } else { + // TODO: It is expensive to get the right child, since we have to build reps for + // all of the opaque children. But in principle, we don't really need them. Could + // we potentially add this element as a right child, leaving its left sibling + // opaque? We would at minimum need to update leftSibling, which currently assumes + // that your left sibling is never opaque. But adding new Elements to the end is a + // quite common operation, so it would be nice if we could do this efficiently. + Element rc = rightChild(); + if (rc.ok()) + return rc.addSiblingRight(e); + } + + // It must be the case that we have no children, so the new element becomes both the + // right and left child of this node. + dassert((thisRep.child.left == kInvalidRepIdx) && (thisRep.child.right == kInvalidRepIdx)); + thisRep.child.left = thisRep.child.right = e._repIdx; + newRep.parent = _repIdx; + impl.deserialize(_repIdx); + return Status::OK(); + } + + Status Element::setValue(const Element::RepIdx newValueIdx) { + // No need to verify(ok()) since we are only called from methods that have done so. + dassert(ok()); + + if (_repIdx == kRootRepIdx) + return Status(ErrorCodes::IllegalOperation, "Cannot call setValue on the root object"); + + Document::Impl& impl = getDocument().getImpl(); + + // Establish our right sibling in case it is opaque. Otherwise, we would lose the + // ability to do so after the modifications below. It is important that this occur + // before we acquire thisRep and valueRep since otherwise we would potentially + // invalidate them. + impl.resolveRightSibling(_repIdx); + + ElementRep& thisRep = impl.getElementRep(_repIdx); + ElementRep& valueRep = impl.getElementRep(newValueIdx); + + bool inPlace = false; + if (impl.canUpdateInPlace(valueRep) && impl.isInPlaceModeEnabled()) { + + // In place updates are currently enabled. We can do an in-place update to an + // element that is serialized and is not in the leaf heap. + const bool inLeafHeap = (thisRep.objIdx == kLeafObjIdx); + const bool hasValue = impl.hasValue(thisRep); + + // TODO: In the future, we can replace values in the leaf heap if they are of the + // same size as the origin was. For now, we don't support that. + if (hasValue && !inLeafHeap) { + + // See if the new Element can be recorded as an in-place update. + dassert(impl.hasValue(valueRep)); + + // Get the BSONElement representations of the existing and new value, so we can + // check if they are size compatible. + BSONElement thisElt = impl.getSerializedElement(thisRep); + BSONElement valueElt = impl.getSerializedElement(valueRep); + + if (thisElt.size() == valueElt.size()) { + + // The old and new elements are size compatible. Compute the base offsets + // of each BSONElement in the object in which it resides. We use these to + // calculate the source and target offsets in the damage entries we are + // going to write. + + const DamageEvent::OffsetSizeType targetBaseOffset = + getElementOffset(impl.getObject(thisRep.objIdx), thisElt); + + const DamageEvent::OffsetSizeType sourceBaseOffset = + getElementOffset(impl.getObject(valueRep.objIdx), valueElt); + + // If this is a type change, record a damage event for the new type. + if (thisElt.type() != valueElt.type()) { + impl.recordDamageEvent(targetBaseOffset, sourceBaseOffset, 1); + } + + dassert(thisElt.fieldNameSize() == valueElt.fieldNameSize()); + dassert(thisElt.valuesize() == valueElt.valuesize()); + + // Record a damage event for the new value data. + impl.recordDamageEvent( + targetBaseOffset + thisElt.fieldNameSize() + 1, + sourceBaseOffset + thisElt.fieldNameSize() + 1, + thisElt.valuesize()); + + inPlace = true; + } + } + } + + if (!inPlace) + getDocument().disableInPlaceUpdates(); + + // If we are not rootish, then wire in the new value among our relations. + if (thisRep.parent != kInvalidRepIdx) { + valueRep.parent = thisRep.parent; + valueRep.sibling.left = thisRep.sibling.left; + valueRep.sibling.right = thisRep.sibling.right; + } + + // Copy the rep for value to our slot so that our repIdx is unmodified. + thisRep = valueRep; + + // Be nice and clear out the source rep to make debugging easier. + valueRep = ElementRep(); + + impl.deserialize(thisRep.parent); + return Status::OK(); + } + + + namespace { + + // A helper for Element::writeElement below. For cases where we are building inside an + // array, we want to ignore field names. So the specialization for BSONArrayBuilder ignores + // the third parameter. + template + struct SubBuilder; + + template<> + struct SubBuilder { + SubBuilder(BSONObjBuilder* builder, BSONType type, const StringData& fieldName) + : buffer( + (type == mongo::Array) ? + builder->subarrayStart(fieldName) : + builder->subobjStart(fieldName)) {} + BufBuilder& buffer; + }; + + template<> + struct SubBuilder { + SubBuilder(BSONArrayBuilder* builder, BSONType type, const StringData&) + : buffer( + (type == mongo::Array) ? + builder->subarrayStart() : + builder->subobjStart()) {} + BufBuilder& buffer; + }; + + } // namespace + + template + void Document::Impl::writeElement(Element::RepIdx repIdx, Builder* builder, + const StringData* fieldName) const { + + const ElementRep& rep = getElementRep(repIdx); + + if (hasValue(rep)) { + const BSONElement element = getSerializedElement(rep); + if (fieldName) + builder->appendAs(element, *fieldName); + else + builder->append(element); + } else { + const BSONType type = getType(rep); + const StringData subName = fieldName ? *fieldName : getFieldName(rep); + SubBuilder subBuilder(builder, type, subName); + + // Otherwise, this is a 'dirty leaf', which is impossible. + dassert((type == mongo::Array) || (type == mongo::Object)); + + if (type == mongo::Array) { + BSONArrayBuilder child_builder(subBuilder.buffer); + writeChildren(repIdx, &child_builder); + child_builder.doneFast(); + } else { + BSONObjBuilder child_builder(subBuilder.buffer); + writeChildren(repIdx, &child_builder); + child_builder.doneFast(); + } + } + } + + template + void Document::Impl::writeChildren(Element::RepIdx repIdx, Builder* builder) const { + + // TODO: In theory, I think we can walk rightwards building a write region from all + // serialized embedded children that share an obj id and form a contiguous memory + // region. For arrays we would need to know something about how many elements we wrote + // that way so that the indexes would come out right. + // + // However, that involves walking the memory twice: once to build the copy region, and + // another time to actually copy it. It is unclear if this is better than just walking + // it once with the recursive solution. + + const ElementRep& rep = getElementRep(repIdx); + + // OK, need to resolve left if we haven't done that yet. + Element::RepIdx current = rep.child.left; + if (current == Element::kOpaqueRepIdx) + current = const_cast(this)->resolveLeftChild(repIdx); + + // We need to write the element, and then walk rightwards. + while (current != Element::kInvalidRepIdx) { + writeElement(current, builder); + + // If we have an opaque region to the right, and we are not in an array, then we + // can bulk copy from the end of the element we just wrote to the end of our + // parent. + const ElementRep& currentRep = getElementRep(current); + + if (currentRep.sibling.right == Element::kOpaqueRepIdx) { + + // Obtain the current parent, so we can see if we can bulk copy the right + // siblings. + const ElementRep& parentRep = getElementRep(currentRep.parent); + + // Bulk copying right only works on objects + if ((getType(parentRep) == mongo::Object) && + (currentRep.objIdx != kInvalidObjIdx) && + (currentRep.objIdx == parentRep.objIdx)) { + + BSONElement currentElt = getSerializedElement(currentRep); + const uint32_t currentSize = currentElt.size(); + + const BSONObj parentObj = (currentRep.parent == kRootRepIdx) ? + getObject(parentRep.objIdx) : + getSerializedElement(parentRep).Obj(); + const uint32_t parentSize = parentObj.objsize(); + + const uint32_t currentEltOffset = getElementOffset(parentObj, currentElt); + const uint32_t nextEltOffset = currentEltOffset + currentSize; + + const char* copyBegin = parentObj.objdata() + nextEltOffset; + const uint32_t copyBytes = parentSize - nextEltOffset; + + // The -1 is because we don't want to copy in the terminal EOO. + builder->bb().appendBuf(copyBegin, copyBytes - 1); + + // We are done with all children. + break; + } + + // We couldn't bulk copy, and our right sibling is opaque. We need to + // resolve. Note that the call to resolve may invalidate 'currentRep', so + // rather than falling through and acquiring the index by examining currentRep, + // update it with the return value of resolveRightSibling and restart the loop. + current = const_cast(this)->resolveRightSibling(current); + continue; + } + + current = currentRep.sibling.right; + } + } + + Document::Document() + : _impl(new Impl(Document::kInPlaceDisabled)) + , _root(makeRootElement()) { + dassert(_root._repIdx == kRootRepIdx); + } + + Document::Document(const BSONObj& value, InPlaceMode inPlaceMode) + : _impl(new Impl(inPlaceMode)) + , _root(makeRootElement(value)) { + dassert(_root._repIdx == kRootRepIdx); + } + + void Document::reset() { + _impl->reset(Document::kInPlaceDisabled); + MONGO_COMPILER_VARIABLE_UNUSED const Element newRoot = makeRootElement(); + dassert(newRoot._repIdx == _root._repIdx); + dassert(_root._repIdx == kRootRepIdx); + } + + void Document::reset(const BSONObj& value, InPlaceMode inPlaceMode) { + _impl->reset(inPlaceMode); + MONGO_COMPILER_VARIABLE_UNUSED const Element newRoot = makeRootElement(value); + dassert(newRoot._repIdx == _root._repIdx); + dassert(_root._repIdx == kRootRepIdx); + } + + Document::~Document() {} + + void Document::reserveDamageEvents(size_t expectedEvents) { + return getImpl().reserveDamageEvents(expectedEvents); + } + + bool Document::getInPlaceUpdates(DamageVector* damages, + const char** source, size_t* size) { + return getImpl().getInPlaceUpdates(damages, source, size); + } + + void Document::disableInPlaceUpdates() { + return getImpl().disableInPlaceUpdates(); + } + + Document::InPlaceMode Document::getCurrentInPlaceMode() const { + return getImpl().getCurrentInPlaceMode(); + } + + Element Document::makeElementDouble(const StringData& fieldName, const double value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.append(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementString(const StringData& fieldName, const StringData& value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + dassert(impl.doesNotAlias(value)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.append(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementObject(const StringData& fieldName) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + Element::RepIdx newEltIdx; + ElementRep& newElt = impl.makeNewRep(&newEltIdx); + impl.insertFieldName(newElt, fieldName); + return Element(this, newEltIdx); + } + + Element Document::makeElementObject(const StringData& fieldName, const BSONObj& value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAliasLeafBuilder(fieldName)); + dassert(impl.doesNotAlias(value)); + + // Copy the provided values into the leaf builder. + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.append(fieldName, value); + Element::RepIdx newEltIdx = impl.insertLeafElement(leafRef, fieldName.size() + 1); + ElementRep& newElt = impl.getElementRep(newEltIdx); + + newElt.child.left = Element::kOpaqueRepIdx; + newElt.child.right = Element::kOpaqueRepIdx; + + return Element(this, newEltIdx); + } + + Element Document::makeElementArray(const StringData& fieldName) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + Element::RepIdx newEltIdx; + ElementRep& newElt = impl.makeNewRep(&newEltIdx); + newElt.array = true; + impl.insertFieldName(newElt, fieldName); + return Element(this, newEltIdx); + } + + Element Document::makeElementArray(const StringData& fieldName, const BSONObj& value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAliasLeafBuilder(fieldName)); + dassert(impl.doesNotAlias(value)); + + // Copy the provided array values into the leaf builder. + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendArray(fieldName, value); + Element::RepIdx newEltIdx = impl.insertLeafElement(leafRef, fieldName.size() + 1); + ElementRep& newElt = impl.getElementRep(newEltIdx); + newElt.child.left = Element::kOpaqueRepIdx; + newElt.child.right = Element::kOpaqueRepIdx; + return Element(this, newEltIdx); + } + + Element Document::makeElementBinary(const StringData& fieldName, + const uint32_t len, + const mongo::BinDataType binType, + const void* const data) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + // TODO: Alias check 'data'? + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendBinData(fieldName, len, binType, data); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementUndefined(const StringData& fieldName) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendUndefined(fieldName); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementNewOID(const StringData& fieldName) { + OID newOID; + newOID.init(); + return makeElementOID(fieldName, newOID); + } + + Element Document::makeElementOID(const StringData& fieldName, const OID value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.append(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementBool(const StringData& fieldName, const bool value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendBool(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementDate(const StringData& fieldName, const Date_t value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendDate(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementNull(const StringData& fieldName) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendNull(fieldName); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementRegex(const StringData& fieldName, + const StringData& re, + const StringData& flags) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + dassert(impl.doesNotAlias(re)); + dassert(impl.doesNotAlias(flags)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendRegex(fieldName, re, flags); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementDBRef(const StringData& fieldName, + const StringData& ns, const OID value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendDBRef(fieldName, ns, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementCode(const StringData& fieldName, const StringData& value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + dassert(impl.doesNotAlias(value)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendCode(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementSymbol(const StringData& fieldName, const StringData& value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + dassert(impl.doesNotAlias(value)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendSymbol(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementCodeWithScope(const StringData& fieldName, + const StringData& code, const BSONObj& scope) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + dassert(impl.doesNotAlias(code)); + dassert(impl.doesNotAlias(scope)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendCodeWScope(fieldName, code, scope); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementInt(const StringData& fieldName, const int32_t value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.append(fieldName, value); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementTimestamp(const StringData& fieldName, const OpTime value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendTimestamp(fieldName, value.asDate()); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementLong(const StringData& fieldName, const int64_t value) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.append(fieldName, static_cast(value)); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementMinKey(const StringData& fieldName) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendMinKey(fieldName); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElementMaxKey(const StringData& fieldName) { + Impl& impl = getImpl(); + dassert(impl.doesNotAlias(fieldName)); + + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendMaxKey(fieldName); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + + Element Document::makeElement(const BSONElement& value) { + Impl& impl = getImpl(); + + // Attempts to create an EOO element are translated to returning an invalid + // Element. For array and object nodes, we flow through the custom + // makeElement{Object|Array} methods, since they have special logic to deal with + // opaqueness. Otherwise, we can just insert via appendAs. + if (value.type() == mongo::EOO) + return end(); + else if(value.type() == mongo::Object) + return makeElementObject(value.fieldNameStringData(), value.Obj()); + else if(value.type() == mongo::Array) + return makeElementArray(value.fieldNameStringData(), value.Obj()); + else { + dassert(impl.doesNotAlias(value)); + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.append(value); + return Element(this, impl.insertLeafElement(leafRef, value.fieldNameSize())); + } + } + + Element Document::makeElementWithNewFieldName(const StringData& fieldName, + const BSONElement& value) { + Impl& impl = getImpl(); + + // See the above makeElement for notes on these cases. + if (value.type() == mongo::EOO) + return end(); + else if(value.type() == mongo::Object) + return makeElementObject(fieldName, value.Obj()); + else if(value.type() == mongo::Array) + return makeElementArray(fieldName, value.Obj()); + else { + dassert(getImpl().doesNotAliasLeafBuilder(fieldName)); + dassert(getImpl().doesNotAlias(value)); + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + builder.appendAs(value, fieldName); + return Element(this, impl.insertLeafElement(leafRef, fieldName.size() + 1)); + } + } + + Element Document::makeElementSafeNum(const StringData& fieldName, SafeNum value) { + + dassert(getImpl().doesNotAlias(fieldName)); + + switch (value.type()) { + case mongo::NumberInt: + return makeElementInt(fieldName, value._value.int32Val); + case mongo::NumberLong: + return makeElementLong(fieldName, value._value.int64Val); + case mongo::NumberDouble: + return makeElementDouble(fieldName, value._value.doubleVal); + default: + // Return an invalid element to indicate that we failed. + return end(); + } + } + + Element Document::makeElement(ConstElement element) { + return makeElement(element, NULL); + } + + Element Document::makeElementWithNewFieldName(const StringData& fieldName, + ConstElement element) { + return makeElement(element, &fieldName); + } + + Element Document::makeRootElement() { + return makeElementObject(StringData(kRootFieldName, StringData::LiteralTag())); + } + + Element Document::makeRootElement(const BSONObj& value) { + Impl& impl = getImpl(); + Element::RepIdx newEltIdx = Element::kInvalidRepIdx; + ElementRep* newElt = &impl.makeNewRep(&newEltIdx); + + // A BSONObj provided for the root Element is stored in _objects rather than being + // copied like all other BSONObjs. + newElt->objIdx = impl.insertObject(value); + impl.insertFieldName(*newElt, kRootFieldName); + + // Strictly, the following is a lie: the root isn't serialized, because it doesn't + // have a contiguous fieldname. However, it is a useful fiction to pretend that it + // is, so we can easily check if we have a 'pristine' document state by checking if + // the root is marked as serialized. + newElt->serialized = true; + + // If the provided value is empty, mark it as having no children, otherwise mark the + // children as opaque. + if (value.isEmpty()) + newElt->child.left = Element::kInvalidRepIdx; + else + newElt->child.left = Element::kOpaqueRepIdx; + newElt->child.right = newElt->child.left; + + return Element(this, newEltIdx); + } + + Element Document::makeElement(ConstElement element, const StringData* fieldName) { + + Impl& impl = getImpl(); + + if (this == &element.getDocument()) { + + // If the Element that we want to build from belongs to this Document, then we have + // to first copy it to the side, and then back in, since otherwise we might be + // attempting both read to and write from the underlying BufBuilder simultaneously, + // which will not work. + BSONObjBuilder builder; + impl.writeElement(element.getIdx(), &builder, fieldName); + BSONObj built = builder.done(); + BSONElement newElement = built.firstElement(); + return makeElement(newElement); + + } else { + + // If the Element belongs to another document, then we can just stream it into our + // builder. We still do need to dassert that the field name doesn't alias us + // somehow. + if (fieldName) { + dassert(impl.doesNotAlias(*fieldName)); + } + BSONObjBuilder& builder = impl.leafBuilder(); + const int leafRef = builder.len(); + + const Impl& oImpl = element.getDocument().getImpl(); + oImpl.writeElement(element.getIdx(), &builder, fieldName); + return Element(this, impl.insertLeafElement(leafRef)); + } + } + + inline Document::Impl& Document::getImpl() { + // Don't use scoped_ptr::operator* since it may generate assertions that the + // pointer is non-null, but we already know that to be always and forever true, and + // otherwise the assertion code gets spammed into every method that inlines the call to + // this function. We just dereference the pointer returned from 'get' ourselves. + return *_impl.get(); + } + + inline const Document::Impl& Document::getImpl() const { + return *_impl.get(); + } + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/document.h b/src/mongo/bson/mutable/document.h new file mode 100644 index 00000000000..b39fcf8b2ac --- /dev/null +++ b/src/mongo/bson/mutable/document.h @@ -0,0 +1,509 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/const_element.h" +#include "mongo/bson/mutable/damage_vector.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/cstdint.h" +#include "mongo/util/safe_num.h" + +namespace mongo { +namespace mutablebson { + + /** Mutable BSON Overview + * + * Mutable BSON provides classes to facilitate the manipulation of existing BSON objects + * or the construction of new BSON objects from scratch in an incremental fashion. The + * operations (including additions, deletions, renamings, type changes and value + * modification) that are to be performed do not need to be known ahead of time, and do + * not need to occur in any particular order. This is in contrast to BSONObjBuilder and + * BSONArrayBuilder which offer only serialization and cannot revise already serialized + * data. If you need to build a BSONObj but you know upfront what you need to build then + * you should use BSONObjBuilder and BSONArrayBuilder directly as they will be faster and + * less resource intensive. + * + * The classes in this library (Document, Element, and ConstElement) present a tree-like + * (or DOM like) interface. Elements are logically equivalent to BSONElements: they carry + * a type, a field name, and a value. Every Element belongs to a Document, which roots the + * tree, and Elements of proper type (mongo::Object or mongo::Array) may have child + * Elements of their own. Given an Element, you may navigate to the Element's parent, to + * its siblings to the left or right of the Element in the tree, and to the leftmost or + * rightmost children of the Element. Note that some Elements may not offer all of these + * relationships: An Element that represents a terminal BSON value (like an integer) will + * not have children (though it may well have siblings). Similarly, an Element that is an + * 'only child' will not have any left or right siblings. Given a Document, you may begin + * navigating by obtaining the root Element of the tree by calling Document::root. See the + * documentation for the Element class for the specific navigation methods that will be + * available from the root Element. + * + * Elements within the Document may be modified in various ways: the value of the Element + * may be changed, the Element may be removed, it may be renamed, and if it is eligible + * for children (i.e. it represents a mongo::Array or mongo::Object) it may have child + * Elements added to it. Once you have completed building or modifying the Document, you + * may write it back out to a BSONObjBuilder by calling Document::writeTo. You may also + * serialize individual Elements within the Document to BSONObjBuilder or BSONArrayBuilder + * objects by calling Element::writeTo or Element::writeArrayTo. + * + * In addition to the above capabilities, there are algorithms provided in 'algorithm.h' + * to help with tasks like searching for Elements that match a predicate or for sorting + * the children of an Object Element. + * + * Example 1: Building up a document from scratch, reworking it, and then serializing it: + + namespace mmb = mongo::mutablebson; + // Create a new document + mmb::Document doc; + // doc contents: '{}' + + // Get the root of the document. + mmb::Element root = doc.root(); + + // Create a new mongo::NumberInt typed Element to represent life, the universe, and + // everything, then push that Element into the root object, making it a child of root. + mmb::Element e0 = doc.makeElementInt("ltuae", 42); + root.pushBack(e0); + // doc contents: '{ ltuae : 42 }' + + // Create a new empty mongo::Object-typed Element named 'magic', and push it back as a + // child of the root, making it a sibling of e0. + mmb::Element e1 = doc.makeElementObject("magic"); + root.pushBack(e1); + // doc contents: '{ ltuae : 42, magic : {} }' + + // Create a new mongo::NumberDouble typed Element to represent Pi, and insert it as child + // of the new object we just created. + mmb::Element e3 = doc.makeElementDouble("pi", 3.14); + e1.pushBack(e3); + // doc contents: '{ ltuae : 42, magic : { pi : 3.14 } }' + + // Create a new mongo::NumberDouble to represent Plancks constant in electrovolt + // micrometers, and add it as a child of the 'magic' object. + mmb::Element e4 = doc.makeElementDouble("hbar", 1.239); + e1.pushBack(e4); + // doc contents: '{ ltuae : 42, magic : { pi : 3.14, hbar : 1.239 } }' + + // Rename the parent element of 'hbar' to be 'constants'. + e4.parent().rename("constants"); + // doc contents: '{ ltuae : 42, constants : { pi : 3.14, hbar : 1.239 } }' + + // Rename 'ltuae' to 'answer' by accessing it as the root objects left child. + doc.root().leftChild().rename("answer"); + // doc contents: '{ answer : 42, constants : { pi : 3.14, hbar : 1.239 } }' + + // Sort the constants by name. + mmb::sortChildren(doc.root().rightChild(), mmb::FieldNameLessThan()); + // doc contents: '{ answer : 42, constants : { hbar : 1.239, pi : 3.14 } }' + + mongo::BSONObjBuilder builder; + doc.writeTo(&builder); + mongo::BSONObj result = builder.obj(); + // result contents: '{ answer : 42, constants : { hbar : 1.239, pi : 3.14 } }' + + * While you can use this library to build Documents from scratch, its real purpose is to + * manipulate existing BSONObjs. A BSONObj may be passed to the Document constructor or to + * Document::make[Object|Array]Element, in which case the Document or Element will reflect + * the values contained within the provided BSONObj. Modifications will not alter the + * underlying BSONObj: they are held off to the side within the Document. However, when + * the Document is subsequently written back out to a BSONObjBuilder, the modifications + * applied to the Document will be reflected in the serialized version. + * + * Example 2: Modifying an existing BSONObj (some error handling removed for length) + + namespace mmb = mongo::mutablebson; + + static const char inJson[] = + "{" + " 'whale': { 'alive': true, 'dv': -9.8, 'height': 50, attrs : [ 'big' ] }," + " 'petunias': { 'alive': true, 'dv': -9.8, 'height': 50 } " + "}"; + mongo::BSONObj obj = mongo::fromjson(inJson); + + // Create a new document representing the BSONObj with the above contents. + mmb::Document doc(obj); + + // The whale hits the planet and dies. + mmb::Element whale = mmb::findFirstChildNamed(doc.root(), "whale"); + // Find the 'dv' field in the whale. + mmb::Element whale_deltav = mmb::findFirstChildNamed(whale, "dv"); + // Set the dv field to zero. + whale_deltav.setValueDouble(0.0); + // Find the 'height' field in the whale. + mmb::Element whale_height = mmb::findFirstChildNamed(whale, "height"); + // Set the height field to zero. + whale_deltav.setValueDouble(0); + // Find the 'alive' field, and set it to false. + mmb::Element whale_alive = mmb::findFirstChildNamed(whale, "alive"); + whale_alive.setValueBool(false); + + // The petunias survive, update its fields much like we did above. + mmb::Element petunias = mmb::findFirstChildNamed(doc.root(), "petunias"); + mmb::Element petunias_deltav = mmb::findFirstChildNamed(petunias, "dv"); + petunias_deltav.setValueDouble(0.0); + mmb::Element petunias_height = mmb::findFirstChildNamed(petunias, "height"); + petunias_deltav.setValueDouble(0); + + // Replace the whale by its wreckage, saving only its attributes: + // Construct a new mongo::Object element for the ex-whale. + mmb::Element ex_whale = doc.makeElementObject("ex-whale"); + doc.root().pushBack(ex_whale); + // Find the attributes of the old 'whale' element. + mmb::Element whale_attrs = mmb::findFirstChildNamed(whale, "attrs"); + // Remove the attributes from the whale (they remain valid, but detached). + whale_attrs.remove(); + // Add the attributes into the ex-whale. + ex_whale.pushBack(whale_attrs); + // Remove the whale object. + whale.remove(); + + // Current state of document: + "{" + " 'petunias': { 'alive': true, 'dv': 0.0, 'height': 50 }," + " 'ex-whale': { 'attrs': [ 'big' ] } })" + "}"; + + * Both of the above examples are derived from tests in mutable_bson_test.cpp, see the + * tests Example1 and Example2 if you would like to play with the code. + * + * Additional details on Element and Document are available in their class and member + * comments. + */ + + /** Document is the entry point into the mutable BSON system. It has a fairly simple + * API. It acts as an owner for the Element resources of the document, provides a + * pre-constructed designated root Object Element, and acts as a factory for new Elements, + * which may then be attached to the root or to other Elements by calling the appropriate + * topology mutation methods in Element. + * + * The default constructor builds an empty Document which you may then extend by creating + * new Elements and manipulating the tree topology. It is also possible to build a + * Document that derives its initial values from a BSONObj. The given BSONObj will not be + * modified, but it also must not be modified elsewhere while Document is using it. Unlike + * all other calls in this library where a BSONObj is passed in, the one argument Document + * constructor *does not copy* the BSONObj's contents, so they must remain valid for the + * duration of Documents lifetime. Document does hold a copy of the BSONObj itself, so it + * will up the refcount if the BSONObj internals are counted. + * + * Newly constructed Elements formed by calls to 'makeElement[Type]' methods are not + * attached to the root of the document. You must explicitly attach them somewhere. If you + * lose the Element value that is returned to you from a 'makeElement' call before you + * attach it to the tree then the value will be unreachable. Elements in a document do not + * outlive the Document. + * + * Document provides a convenience method to serialize all of the Elements in the tree + * that are reachable from the root element to a BSONObjBuilder. In general you should use + * this in preference to root().writeTo() if you mean to write the entire + * Document. Similarly, Document provides wrappers for comparisons that simply delegate to + * comparison operations on the root Element. + * + * A 'const Document' is very limited: you may only write its contents out or obtain a + * ConstElement for the root. ConstElement is much like Element, but does not permit + * mutations. See the class comment for ConstElement for more information. + */ + class Document { + + // TODO: In principle there is nothing that prevents implementing a deep copy for + // Document, but for now it is not permitted. + MONGO_DISALLOW_COPYING(Document); + + public: + + // + // Lifecycle + // + + /** Construct a new empty document. */ + Document(); + + enum InPlaceMode { + kInPlaceDisabled = 0, + kInPlaceEnabled = 1, + }; + + /** Construct new document for the given BSONObj. The data in 'value' is NOT copied. By + * default, queueing of in-place modifications against the underlying document is + * permitted. To disable this behavior, explicitly pass kInPlaceDisabled. + */ + explicit Document(const BSONObj& value, InPlaceMode inPlaceMode = kInPlaceEnabled); + + /** Abandon all internal state associated with this Document, and return to a state + * semantically equivalent to that yielded by a call to the default constructor. All + * objects associated with the current document state are invalidated (e.g. Elements, + * BSONElements, BSONObj's values, field names, etc.). This method is useful because + * it may (though it is not required to) preserve the memory allocation of the + * internal data structures of Document. If you need to logically create and destroy + * many Documents in serial, it may be faster to reset. + */ + void reset(); + + /** As the no argument 'reset', but returns to a state semantically equivalent to that + * yielded by a call to the two argument constructor with the arguments provided + * here. As with the other 'reset' call, all associated objects are invalidated. */ + void reset(const BSONObj& value, InPlaceMode inPlaceMode = kInPlaceEnabled); + + /** Destroy this document permanently */ + ~Document(); + + + // + // Comparison API + // + + /** Compare this Document to 'other' with the semantics of BSONObj::woCompare. */ + inline int compareWith(const Document& other, bool considerFieldName = true) const; + + /** Compare this Document to 'other' with the semantics of BSONObj::woCompare. */ + inline int compareWithBSONObj(const BSONObj& other, bool considerFieldName = true) const; + + + // + // Serialization API + // + + /** Serialize the Elements reachable from the root Element of this Document to the + * provided builder. + */ + inline void writeTo(BSONObjBuilder* builder) const; + + /** Serialize the Elements reachable from the root Element of this Document and return + * the result as a BSONObj. + */ + inline BSONObj getObject() const; + + + // + // Element creation API. + // + // Newly created elements are not attached to the tree (effectively, they are + // 'roots'). You must call one of the topology management methods in 'Element' to + // connect the newly created Element to another Element in the Document, possibly the + // Element referenced by Document::root. Elements do not outlive the Document. + // + + /** Create a new double Element with the given value and field name. */ + Element makeElementDouble(const StringData& fieldName, double value); + + /** Create a new string Element with the given value and field name. */ + Element makeElementString(const StringData& fieldName, const StringData& value); + + /** Create a new empty object Element with the given field name. */ + Element makeElementObject(const StringData& fieldName); + + /** Create a new object Element with the given field name. The data in 'value' is + * copied. + */ + Element makeElementObject(const StringData& fieldName, const BSONObj& value); + + /** Create a new empty array Element with the given field name. */ + Element makeElementArray(const StringData& fieldName); + + /** Create a new array Element with the given field name. The data in 'value' is + * copied. + */ + Element makeElementArray(const StringData& fieldName, const BSONObj& value); + + /** Create a new binary Element with the given data and field name. */ + Element makeElementBinary( + const StringData& fieldName, uint32_t len, BinDataType binType, const void* data); + + /** Create a new undefined Element with the given field name. */ + Element makeElementUndefined(const StringData& fieldName); + + /** Create a new OID + Element with the given field name. */ + Element makeElementNewOID(const StringData& fieldName); + + /** Create a new OID Element with the given value and field name. */ + Element makeElementOID(const StringData& fieldName, mongo::OID value); + + /** Create a new bool Element with the given value and field name. */ + Element makeElementBool(const StringData& fieldName, bool value); + + /** Create a new date Element with the given value and field name. */ + Element makeElementDate(const StringData& fieldName, Date_t value); + + /** Create a new null Element with the given field name. */ + Element makeElementNull(const StringData& fieldName); + + /** Create a new regex Element with the given data and field name. */ + Element makeElementRegex( + const StringData& fieldName, const StringData& regex, const StringData& flags); + + /** Create a new DBRef Element with the given data and field name. */ + Element makeElementDBRef( + const StringData& fieldName, const StringData& ns, mongo::OID oid); + + /** Create a new code Element with the given value and field name. */ + Element makeElementCode(const StringData& fieldName, const StringData& value); + + /** Create a new symbol Element with the given value and field name. */ + Element makeElementSymbol(const StringData& fieldName, const StringData& value); + + /** Create a new scoped code Element with the given data and field name. */ + Element makeElementCodeWithScope( + const StringData& fieldName, const StringData& code, const BSONObj& scope); + + /** Create a new integer Element with the given value and field name. */ + Element makeElementInt(const StringData& fieldName, int32_t value); + + /** Create a new timetamp Element with the given value and field name. */ + Element makeElementTimestamp(const StringData& fieldName, OpTime value); + + /** Create a new long integer Element with the given value and field name. */ + Element makeElementLong(const StringData& fieldName, int64_t value); + + /** Create a new min key Element with the given field name. */ + Element makeElementMinKey(const StringData& fieldName); + + /** Create a new max key Element with the given field name. */ + Element makeElementMaxKey(const StringData& fieldName); + + + // + // Element creation methods from variant types + // + + /** Construct a new Element with the same name, type, and value as the provided + * BSONElement. The value is copied. + */ + Element makeElement(const BSONElement& elt); + + /** Construct a new Element with the same type and value as the provided BSONElement, + * but with a new name. The value is copied. + */ + Element makeElementWithNewFieldName(const StringData& fieldName, const BSONElement& elt); + + /** Create a new element of the appopriate type to hold the given value, with the given + * field name. + */ + Element makeElementSafeNum(const StringData& fieldName, SafeNum value); + + /** Construct a new element with the same name, type, and value as the provided mutable + * Element. The data is copied from the given Element. Unlike most methods in this + * class the provided Element may be from a different Document. + */ + Element makeElement(ConstElement elt); + + /** Construct a new Element with the same type and value as the provided mutable + * Element, but with a new field name. The data is copied from the given + * Element. Unlike most methods in this class the provided Element may be from a + * different Document. + */ + Element makeElementWithNewFieldName(const StringData& fieldName, ConstElement elt); + + // + // Accessors + // + + /** Returns the root element for this document. */ + inline Element root(); + + /** Returns the root element for this document. */ + inline ConstElement root() const; + + /** Returns an element that will compare equal to a non-ok element. */ + inline Element end(); + + /** Returns an element that will compare equal to a non-ok element. */ + inline ConstElement end() const; + + inline std::string toString() const; + + // + // In-place API. + // + + /** Ensure that at least 'expectedEvents' damage events can be recorded for in-place + * mutations without reallocation. This call is ignored if damage events are disabled. + */ + void reserveDamageEvents(size_t expectedEvents); + + /** Request a vector of damage events describing in-place updates to this Document. If + * the modifications to this Document were not all able to be achieved in-place, then + * a non-OK Status is returned, and the provided damage vector will be made empty and + * *source set equal to NULL. Otherwise, the provided damage vector is populated, and + * the 'source' argument is set to point to a region from which bytes can be read. The + * 'source' offsets in the damage vector are to be interpreted as offsets within this + * region. If the 'size' parameter is non-null and 'source' is set to a non-NULL + * value, then size will be filled in with the size of the 'source' region to + * facilitate making an owned copy of the source data, in the event that that is + * needed. + * + * The lifetime of the source region should be considered to extend only from the + * return from this call to before the next API call on this Document or any of its + * member Elements. That is almost certainly overly conservative: some read only calls + * are undoubtedly fine. But it is very easy to invalidate 'source' by calling any + * mutating operation, so proceed with due caution. + * + * It is expected, though, that in normal modes of operation obtainin the damage + * vector is one of the last operations performed on a Document before its + * destruction, so this is not so great a restriction. + * + * The destination offsets in the damage events are implicitly offsets into the + * BSONObj used to construct this Document. + */ + bool getInPlaceUpdates(DamageVector* damages, + const char** source, + size_t* size = NULL); + + /** Drop the queue of in-place update damage events, and do not queue new operations + * that would otherwise have been in-place. Use this if you know that in-place updates + * will not continue to be possible and do not want to pay the overhead of + * speculatively queueing them. After calling this method, getInPlaceUpdates will + * return a non-OK Status. It is not possible to re-enable in-place updates once + * disabled. + */ + void disableInPlaceUpdates(); + + /** Returns the current in-place mode for the document. Note that for some documents, + * like those created without any backing BSONObj, this will always return kForbidden, + * since in-place updates make no sense for such an object. In other cases, an object + * which started in kInPlacePermitted mode may transition to kInPlaceForbidden if a + * topology mutating operation is applied. + */ + InPlaceMode getCurrentInPlaceMode() const; + + /** A convenience routine, this returns true if the current in-place mode is + * kInPlaceEnabled, and false otherwise. + */ + inline bool isInPlaceModeEnabled() const; + + private: + friend class Element; + + // For now, the implementation of Document is firewalled. + class Impl; + inline Impl& getImpl(); + inline const Impl& getImpl() const; + + Element makeRootElement(); + Element makeRootElement(const BSONObj& value); + Element makeElement(ConstElement element, const StringData* fieldName); + + const boost::scoped_ptr _impl; + + // The root element of this document. + const Element _root; + }; + +} // namespace mutablebson +} // namespace mongo + +#include "mongo/bson/mutable/document-inl.h" diff --git a/src/mongo/bson/mutable/element-inl.h b/src/mongo/bson/mutable/element-inl.h new file mode 100644 index 00000000000..97a3f06426e --- /dev/null +++ b/src/mongo/bson/mutable/element-inl.h @@ -0,0 +1,144 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { +namespace mutablebson { + + inline Element Element::operator[](size_t n) const { + return findNthChild(n); + } + + inline Element Element::operator[](const StringData& name) const { + return findFirstChildNamed(name); + } + + inline double Element::getValueDouble() const { + dassert(hasValue() && isType(mongo::NumberDouble)); + return getValue()._numberDouble(); + } + + inline StringData Element::getValueString() const { + dassert(hasValue() && isType(mongo::String)); + return getValueStringOrSymbol(); + } + + inline BSONObj Element::getValueObject() const { + dassert(hasValue() && isType(mongo::Object)); + return getValue().Obj(); + } + + inline BSONArray Element::getValueArray() const { + dassert(hasValue() && isType(mongo::Array)); + return BSONArray(getValue().Obj()); + } + + inline bool Element::isValueUndefined() const { + return isType(mongo::Undefined); + } + + inline OID Element::getValueOID() const { + dassert(hasValue() && isType(mongo::jstOID)); + return getValue().__oid(); + } + + inline bool Element::getValueBool() const { + dassert(hasValue() && isType(mongo::Bool)); + return getValue().boolean(); + } + + inline Date_t Element::getValueDate() const { + dassert(hasValue() && isType(mongo::Date)); + return getValue().date(); + } + + inline bool Element::isValueNull() const { + return isType(mongo::jstNULL); + } + + inline StringData Element::getValueSymbol() const { + dassert(hasValue() && isType(mongo::Symbol)); + return getValueStringOrSymbol(); + } + + inline int32_t Element::getValueInt() const { + dassert(hasValue() && isType(mongo::NumberInt)); + return getValue()._numberInt(); + } + + inline OpTime Element::getValueTimestamp() const { + dassert(hasValue() && isType(mongo::Timestamp)); + return getValue()._opTime(); + } + + inline int64_t Element::getValueLong() const { + dassert(hasValue() && isType(mongo::NumberLong)); + return getValue()._numberLong(); + } + + inline bool Element::isValueMinKey() const { + return isType(mongo::MinKey); + } + + inline bool Element::isValueMaxKey() const { + return isType(mongo::MaxKey); + } + + inline bool Element::ok() const { + dassert(_doc != NULL); + return _repIdx <= kMaxRepIdx; + } + + inline Document& Element::getDocument() { + return *_doc; + } + + inline const Document& Element::getDocument() const { + return *_doc; + } + + inline bool Element::isType(BSONType type) const { + return (getType() == type); + } + + inline Element::RepIdx Element::getIdx() const { + return _repIdx; + } + + inline Element::Element(Document* doc, RepIdx repIdx) + : _doc(doc) + , _repIdx(repIdx) { + dassert(_doc != NULL); + } + + inline StringData Element::getValueStringOrSymbol() const { + const BSONElement value = getValue(); + const char* str = value.valuestr(); + const size_t size = value.valuestrsize() - 1; + return StringData(str, size); + } + + inline bool operator==(const Element& l, const Element& r) { + return (l._doc == r._doc) && (l._repIdx == r._repIdx); + } + + inline bool operator!=(const Element& l, const Element& r) { + return !(l == r); + } + + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/element.cpp b/src/mongo/bson/mutable/element.cpp new file mode 100644 index 00000000000..b835d84d4f0 --- /dev/null +++ b/src/mongo/bson/mutable/element.cpp @@ -0,0 +1,173 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/mutable/element.h" + +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" + +namespace mongo { +namespace mutablebson { + + // Many of the methods of Element are actually implemented in document.cpp, since they need + // access to the firewalled implementation of Document. + + Status Element::pushFront(Element e) { + return addChild(e, true); + } + + Status Element::pushBack(Element e) { + return addChild(e, false); + } + + Status Element::popFront() { + Element left = leftChild(); + if (!left.ok()) + return Status(ErrorCodes::EmptyArrayOperation, "popFront on empty"); + return left.remove(); + } + + Status Element::popBack() { + Element right = rightChild(); + if (!right.ok()) + return Status(ErrorCodes::EmptyArrayOperation, "popBack on empty"); + return right.remove(); + } + + Status Element::appendDouble(const StringData& fieldName, double value) { + return pushBack(getDocument().makeElementDouble(fieldName, value)); + } + + Status Element::appendString(const StringData& fieldName, const StringData& value) { + return pushBack(getDocument().makeElementString(fieldName, value)); + } + + Status Element::appendObject(const StringData& fieldName, const BSONObj& value) { + return pushBack(getDocument().makeElementObject(fieldName, value)); + } + + Status Element::appendArray(const StringData& fieldName, const BSONObj& value) { + return pushBack(getDocument().makeElementArray(fieldName, value)); + } + + Status Element::appendBinary(const StringData& fieldName, + uint32_t len, mongo::BinDataType binType, + const void* data) { + return pushBack(getDocument().makeElementBinary(fieldName, len, binType, data)); + } + + Status Element::appendUndefined(const StringData& fieldName) { + return pushBack(getDocument().makeElementUndefined(fieldName)); + } + + Status Element::appendOID(const StringData& fieldName, const OID value) { + return pushBack(getDocument().makeElementOID(fieldName, value)); + } + + Status Element::appendBool(const StringData& fieldName, bool value) { + return pushBack(getDocument().makeElementBool(fieldName, value)); + } + + Status Element::appendDate(const StringData& fieldName, Date_t value) { + return pushBack(getDocument().makeElementDate(fieldName, value)); + } + + Status Element::appendNull(const StringData& fieldName) { + return pushBack(getDocument().makeElementNull(fieldName)); + } + + Status Element::appendRegex(const StringData& fieldName, + const StringData& re, const StringData& flags) { + return pushBack(getDocument().makeElementRegex(fieldName, re, flags)); + } + + Status Element::appendDBRef(const StringData& fieldName, + const StringData& ns, const OID oid) { + return pushBack(getDocument().makeElementDBRef(fieldName, ns, oid)); + } + + Status Element::appendCode(const StringData& fieldName, const StringData& value) { + return pushBack(getDocument().makeElementCode(fieldName, value)); + } + + Status Element::appendSymbol(const StringData& fieldName, const StringData& value) { + return pushBack(getDocument().makeElementSymbol(fieldName, value)); + } + + Status Element::appendCodeWithScope(const StringData& fieldName, + const StringData& code, const BSONObj& scope) { + return pushBack(getDocument().makeElementCodeWithScope(fieldName, code, scope)); + } + + Status Element::appendInt(const StringData& fieldName, int32_t value) { + return pushBack(getDocument().makeElementInt(fieldName, value)); + } + + Status Element::appendTimestamp(const StringData& fieldName, OpTime value) { + return pushBack(getDocument().makeElementTimestamp(fieldName, value)); + } + + Status Element::appendLong(const StringData& fieldName, int64_t value) { + return pushBack(getDocument().makeElementLong(fieldName, value)); + } + + Status Element::appendMinKey(const StringData& fieldName) { + return pushBack(getDocument().makeElementMinKey(fieldName)); + } + + Status Element::appendMaxKey(const StringData& fieldName) { + return pushBack(getDocument().makeElementMaxKey(fieldName)); + } + + Status Element::appendElement(const BSONElement& value) { + return pushBack(getDocument().makeElement(value)); + } + + Status Element::appendSafeNum(const StringData& fieldName, SafeNum value) { + return pushBack(getDocument().makeElementSafeNum(fieldName, value)); + } + + std::string Element::toString() const { + if (!ok()) + return "INVALID-MUTABLE-ELEMENT"; + + if (hasValue()) + return getValue().toString(); + + const BSONType type = getType(); + + // The only types that sometimes don't have a value are Object and Array nodes. + dassert((type == mongo::Object) || (type == mongo::Array)); + + if (type == mongo::Object) { + BSONObjBuilder builder; + writeTo(&builder); + BSONObj obj = builder.obj(); + return obj.firstElement().toString(); + } else { + // It must be an array. + BSONObjBuilder builder; + BSONArrayBuilder arrayBuilder(builder.subarrayStart(getFieldName())); + writeArrayTo(&arrayBuilder); + arrayBuilder.done(); + BSONObj obj = builder.obj(); + return obj.firstElement().toString(); + } + } + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/element.h b/src/mongo/bson/mutable/element.h new file mode 100644 index 00000000000..189b840068c --- /dev/null +++ b/src/mongo/bson/mutable/element.h @@ -0,0 +1,609 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/cstdint.h" +#include "mongo/util/safe_num.h" + +namespace mongo { +namespace mutablebson { + + /** For an overview of mutable BSON, please see the file document.h in this directory. */ + + class ConstElement; + class Document; + + /** Element represents a BSON value or object in a mutable BSON Document. The lifetime of + * an Element is a subset of the Document to which it belongs. Much like a BSONElement, an + * Element has a type, a field name, and (usually) a value. An Element may be used to read + * or modify the value (including changing its type), to navigate to related Elements in + * the Document tree, or for a number of topological changes to the Document + * structure. Element also offers the ability to compare its value to that of other + * Elements, and to serialize its value to a BSONObjBuilder or BSONArrayBuilder. + * + * Elements have reference or iterator like semantics, and are very lightweight. You + * should not worry about the cost of passing an Element by value, copying an Element, or + * similar operations. Such operations do not mean that the logical element in the + * underlying Document is duplicated. Only the reference is duplicated. + * + * The API for Element is broken into several sections: + * + * - Topology mutation: These methods are to either add other Elements to the Document + * tree as siblings or children (when applicable) of the current Element, to remove the + * Element from the tree, or to remove children of the Element (when applicable). + * + * - Navigation: These methods are used to navigate the Document tree by returning other + * Elements in specified relationships to the current Element. In this regard, Elements + * act much like STL iterators that walk over the Document tree. One important + * difference is that Elements are never invalidated, even when 'remove' is called. If + * you have two Elements that alias the same element in the Document tree, modifications + * through one Element will be visible via the other. + * + * - Value access: These methods provide access to the value in the Document tree that the + * current Element represents. All leaf (a.k.a. 'primitive', or non-Object and + * non-Array) like Elements will always be able to provide a value. However, there are + * cases where non-leaf Elements (representing Objects or Arrays) cannot provide a + * value. Therefore, you must always call 'hasValue' to determine if the value is + * available before calling 'getValue'. Similarly, you must determine the type of the + * Element by calling getType() and only call the matching typed getValue. + * + * - Comparison: It is possible to compare one Element with another to determine ordering + * or equality as defined by woCompare. Similarly, it is possible to directly compare an + * Element with a BSONElement. It is legal to compare two Elements which belong to + * different Documents. + * + * - Serialization: Elements may be serialized to BSONObjBuilder or to BSONArrayBuilder + * objects when appropriate. One detail to consider is that writeTo for the root Element + * behaves differently than the others: it does not start a new subobj scope in the + * builder, so all of its children will be added at the current level to the + * builder. The provided builder does not have its 'done' method called automatically. + * + * - Value mutation: You may freely modify the value of an Element, including + * modifications that change the type of the Element and setting the value of the + * Element to the value of another BSONObj. You may also set the value from a SafeNum or + * from a BSONElement. + * + * - Accessors: These provide access to various properties of the Element, like the + * Document to which the Element belongs, the BSON type and field name of the Element, + * etc. One critical accessor is 'ok'. When using the topology API to navigate a + * document, it is possible to request an Element which does not exist, like the parent + * of the root element, or the left child of an integer, or the right sibling of the + * last element in an array. In these cases, the topology API will return an Element for + * which the 'ok' accessor will return 'false', which is roughly analagous to an 'end' + * valued STL iterator. It is illegal to call any method (other than 'ok') on a non-OK + * Element. + * + * - Streaming API: As a convenience for when you are building Documents from scratch, an + * API is provided that combines the effects of calling makeElement on the Document with + * calling pushBack on the current Element. The effect is to create the element and make + * it the new rightmost child of this Element. Use of this API is discouraged and it may + * be removed. + */ + + class Element { + public: + typedef uint32_t RepIdx; + + // Some special RepIdx values. These are really implementation details, but they are + // here so that we can inline Element::OK, which gets called very frequently, and they + // need to be public so some free functions in document.cpp can use them. You must not + // use these values explicitly. + + // Used to signal an invalid Element. + static const RepIdx kInvalidRepIdx = RepIdx(-1); + + // A rep that points to an unexamined entity + static const RepIdx kOpaqueRepIdx = RepIdx(-2); + + // This is the highest valid rep that does not overlap flag values. + static const RepIdx kMaxRepIdx = RepIdx(-3); + + // + // Topology mutation API. Element arguments must belong to the same Document. + // + + /** Add the provided Element to the left of this Element. The added Element must be + * 'ok', and this Element must have a parent. + */ + Status addSiblingLeft(Element e); + + /** Add the provided Element to the right of this Element. The added Element must be + * 'ok', and this Element must have a parent. + */ + Status addSiblingRight(Element e); + + /** 'Remove' this Element by detaching it from its parent and siblings. The Element + * continues to exist and may be manipulated, but cannot be re-obtained by navigating + * from the root. + */ + Status remove(); + + /** If this Element is empty, add 'e' as the first child. Otherwise, add 'e' as the new + * left child. + */ + Status pushFront(Element e); + + /** If this Element is empty, add 'e' as the first child. Otherwise, add 'e' as the new + * right child. + */ + Status pushBack(Element e); + + /** Remove the leftmost child Element if it exists, otherwise return an error. */ + Status popFront(); + + /** Remove the rightmost child Element if it exists, otherwise return an error. */ + Status popBack(); + + /** Rename this Element to the provided name. */ + Status rename(const StringData& newName); + + + // + // Navigation API. + // + + /** Returns either this Element's left child, or a non-ok Element if no left child + * exists. + */ + Element leftChild() const; + + /** Returns either this Element's right child, or a non-ok Element if no right child + * exists. Note that obtaining the right child may require realizing all immediate + * child nodes of a document that is being consumed lazily. + */ + Element rightChild() const; + + /** Returns true if this element has children. Always returns false if this Element is + * not an Object or Array. + */ + bool hasChildren() const; + + /** Returns either this Element's sibling 'distance' elements to the left, or a non-ok + * Element if no such left sibling exists. + */ + Element leftSibling(size_t distance = 1) const; + + /** Returns either this Element's sibling 'distance' Elements to the right, or a non-ok + * Element if no such right sibling exists. + */ + Element rightSibling(size_t distance = 1) const; + + /** Returns this Element's parent, or a non-ok Element if this Element has no parent + * (is a root). + */ + Element parent() const; + + /** Returns the nth child, if any, of this Element. If no such element exists, a non-ok + * Element is returned. This is not a constant time operation. This method is also + * available as operator[] taking a size_t for convenience. + */ + Element findNthChild(size_t n) const; + inline Element operator[](size_t n) const; + + /** Returns the first child, if any, of this Element named 'name'. If no such Element + * exists, a non-ok Element is returned. This is not a constant time operation. This + * method is also available as operator[] taking a StringData for convenience. + */ + Element findFirstChildNamed(const StringData& name) const; + inline Element operator[](const StringData& name) const; + + /** Returns the first element found named 'name', starting the search at the current + * Element, and walking right. If no such Element exists, a non-ok Element is + * returned. This is not a constant time operation. This implementation is used in the + * specialized implementation of findElement. + */ + Element findElementNamed(const StringData& name) const; + + // + // Counting API. + // + + /** Returns the number of valid siblings to the left of this Element. */ + size_t countSiblingsLeft() const; + + /** Returns the number of valid siblings to the right of this Element. */ + size_t countSiblingsRight() const; + + /** Return the number of children of this Element. */ + size_t countChildren() const; + + // + // Value access API. + // + // We only provide accessors for BSONElement and for simple types. For more complex + // types like regex you should obtain the BSONElement and use that API to extract the + // components. + // + // Note that the getValueX methods are *unchecked* in release builds: You are + // responsible for calling hasValue() to ensure that this element has a value + // representation, and for calling getType to ensure that the Element is of the proper + // type. + // + // As usual, methods here are in bsonspec type order, please keep them that way. + // + + /** Returns true if 'getValue' can return a valid BSONElement from which a value may be + * extracted. See the notes for 'getValue' to understand the conditions under which an + * Element can provide a BSONElement. + */ + bool hasValue() const; + + /** Returns true if this element is a numeric type (e.g. NumberLong). Currently, the + * only numeric BSON types are NumberLong, NumberInt, and NumberDouble. + */ + bool isNumeric() const; + + /** Returns true if this element is one of the integral numeric types (e.g. NumberLong + * or NumberInt). + */ + bool isIntegral() const; + + /** Get the value of this element if available. Note that not all elements have a + * representation as a BSONElement. For elements that do have a representation, this + * will return it. For elements that do not this method returns an eoo + * BSONElement. All 'value-ish' Elements will have a BSONElement + * representation. 'Tree-ish' Elements may or may not have a BSONElement + * representation. Mutations may cause elements to change whether or not they have a + * value and may invalidate previously returned values. + * + * Please note that a const BSONElement allows retrieval of a non-const + * BSONObj. However, the contents of the BSONElement returned here must be treated as + * const. + */ + const BSONElement getValue() const; + + /** Get the value from a double valued Element. */ + inline double getValueDouble() const; + + /** Get the value from a string valued Element. */ + inline StringData getValueString() const; + + /** Get the value from an object valued Element. Note that this may not always be + * possible! + */ + inline BSONObj getValueObject() const; + + /** Get the value from an object valued Element. Note that this may not always be + * possible! + */ + inline BSONArray getValueArray() const; + + /** Returns true if this Element is the undefined type. */ + inline bool isValueUndefined() const; + + /** Get the value from an OID valued Element. */ + inline OID getValueOID() const; + + /** Get the value from a bool valued Element. */ + inline bool getValueBool() const; + + /** Get the value from a date valued Element. */ + inline Date_t getValueDate() const; + + /** Returns true if this Element is the null type. */ + inline bool isValueNull() const; + + /** Get the value from a symbol valued Element. */ + inline StringData getValueSymbol() const; + + /** Get the value from an int valued Element. */ + inline int32_t getValueInt() const; + + /** Get the value from a timestamp valued Element. */ + inline OpTime getValueTimestamp() const; + + /** Get the value from a long valued Element. */ + inline int64_t getValueLong() const; + + /** Returns true if this Element is the min key type. */ + inline bool isValueMinKey() const; + + /** Returns true if this Element is the max key type. */ + inline bool isValueMaxKey() const; + + /** Returns the numeric value as a SafeNum */ + SafeNum getValueSafeNum() const; + + + // + // Comparision API. + // + + /** Compare this Element with Element 'other'. The two Elements may belong to different + * Documents. You should not call this on the root Element of the Document because the + * root Element does not have a field name. Use compareWithBSONObj to handle that + * case. + * + * Returns -1 if this < other according to BSONElement::woCompare + * Returns 0 if this == other either tautologically, or according to woCompare. + * Returns 1 if this > other according to BSONElement::woCompare + */ + int compareWithElement(const ConstElement& other, bool considerFieldName = true) const; + + /** Compare this Element with BSONElement 'other'. You should not call this on the root + * Element of the Document because the root Element does not have a field name. Use + * compareWithBSONObj to handle that case. + * + * Returns -1 if this < other according to BSONElement::woCompare + * Returns 0 if this == other either tautologically, or according to woCompare. + * Returns 1 if this > other according to BSONElement::woCompare + */ + int compareWithBSONElement(const BSONElement& other, bool considerFieldName = true) const; + + /** Compare this Element, which must be an Object or an Array, with 'other'. + * + * Returns -1 if this object < other according to BSONElement::woCompare + * Returns 0 if this object == other either tautologically, or according to woCompare. + * Returns 1 if this object > other according to BSONElement::woCompare + */ + int compareWithBSONObj(const BSONObj& other, bool considerFieldName = true) const; + + + // + // Serialization API. + // + + /** Write this Element to the provided object builder. */ + void writeTo(BSONObjBuilder* builder) const; + + /** Write this Element to the provided array builder. This Element must be of type + * mongo::Array. + */ + void writeArrayTo(BSONArrayBuilder* builder) const; + + + // + // Value mutation API. Please note that the types are ordered according to bsonspec.org + // ordering. Please keep them that way. + // + + /** Set the value of this Element to the given double. */ + Status setValueDouble(double value); + + /** Set the value of this Element to the given string. */ + Status setValueString(const StringData& value); + + /** Set the value of this Element to the given object. The data in 'value' is + * copied. + */ + Status setValueObject(const BSONObj& value); + + /** Set the value of this Element to the given object. The data in 'value' is + * copied. + */ + Status setValueArray(const BSONObj& value); + + /** Set the value of this Element to the given binary data. */ + Status setValueBinary(uint32_t len, mongo::BinDataType binType, const void* data); + + /** Set the value of this Element to Undefined. */ + Status setValueUndefined(); + + /** Set the value of this Element to the given OID. */ + Status setValueOID(OID value); + + /** Set the value of this Element to the given boolean. */ + Status setValueBool(bool value); + + /** Set the value of this Element to the given date. */ + Status setValueDate(Date_t value); + + /** Set the value of this Element to Null. */ + Status setValueNull(); + + /** Set the value of this Element to the given regex parameters. */ + Status setValueRegex(const StringData& re, const StringData& flags); + + /** Set the value of this Element to the given db ref parameters. */ + Status setValueDBRef(const StringData& ns, OID oid); + + /** Set the value of this Element to the given code data. */ + Status setValueCode(const StringData& value); + + /** Set the value of this Element to the given symbol. */ + Status setValueSymbol(const StringData& value); + + /** Set the value of this Element to the given code and scope data. */ + Status setValueCodeWithScope(const StringData& code, const BSONObj& scope); + + /** Set the value of this Element to the given integer. */ + Status setValueInt(int32_t value); + + /** Set the value of this Element to the given timestamp. */ + Status setValueTimestamp(OpTime value); + + /** Set the value of this Element to the given long integer */ + Status setValueLong(int64_t value); + + /** Set the value of this Element to MinKey. */ + Status setValueMinKey(); + + /** Set the value of this Element to MaxKey. */ + Status setValueMaxKey(); + + + // + // Value mutation API from variant types. + // + + /** Set the value of this element to equal the value of the provided BSONElement + * 'value'. The name of this Element is not modified. + * + * The contents of value are copied. + */ + Status setValueBSONElement(const BSONElement& value); + + /** Set the value of this Element to a numeric type appropriate to hold the given + * SafeNum value. + */ + Status setValueSafeNum(const SafeNum value); + + + // + // Accessors + // + + /** Returns true if this Element represents a valid part of the Document. */ + inline bool ok() const; + + /** Returns the Document to which this Element belongs. */ + inline Document& getDocument(); + + /** Returns the Document to which this Element belongs. */ + inline const Document& getDocument() const; + + /** Returns the BSONType of this Element. */ + BSONType getType() const; + + /** Returns true if this Element is of the specified type */ + inline bool isType(BSONType type) const; + + /** Returns the field name of this Element. Note that the value returned here is not + * stable across mutations, since the storage for fieldNames may be reallocated. If + * you need a stable version of the fieldName, you must call toString on the returned + * StringData. + */ + StringData getFieldName() const; + + /** Returns the opaque ID for this element. This is unlikely to be useful to a caller + * and is mostly for testing. + */ + inline RepIdx getIdx() const; + + + // + // Stream API - BSONObjBuilder like API, but methods return a Status. These are + // strictly a convenience API. You don't need to use them if you would rather be more + // explicit. + // + + /** Append the provided double value as a new field with the provided name. */ + Status appendDouble(const StringData& fieldName, double value); + + /** Append the provided string value as a new field with the provided name. */ + Status appendString(const StringData& fieldName, const StringData& value); + + /** Append the provided object as a new field with the provided name. The data in + * 'value' is copied. + */ + Status appendObject(const StringData& fieldName, const BSONObj& value); + + /** Append the provided array object as a new field with the provided name. The data in + * value is copied. + */ + Status appendArray(const StringData& fieldName, const BSONObj& value); + + /** Append the provided binary data as a new field with the provided name. */ + Status appendBinary(const StringData& fieldName, + uint32_t len, mongo::BinDataType binType, const void* data); + + /** Append an undefined value as a new field with the provided name. */ + Status appendUndefined(const StringData& fieldName); + + /** Append the provided OID as a new field with the provided name. */ + Status appendOID(const StringData& fieldName, mongo::OID value); + + /** Append the provided bool as a new field with the provided name. */ + Status appendBool(const StringData& fieldName, bool value); + + /** Append the provided date as a new field with the provided name. */ + Status appendDate(const StringData& fieldName, Date_t value); + + /** Append a null as a new field with the provided name. */ + Status appendNull(const StringData& fieldName); + + /** Append the provided regex data as a new field with the provided name. */ + Status appendRegex(const StringData& fieldName, + const StringData& re, const StringData& flags); + + /** Append the provided DBRef data as a new field with the provided name. */ + Status appendDBRef(const StringData& fieldName, + const StringData& ns, mongo::OID oid); + + /** Append the provided code data as a new field with the iven name. */ + Status appendCode(const StringData& fieldName, const StringData& value); + + /** Append the provided symbol data as a new field with the provided name. */ + Status appendSymbol(const StringData& fieldName, const StringData& value); + + /** Append the provided code and scope data as a new field with the provided name. */ + Status appendCodeWithScope(const StringData& fieldName, + const StringData& code, const BSONObj& scope); + + /** Append the provided integer as a new field with the provided name. */ + Status appendInt(const StringData& fieldName, int32_t value); + + /** Append the provided timestamp as a new field with the provided name. */ + Status appendTimestamp(const StringData& fieldName, OpTime value); + + /** Append the provided long integer as a new field with the provided name. */ + Status appendLong(const StringData& fieldName, int64_t value); + + /** Append a max key as a new field with the provided name. */ + Status appendMinKey(const StringData& fieldName); + + /** Append a min key as a new field with the provided name. */ + Status appendMaxKey(const StringData& fieldName); + + /** Append the given BSONElement. The data in 'value' is copied. */ + Status appendElement(const BSONElement& value); + + /** Append the provided number as field of the appropriate numeric type with the + * provided name. + */ + Status appendSafeNum(const StringData& fieldName, SafeNum value); + + /** Convert this element to its JSON representation if ok(), + * otherwise return !ok() message */ + std::string toString() const; + + private: + friend class Document; + friend class ConstElement; + + friend bool operator==(const Element&, const Element&); + + inline Element(Document* doc, RepIdx repIdx); + + Status addChild(Element e, bool front); + + StringData getValueStringOrSymbol() const; + + Status setValue(Element::RepIdx newValueIdx); + + Document* _doc; + RepIdx _repIdx; + }; + + /** Element comparison support. Comparison is like STL iterator comparision: equal Elements + * refer to the same underlying data. The equality does *not* mean that the underlying + * values are equivalent. Use the Element::compareWith methods to compare the represented + * data. + */ + + /** Returns true if l and r refer to the same data, false otherwise. */ + inline bool operator==(const Element& l, const Element& r); + + /** Returns false if l and r refer to the same data, true otherwise. */ + inline bool operator!=(const Element& l, const Element& r); + +} // namespace mutablebson +} // namespace mongo + +#include "mongo/bson/mutable/element-inl.h" diff --git a/src/mongo/bson/mutable/mutable_bson.cpp b/src/mongo/bson/mutable/mutable_bson.cpp deleted file mode 100644 index 0e5d91e95fb..00000000000 --- a/src/mongo/bson/mutable/mutable_bson.cpp +++ /dev/null @@ -1,1035 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/bson/mutable/mutable_bson.h" - -#include -#include -#include -#include - -#include "mongo/bson/mutable/mutable_bson_heap.h" -#include "mongo/bson/mutable/mutable_bson_internal.h" - -namespace mongo { -namespace mutablebson { - -namespace { - const int32_t SHORTBIT = (1<<16); -} // namespace - - // - // Element navigation - // - - Element Element::leftChild() const { - return Element(_doc, _doc->_elements->_vec[_rep]._child._left); - } - - Element Element::rightChild() const { - return Element(_doc, _doc->_elements->_vec[_rep]._child._right); - } - - Element Element::leftSibling() const { - return Element(_doc, _doc->_elements->_vec[_rep]._sibling._left); - } - - Element Element::rightSibling() const { - return Element(_doc, _doc->_elements->_vec[_rep]._sibling._right); - } - - Element Element::parent() const { - return Element(_doc, _doc->_elements->_vec[_rep]._parent); - } - - SiblingIterator Element::children() { - return SiblingIterator(leftChild()); - } - - StringData Element::getFieldName() const { - return _doc->getHeap()->getStringBuffer(_doc->_elements->_vec[_rep]._nameref); - } - - // - // Element update API - // - - Status Element::addChild(Element e) { - - if (isNull()) { - return Status(ErrorCodes::IllegalOperation, "trying to add child to null node"); - } - if (isSimpleType()) { - return Status(ErrorCodes::IllegalOperation, "trying to add child to atomic node"); - } - - ElementRep& thisRep = _doc->_elements->_vec[_rep]; - ElementRep& newRep = e._doc->_elements->_vec[e._rep]; - - /* check that new element roots a clean subtree, no dangling references */ - Status s = checkSubtreeIsClean(e); - if (s.code() != ErrorCodes::OK) return s; - - newRep._parent = _rep; - - /* link to end of the existing sibling list */ - newRep._sibling._left = thisRep._child._right; - if (thisRep._child._right != EMPTY_REP) { - ElementRep& rightRep = _doc->_elements->_vec[thisRep._child._right]; - rightRep._sibling._right = e._rep; - } - else { - thisRep._child._left = e._rep; - } - - /* link as new right child */ - thisRep._child._right = e._rep; - return Status::OK(); - } - - Status Element::addSiblingAfter(Element e) { - - if (isNull()) { - return Status(ErrorCodes::IllegalOperation, "trying to add sibling to null node"); - } - if (e.isNull()) { - return Status(ErrorCodes::IllegalOperation, "trying to add null node as sibling"); - } - - ElementRep& thisRep = _doc->_elements->_vec[_rep]; - ElementRep& newRep = e._doc->_elements->_vec[e._rep]; - - /* check that new element roots a clean subtree, no dangling references */ - Status s = checkSubtreeIsClean(e); - if (s.code() != ErrorCodes::OK) return s; - - /* link in new node */ - newRep._parent = thisRep._parent; - newRep._sibling._left = _rep; - newRep._sibling._right = thisRep._sibling._right; - thisRep._sibling._right = e.getRep(); - if (newRep._sibling._right != EMPTY_REP) { - ElementRep& rightRep = _doc->_elements->_vec[newRep._sibling._right]; - rightRep._sibling._left = e.getRep(); - } - - /* fix parent right child */ - if (thisRep._parent != EMPTY_REP) { - ElementRep& parentRep = _doc->_elements->_vec[thisRep._parent]; - if (parentRep._child._right ==_rep) { - parentRep._child._right=e.getRep(); - } - } - - return Status::OK(); - } - - Status Element::addSiblingBefore(Element e) { - - if (isNull()) { - return Status(ErrorCodes::IllegalOperation, "trying to add sibling to null node"); - } - if (e.isNull()) { - return Status(ErrorCodes::IllegalOperation, "trying to add null node as sibling"); - } - - ElementRep& thisRep = _doc->_elements->_vec[_rep]; - ElementRep& newRep = e._doc->_elements->_vec[e._rep]; - - /* check that new element roots a clean subtree, no dangling references */ - Status s = checkSubtreeIsClean(e); - if (s.code() != ErrorCodes::OK) return s; - - /* link in new node */ - newRep._parent = thisRep._parent; - newRep._sibling._right = _rep; - newRep._sibling._left = thisRep._sibling._left; - thisRep._sibling._left = e.getRep(); - if (newRep._sibling._left != EMPTY_REP) { - ElementRep& leftRep = _doc->_elements->_vec[newRep._sibling._left]; - leftRep._sibling._right = e.getRep(); - } - - /* fix parent left child */ - if (thisRep._parent != EMPTY_REP) { - ElementRep& parentRep = _doc->_elements->_vec[thisRep._parent]; - if (parentRep._child._left == _rep) { - parentRep._child._left=e.getRep(); - } - } - - return Status::OK(); - } - - Status Element::remove() { - - ElementRep& thisRep = _doc->_elements->_vec[_rep]; - if (thisRep._parent == EMPTY_REP) { - return Status(ErrorCodes::IllegalOperation, "trying to remove document root node"); - } - - /* skip over the element being deleted */ - if (thisRep._sibling._right != EMPTY_REP) { - ElementRep& rightRep = _doc->_elements->_vec[thisRep._sibling._right]; - rightRep._sibling._left = thisRep._sibling._left; - } - if (thisRep._sibling._left != EMPTY_REP) { - ElementRep& leftRep = _doc->_elements->_vec[thisRep._sibling._left]; - leftRep._sibling._right = thisRep._sibling._right; - } - - /* fix parent right child, if needed */ - ElementRep& parentRep = _doc->_elements->_vec[thisRep._parent]; - if (parentRep._child._right == getRep()) { - parentRep._child._right = thisRep._sibling._left; - } - - /* fix parent left child, if needed */ - if (parentRep._child._left == getRep()) { - parentRep._child._left = thisRep._sibling._right; - } - - /* clear links */ - thisRep._parent = EMPTY_REP; - thisRep._sibling._left = EMPTY_REP; - thisRep._sibling._right = EMPTY_REP; - - return Status::OK(); - } - - Status Element::rename(const StringData& newName) { - ElementRep& thisRep = _doc->_elements->_vec[_rep]; - thisRep._nameref = _doc->getHeap()->putString(newName); - return Status::OK(); - } - - Status Element::move(Element newParent) { - Status s = remove(); - if (s.code() != ErrorCodes::OK) return s; - s = newParent.addChild(*this); - if (s.code() != ErrorCodes::OK) return s; - return Status::OK(); - } - - - // - // Element array API - // - - Status Element::arraySize(uint32_t* i) { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "size on non-array"); - } - ElementRep& er = _doc->_elements->_vec[_rep]; - if (er._child._left == EMPTY_REP) { - *i = 0; - } - else { - Element e(_doc, er._child._left); - uint32_t n = 0; - for (SiblingIterator sibIt(e); !sibIt.done(); ++sibIt) ++n; - *i = n; - } - return Status::OK(); - } - - Status Element::peekBack(Element* ep) { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "peekBack on non-array"); - } - ElementRep& er = _doc->_elements->_vec[_rep]; - if (er._child._right == EMPTY_REP) { - return Status(ErrorCodes::EmptyArrayOperation, "peekBack on empty array"); - } - *ep = Element(_doc, er._child._right); - return Status::OK(); - } - - Status Element::pushBack(Element e) { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "pushBack on non-array"); - } - addChild(e); - return Status::OK(); - } - - Status Element::popBack() { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "popBack on non-array"); - } - ElementRep& er = _doc->_elements->_vec[_rep]; - if (er._child._right == EMPTY_REP) { - return Status(ErrorCodes::EmptyArrayOperation, "popBack on empty array"); - } - rightChild().remove(); - return Status::OK(); - } - - Status Element::peekFront(Element* ep) { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "peekFront on non-array"); - } - ElementRep& er = _doc->_elements->_vec[_rep]; - if (er._child._left == EMPTY_REP) { - return Status(ErrorCodes::EmptyArrayOperation, "peekFront on empty array"); - } - *ep = Element(_doc, er._child._left); - return Status::OK(); - } - - Status Element::pushFront(Element e) { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "pushFront on non-array"); - } - ElementRep& er = _doc->_elements->_vec[_rep]; - if (er._child._left == EMPTY_REP) { - addChild(e); - } - else { - leftChild().addSiblingBefore(e); - } - return Status::OK(); - } - - Status Element::popFront() { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "popFront on non-array"); - } - ElementRep& er = _doc->_elements->_vec[_rep]; - if (er._child._left == EMPTY_REP) { - return Status(ErrorCodes::EmptyArrayOperation, "popFront on empty array"); - } - leftChild().remove(); - return Status::OK(); - } - - Status Element::get(uint32_t index, Element* ep) { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "get(index, &e) on non-array"); - } - ElementRep& thisRep = _doc->_elements->_vec[_rep]; - if (thisRep._child._left == EMPTY_REP) { - return Status(ErrorCodes::IllegalOperation, "get(index, &e) on empty array"); - } - Element eLeft(_doc, thisRep._child._left); - uint32_t n = 0; - SiblingIterator sibIt(eLeft); - for (; !sibIt.done() && n < index; ++sibIt) ++n; - if (sibIt.done()) { - *ep = Element(_doc, EMPTY_REP); - return Status(ErrorCodes::IllegalOperation, "get(index, &e) out of bounds"); - } - *ep = (*sibIt); - return Status::OK(); - } - - Status Element::set(uint32_t index, Element e) { - if (type() != mongo::Array) { - return Status(ErrorCodes::IllegalOperation, "set(index, e) on non-array"); - } - if (!e.isSimpleType()) { - return Status(ErrorCodes::IllegalOperation, - "set(index, e) source is non-simple type node"); - } - ElementRep& thisRep = _doc->_elements->_vec[_rep]; - if (thisRep._child._left == EMPTY_REP) { - return Status(ErrorCodes::IllegalOperation, "set(index, e) on empty array"); - } - Element eLeft(_doc, thisRep._child._left); - if (!eLeft.isSimpleType()) { - return Status(ErrorCodes::IllegalOperation, - "set(index, e) target is non-simple type node"); - } - uint32_t n = 0; - SiblingIterator sibIt(eLeft); - for (; !sibIt.done() && n < index; ++sibIt) ++n; - if (sibIt.done()) { - return Status(ErrorCodes::IllegalOperation, "get(index, e) out of bounds"); - } -ElementRep& dstRep = _doc->_elements->_vec[(*sibIt)._rep]; - ElementRep& srcRep = _doc->_elements->_vec[e._rep]; - dstRep._value = srcRep._value; - return Status::OK(); - } - - // - // Element get/set value interface - // - - bool Element::getBoolValue() const { - return _doc->_elements->_vec[_rep]._value.boolVal; - } - int32_t Element::getIntValue() const { - return _doc->_elements->_vec[_rep]._value.intVal; - } - int64_t Element::getLongValue() const { - return _doc->_elements->_vec[_rep]._value.longVal; - } - OpTime Element::getTSValue() const { - return OpTime(_doc->_elements->_vec[_rep]._value.tsVal); - } - int64_t Element::getDateValue() const { - return _doc->_elements->_vec[_rep]._value.dateVal; - } - double Element::getDoubleValue() const { - return _doc->_elements->_vec[_rep]._value.doubleVal; - } - OID Element::getOIDValue() const { - return OID( - reinterpret_cast( - _doc->_elements->_vec[_rep]._value.shortStr)); - } - const char* Element::getRegexValue() const { - return getStringValue(); - } - - const char* Element::getStringValue() const { - if (isInlineType()) { - return &_doc->_elements->_vec[_rep]._value.shortStr[0]; - } - else { - return _doc->getHeap()->getStringBuffer(_doc->_elements->_vec[_rep]._value.valueRef); - } - } - - SafeNum Element::getSafeNumValue() const { - switch (_doc->_elements->_vec[_rep]._type) { - case mongo::NumberInt: - return SafeNum(_doc->_elements->_vec[_rep]._value.intVal); - case mongo::NumberLong: - return SafeNum(static_cast(_doc->_elements->_vec[_rep]._value.longVal)); - case mongo::NumberDouble: - return SafeNum(_doc->_elements->_vec[_rep]._value.doubleVal); - default: - return SafeNum(); - } - } - - - void Element::setBoolValue(bool boolVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::Bool; - e._value.boolVal = boolVal; - } - - void Element::setIntValue(int32_t intVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::NumberInt; - e._value.intVal = intVal; - } - - void Element::setLongValue(int64_t longVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::NumberLong; - e._value.longVal = longVal; - } - - void Element::setTSValue(OpTime tsVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::Timestamp; - e._value.tsVal = tsVal.asDate(); - } - - void Element::setDateValue(int64_t dateVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::Date; - e._value.dateVal = dateVal; - } - - void Element::setDoubleValue(double doubleVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::NumberDouble; - e._value.doubleVal = doubleVal; - } - - void Element::setOIDValue(const OID& oid) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::jstOID; - BOOST_STATIC_ASSERT(mongo::OID::kOIDSize <= sizeof(e._value.shortStr)); - std::memcpy(e._value.shortStr, reinterpret_cast(oid.getData()), OID::kOIDSize); - } - - void Element::setRegexValue(const StringData& re) { - // type is set to "string" in setStringValue - setStringValue(re); - _doc->_elements->_vec[_rep]._type = mongo::RegEx; - } - - void Element::setStringValue(const StringData& stringVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::String; - if (stringVal.size() < SHORT_LIMIT) { - stringVal.copyTo( e._value.shortStr, true ); - e._type |= SHORTBIT; - } - else { - e._value.valueRef = _doc->_heap->putString(stringVal); - } - } - - void Element::setSafeNumValue(const SafeNum& safeNumVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = safeNumVal.type(); - switch (e._type) { - case mongo::NumberInt: - e._value.intVal = safeNumVal._value.int32Val; - break; - case mongo::NumberLong: - e._value.longVal = safeNumVal._value.int64Val; - break; - case mongo::NumberDouble: - e._value.doubleVal = safeNumVal._value.doubleVal; - break; - default: - // Invalid type - e._type was set to EOO above, so we're done - break; - } - } - - void Element::setMinKey() { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::MinKey; - } - - void Element::setMaxKey() { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::MinKey; - } - - void Element::setUndefined() { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::Undefined; - } - - void Element::setNull() { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::jstNULL; - } - - void Element::setSymbol(const StringData& symbolVal) { - ElementRep& e = _doc->_elements->_vec[_rep]; - e._type = mongo::Symbol; - if (symbolVal.size() < SHORT_LIMIT) { - symbolVal.copyTo( e._value.shortStr, true ); - e._type |= SHORTBIT; - } - else { - e._value.valueRef = _doc->_heap->putString(symbolVal); - } - } - - void Element::setValueFromBSONElement(const BSONElement& val) { - switch(val.type()) { - case MinKey: - setMinKey(); - break; - case EOO: - verify(false); - break; - case NumberDouble: - setDoubleValue(val._numberDouble()); - break; - case String: - setStringValue(StringData(val.valuestr(), val.valuestrsize())); - break; - case Object: - verify(false); - break; - case Array: - verify(false); - break; - case BinData: - verify(false); - break; - case Undefined: - setUndefined(); - break; - case jstOID: - setOIDValue(val.__oid()); - break; - case Bool: - setBoolValue(val.boolean()); - break; - case Date: - setDateValue(val.date()); - break; - case jstNULL: - setNull(); - break; - case RegEx: - verify(false); - break; - case DBRef: - verify(false); - break; - case Code: - verify(false); - break; - case Symbol: - setSymbol(StringData(val.valuestr(), val.valuestrsize())); - break; - case CodeWScope: - verify(false); - break; - case NumberInt: - setIntValue(val._numberInt()); - break; - case Timestamp: - setTSValue(val._opTime()); - break; - case NumberLong: - setLongValue(val._numberLong()); - break; - case MaxKey: - setMaxKey(); - break; - default: - verify(false); - break; - } - } - - // - // decoders - // - - Status Element::prefix(std::string* result, char delim) const { - std::string s = getStringValue(); - size_t n = s.find(delim); - if (n == std::string::npos) { - return Status(ErrorCodes::IllegalOperation, "expecting regex format /PAT/flags"); - } - size_t m = s.find_last_of(delim); - if (m == std::string::npos || m == n) { - return Status(ErrorCodes::IllegalOperation, "expecting regex format /pat/FLAGS"); - } - result->append(s.substr(n+1,m-n-1)); - return Status::OK(); - } - - Status Element::suffix(std::string* result, char delim) const { - std::string s = getStringValue(); - size_t n = s.find(delim); - if (n == std::string::npos) { - return Status(ErrorCodes::IllegalOperation, "expecting regex format ./.pat/flags"); - } - size_t m = s.find_last_of(delim); - if (m == std::string::npos || m == n) { - return Status(ErrorCodes::IllegalOperation, "expecting regex format /pat./.flags"); - } - result->append(s.substr(m+1)); - return Status::OK(); - } - - Status Element::regex(std::string* result) const { - return prefix(result, '/'); - } - - Status Element::regexFlags(std::string* result) const { - return suffix(result, '/'); - } - - Status Element::dbrefNS(std::string* result) const { - return prefix(result, ':'); - } - - Status Element::dbrefOID(std::string* result) const { - return suffix(result, ':'); - } - - Status Element::codeWScopeCode(std::string* result) const { - return prefix(result, '|'); - } - - Status Element::codeWScopeScope(std::string* result) const { - return suffix(result, '|'); - } - - // - // Element predicates - // - - bool Element::isBoolean() const { - return (_doc->_elements->_vec[_rep]._type == mongo::Bool); - } - - bool Element::isNonAtomic() const { - return !isSimpleType(); - } - - bool Element::isArray() const { - return (type() == mongo::Array); - } - - bool Element::isSimpleType() const { - switch(type()) { - case mongo::NumberLong: - case mongo::NumberDouble: - case mongo::NumberInt: - case mongo::String: - case mongo::Bool: - case mongo::Date: - case jstOID: - return true; - default: - return false; - } - } - - bool Element::isNumber() const { - BSONType t = type(); - if (t == mongo::NumberDouble || - t == mongo::NumberInt || - t == mongo::NumberLong) return true; - return false; - } - - bool Element::isNull() const { - return (type() == mongo::jstNULL); - } - - BSONType Element::type() const { - ElementRep& e = _doc->_elements->_vec[_rep]; - int32_t t = e._type; - if ((t >= 0) && (t & SHORTBIT)) t ^= SHORTBIT; - return static_cast(t); - } - - bool Element::isInlineType() const { - ElementRep& e = _doc->_elements->_vec[_rep]; - int32_t t = e._type; - if ((t >= 0) && (t & SHORTBIT)) return true; - return false; - } - - // TODO: These should be probably be made inline. - Status Element::appendBool(const StringData& fieldName, bool boolVal) { - return addChild(_doc->makeBoolElement(fieldName, boolVal)); - } - - Status Element::appendInt(const StringData& fieldName, int32_t intVal) { - return addChild(_doc->makeIntElement(fieldName, intVal)); - } - - Status Element::appendLong(const StringData& fieldName, int64_t longVal) { - return addChild(_doc->makeLongElement(fieldName, longVal)); - } - - Status Element::appendTS(const StringData& fieldName, OpTime tsVal) { - return addChild(_doc->makeTSElement(fieldName, tsVal)); - } - - Status Element::appendDate(const StringData& fieldName, int64_t millis) { - return addChild(_doc->makeDateElement(fieldName, millis)); - } - - Status Element::appendDouble(const StringData& fieldName, double doubleVal) { - return addChild(_doc->makeDoubleElement(fieldName, doubleVal)); - } - - Status Element::appendOID(const StringData& fieldName, const mongo::OID& oid) { - return addChild(_doc->makeOIDElement(fieldName, oid)); - } - - Status Element::appendString(const StringData& fieldName, const StringData& stringVal) { - return addChild(_doc->makeStringElement(fieldName, stringVal)); - } - - Status Element::appendCode(const StringData& fieldName, const StringData& code) { - return addChild(_doc->makeCodeElement(fieldName, code)); - } - - Status Element::appendSymbol(const StringData& fieldName, const StringData& symbol) { - return addChild(_doc->makeSymbolElement(fieldName, symbol)); - } - - Status Element::appendNull(const StringData& fieldName) { - return addChild(_doc->makeNullElement(fieldName)); - } - - Status Element::appendMinKey(const StringData& fieldName) { - return addChild(_doc->makeMinKeyElement(fieldName)); - } - - Status Element::appendMaxKey(const StringData& fieldName) { - return addChild(_doc->makeMaxKeyElement(fieldName)); - } - - Status Element::appendRegex( const StringData& fieldName, - const StringData& re, const StringData& flags) { - return addChild(_doc->makeRegexElement(fieldName, re, flags)); - } - - Status Element::appendCodeWScope( const StringData& fieldName, - const StringData& code, const StringData& scope) { - return addChild(_doc->makeCodeWScopeElement(fieldName, code, scope)); - } - - Status Element::appendDBRef( const StringData& fieldName, - const StringData& ns, const mongo::OID& oid) { - return addChild(_doc->makeDBRefElement(fieldName, ns, oid)); - } - - Status Element::appendBinary( const StringData& fieldName, - uint32_t len, mongo::BinDataType binType, const void* data) { - return addChild(_doc->makeBinaryElement(fieldName, len, binType, data)); - } - - Status Element::appendSafeNum(const StringData& fieldName, const SafeNum num) { - return addChild(_doc->makeSafeNumElement(fieldName, num)); - } - - // - // Element helper methods - // - - /** check that new element roots a clean subtree, no dangling references */ - inline Status Element::checkSubtreeIsClean(Element e) { - ElementRep& rep = e._doc->_elements->_vec[e._rep]; - if (rep._sibling._left != EMPTY_REP) { - return Status(ErrorCodes::IllegalOperation, "addChild: dangling left sibling"); - } - if (rep._sibling._right != EMPTY_REP) { - return Status(ErrorCodes::IllegalOperation, "addChild: dangling right sibling"); - } - if (rep._parent != EMPTY_REP) { - return Status(ErrorCodes::IllegalOperation, "addChild: dangling parent"); - } - return Status::OK(); - } - - - // - // Document Implementation - // - - Document::Document(Heap* heap) : - _heap(heap), - _elements(new ElementVector), - _root(makeObjElement(StringData("", StringData::LiteralTag()))) { - } - - Document::~Document() {} - - // Document factory methods - - Element Document::makeObjElement(const StringData& fieldName) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - _elements->push_back(ElementRep(mongo::Object, nameref)); - return Element(this, rep); - } - - Element Document::makeArrayElement(const StringData& fieldName) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - _elements->push_back(ElementRep(mongo::Array, nameref)); - return Element(this, rep); - } - - Element Document::makeNullElement(const StringData& fieldName) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - _elements->push_back(ElementRep(mongo::jstNULL, nameref)); - return Element(this, rep); - } - - Element Document::makeMinKeyElement(const StringData& fieldName) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - _elements->push_back(ElementRep(mongo::MinKey, nameref)); - return Element(this, rep); - } - - Element Document::makeMaxKeyElement(const StringData& fieldName) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - _elements->push_back(ElementRep(mongo::MaxKey, nameref)); - return Element(this, rep); - } - - Element Document::makeBoolElement(const StringData& fieldName, bool b) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.boolVal = b; - _elements->push_back(ElementRep(mongo::Bool, nameref, val)); - return Element(this, rep); - } - - Element Document::makeIntElement(const StringData& fieldName, int32_t i) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.intVal = i; - _elements->push_back(ElementRep(mongo::NumberInt, nameref, val)); - return Element(this, rep); - } - - Element Document::makeLongElement(const StringData& fieldName, int64_t j) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.longVal = j; - _elements->push_back(ElementRep(mongo::NumberLong, nameref, val)); - return Element(this, rep); - } - - Element Document::makeTSElement(const StringData& fieldName, OpTime ts) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.tsVal = ts.asDate(); - _elements->push_back(ElementRep(mongo::Timestamp, nameref, val)); - return Element(this, rep); - } - - Element Document::makeDateElement(const StringData& fieldName, int64_t date) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.dateVal = date; - _elements->push_back(ElementRep(mongo::Date, nameref, val)); - return Element(this, rep); - } - - Element Document::makeDoubleElement(const StringData& fieldName, double d) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.doubleVal = d; - _elements->push_back(ElementRep(mongo::NumberDouble, nameref, val)); - return Element(this, rep); - } - - Element Document::makeOIDElement(const StringData& fieldName, const mongo::OID& oid) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - memcpy(val.shortStr, (char*)oid.getData(), 12); - _elements->push_back(ElementRep(mongo::jstOID, nameref, val)); - return Element(this, rep); - } - - Element Document::makeStringElement(const StringData& fieldName, const StringData& stringVal) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - if (stringVal.size() < SHORT_LIMIT) { - stringVal.copyTo( val.shortStr, true ); - _elements->push_back(ElementRep(mongo::String|SHORTBIT, nameref, val)); - } - else { - val.valueRef = _heap->putString(stringVal); - _elements->push_back(ElementRep(mongo::String, nameref, val)); - } - return Element(this, rep); - } - - Element Document::makeRegexElement( - const StringData& fieldName, const StringData& re, const StringData& flags) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.valueRef = _heap->putString(re); - _elements->push_back(ElementRep(mongo::String, nameref, val)); - return Element(this, rep); - } - - Element Document::makeCodeElement( - const StringData& fieldName, const StringData& code) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.valueRef = _heap->putString(code); - _elements->push_back(ElementRep(mongo::Code, nameref, val)); - return Element(this, rep); - } - - Element Document::makeSymbolElement( const StringData& fieldName, const StringData& symbol) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - val.valueRef = _heap->putString(symbol); - _elements->push_back(ElementRep(mongo::Symbol, nameref, val)); - return Element(this, rep); - } - - /** Question: Are these three cases sub-objects or primitive types? - Existing code appears to form strange concatenated strings with embedded nulls. - Stubbed for now, pending better understanding. */ - - Element Document::makeCodeWScopeElement( const StringData& fieldName, - const StringData& theCode, const StringData& theScope) { - return Element(this,EMPTY_REP); - } - - Element Document::makeDBRefElement( const StringData& fieldName, - const StringData& ns, const mongo::OID& oid) { - return Element(this,EMPTY_REP); - } - - Element Document::makeBinaryElement( const StringData& fieldName, - uint32_t len, mongo::BinDataType binType, const void* data) { - return Element(this,EMPTY_REP); - } - - Element Document::makeSafeNumElement(const StringData& fieldName, const SafeNum& safeNum) { - uint32_t rep = _elements->size(); - uint32_t nameref = _heap->putString(fieldName); - ValueType val; - - switch (safeNum.type()) { - case mongo::NumberInt: - val.intVal = safeNum._value.int32Val; - break; - case mongo::NumberLong: - val.longVal = safeNum._value.int64Val; - break; - case mongo::NumberDouble: - val.doubleVal = safeNum._value.doubleVal; - break; - default: - // Invalid SafeNum - type is set to EOO below - break; - } - - _elements->push_back(ElementRep(safeNum.type(), nameref, val)); - return Element(this, rep); - } - - // - // Iterator base class - // - - Iterator::Iterator() : _doc(NULL), _theRep(EMPTY_REP) {} - - // - // SiblingIterator - // - - bool SiblingIterator::done() const { return (_theRep == EMPTY_REP); } - - bool SiblingIterator::advance() { - if (_theRep == EMPTY_REP) return false; - ElementRep& e = _doc->_elements->_vec[_theRep]; - _theRep = e._sibling._right; - return (_theRep == EMPTY_REP); - } - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson.h b/src/mongo/bson/mutable/mutable_bson.h deleted file mode 100644 index c7f0114d4a0..00000000000 --- a/src/mongo/bson/mutable/mutable_bson.h +++ /dev/null @@ -1,343 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/status.h" -#include "mongo/db/jsobj.h" -#include "mongo/platform/cstdint.h" -#include "mongo/util/safe_num.h" - -namespace mongo { -namespace mutablebson { - - class Document; - class ElementVector; - class Heap; - class SiblingIterator; - struct ElementRep; - - /* - * The Element class represents one node in the document tree. A document is - * identified with its root node. - * - * Elements are created with the help of the Document class. Once created, an - * element can be introduced in any position in a document tree with methods - * provided here. - * - * Example Usage: - * - * // creation - * mutablebson::BasicHeap myHeap; - * mutablebson::Document doc(&myHeap); - * // Document is: {} - * - * mutablebson::Element e0 = doc.makeObjElement("e0"); - * doc.root().addChild(e0); - * // Document is: { e0 : {} } - * - * mutablebson::Element e1 = doc.makeObjElement("e1"); - * e0.addChild(e1); - * // Document is: { e0 : { e1 : {} } } - * - * TODO: Add iteration and search examples - * - */ - class Element { - public: - Element(Document* doc, uint32_t rep) - : _doc(doc) - , _rep(rep) {} - - ~Element() {} - - // - // navigation API - // - - Element leftChild() const; - Element rightChild() const; - Element leftSibling() const; - Element rightSibling() const; - Element parent() const; - - /** Iterate children of this node */ - SiblingIterator children(); - - // - // update API - // - - Status addChild(Element e); - Status addSiblingBefore(Element e); - Status addSiblingAfter(Element e); - Status remove(); - Status rename(const StringData& newName); - Status move(Element newParent); - - // - // array API - // - - Status arraySize(uint32_t* size); - Status peekBack(Element* ep); - Status pushBack(Element e); - Status popBack(); - Status peekFront(Element* ep); - Status pushFront(Element e); - Status popFront(); - Status get(uint32_t index, Element* ep); - Status set(uint32_t index, Element e); - - // - // accessors - // - - /** Returns true if type is inlined */ - bool isInlineType() const; - - /** Returns the BSONTypeof this element */ - BSONType type() const; - - bool getBoolValue() const; - int32_t getIntValue() const; - int64_t getLongValue() const; - OpTime getTSValue() const; - int64_t getDateValue() const; - double getDoubleValue() const; - OID getOIDValue() const; - - /** - The methods returning const char* all have the property that the returned value is - only valid until the next non-const method is called on the underlying heap, because - those operations might invalidate the iterators from which these values are implicitly - derived. This happens in practice if the heap's underlying vector grows. (andy) - */ - const char* getStringValue() const; - const char* getRegexValue() const; - - SafeNum getSafeNumValue() const; - - void setBoolValue(bool boolVal); - void setIntValue(int32_t intVal); - void setLongValue(int64_t longVal); - void setTSValue(OpTime tsVal); - void setDateValue(int64_t millis); - void setDoubleValue(double doubleVal); - void setOIDValue(const OID& oid); - void setStringValue(const StringData& stringVal); - void setRegexValue(const StringData& re); - void setSafeNumValue(const SafeNum& safeNum); - void setMinKey(); - void setMaxKey(); - void setUndefined(); - void setNull(); - void setSymbol(const StringData& symbolVal); - - /** Set the value of this element to equal the value of the - * provided BSONElement 'val'. The name of this Element is - * not modified. - */ - void setValueFromBSONElement(const BSONElement& val); - - // additional methods needed for BSON decoding - Status prefix(std::string* result, char delim) const; - Status suffix(std::string* result, char delim) const; - Status regex(std::string* result) const; - Status regexFlags(std::string* result) const; - Status dbrefNS(std::string* result) const; - Status dbrefOID(std::string* result) const; - Status codeWScopeCode(std::string* result) const; - Status codeWScopeScope(std::string* result) const; - - bool isBoolean() const; - bool isSimpleType() const; - bool isNumber() const; - bool isArray() const; - bool isNull() const; - bool isNonAtomic() const; - - // - // stream API - BSONObjBuilder like API, but methods return a Status. - // - - Status appendBool(const StringData& fieldName, bool boolVal); - Status appendInt(const StringData& fieldName, int32_t intVal); - Status appendLong(const StringData& fieldName, int64_t longVal); - Status appendTS(const StringData& fieldName, OpTime tsVal); - Status appendDate(const StringData& fieldName, int64_t millis); - Status appendDouble(const StringData& fieldName, double doubleVal); - Status appendOID(const StringData& fieldName, const mongo::OID& oid); - Status appendString(const StringData& fieldName, const StringData& stringVal); - Status appendCode(const StringData& fieldName, const StringData& code); - Status appendSymbol(const StringData& fieldName, const StringData& symbol); - Status appendNull(const StringData& fieldName); - Status appendMinKey(const StringData& fieldName); - Status appendMaxKey(const StringData& fieldName); - - Status appendDBRef( - const StringData& fieldName, const StringData& ns, const mongo::OID& oid); - Status appendRegex( - const StringData& fieldName, const StringData& re, const StringData& flags); - Status appendCodeWScope( - const StringData& fieldName, const StringData& code, const StringData& scope); - Status appendBinary( - const StringData& fieldName, uint32_t len, BinDataType t, const void* bin); - - Status appendSafeNum(const StringData& fieldName, const SafeNum num); - - // - // encapsulate state - // - - uint32_t getRep() const { return _rep; }; - Document* getDocument() const { return _doc; } - - StringData getFieldName() const; - - private: - inline Status checkSubtreeIsClean(Element e); - - // We carry the document in every element. The document determines the element: - // '_rep' is resolved through the document ElementVector and Heap - Document* _doc; - uint32_t _rep; - }; - - /** - * Document contains the root node, and factory methods for - * creating new nodes. Storage is allocated for the nodes within - * the Content heap, and the document tree structure is stored in - * the document ElementVector. - * - * For example usage, see class Element. - */ - class Document { - MONGO_DISALLOW_COPYING(Document); - - public: - explicit Document(Heap* heap); - ~Document(); - - // - // getters, (setters?) - // - - Heap* getHeap() { return _heap; } - const Heap* getHeap() const { return _heap; } - - // - // The distinguished root Element of the document, which is - // always an Object element. - // - Element& root() { return _root; } - const Element& root() const { return _root; } - - // - // factory methods - // - - Element makeObjElement(const StringData& fieldName); - Element makeArrayElement(const StringData& fieldName); - Element makeNullElement(const StringData& fieldName); - Element makeMinKeyElement(const StringData& fieldName); - Element makeMaxKeyElement(const StringData& fieldName); - Element makeBoolElement(const StringData& fieldName, bool boolVal); - Element makeIntElement(const StringData& fieldName, int32_t intVal); - Element makeLongElement(const StringData& fieldName, int64_t longVal); - Element makeTSElement(const StringData& fieldName, OpTime tsVal); - Element makeDateElement(const StringData& fieldName, int64_t dateVal); - Element makeDoubleElement(const StringData& fieldName, double millis); - Element makeOIDElement(const StringData& fieldName, const mongo::OID& oid); - Element makeStringElement(const StringData& fieldName, const StringData& stringVal); - Element makeCodeElement(const StringData& fieldName, const StringData& code); - Element makeSymbolElement(const StringData& fieldName, const StringData& symbol); - Element makeSafeNumElement(const StringData& fieldName, const SafeNum& safeNum); - - Element makeRegexElement( - const StringData& fieldName, const StringData& regex, const StringData& flags); - Element makeCodeWScopeElement( - const StringData& fieldName, const StringData& code, const StringData& scope); - Element makeDBRefElement( - const StringData& fieldName, const StringData& ns, const mongo::OID& oid); - Element makeBinaryElement( - const StringData& fieldName, uint32_t len, BinDataType binType, const void* data); - - private: - friend class Element; - friend class SiblingIterator; - - Heap* const _heap; - boost::scoped_ptr _elements; - Element _root; - }; - - // - // iteration support - // - - class Iterator { - public: - Iterator(); - - explicit Iterator(Element e) - : _doc(e.getDocument()) - , _theRep(e.getRep()) {} - - virtual ~Iterator() {} - - // iterator interface - virtual Iterator& operator++() = 0; - virtual bool done() const = 0; - - Element operator*() { return Element(getDocument(), getRep()); } - - // acessors - Document* getDocument() const { return _doc; } - uint32_t getRep() const { return _theRep; } - - protected: - Document* _doc; - uint32_t _theRep; - }; - - /** implementation: sibling iterator */ - class SiblingIterator : public Iterator { - public: - SiblingIterator() - : Iterator() {} - - explicit SiblingIterator(Element e) - : Iterator(e) {} - - virtual ~SiblingIterator() {} - - virtual SiblingIterator& operator++() { - advance(); - return *this; - } - - virtual bool done() const; - - private: - bool advance(); - }; - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_algo.h b/src/mongo/bson/mutable/mutable_bson_algo.h deleted file mode 100644 index 118f5f04914..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_algo.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "mongo/bson/mutable/mutable_bson.h" - -namespace mongo { -namespace mutablebson { - - /** Searches forward among the Elements iterated by 'first', - * returning an iterator to the first item matching the predicate - * 'p'. If no element matches 'p', then the 'done' method on the - * returned iterator will return true. Note that this method - * takes SiblingIterators, so it does not search into - * subdocuments, only within siblings. - */ - template - inline SiblingIterator findElement(SiblingIterator first, Predicate predicate) { - while (!first.done() && !predicate(*first)) - ++first; - return first; - } - - /** A predicate for findElement that matches on the field name of - * Elements. - */ - class FieldNameEquals { - public: - /** The lifetime of this object must be a subset of the - * lifetime of 'fieldName'. - */ - explicit FieldNameEquals(const StringData& fieldName) - : _fieldName(fieldName) {} - - bool operator()(const Element& element) const { - return (_fieldName == element.getFieldName()); - } - - private: - const StringData& _fieldName; - }; - - /** - * A convenience wrapper around findElement. - */ - inline SiblingIterator findElementNamed(SiblingIterator first, const StringData& fieldName) { - return findElement(first, FieldNameEquals(fieldName)); - } - - /** - * Finds the first child under 'parent' that matches the given - * field name, or returns a done Iterator. - */ - inline SiblingIterator findFirstChildNamed(Element parent, const StringData& fieldName) { - return findElementNamed(parent.children(), fieldName); - } - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_algo_test.cpp b/src/mongo/bson/mutable/mutable_bson_algo_test.cpp index a660dddcf49..132b3b29a2e 100644 --- a/src/mongo/bson/mutable/mutable_bson_algo_test.cpp +++ b/src/mongo/bson/mutable/mutable_bson_algo_test.cpp @@ -15,9 +15,12 @@ #include "mongo/platform/basic.h" -#include "mongo/bson/mutable/mutable_bson_algo.h" +#include "mongo/bson/mutable/algorithm.h" -#include "mongo/bson/mutable/mutable_bson_heap.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/json.h" +#include "mongo/platform/basic.h" #include "mongo/unittest/unittest.h" namespace { @@ -28,23 +31,21 @@ namespace { class DocumentTest : public mongo::unittest::Test { public: DocumentTest() - : _heap() - , _doc(&_heap) {} + : _doc() {} Document& doc() { return _doc; } private: - BasicHeap _heap; Document _doc; }; TEST_F(DocumentTest, FindInEmptyObject) { - const SiblingIterator children = doc().root().children(); - ASSERT_TRUE(children.done()); - SiblingIterator next = findElementNamed(children, "X"); - ASSERT_TRUE(next.done()); - ASSERT_EQUALS(children.getDocument(), next.getDocument()); - ASSERT_EQUALS(children.getRep(), next.getRep()); + Element leftChild = doc().root().leftChild(); + ASSERT_FALSE(leftChild.ok()); + Element found = findElementNamed(leftChild, "X"); + ASSERT_FALSE(found.ok()); + ASSERT_EQUALS(&leftChild.getDocument(), &found.getDocument()); + ASSERT_EQUALS(leftChild.getIdx(), found.getIdx()); } class OneChildTest : public DocumentTest { @@ -54,23 +55,23 @@ namespace { }; TEST_F(OneChildTest, FindNoMatch) { - const SiblingIterator children = doc().root().children(); - ASSERT_FALSE(children.done()); - SiblingIterator next = findElementNamed(children, "f"); - ASSERT_TRUE(next.done()); - ASSERT_EQUALS(children.getDocument(), next.getDocument()); + Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + Element found = findElementNamed(leftChild, "f"); + ASSERT_FALSE(found.ok()); + ASSERT_EQUALS(&leftChild.getDocument(), &found.getDocument()); } TEST_F(OneChildTest, FindMatch) { - const SiblingIterator children = doc().root().children(); - ASSERT_FALSE(children.done()); - SiblingIterator next = findElementNamed(children, "t"); - ASSERT_FALSE(next.done()); - ASSERT_EQUALS(children.getDocument(), next.getDocument()); - ASSERT_EQUALS((*next).getFieldName(), "t"); - next = findElementNamed(++next, "t"); - ASSERT_TRUE(next.done()); - ASSERT_EQUALS(children.getDocument(), next.getDocument()); + Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + Element found = findElementNamed(leftChild, "t"); + ASSERT_TRUE(found.ok()); + ASSERT_EQUALS(&leftChild.getDocument(), &found.getDocument()); + ASSERT_EQUALS(found.getFieldName(), "t"); + found = findElementNamed(found.rightSibling(), "t"); + ASSERT_FALSE(found.ok()); + ASSERT_EQUALS(&leftChild.getDocument(), &found.getDocument()); } class ManyChildrenTest : public DocumentTest { @@ -87,78 +88,232 @@ namespace { TEST_F(ManyChildrenTest, FindAtStart) { static const char kName[] = "begin"; - const SiblingIterator children = doc().root().children(); - ASSERT_FALSE(children.done()); - SiblingIterator next = findElementNamed(children, kName); - ASSERT_FALSE(next.done()); - ASSERT_EQUALS((*next).getFieldName(), kName); - ASSERT_EQUALS(children.getDocument(), next.getDocument()); - ASSERT_TRUE(findElementNamed(++next, kName).done()); + Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + Element found = findElementNamed(leftChild, kName); + ASSERT_TRUE(found.ok()); + ASSERT_EQUALS(found.getFieldName(), kName); + ASSERT_EQUALS(&leftChild.getDocument(), &found.getDocument()); + ASSERT_FALSE(findElementNamed(found.rightSibling(), kName).ok()); } TEST_F(ManyChildrenTest, FindInMiddle) { static const char kName[] = "middle"; - const SiblingIterator children = doc().root().children(); - ASSERT_FALSE(children.done()); - SiblingIterator next = findElementNamed(children, kName); - ASSERT_FALSE(next.done()); - ASSERT_EQUALS((*next).getFieldName(), kName); - ASSERT_EQUALS(children.getDocument(), next.getDocument()); - ASSERT_TRUE(findElementNamed(++next, kName).done()); + Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + Element found = findElementNamed(leftChild, kName); + ASSERT_TRUE(found.ok()); + ASSERT_EQUALS(found.getFieldName(), kName); + ASSERT_EQUALS(&leftChild.getDocument(), &found.getDocument()); + ASSERT_FALSE(findElementNamed(found.rightSibling(), kName).ok()); } TEST_F(ManyChildrenTest, FindAtEnd) { static const char kName[] = "end"; - const SiblingIterator children = doc().root().children(); - ASSERT_FALSE(children.done()); - SiblingIterator next = findElementNamed(children, kName); - ASSERT_FALSE(next.done()); - ASSERT_EQUALS((*next).getFieldName(), kName); - ASSERT_EQUALS(children.getDocument(), next.getDocument()); - ASSERT_TRUE(findElementNamed(++next, kName).done()); + Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + Element found = findElementNamed(leftChild, kName); + ASSERT_TRUE(found.ok()); + ASSERT_EQUALS(found.getFieldName(), kName); + ASSERT_EQUALS(&leftChild.getDocument(), &found.getDocument()); + ASSERT_FALSE(findElementNamed(found.rightSibling(), kName).ok()); } TEST_F(ManyChildrenTest, FindRepeatedSparse) { static const char kName[] = "repeated_sparse"; - const SiblingIterator children = doc().root().children(); - ASSERT_FALSE(children.done()); - SiblingIterator first = findElementNamed(children, kName); - ASSERT_FALSE(first.done()); - ASSERT_EQUALS((*first).getFieldName(), kName); - ASSERT_EQUALS(children.getDocument(), first.getDocument()); - SiblingIterator second = findElementNamed(++SiblingIterator(first), kName); - ASSERT_FALSE(second.done()); - ASSERT_EQUALS(first.getDocument(), second.getDocument()); - ASSERT_NOT_EQUALS(first.getRep(), second.getRep()); - SiblingIterator none = findElementNamed(++SiblingIterator(second), kName); - ASSERT_TRUE(none.done()); + Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + Element first = findElementNamed(leftChild, kName); + ASSERT_TRUE(first.ok()); + ASSERT_EQUALS(first.getFieldName(), kName); + ASSERT_EQUALS(&leftChild.getDocument(), &first.getDocument()); + Element second = findElementNamed(first.rightSibling(), kName); + ASSERT_TRUE(second.ok()); + ASSERT_EQUALS(&first.getDocument(), &second.getDocument()); + ASSERT_NOT_EQUALS(first.getIdx(), second.getIdx()); + Element none = findElementNamed(second.rightSibling(), kName); + ASSERT_FALSE(none.ok()); } TEST_F(ManyChildrenTest, FindRepeatedDense) { static const char kName[] = "repeated_dense"; - const SiblingIterator children = doc().root().children(); - ASSERT_FALSE(children.done()); - SiblingIterator first = findElementNamed(children, kName); - ASSERT_FALSE(first.done()); - ASSERT_EQUALS((*first).getFieldName(), kName); - ASSERT_EQUALS(children.getDocument(), first.getDocument()); - SiblingIterator second = findElementNamed(++SiblingIterator(first), kName); - ASSERT_FALSE(second.done()); - ASSERT_EQUALS(first.getDocument(), second.getDocument()); - ASSERT_NOT_EQUALS(first.getRep(), second.getRep()); - SiblingIterator none = findElementNamed(++SiblingIterator(second), kName); - ASSERT_TRUE(none.done()); + Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + Element first = findElementNamed(leftChild, kName); + ASSERT_TRUE(first.ok()); + ASSERT_EQUALS(first.getFieldName(), kName); + ASSERT_EQUALS(&leftChild.getDocument(), &first.getDocument()); + Element second = findElementNamed(first.rightSibling(), kName); + ASSERT_TRUE(second.ok()); + ASSERT_EQUALS(&first.getDocument(), &second.getDocument()); + ASSERT_NOT_EQUALS(first.getIdx(), second.getIdx()); + Element none = findElementNamed(second.rightSibling(), kName); + ASSERT_FALSE(none.ok()); } TEST_F(ManyChildrenTest, FindDoesNotSearchWithinChildren) { static const char kName[] = "in_child"; - SiblingIterator found_before_add = findElementNamed(doc().root().children(), kName); - ASSERT_TRUE(found_before_add.done()); - Element subdoc = doc().makeObjElement("child"); - ASSERT_EQUALS(Status::OK(), doc().root().addChild(subdoc)); + Element found_before_add = findElementNamed(doc().root().leftChild(), kName); + ASSERT_FALSE(found_before_add.ok()); + Element subdoc = doc().makeElementObject("child"); + ASSERT_EQUALS(Status::OK(), doc().root().pushBack(subdoc)); ASSERT_EQUALS(Status::OK(), subdoc.appendBool(kName, true)); - SiblingIterator found_after_add = findElementNamed(doc().root().children(), kName); - ASSERT_TRUE(found_after_add.done()); + Element found_after_add = findElementNamed(doc().root().leftChild(), kName); + ASSERT_FALSE(found_after_add.ok()); + } + + TEST_F(ManyChildrenTest, getNthSibling) { + const Element leftChild = doc().root().leftChild(); + ASSERT_TRUE(leftChild.ok()); + const Element rightChild = doc().root().rightChild(); + ASSERT_TRUE(rightChild.ok()); + + // Check that moving zero is a no-op + Element zeroAway = getNthSibling(leftChild, 0); + ASSERT_TRUE(zeroAway.ok()); + ASSERT_EQUALS(leftChild, zeroAway); + zeroAway = getNthSibling(rightChild, 0); + ASSERT_TRUE(zeroAway.ok()); + ASSERT_EQUALS(rightChild, zeroAway); + + // Check that moving left of leftmost gets a not-ok element. + Element badLeft = getNthSibling(leftChild, -1); + ASSERT_FALSE(badLeft.ok()); + + // Check that moving right of rightmost gets a non-ok element. + Element badRight = getNthSibling(rightChild, 1); + ASSERT_FALSE(badRight.ok()); + + // Check that the moving one right from leftmost gets us the expected element. + Element target = leftChild.rightSibling(); + ASSERT_TRUE(target.ok()); + Element query = getNthSibling(leftChild, 1); + ASSERT_TRUE(target.ok()); + ASSERT_EQUALS(target, query); + + // And the same from the other side + target = rightChild.leftSibling(); + ASSERT_TRUE(target.ok()); + query = getNthSibling(rightChild, -1); + ASSERT_TRUE(target.ok()); + ASSERT_EQUALS(target, query); + + // Ensure that walking more chidren than we have gets us past the end + const int children = countChildren(doc().root()); + query = getNthSibling(leftChild, children); + ASSERT_FALSE(query.ok()); + query = getNthSibling(rightChild, -children); + ASSERT_FALSE(query.ok()); + + // Ensure that walking all the children in either direction gets + // us to the other right/left child. + query = getNthSibling(leftChild, children - 1); + ASSERT_TRUE(query.ok()); + ASSERT_EQUALS(rightChild, query); + query = getNthSibling(rightChild, -(children - 1)); + ASSERT_TRUE(query.ok()); + ASSERT_EQUALS(leftChild, query); + } + + class CountTest : public DocumentTest { + virtual void setUp() { + Element root = doc().root(); + + ASSERT_OK(root.appendInt("leaf", 0)); + + Element one = doc().makeElementObject("oneChild"); + ASSERT_TRUE(one.ok()); + ASSERT_OK(one.appendInt("one", 1)); + ASSERT_OK(root.pushBack(one)); + + Element threeChildren = doc().makeElementObject("threeChildren"); + ASSERT_TRUE(one.ok()); + ASSERT_OK(threeChildren.appendInt("one", 1)); + ASSERT_OK(threeChildren.appendInt("two", 2)); + ASSERT_OK(threeChildren.appendInt("three", 3)); + ASSERT_OK(root.pushBack(threeChildren)); + } + }; + + TEST_F(CountTest, EmptyDocument) { + // Doesn't use the fixture but belongs in the same group of tests. + Document doc; + ASSERT_EQUALS(countChildren(doc.root()), 0u); + } + + TEST_F(CountTest, EmptyElement) { + Element leaf = findFirstChildNamed(doc().root(), "leaf"); + ASSERT_TRUE(leaf.ok()); + ASSERT_EQUALS(countChildren(leaf), 0u); + } + + TEST_F(CountTest, OneChildElement) { + Element oneChild = findFirstChildNamed(doc().root(), "oneChild"); + ASSERT_TRUE(oneChild.ok()); + ASSERT_EQUALS(countChildren(oneChild), 1u); + } + + TEST_F(CountTest, ManyChildren) { + Element threeChildren = findFirstChildNamed(doc().root(), "threeChildren"); + ASSERT_TRUE(threeChildren.ok()); + ASSERT_EQUALS(countChildren(threeChildren), 3u); + } + + TEST_F(CountTest, CountSiblingsNone) { + ConstElement current = findFirstChildNamed(doc().root(), "oneChild"); + ASSERT_TRUE(current.ok()); + + current = current.leftChild(); + ASSERT_TRUE(current.ok()); + + ASSERT_EQUALS(0U, countSiblingsLeft(current)); + ASSERT_EQUALS(0U, countSiblingsRight(current)); + } + + TEST_F(CountTest, CountSiblingsMany) { + ConstElement current = findFirstChildNamed(doc().root(), "threeChildren"); + ASSERT_TRUE(current.ok()); + + current = current.leftChild(); + ASSERT_TRUE(current.ok()); + + ASSERT_EQUALS(0U, countSiblingsLeft(current)); + ASSERT_EQUALS(2U, countSiblingsRight(current)); + + current = current.rightSibling(); + ASSERT_TRUE(current.ok()); + ASSERT_EQUALS(1U, countSiblingsLeft(current)); + ASSERT_EQUALS(1U, countSiblingsRight(current)); + + current = current.rightSibling(); + ASSERT_TRUE(current.ok()); + ASSERT_EQUALS(2U, countSiblingsLeft(current)); + ASSERT_EQUALS(0U, countSiblingsRight(current)); + + current = current.rightSibling(); + ASSERT_FALSE(current.ok()); + } + + TEST(DeduplicateTest, ManyDuplicates) { + Document doc(mongo::fromjson("{ x : [ 1, 2, 2, 3, 3, 3, 4, 4, 4 ] }")); + deduplicateChildren(doc.root().leftChild(), woEqual(false)); + ASSERT_TRUE(checkDoc(doc, mongo::fromjson("{x : [ 1, 2, 3, 4 ]}"))); + } + + TEST(FullNameTest, RootField) { + Document doc(mongo::fromjson("{ x : 1 }")); + ASSERT_EQUALS("x", getFullName(doc.root().leftChild())); + } + + TEST(FullNameTest, OneLevel) { + Document doc(mongo::fromjson("{ x : { y: 1 } }")); + ASSERT_EQUALS("x.y", getFullName(doc.root().leftChild().leftChild())); + } + + TEST(FullNameTest, InsideArray) { + Document doc(mongo::fromjson("{ x : { y: [ 1 , 2 ] } }")); + ASSERT_EQUALS("x.y.1", + getFullName(doc.root().leftChild().leftChild().leftChild().rightSibling())); } } // namespace diff --git a/src/mongo/bson/mutable/mutable_bson_builder.cpp b/src/mongo/bson/mutable/mutable_bson_builder.cpp deleted file mode 100644 index 11175c9a07e..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_builder.cpp +++ /dev/null @@ -1,298 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License-> - * You may obtain a copy of the License at - * - * http://www.apache->org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License-> - */ - -#include "mongo/platform/basic.h" - -#include "mongo/bson/mutable/mutable_bson_builder.h" - -#include - -#include "mongo/base/error_codes.h" -#include "mongo/base/status.h" - -namespace mongo { -namespace mutablebson { - - // - // ElementBuilder - // - - Status ElementBuilder::parse(const BSONObj& src, Element* dst) { - Status result(Status::OK()); - Document& doc = *dst->getDocument(); - - BSONObjIterator it(src); - for (; it.more(); ++it) { - BSONElement bsonElem = *it; - - // TODO: We should have a way to get the fieldName as a StringData. - const char* fieldName = bsonElem.fieldName(); - - switch (bsonElem.type()) { - case MinKey: { - result = dst->appendMinKey(fieldName); - break; - } - case NumberDouble: { - result = dst->appendDouble(fieldName, bsonElem.Double()); - break; - } - case String: { - result = dst->appendString(fieldName, bsonElem.String().c_str()); - break; - } - case Object: { - Element e0 = doc.makeObjElement(fieldName); - result = ElementBuilder::parse(bsonElem.Obj(), &e0); - if (result.isOK()) - result = dst->addChild(e0); - break; - } - case Array: { - Element e0 = doc.makeArrayElement(fieldName); - result = ElementBuilder::parse(bsonElem.Obj(), &e0); - if (result.isOK()) - result = dst->addChild(e0); - break; - } - case BinData: { // a stub - break; - } - case Undefined: { // a stub - break; - } - case jstOID: { - result = dst->appendOID(fieldName, bsonElem.OID()); - break; - } - case Bool: { - result = dst->appendBool(fieldName, bsonElem.Bool()); - break; - } - case Date: { - result = dst->appendDate(fieldName, bsonElem.Date().millis); - break; - } - case jstNULL: { - result = dst->appendNull(fieldName); - break; - } - case RegEx: { - result = dst->appendRegex(fieldName, bsonElem.regex(), bsonElem.regexFlags()); - break; - } - case DBRef: { - result = dst->appendDBRef(fieldName, bsonElem.dbrefNS(), bsonElem.dbrefOID()); - break; - } - /** Note: The following three cases seem fishy. They need better understanding. */ - case Code: { - result = dst->appendCode(fieldName, bsonElem.str().c_str()); - break; - } - case Symbol: { - result = dst->appendSymbol(fieldName, bsonElem.str().c_str()); - break; - } - case CodeWScope: { - result = dst->appendCodeWScope( - fieldName, - bsonElem.codeWScopeCode(), - bsonElem.codeWScopeScopeData()); - break; - } - case NumberInt: { - result = dst->appendInt(fieldName, bsonElem.Int()); - break; - } - case Timestamp: { - result = dst->appendTS(fieldName, bsonElem.timestampTime().millis); - break; - } - case NumberLong: { - result = dst->appendLong(fieldName, bsonElem.Long()); - break; - } - case MaxKey: { - result = dst->appendMaxKey(fieldName); - break; - } - default: { - } - } - - if (!result.isOK()) - break; - } - - return result; - } - - - // - // BSONBuilder - // - - void BSONBuilder::buildFromElement(Element src, BSONObjBuilder* dst) { - - const StringData srcFieldName = src.getFieldName(); - - switch (src.type()) { - case MinKey: { - dst->appendMinKey(srcFieldName); - break; - } - case EOO: { - break; - } - case NumberDouble: { - dst->appendNumber(srcFieldName, src.getDoubleValue()); - break; - } - case String: { - dst->append(srcFieldName, src.getStringValue()); - break; - } - case Object: { - BSONObjBuilder subBuilder(dst->subobjStart(srcFieldName)); - BSONBuilder::build(src, &subBuilder); - subBuilder.doneFast(); - break; - } - case Array: { - BSONObjBuilder subBuilder(dst->subarrayStart(srcFieldName)); - SiblingIterator arrayIt = src.children(); - for (uint32_t n=0; !arrayIt.done(); ++arrayIt,++n) { - Element e0 = *arrayIt; - ostringstream oss; oss << "" << n; - e0.rename(oss.str()); - BSONBuilder::buildFromElement(e0, &subBuilder); - } - subBuilder.doneFast(); - break; - } - case BinData: { - uint32_t len(0); - BinDataType subType(mongo::BinDataGeneral); - dst->appendBinData(srcFieldName, len, subType, src.getStringValue()); - break; - } - case Undefined: { - dst->appendUndefined(srcFieldName); - break; - } - case jstOID: { - OID oid = src.getOIDValue(); - dst->appendOID(srcFieldName, &oid); - break; - } - case Bool: { - dst->appendBool(srcFieldName, src.getBoolValue()); - break; - } - case Date: { - dst->appendDate(srcFieldName, src.getDateValue()); - break; - } - case jstNULL: { - dst->appendNull(srcFieldName); - break; - } - case RegEx: { - string re(""); - Status err = src.regex(&re); - if (err.code() != ErrorCodes::OK) { - break; - } - string flags(""); - err = src.regexFlags(&flags); - if (err.code() != ErrorCodes::OK) { - break; - } - dst->appendRegex(srcFieldName, re, flags); - break; - } - case DBRef: { - string ns(""); - Status err = src.dbrefNS(&ns); - if (err.code() != ErrorCodes::OK) { - break; - } - string oidStr(""); - err = src.dbrefOID(&oidStr); - if (err.code() != ErrorCodes::OK) { - break; - } - mongo::OID oid(oidStr); - dst->appendDBRef(srcFieldName, ns, oid); - break; - } - - case Code: { - dst->appendCode(srcFieldName, src.getStringValue()); - break; - } - case Symbol: { - dst->appendSymbol(srcFieldName, src.getStringValue()); - break; - } - case CodeWScope: { - string code(""); - Status err = src.codeWScopeCode(&code); - if (err.code() != ErrorCodes::OK) { - break; - } - string scope(""); - err = src.codeWScopeScope(&scope); - if (err.code() != ErrorCodes::OK) { - break; - } - dst->appendCode(code, scope); - break; - } - - case NumberInt: { - dst->appendNumber(srcFieldName, src.getIntValue()); - break; - } - case Timestamp: { - dst->appendTimeT(srcFieldName, src.getLongValue()); - break; - } - case NumberLong: { - dst->appendNumber(srcFieldName, static_cast(src.getLongValue())); - break; - } - case MaxKey: { - dst->appendMaxKey(srcFieldName); - break; - } - default: { - } - } - } - - void BSONBuilder::build(Element src, BSONObjBuilder* dst) { - - SiblingIterator it = src.children(); - - for (; !it.done(); ++it) { - Element elem = *it; - BSONBuilder::buildFromElement(elem, dst); - } - } - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_builder.h b/src/mongo/bson/mutable/mutable_bson_builder.h deleted file mode 100644 index 7f8c29d857b..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_builder.h +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "mongo/bson/mutable/mutable_bson.h" -#include "mongo/db/jsobj.h" - -namespace mongo { -namespace mutablebson { - - /** static method for creating a MutableBSON tree from a BSONObj. */ - class ElementBuilder { - public: - static Status parse(const BSONObj& src, Element* dst); - - static Status parse(const BSONObj& src, Document* dst) { - return parse(src, &dst->root()); - } - }; - - /** static method for creating BSONObj from MutableBSON */ - class BSONBuilder { - public: - static void buildFromElement(Element src, BSONObjBuilder* dst); - static void build(Element src, BSONObjBuilder* dst); - - static void build(const Document& src, BSONObjBuilder* dst) { - return build(src.root(), dst); - } - - }; - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_builder_test.cpp b/src/mongo/bson/mutable/mutable_bson_builder_test.cpp deleted file mode 100644 index 004e58fd495..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_builder_test.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/bson/mutable/mutable_bson_builder.h" - -#include "mongo/base/status.h" -#include "mongo/bson/bsonobj.h" -#include "mongo/bson/mutable/mutable_bson.h" -#include "mongo/bson/mutable/mutable_bson_heap.h" -#include "mongo/bson/mutable/mutable_bson_internal.h" -#include "mongo/db/json.h" -#include "mongo/unittest/unittest.h" - -namespace { - - // TODO: This object should contain a representative for every - // BSON type, including deprecated values. - static const char jsonSample[] = - "{_id:ObjectId(\"47cc67093475061e3d95369d\")," - "query:\"kate hudson\"," - "owner:1234567887654321," - "date:\"2011-05-13T14:22:46.777Z\"," - "score:123.456," - "field1:Infinity," - "\"field2\":-Infinity," - "\"field3\":NaN," - "users:[" - "{uname:\"@aaaa\",editid:\"1234\",date:1303959350,yes_votes:0,no_votes:0}," - "{uname:\"@bbbb\",editid:\"5678\",date:1303959350,yes_votes:0,no_votes:0}]," - "pattern:/match.*this/," - "lastfield:\"last\"}"; - - TEST(BuilderAPI, RoundTrip) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); - - int len; - mongo::BSONObj obj = mongo::fromjson(jsonSample, &len); - mongo::mutablebson::ElementBuilder::parse(obj, &doc); - - mongo::BSONObjBuilder builder; - mongo::mutablebson::BSONBuilder::build(doc.root(), &builder); - mongo::BSONObj built = builder.done(); - - // TODO: When both builders are feature complete, add a an assert - // that the round tripped objects are equivalent. - } - -} // unnamed namespace diff --git a/src/mongo/bson/mutable/mutable_bson_heap.cpp b/src/mongo/bson/mutable/mutable_bson_heap.cpp deleted file mode 100644 index 069c08dffcc..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_heap.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/bson/mutable/mutable_bson_heap.h" - -namespace mongo { -namespace mutablebson { - - // - // BasicHeap implementation - // - - BasicHeap::BasicHeap() {} - - BasicHeap::~BasicHeap() {} - - uint32_t BasicHeap::alloc(uint32_t size) { - uint32_t result = _heap.size(); - _heap.reserve(result + size); - _heap.resize(result + size); - return result; - } - - template T* BasicHeap::deref(uint32_t index) { - std::vector::const_iterator it = _heap.begin(); - return (T*)(&*it + index); - } - - template const T* BasicHeap::deref(uint32_t index) const { - std::vector::const_iterator it = _heap.begin(); - return (const T*)(&*it + index); - } - - std::string BasicHeap::getString(uint32_t index) { - return string(BasicHeap::deref(index)); - } - - char* BasicHeap::getStringBuffer(uint32_t index) { - return BasicHeap::deref(index); - } - - uint32_t BasicHeap::putString(const StringData& s) { - uint32_t index = alloc(s.size() + 1); - char* buf = deref(index); - s.copyTo( buf, true ); - return index; - } - - // stubs: - mongo::OID BasicHeap::getOID(uint32_t index) { - return mongo::OID(); - } - - const char* BasicHeap::getBinary(uint32_t index) { - return NULL; - } - - uint32_t BasicHeap::putOID(const mongo::OID& id) { - return 0; - } - - uint32_t BasicHeap::putBinary(const char* buf) { - return 0; - } - - // - // BSONObjHeap implementation - // - - // BSON internal format reminder: - // {}* EOO - // - // Data: - // Bool: - // EOO: nothing follows - // Undefined: nothing follows - // OID: an OID object - // NumberDouble: - // NumberInt: - // String: - // Date: <8bytes> - // Regex: - // Object: a nested object, leading with its entire size, which terminates with EOO. - // Array: same as object - // DBRef: - // DBRef: a database reference: basically a collection name plus an Object ID - // BinData: - // Code: a function (not a closure): same format as String. - // Symbol: a language symbol (say a python symbol). same format as String. - // Code With Scope: - - BSONObjHeap::BSONObjHeap(const BSONObj* obj) : - _obj(obj) { - } - - BSONObjHeap::~BSONObjHeap() { - } - - uint32_t BSONObjHeap::alloc(uint32_t size) { - uint32_t result = _heap.size(); - _heap.reserve(result + size); - _heap.resize(result + size); - return (_obj->objsize()+result); - } - - template T* BSONObjHeap::deref(uint32_t offset) { - if ((int)offset < _obj->objsize()) return (T*)(_obj->objdata()+offset); - uint32_t offset0 = (offset - _obj->objsize()); - std::vector::const_iterator it = _heap.begin(); - return (T*)(&*it + offset0); - } - - template const T* BSONObjHeap::deref(uint32_t offset) const { - if ((int)offset < _obj->objsize()) return (const T*)(_obj->objdata()+offset); - uint32_t offset0 = (offset - _obj->objsize()); - std::vector::const_iterator it = _heap.begin(); - return (const T*)(&*it + offset0); - } - - std::string BSONObjHeap::getString(uint32_t offset) { - return string(BSONObjHeap::deref(offset)); - } - - char* BSONObjHeap::getStringBuffer(uint32_t offset) { - return BSONObjHeap::deref(offset); - } - - uint32_t BSONObjHeap::putString(const StringData& s) { - uint32_t offset = alloc(s.size() + 1); - char* buf = deref(offset); - s.copyTo( buf, true ); - return offset; - } - - // stubs: - mongo::OID BSONObjHeap::getOID(uint32_t offset) { - return mongo::OID(); - } - - const char* BSONObjHeap::getBinary(uint32_t offset) { - return NULL; - } - - uint32_t BSONObjHeap::putOID(const mongo::OID& id) { - return 0; - } - - uint32_t BSONObjHeap::putBinary(const char* buf) { - return 0; - } - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_heap.h b/src/mongo/bson/mutable/mutable_bson_heap.h deleted file mode 100644 index 2f49360f8be..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_heap.h +++ /dev/null @@ -1,116 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - - -#include -#include - -#include "mongo/base/string_data.h" -#include "mongo/bson/bsonobj.h" -#include "mongo/bson/oid.h" -#include "mongo/platform/cstdint.h" - -namespace mongo { -namespace mutablebson { - - /* - * Interface for heap implementations - * - * See mutable_bson_internal.h for further details. - */ - class Heap { - public: - virtual ~Heap() {} - - /** Can we make these 'const' functions without triggering a 'const storm'? */ - virtual std::string getString(uint32_t index) = 0; - virtual uint32_t putString(const StringData& s) = 0; - - virtual char* getStringBuffer(uint32_t index) = 0; - - virtual mongo::OID getOID(uint32_t index) = 0; - virtual uint32_t putOID(const mongo::OID& oid) = 0; - - virtual const char* getBinary(uint32_t index) = 0; - virtual uint32_t putBinary(const char* binData) = 0; - }; - - - /* - * Vector implemetation of Heap interface - * - * See mutable_bson_internal.h for further details. - */ - class BasicHeap : public Heap { - public: - BasicHeap(); - ~BasicHeap(); - - std::string getString(uint32_t index); - uint32_t putString(const StringData& s); - - char* getStringBuffer(uint32_t index); - - mongo::OID getOID(uint32_t index); - uint32_t putOID(const mongo::OID& oid); - - const char* getBinary(uint32_t index); - uint32_t putBinary(const char* binData); - - private: - uint32_t alloc(uint32_t size); - template T* deref(uint32_t index); - template const T* deref(uint32_t index) const; - - std::vector _heap; - }; - - /* - * BSONObj implementation of Heap interface - * The design: store offsets directly into a given BSONObj. These offsets can be used for - * read operations and in-place updates. Any update that requires additional storage is - * handled by an auxiliary vector structure. - * - * See mutable_bson_internal.h for further details. - */ - class BSONObjHeap : public Heap { - public: - BSONObjHeap(const BSONObj*); - ~BSONObjHeap(); - - std::string getString(uint32_t index); - uint32_t putString(const StringData& s); - - char* getStringBuffer(uint32_t index); - - mongo::OID getOID(uint32_t index); - uint32_t putOID(const mongo::OID& oid); - - const char* getBinary(uint32_t index); - uint32_t putBinary(const char* binData); - - private: - uint32_t alloc(uint32_t size); - template T* deref(uint32_t index); - template const T* deref(uint32_t index) const; - - const BSONObj* _obj; // static size, read and in-place updates only - std::vector _heap; // for insertions and non-in-place updates - }; - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_heap_test.cpp b/src/mongo/bson/mutable/mutable_bson_heap_test.cpp deleted file mode 100644 index edd328a039c..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_heap_test.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/bson/mutable/mutable_bson_heap.h" - -#include - -#include "mongo/unittest/unittest.h" - -namespace { - - TEST(BasicHeap, PutAndGetString) { - mongo::mutablebson::BasicHeap heap; - const std::string str = "my string"; - const char* anotherStr = "another string"; - - ASSERT_EQUALS(heap.putString(str), 0U); - ASSERT_EQUALS(heap.putString(anotherStr), str.size()+1); - ASSERT_EQUALS(heap.getString(0), str); - ASSERT_EQUALS(heap.getString(str.size()+1), std::string(anotherStr)); - } - -} // unnamed namespace diff --git a/src/mongo/bson/mutable/mutable_bson_internal.cpp b/src/mongo/bson/mutable/mutable_bson_internal.cpp deleted file mode 100644 index 37444061a0f..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_internal.cpp +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/bson/mutable/mutable_bson_internal.h" - -namespace mongo { -namespace mutablebson { - - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_internal.h b/src/mongo/bson/mutable/mutable_bson_internal.h deleted file mode 100644 index 540dcda2319..00000000000 --- a/src/mongo/bson/mutable/mutable_bson_internal.h +++ /dev/null @@ -1,215 +0,0 @@ -/* Copyright 2010 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include "mongo/bson/mutable/mutable_bson.h" -#include "mongo/platform/cstdint.h" - -namespace mongo { -namespace mutablebson { - -/* - * Internal representation for a mutable bson. - * - * Mutable BSON represents documents using buffers in a packed - * non-serialized, mutable data structure. The layout is as follows: - * - * heap: [ (en:32, [byte*)* ] - * [03|abc|03|def|04|ghij|... ] - * - * elementVector: [ ElementRep* ] - * [r0|r1|r2|r3|... ] - * - * - * ElementReps are fixed-width data structures that include: - * - * type:32, // 32 bits, regular BSONType, additional bits usage tbd - * name:32, // 32 bits, field name heap reference - * value:64 // 64 bits, either inline atomic or heap reference - * //overloaded as: - * leftChild:32 // 32 bits, ref to leftmost child - * rightChild:32 // 32 bits, ref to rightmost child - * rightSibling:32 // 32 bits, ref to next sibling - * leftSibling:32 // 32 bits, ref to prev sibling - * parent:32 // 32 bits, ref to parent - * pad to 32 bytes // = minimal single cache line - * - * - * ElementRep contains node data - * Element wraps ElementRep by adding document, (.g) heap - * - * Element::find returns an Iterator - * Iterators have undefined behavior after move or removal of current node or - * any ancestor of current node. - * - * Document includes the heap and element vector - * - * BasicHeap implements Heap interface using vector - * BSONObjHeap implements Heap interface using BSON _objdata buffer + vector - * Values of size <= 64 bits may be stored inline in ElementReps - * Embedded objects and arrays are stored as elements with up/down-links - * to/from parent, and doubly linked sibling indexes. - * Larger atomic types and variable-length types are stored in the heap - * and referenced via heap indexes. - * - * For example, the document - * - * { aa:1, bbb:{c:2, d:{e:{f:"xyz", g:[4,5,6]}}} } - * - * is represented as follows: - * - * MutableBSON::ElementVector - * 0 [top| 0| 1 2| .| .| .] { root node left/right children "aa", "bbb" - * 1 [num|00| 0 1| 2| .| 0] "aa":1 leaf node value = 1; right sibling "bbb" - * 2 [obj|06| 3 4| .| 1| 0] "bbb":{..} subtree left/right children "c","d"; left sib "aa" - * 3 [num|13| 0 2| 3| .| 2] "c":2 leaf node value=2; right sibling "d" - * 4 [obj|18| 5 5| .| 4| 2] "d":{..} subtree left/right children "e", "e"; left sib "c" - * 5 [obj|23| 6 7| .| .| 4] "e":{..} subtree left/right children "f", "g" - * 6 [str|28| 0 33| 7| .| 5] "f":"xyz" leaf node value on heap, right sibling "g" - * 7 [obj|40| 8 10| .| 6| 5] "g":{..} sublist left/right children "g.0","g.2"; left sib "f" - * 8 [num|45| 0 4| 9| .| 7] "0":4 leaf node value=4; right sibling "g.1" - * 9 [num|50| 0 5|10| 8| 7] "1":5 leaf node value=5; right sibling "g.2", left sib "g.1" - * 10[num|55| 0 6| .| 9| 7] "2":6 leaf node value=6; left sibling "g.1" - * + + - * |__|__ 64 bit value, or pair of 32-bit left/right child indexes - * - * Heap - * [02|aa|03|bbb|01|c|01|d|01|e|01|f|03|xyz|01|g|01|0|01|1|01|2|.. ] - * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ - * 0 6 13 18 23 28 33 40 45 50 55 - * - */ - - static const uint32_t EMPTY_REP = (uint32_t)-1; - static const uint32_t NULL_REF = (uint32_t)-1; - static const uint32_t SHORT_LIMIT = 16; - - union ValueType { - bool boolVal; - int32_t intVal; - int64_t longVal; - int64_t tsVal; - int64_t dateVal; - double doubleVal; - char shortStr[SHORT_LIMIT]; /* OID and short strings */ - uint64_t valueRef; /* index to heap (strings, binary, regex) */ - }; - - /** - fixed-width node - stored in Document ElementVector - - "rep" is a name coined by Stroustrup in his original - std::string implementation. It means "representative". - Element is the abstraction, ElementRep is the implementation class. - Algorithms manipulate Elements. ElementRep's can be swapped out - without change upstairs. - */ - struct ElementRep { - ElementRep() - : _type(mongo::Undefined) - , _nameref(NULL_REF) - , _parent(EMPTY_REP) { - _child._left = EMPTY_REP; - _child._right = EMPTY_REP; - _sibling._left = EMPTY_REP; - _sibling._right = EMPTY_REP; - } - - ElementRep(int32_t type) - : _type(type) - , _nameref(NULL_REF) - , _parent(EMPTY_REP) { - _child._left = EMPTY_REP; - _child._right = EMPTY_REP; - _sibling._left = EMPTY_REP; - _sibling._right = EMPTY_REP; - } - - ElementRep(int32_t type, uint32_t nameref) - : _type(type) - , _nameref(nameref) - , _parent(EMPTY_REP) { - _child._left = EMPTY_REP; - _child._right = EMPTY_REP; - _sibling._left = EMPTY_REP; - _sibling._right = EMPTY_REP; - } - - ElementRep(int32_t type, uint32_t nameref, ValueType value) - : _type(type) - , _nameref(nameref) - , _value(value) - , _parent(EMPTY_REP) { - _sibling._left = EMPTY_REP; - _sibling._right = EMPTY_REP; - } - - ElementRep(int32_t type, uint32_t nameref, ValueType value, uint32_t parentref) - : _type(type) - , _nameref(nameref) - , _value(value) - , _parent(parentref) { - _sibling._left = EMPTY_REP; - _sibling._right = EMPTY_REP; - } - - void clearSiblings() { - _sibling._left = EMPTY_REP; - _sibling._right = EMPTY_REP; - } - - void clearParent() { - _parent = EMPTY_REP; - } - - int32_t _type; - uint32_t _nameref; - - union { - ValueType _value; - struct { - uint32_t _left; - uint32_t _right; - } _child; - }; - struct { - uint32_t _left; - uint32_t _right; - } _sibling; - uint32_t _parent; - }; - - - /** node store - as vector of ElementRep's */ - class ElementVector { - public: - // vector interface - void push_back(const ElementRep& elemRep) { _vec.push_back(elemRep); } - ElementRep& operator[](uint32_t index) { return _vec[index]; } - const ElementRep& operator[](uint32_t index) const { return _vec[index]; } - uint32_t size() const { return _vec.size(); } - - private: - friend class Element; - friend class SiblingIterator; - - std::vector _vec; - }; - -} // namespace mutablebson -} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_test.cpp b/src/mongo/bson/mutable/mutable_bson_test.cpp index f04a441c32e..3f8b2f40c74 100644 --- a/src/mongo/bson/mutable/mutable_bson_test.cpp +++ b/src/mongo/bson/mutable/mutable_bson_test.cpp @@ -1,4 +1,4 @@ -/* Copyright 2010 10gen Inc. +/* Copyright 2012 10gen Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,20 @@ #include "mongo/platform/basic.h" -#include "mongo/bson/mutable/mutable_bson.h" +#include "mongo/bson/mutable/document.h" #include "mongo/base/status.h" #include "mongo/bson/bsonobj.h" -#include "mongo/bson/mutable/mutable_bson_algo.h" -#include "mongo/bson/mutable/mutable_bson_builder.h" -#include "mongo/bson/mutable/mutable_bson_heap.h" -#include "mongo/bson/mutable/mutable_bson_internal.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/bson/mutable/damage_vector.h" #include "mongo/db/json.h" #include "mongo/unittest/unittest.h" namespace { + namespace mmb = mongo::mutablebson; + TEST(TopologyBuilding, TopDownFromScratch) { /* [ e0 ] @@ -42,21 +43,20 @@ namespace { [ e4 ]..[ e5 ] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element e0 = doc.makeObjElement("e0"); - mongo::mutablebson::Element e1 = doc.makeObjElement("e1"); - mongo::mutablebson::Element e2 = doc.makeObjElement("e2"); - mongo::mutablebson::Element e3 = doc.makeObjElement("e3"); - mongo::mutablebson::Element e4 = doc.makeObjElement("e4"); - mongo::mutablebson::Element e5 = doc.makeObjElement("e5"); + mmb::Element e0 = doc.makeElementObject("e0"); + mmb::Element e1 = doc.makeElementObject("e1"); + mmb::Element e2 = doc.makeElementObject("e2"); + mmb::Element e3 = doc.makeElementObject("e3"); + mmb::Element e4 = doc.makeElementObject("e4"); + mmb::Element e5 = doc.makeElementObject("e5"); - ASSERT_EQUALS(e0.addChild(e1), mongo::Status::OK()); - ASSERT_EQUALS(e0.addChild(e2), mongo::Status::OK()); - ASSERT_EQUALS(e2.addChild(e3), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e4), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e5), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e1), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e2), mongo::Status::OK()); + ASSERT_EQUALS(e2.pushBack(e3), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e4), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e5), mongo::Status::OK()); ASSERT_EQUALS("e0", e0.getFieldName()); ASSERT_EQUALS("e1", e0.leftChild().getFieldName()); @@ -88,19 +88,18 @@ namespace { [ e3 ]..[ e4 ] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element e0 = doc.makeObjElement("e0"); - mongo::mutablebson::Element e1 = doc.makeObjElement("e1"); - mongo::mutablebson::Element e2 = doc.makeObjElement("e2"); - mongo::mutablebson::Element e3 = doc.makeObjElement("e3"); - mongo::mutablebson::Element e4 = doc.makeObjElement("e4"); + mmb::Element e0 = doc.makeElementObject("e0"); + mmb::Element e1 = doc.makeElementObject("e1"); + mmb::Element e2 = doc.makeElementObject("e2"); + mmb::Element e3 = doc.makeElementObject("e3"); + mmb::Element e4 = doc.makeElementObject("e4"); - ASSERT_EQUALS(e0.addChild(e1), mongo::Status::OK()); - ASSERT_EQUALS(e0.addChild(e2), mongo::Status::OK()); - ASSERT_EQUALS(e2.addChild(e3), mongo::Status::OK()); - ASSERT_EQUALS(e3.addSiblingAfter(e4), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e1), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e2), mongo::Status::OK()); + ASSERT_EQUALS(e2.pushBack(e3), mongo::Status::OK()); + ASSERT_EQUALS(e3.addSiblingRight(e4), mongo::Status::OK()); ASSERT_EQUALS("e4", e3.rightSibling().getFieldName()); ASSERT_EQUALS("e3", e4.leftSibling().getFieldName()); @@ -120,21 +119,20 @@ namespace { [ e7 ]..[ e5 ]..[ e6 ] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element e2 = doc.makeObjElement("e2"); - mongo::mutablebson::Element e3 = doc.makeObjElement("e3"); - mongo::mutablebson::Element e4 = doc.makeObjElement("e4"); - mongo::mutablebson::Element e5 = doc.makeObjElement("e5"); - mongo::mutablebson::Element e6 = doc.makeObjElement("e6"); - mongo::mutablebson::Element e7 = doc.makeObjElement("e7"); + mmb::Element e2 = doc.makeElementObject("e2"); + mmb::Element e3 = doc.makeElementObject("e3"); + mmb::Element e4 = doc.makeElementObject("e4"); + mmb::Element e5 = doc.makeElementObject("e5"); + mmb::Element e6 = doc.makeElementObject("e6"); + mmb::Element e7 = doc.makeElementObject("e7"); - ASSERT_EQUALS(e2.addChild(e3), mongo::Status::OK()); - ASSERT_EQUALS(e2.addChild(e4), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e5), mongo::Status::OK()); - ASSERT_EQUALS(e5.addSiblingAfter(e6), mongo::Status::OK()); - ASSERT_EQUALS(e5.addSiblingBefore(e7), mongo::Status::OK()); + ASSERT_EQUALS(e2.pushBack(e3), mongo::Status::OK()); + ASSERT_EQUALS(e2.pushBack(e4), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e5), mongo::Status::OK()); + ASSERT_EQUALS(e5.addSiblingRight(e6), mongo::Status::OK()); + ASSERT_EQUALS(e5.addSiblingLeft(e7), mongo::Status::OK()); ASSERT_EQUALS("e5", e7.rightSibling().getFieldName()); ASSERT_EQUALS("e7", e5.leftSibling().getFieldName()); @@ -161,23 +159,22 @@ namespace { / \ [ e9 ]..[ e10] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); - - mongo::mutablebson::Element e3 = doc.makeObjElement("e3"); - mongo::mutablebson::Element e4 = doc.makeObjElement("e4"); - mongo::mutablebson::Element e5 = doc.makeObjElement("e5"); - mongo::mutablebson::Element e6 = doc.makeObjElement("e6"); - mongo::mutablebson::Element e8 = doc.makeObjElement("e8"); - mongo::mutablebson::Element e9 = doc.makeObjElement("e9"); - mongo::mutablebson::Element e10 = doc.makeObjElement("e10"); - - ASSERT_EQUALS(e3.addChild(e4), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e5), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e6), mongo::Status::OK()); - ASSERT_EQUALS(e8.addChild(e9), mongo::Status::OK()); - ASSERT_EQUALS(e8.addChild(e10), mongo::Status::OK()); - ASSERT_EQUALS(e5.addChild(e8), mongo::Status::OK()); + mmb::Document doc; + + mmb::Element e3 = doc.makeElementObject("e3"); + mmb::Element e4 = doc.makeElementObject("e4"); + mmb::Element e5 = doc.makeElementObject("e5"); + mmb::Element e6 = doc.makeElementObject("e6"); + mmb::Element e8 = doc.makeElementObject("e8"); + mmb::Element e9 = doc.makeElementObject("e9"); + mmb::Element e10 = doc.makeElementObject("e10"); + + ASSERT_EQUALS(e3.pushBack(e4), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e5), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e6), mongo::Status::OK()); + ASSERT_EQUALS(e8.pushBack(e9), mongo::Status::OK()); + ASSERT_EQUALS(e8.pushBack(e10), mongo::Status::OK()); + ASSERT_EQUALS(e5.pushBack(e8), mongo::Status::OK()); ASSERT_EQUALS("e8", e9.parent().getFieldName()); ASSERT_EQUALS("e8", e10.parent().getFieldName()); @@ -197,15 +194,14 @@ namespace { / \ \ [ e1 ] [ e2 ] [ e2 ] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element e0 = doc.makeObjElement("e0"); - mongo::mutablebson::Element e1 = doc.makeObjElement("e1"); - mongo::mutablebson::Element e2 = doc.makeObjElement("e2"); + mmb::Element e0 = doc.makeElementObject("e0"); + mmb::Element e1 = doc.makeElementObject("e1"); + mmb::Element e2 = doc.makeElementObject("e2"); - ASSERT_EQUALS(e0.addChild(e1), mongo::Status::OK()); - ASSERT_EQUALS(e0.addChild(e2), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e1), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e2), mongo::Status::OK()); ASSERT_EQUALS(e1.remove(), mongo::Status::OK()); ASSERT_EQUALS("e2", e0.leftChild().getFieldName()); @@ -225,24 +221,23 @@ namespace { / \ [ e9 ]..[ e10] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); - - mongo::mutablebson::Element e3 = doc.makeObjElement("e3"); - mongo::mutablebson::Element e4 = doc.makeObjElement("e4"); - mongo::mutablebson::Element e5 = doc.makeObjElement("e5"); - mongo::mutablebson::Element e6 = doc.makeObjElement("e6"); - mongo::mutablebson::Element e8 = doc.makeObjElement("e8"); - mongo::mutablebson::Element e9 = doc.makeObjElement("e9"); - mongo::mutablebson::Element e10 = doc.makeObjElement("e10"); - - ASSERT_EQUALS(e3.addChild(e4), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e5), mongo::Status::OK()); - ASSERT_EQUALS(e5.addSiblingAfter(e6), mongo::Status::OK()); - - ASSERT_EQUALS(e8.addChild(e9), mongo::Status::OK()); - ASSERT_EQUALS(e8.addChild(e10), mongo::Status::OK()); - ASSERT_EQUALS(e5.addChild(e8), mongo::Status::OK()); + mmb::Document doc; + + mmb::Element e3 = doc.makeElementObject("e3"); + mmb::Element e4 = doc.makeElementObject("e4"); + mmb::Element e5 = doc.makeElementObject("e5"); + mmb::Element e6 = doc.makeElementObject("e6"); + mmb::Element e8 = doc.makeElementObject("e8"); + mmb::Element e9 = doc.makeElementObject("e9"); + mmb::Element e10 = doc.makeElementObject("e10"); + + ASSERT_EQUALS(e3.pushBack(e4), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e5), mongo::Status::OK()); + ASSERT_EQUALS(e5.addSiblingRight(e6), mongo::Status::OK()); + + ASSERT_EQUALS(e8.pushBack(e9), mongo::Status::OK()); + ASSERT_EQUALS(e8.pushBack(e10), mongo::Status::OK()); + ASSERT_EQUALS(e5.pushBack(e8), mongo::Status::OK()); ASSERT_EQUALS(e5.remove(), mongo::Status::OK()); ASSERT_EQUALS("e3", e4.parent().getFieldName()); @@ -260,17 +255,15 @@ namespace { [ e1 ]..[ e2 ] [ e1 ]..[ e2 ] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element e0 = doc.makeObjElement("e0"); - mongo::mutablebson::Element e1 = doc.makeObjElement("e1"); - mongo::mutablebson::Element e2 = doc.makeObjElement("e2"); + mmb::Element e0 = doc.makeElementObject("e0"); + mmb::Element e1 = doc.makeElementObject("e1"); + mmb::Element e2 = doc.makeElementObject("e2"); - ASSERT_EQUALS(e0.addChild(e1), mongo::Status::OK()); - ASSERT_EQUALS(e0.addChild(e2), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e1), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e2), mongo::Status::OK()); ASSERT_EQUALS(e0.rename("f0"), mongo::Status::OK()); - ASSERT_EQUALS("f0", e0.getFieldName()); } @@ -288,21 +281,20 @@ namespace { [ e4 ]..[ e5 ] */ - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element e0 = doc.makeObjElement("e0"); - mongo::mutablebson::Element e1 = doc.makeObjElement("e1"); - mongo::mutablebson::Element e2 = doc.makeObjElement("e2"); - mongo::mutablebson::Element e3 = doc.makeObjElement("e3"); - mongo::mutablebson::Element e4 = doc.makeObjElement("e4"); - mongo::mutablebson::Element e5 = doc.makeObjElement("e5"); + mmb::Element e0 = doc.makeElementObject("e0"); + mmb::Element e1 = doc.makeElementObject("e1"); + mmb::Element e2 = doc.makeElementObject("e2"); + mmb::Element e3 = doc.makeElementObject("e3"); + mmb::Element e4 = doc.makeElementObject("e4"); + mmb::Element e5 = doc.makeElementObject("e5"); - ASSERT_EQUALS(e0.addChild(e1), mongo::Status::OK()); - ASSERT_EQUALS(e0.addChild(e2), mongo::Status::OK()); - ASSERT_EQUALS(e2.addChild(e3), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e4), mongo::Status::OK()); - ASSERT_EQUALS(e3.addChild(e5), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e1), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e2), mongo::Status::OK()); + ASSERT_EQUALS(e2.pushBack(e3), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e4), mongo::Status::OK()); + ASSERT_EQUALS(e3.pushBack(e5), mongo::Status::OK()); ASSERT_EQUALS("e0", e0.getFieldName()); ASSERT_EQUALS("e1", e0.leftChild().getFieldName()); @@ -317,7 +309,9 @@ namespace { ASSERT_EQUALS("e5", e3.rightChild().getFieldName()); ASSERT_EQUALS("e5", e4.rightSibling().getFieldName()); ASSERT_EQUALS("e4", e5.leftSibling().getFieldName()); - ASSERT_EQUALS(e3.move(e0), mongo::Status::OK()); + + ASSERT_EQUALS(e3.remove(), mongo::Status::OK()); + ASSERT_EQUALS(e0.pushBack(e3), mongo::Status::OK()); ASSERT_EQUALS("e0", e3.parent().getFieldName()); ASSERT_EQUALS("e1", e0.leftChild().getFieldName()); @@ -328,8 +322,116 @@ namespace { ASSERT_EQUALS("e5", e3.rightChild().getFieldName()); } + TEST(TopologyBuilding, CantAddAttachedAsLeftSibling) { + mmb::Document doc; + ASSERT_OK(doc.root().appendString("foo", "foo")); + mmb::Element foo = doc.root().rightChild(); + ASSERT_TRUE(foo.ok()); + ASSERT_OK(doc.root().appendString("bar", "bar")); + mmb::Element bar = doc.root().rightChild(); + ASSERT_TRUE(bar.ok()); + ASSERT_NOT_OK(foo.addSiblingLeft(bar)); + } + + TEST(TopologyBuilding, CantAddAttachedAsRightSibling) { + mmb::Document doc; + ASSERT_OK(doc.root().appendString("foo", "foo")); + mmb::Element foo = doc.root().rightChild(); + ASSERT_TRUE(foo.ok()); + ASSERT_OK(doc.root().appendString("bar", "bar")); + mmb::Element bar = doc.root().rightChild(); + ASSERT_TRUE(bar.ok()); + ASSERT_NOT_OK(foo.addSiblingRight(bar)); + } + + TEST(TopologyBuilding, CantAddAttachedAsChild) { + mmb::Document doc; + mmb::Element foo = doc.makeElementObject("foo"); + ASSERT_TRUE(foo.ok()); + ASSERT_OK(doc.root().pushBack(foo)); + ASSERT_OK(doc.root().appendString("bar", "bar")); + mmb::Element bar = doc.root().rightChild(); + ASSERT_TRUE(bar.ok()); + ASSERT_NOT_OK(foo.pushFront(bar)); + ASSERT_NOT_OK(foo.pushBack(bar)); + } + + TEST(TopologyBuilding, CantAddChildrenToNonObject) { + mmb::Document doc; + ASSERT_OK(doc.root().appendString("foo", "bar")); + mmb::Element foo = doc.root().rightChild(); + ASSERT_TRUE(foo.ok()); + mmb::Element bar = doc.makeElementString("bar", "bar"); + ASSERT_TRUE(bar.ok()); + ASSERT_NOT_OK(foo.pushFront(bar)); + ASSERT_NOT_OK(foo.pushBack(bar)); + } + + TEST(TopologyBuilding, CantAddLeftSiblingToDetached) { + mmb::Document doc; + ASSERT_OK(doc.root().appendString("foo", "foo")); + mmb::Element foo = doc.root().rightChild(); + ASSERT_TRUE(foo.ok()); + ASSERT_OK(foo.remove()); + ASSERT_FALSE(foo.parent().ok()); + mmb::Element bar = doc.makeElementString("bar", "bar"); + ASSERT_TRUE(bar.ok()); + ASSERT_NOT_OK(foo.addSiblingLeft(bar)); + } + + TEST(TopologyBuilding, CantAddRightSiblingToDetached) { + mmb::Document doc; + ASSERT_OK(doc.root().appendString("foo", "foo")); + mmb::Element foo = doc.root().rightChild(); + ASSERT_TRUE(foo.ok()); + ASSERT_OK(foo.remove()); + ASSERT_FALSE(foo.parent().ok()); + mmb::Element bar = doc.makeElementString("bar", "bar"); + ASSERT_TRUE(bar.ok()); + ASSERT_NOT_OK(foo.addSiblingRight(bar)); + } + + TEST(TopologyBuilding, AddSiblingLeftIntrusion) { + mmb::Document doc; + ASSERT_OK(doc.root().appendString("first", "first")); + mmb::Element first = doc.root().rightChild(); + ASSERT_TRUE(first.ok()); + ASSERT_OK(doc.root().appendString("last", "last")); + mmb::Element last = doc.root().rightChild(); + ASSERT_TRUE(last.ok()); + + ASSERT_EQUALS(first, last.leftSibling()); + ASSERT_EQUALS(last, first.rightSibling()); + + mmb::Element middle = doc.makeElementString("middle", "middle"); + ASSERT_TRUE(middle.ok()); + ASSERT_OK(last.addSiblingLeft(middle)); + + ASSERT_EQUALS(middle, first.rightSibling()); + ASSERT_EQUALS(middle, last.leftSibling()); + + ASSERT_EQUALS(first, middle.leftSibling()); + ASSERT_EQUALS(last, middle.rightSibling()); + } + + TEST(TopologyBuilding, AddSiblingRightIntrusion) { + mmb::Document doc; + ASSERT_OK(doc.root().appendString("first", "first")); + mmb::Element first = doc.root().rightChild(); + ASSERT_TRUE(first.ok()); + ASSERT_OK(doc.root().appendString("last", "last")); + mmb::Element last = doc.root().rightChild(); + ASSERT_TRUE(last.ok()); + mmb::Element middle = doc.makeElementString("middle", "middle"); + ASSERT_TRUE(middle.ok()); + ASSERT_OK(first.addSiblingRight(middle)); + ASSERT_EQUALS(first, middle.leftSibling()); + ASSERT_EQUALS(last, middle.rightSibling()); + } + TEST(ArrayAPI, SimpleNumericArray) { /* + { a : [] } create { a : [10] } pushBack { a : [10, 20] } pushBack { a : [10, 20, 30] } pushBack @@ -341,25 +443,29 @@ namespace { { a : [10, 20, 30] } popFront { a : [10, 20] } popBack { a : [20] } popFront + { a : [100] } set + { a : [] } popFront */ - - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); - - mongo::mutablebson::Element e1 = doc.makeArrayElement("a"); - mongo::mutablebson::Element e2 = doc.makeIntElement("", 10); - ASSERT_EQUALS(10, e2.getIntValue()); - mongo::mutablebson::Element e3 = doc.makeIntElement("", 20); - ASSERT_EQUALS(20, e3.getIntValue()); - mongo::mutablebson::Element e4 = doc.makeIntElement("", 30); - ASSERT_EQUALS(30, e4.getIntValue()); - mongo::mutablebson::Element e5 = doc.makeIntElement("", 40); - ASSERT_EQUALS(40, e5.getIntValue()); - mongo::mutablebson::Element e6 = doc.makeIntElement("", 5); - ASSERT_EQUALS(5, e6.getIntValue()); - mongo::mutablebson::Element e7 = doc.makeIntElement("", 0); - ASSERT_EQUALS(0, e7.getIntValue()); - + + mmb::Document doc; + + mmb::Element e1 = doc.makeElementArray("a"); + ASSERT_FALSE(e1[0].ok()); + ASSERT_FALSE(e1[1].ok()); + + mmb::Element e2 = doc.makeElementInt("", 10); + ASSERT_EQUALS(10, e2.getValueInt()); + mmb::Element e3 = doc.makeElementInt("", 20); + ASSERT_EQUALS(20, e3.getValueInt()); + mmb::Element e4 = doc.makeElementInt("", 30); + ASSERT_EQUALS(30, e4.getValueInt()); + mmb::Element e5 = doc.makeElementInt("", 40); + ASSERT_EQUALS(40, e5.getValueInt()); + mmb::Element e6 = doc.makeElementInt("", 5); + ASSERT_EQUALS(5, e6.getValueInt()); + mmb::Element e7 = doc.makeElementInt("", 0); + ASSERT_EQUALS(0, e7.getValueInt()); + ASSERT_EQUALS(e1.pushBack(e2), mongo::Status::OK()); ASSERT_EQUALS(e1.pushBack(e3), mongo::Status::OK()); ASSERT_EQUALS(e1.pushBack(e4), mongo::Status::OK()); @@ -367,252 +473,2541 @@ namespace { ASSERT_EQUALS(e1.pushFront(e6), mongo::Status::OK()); ASSERT_EQUALS(e1.pushFront(e7), mongo::Status::OK()); - uint32_t n; - ASSERT_EQUALS(e1.arraySize(&n), mongo::Status::OK()); - ASSERT_EQUALS(6, (int)n); - - mongo::mutablebson::Element e(&doc, mongo::mutablebson::EMPTY_REP); - ASSERT_EQUALS(e1.get(0, &e), mongo::Status::OK()); - ASSERT_EQUALS(0, e.getIntValue()); - ASSERT_EQUALS(e1.get(1, &e), mongo::Status::OK()); - ASSERT_EQUALS(5, e.getIntValue()); - ASSERT_EQUALS(e1.get(2, &e), mongo::Status::OK()); - ASSERT_EQUALS(10, e.getIntValue()); - ASSERT_EQUALS(e1.get(3, &e), mongo::Status::OK()); - ASSERT_EQUALS(20, e.getIntValue()); - ASSERT_EQUALS(e1.get(4, &e), mongo::Status::OK()); - ASSERT_EQUALS(30, e.getIntValue()); - ASSERT_EQUALS(e1.get(5, &e), mongo::Status::OK()); - ASSERT_EQUALS(40, e.getIntValue()); - - ASSERT_EQUALS(e1.peekBack(&e), mongo::Status::OK()); - ASSERT_EQUALS(40, e.getIntValue()); + ASSERT_EQUALS(size_t(6), mmb::countChildren(e1)); + ASSERT_EQUALS(0, e1[0].getValueInt()); + ASSERT_EQUALS(5, e1[1].getValueInt()); + ASSERT_EQUALS(10, e1[2].getValueInt()); + ASSERT_EQUALS(20, e1[3].getValueInt()); + ASSERT_EQUALS(30, e1[4].getValueInt()); + ASSERT_EQUALS(40, e1[5].getValueInt()); + ASSERT_EQUALS(40, e1.rightChild().getValueInt()); ASSERT_EQUALS(e1.popBack(), mongo::Status::OK()); - ASSERT_EQUALS(e1.arraySize(&n), mongo::Status::OK()); - ASSERT_EQUALS(5, (int)n); - ASSERT_EQUALS(e1.get(0, &e), mongo::Status::OK()); - ASSERT_EQUALS(0, e.getIntValue()); - ASSERT_EQUALS(e1.get(1, &e), mongo::Status::OK()); - ASSERT_EQUALS(5, e.getIntValue()); - ASSERT_EQUALS(e1.get(2, &e), mongo::Status::OK()); - ASSERT_EQUALS(10, e.getIntValue()); - ASSERT_EQUALS(e1.get(3, &e), mongo::Status::OK()); - ASSERT_EQUALS(20, e.getIntValue()); - ASSERT_EQUALS(e1.get(4, &e), mongo::Status::OK()); - ASSERT_EQUALS(30, e.getIntValue()); - - ASSERT_EQUALS(e1.peekFront(&e), mongo::Status::OK()); - ASSERT_EQUALS(0, e.getIntValue()); + + ASSERT_EQUALS(size_t(5), mmb::countChildren(e1)); + ASSERT_EQUALS(0, e1[0].getValueInt()); + ASSERT_EQUALS(5, e1[1].getValueInt()); + ASSERT_EQUALS(10, e1[2].getValueInt()); + ASSERT_EQUALS(20, e1[3].getValueInt()); + ASSERT_EQUALS(30, e1[4].getValueInt()); + ASSERT_EQUALS(0, e1.leftChild().getValueInt()); ASSERT_EQUALS(e1.popFront(), mongo::Status::OK()); - ASSERT_EQUALS(e1.arraySize(&n), mongo::Status::OK()); - ASSERT_EQUALS(4, (int)n); - ASSERT_EQUALS(e1.get(0, &e), mongo::Status::OK()); - ASSERT_EQUALS(5, e.getIntValue()); - ASSERT_EQUALS(e1.get(1, &e), mongo::Status::OK()); - ASSERT_EQUALS(10, e.getIntValue()); - ASSERT_EQUALS(e1.get(2, &e), mongo::Status::OK()); - ASSERT_EQUALS(20, e.getIntValue()); - ASSERT_EQUALS(e1.get(3, &e), mongo::Status::OK()); - ASSERT_EQUALS(30, e.getIntValue()); - - ASSERT_EQUALS(e1.peekFront(&e), mongo::Status::OK()); - ASSERT_EQUALS(5, e.getIntValue()); + + ASSERT_EQUALS(size_t(4), mmb::countChildren(e1)); + ASSERT_EQUALS(5, e1[0].getValueInt()); + ASSERT_EQUALS(10, e1[1].getValueInt()); + ASSERT_EQUALS(20, e1[2].getValueInt()); + ASSERT_EQUALS(30, e1[3].getValueInt()); + ASSERT_EQUALS(5, e1.leftChild().getValueInt()); ASSERT_EQUALS(e1.popFront(), mongo::Status::OK()); - ASSERT_EQUALS(e1.arraySize(&n), mongo::Status::OK()); - ASSERT_EQUALS(3, (int)n); - ASSERT_EQUALS(e1.get(0, &e), mongo::Status::OK()); - ASSERT_EQUALS(10, e.getIntValue()); - ASSERT_EQUALS(e1.get(1, &e), mongo::Status::OK()); - ASSERT_EQUALS(20, e.getIntValue()); - ASSERT_EQUALS(e1.get(2, &e), mongo::Status::OK()); - ASSERT_EQUALS(30, e.getIntValue()); - - ASSERT_EQUALS(e1.peekBack(&e), mongo::Status::OK()); - ASSERT_EQUALS(30, e.getIntValue()); + + ASSERT_EQUALS(size_t(3), mmb::countChildren(e1)); + ASSERT_EQUALS(10, e1[0].getValueInt()); + ASSERT_EQUALS(20, e1[1].getValueInt()); + ASSERT_EQUALS(30, e1[2].getValueInt()); + ASSERT_EQUALS(30, e1.rightChild().getValueInt()); ASSERT_EQUALS(e1.popBack(), mongo::Status::OK()); - ASSERT_EQUALS(e1.arraySize(&n), mongo::Status::OK()); - ASSERT_EQUALS(2, (int)n); - ASSERT_EQUALS(e1.get(0, &e), mongo::Status::OK()); - ASSERT_EQUALS(10, e.getIntValue()); - ASSERT_EQUALS(e1.get(1, &e), mongo::Status::OK()); - ASSERT_EQUALS(20, e.getIntValue()); - - ASSERT_EQUALS(e1.peekFront(&e), mongo::Status::OK()); - ASSERT_EQUALS(10, e.getIntValue()); + + ASSERT_EQUALS(size_t(2), mmb::countChildren(e1)); + ASSERT_EQUALS(10, e1[0].getValueInt()); + ASSERT_EQUALS(20, e1[1].getValueInt()); + ASSERT_EQUALS(10, e1.leftChild().getValueInt()); + ASSERT_EQUALS(e1.popFront(), mongo::Status::OK()); + + ASSERT_EQUALS(size_t(1), mmb::countChildren(e1)); + ASSERT_EQUALS(20, e1[0].getValueInt()); + + ASSERT_EQUALS(e1[0].setValueInt(100), mongo::Status::OK()); + ASSERT_EQUALS(100, e1[0].getValueInt()); + ASSERT_EQUALS(100, e1.leftChild().getValueInt()); + ASSERT_EQUALS(size_t(1), mmb::countChildren(e1)); ASSERT_EQUALS(e1.popFront(), mongo::Status::OK()); - ASSERT_EQUALS(e1.arraySize(&n), mongo::Status::OK()); - ASSERT_EQUALS(1, (int)n); - ASSERT_EQUALS(e1.get(0, &e), mongo::Status::OK()); - ASSERT_EQUALS(20, e.getIntValue()); - - mongo::mutablebson::Element e8 = doc.makeIntElement("", 100); - ASSERT_EQUALS(100, e8.getIntValue()); - ASSERT_EQUALS(e1.set(0, e8), mongo::Status::OK()); - ASSERT_EQUALS(e1.peekFront(&e), mongo::Status::OK()); - ASSERT_EQUALS(100, e.getIntValue()); + + ASSERT_EQUALS(size_t(0), mmb::countChildren(e1)); + ASSERT_FALSE(e1[0].ok()); + ASSERT_FALSE(e1[1].ok()); } TEST(Element, setters) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeNullElement("t0"); + mmb::Element t0 = doc.makeElementNull("t0"); - t0.setBoolValue(true); - ASSERT_EQUALS(mongo::Bool, t0.type()); + t0.setValueBool(true); + ASSERT_EQUALS(mongo::Bool, t0.getType()); - t0.setIntValue(12345); - ASSERT_EQUALS(mongo::NumberInt, t0.type()); + t0.setValueInt(12345); + ASSERT_EQUALS(mongo::NumberInt, t0.getType()); - t0.setLongValue(12345LL); - ASSERT_EQUALS(mongo::NumberLong, t0.type()); + t0.setValueLong(12345LL); + ASSERT_EQUALS(mongo::NumberLong, t0.getType()); - t0.setTSValue(mongo::OpTime()); - ASSERT_EQUALS(mongo::Timestamp, t0.type()); + t0.setValueTimestamp(mongo::OpTime()); + ASSERT_EQUALS(mongo::Timestamp, t0.getType()); - t0.setDateValue(12345LL); - ASSERT_EQUALS(mongo::Date, t0.type()); + t0.setValueDate(12345LL); + ASSERT_EQUALS(mongo::Date, t0.getType()); - t0.setDoubleValue(123.45); - ASSERT_EQUALS(mongo::NumberDouble, t0.type()); + t0.setValueDouble(123.45); + ASSERT_EQUALS(mongo::NumberDouble, t0.getType()); - t0.setOIDValue(mongo::OID("47cc67093475061e3d95369d")); - ASSERT_EQUALS(mongo::jstOID, t0.type()); + t0.setValueOID(mongo::OID("47cc67093475061e3d95369d")); + ASSERT_EQUALS(mongo::jstOID, t0.getType()); - t0.setRegexValue("[a-zA-Z]?"); - ASSERT_EQUALS(mongo::RegEx, t0.type()); + t0.setValueRegex("[a-zA-Z]?", ""); + ASSERT_EQUALS(mongo::RegEx, t0.getType()); + + t0.setValueString("foo bar baz"); + ASSERT_EQUALS(mongo::String, t0.getType()); + } - t0.setStringValue("foo bar baz"); - ASSERT_EQUALS(mongo::String, t0.type()); + TEST(Element, toString) { + mongo::BSONObj obj = mongo::fromjson("{ a : 1, b : [1, 2, 3], c : { x : 'x' } }"); + mmb::Document doc(obj); + + // Deserialize the 'c' but keep its value the same. + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + ASSERT_OK(c.appendString("y", "y")); + ASSERT_OK(c.popBack()); + + // 'a' + mongo::BSONObjIterator iter(obj); + mmb::Element docChild = doc.root().leftChild(); + ASSERT_TRUE(docChild.ok()); + ASSERT_EQUALS(iter.next().toString(), docChild.toString()); + + // 'b' + docChild = docChild.rightSibling(); + ASSERT_TRUE(docChild.ok()); + ASSERT_TRUE(iter.more()); + ASSERT_EQUALS(iter.next().toString(), mmb::ConstElement(docChild).toString()); + + // 'c' + docChild = docChild.rightSibling(); + ASSERT_TRUE(docChild.ok()); + ASSERT_TRUE(iter.more()); + ASSERT_EQUALS(iter.next().toString(), docChild.toString()); + + // eoo + docChild = docChild.rightSibling(); + ASSERT_FALSE(iter.more()); + ASSERT_FALSE(docChild.ok()); } TEST(TimestampType, createElement) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeTSElement("t0", mongo::OpTime()); - ASSERT(mongo::OpTime() == t0.getTSValue()); + mmb::Element t0 = doc.makeElementTimestamp("t0", mongo::OpTime()); + ASSERT(mongo::OpTime() == t0.getValueTimestamp()); - mongo::mutablebson::Element t1 = doc.makeTSElement("t1", mongo::OpTime(123, 456)); - ASSERT(mongo::OpTime(123, 456) == t1.getTSValue()); + mmb::Element t1 = doc.makeElementTimestamp("t1", mongo::OpTime(123, 456)); + ASSERT(mongo::OpTime(123, 456) == t1.getValueTimestamp()); } TEST(TimestampType, setElement) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeTSElement("t0", mongo::OpTime()); - t0.setTSValue(mongo::OpTime(123, 456)); - ASSERT(mongo::OpTime(123, 456) == t0.getTSValue()); + mmb::Element t0 = doc.makeElementTimestamp("t0", mongo::OpTime()); + t0.setValueTimestamp(mongo::OpTime(123, 456)); + ASSERT(mongo::OpTime(123, 456) == t0.getValueTimestamp()); // Try setting to other types and back to OpTime - t0.setLongValue(1234567890); - ASSERT_EQUALS(1234567890LL, t0.getLongValue()); - t0.setTSValue(mongo::OpTime(789, 321)); - ASSERT(mongo::OpTime(789, 321) == t0.getTSValue()); - - t0.setStringValue("foo bar baz"); - ASSERT_EQUALS(0, strcmp("foo bar baz", t0.getStringValue())); - t0.setTSValue(mongo::OpTime(9876, 5432)); - ASSERT(mongo::OpTime(9876, 5432) == t0.getTSValue()); + t0.setValueLong(1234567890); + ASSERT_EQUALS(1234567890LL, t0.getValueLong()); + t0.setValueTimestamp(mongo::OpTime(789, 321)); + ASSERT(mongo::OpTime(789, 321) == t0.getValueTimestamp()); + + t0.setValueString("foo bar baz"); + ASSERT_EQUALS("foo bar baz", t0.getValueString()); + t0.setValueTimestamp(mongo::OpTime(9876, 5432)); + ASSERT(mongo::OpTime(9876, 5432) == t0.getValueTimestamp()); } TEST(TimestampType, appendElement) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeObjElement("e0"); - t0.appendTS("a timestamp field", mongo::OpTime(1352151971, 471)); + mmb::Element t0 = doc.makeElementObject("e0"); + t0.appendTimestamp("a timestamp field", mongo::OpTime(1352151971, 471)); - mongo::mutablebson::SiblingIterator it = - mongo::mutablebson::findFirstChildNamed(t0, "a timestamp field"); - ASSERT_EQUALS(it.done(), false); - ASSERT(mongo::OpTime(1352151971, 471) == - mongo::mutablebson::Element(&doc, it.getRep()).getTSValue()); + mmb::Element it = + mmb::findFirstChildNamed(t0, "a timestamp field"); + ASSERT_TRUE(it.ok()); + ASSERT(mongo::OpTime(1352151971, 471) == it.getValueTimestamp()); } TEST(SafeNumType, createElement) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeSafeNumElement("t0", mongo::SafeNum(123.456)); - ASSERT_EQUALS(mongo::SafeNum(123.456), t0.getSafeNumValue()); + mmb::Element t0 = doc.makeElementSafeNum("t0", mongo::SafeNum(123.456)); + ASSERT_EQUALS(mongo::SafeNum(123.456), t0.getValueSafeNum()); } // Try getting SafeNums from different types. TEST(SafeNumType, getSafeNum) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeIntElement("t0", 1234567890); - ASSERT_EQUALS(1234567890, t0.getIntValue()); - mongo::SafeNum num = t0.getSafeNumValue(); + mmb::Element t0 = doc.makeElementInt("t0", 1234567890); + ASSERT_EQUALS(1234567890, t0.getValueInt()); + mongo::SafeNum num = t0.getValueSafeNum(); ASSERT_EQUALS(num, 1234567890); - t0.setLongValue(1234567890LL); - ASSERT_EQUALS(1234567890LL, t0.getLongValue()); - num = t0.getSafeNumValue(); + t0.setValueLong(1234567890LL); + ASSERT_EQUALS(1234567890LL, t0.getValueLong()); + num = t0.getValueSafeNum(); ASSERT_EQUALS(num, 1234567890LL); - t0.setDoubleValue(123.456789); - ASSERT_EQUALS(123.456789, t0.getDoubleValue()); - num = t0.getSafeNumValue(); + t0.setValueDouble(123.456789); + ASSERT_EQUALS(123.456789, t0.getValueDouble()); + num = t0.getValueSafeNum(); ASSERT_EQUALS(num, 123.456789); } TEST(SafeNumType, setSafeNum) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeSafeNumElement("t0", mongo::SafeNum(123456)); - t0.setSafeNumValue(mongo::SafeNum(654321)); - ASSERT_EQUALS(mongo::SafeNum(654321), t0.getSafeNumValue()); + mmb::Element t0 = doc.makeElementSafeNum("t0", mongo::SafeNum(123456)); + t0.setValueSafeNum(mongo::SafeNum(654321)); + ASSERT_EQUALS(mongo::SafeNum(654321), t0.getValueSafeNum()); // Try setting to other types and back to SafeNum - t0.setLongValue(1234567890); - ASSERT_EQUALS(1234567890LL, t0.getLongValue()); - t0.setSafeNumValue(mongo::SafeNum(1234567890)); - ASSERT_EQUALS(mongo::SafeNum(1234567890), t0.getSafeNumValue()); + t0.setValueLong(1234567890); + ASSERT_EQUALS(1234567890LL, t0.getValueLong()); + t0.setValueSafeNum(mongo::SafeNum(1234567890)); + ASSERT_EQUALS(mongo::SafeNum(1234567890), t0.getValueSafeNum()); + + t0.setValueString("foo bar baz"); + + mongo::StringData left = "foo bar baz"; + mongo::StringData right = t0.getValueString(); + ASSERT_EQUALS(left, right); - t0.setStringValue("foo bar baz"); - ASSERT_EQUALS("foo bar baz", std::string(t0.getStringValue())); - t0.setSafeNumValue(mongo::SafeNum(12345)); - ASSERT_EQUALS(mongo::SafeNum(12345), t0.getSafeNumValue()); + ASSERT_EQUALS(mongo::StringData("foo bar baz"), t0.getValueString()); + t0.setValueSafeNum(mongo::SafeNum(12345)); + ASSERT_EQUALS(mongo::SafeNum(12345), t0.getValueSafeNum()); } TEST(SafeNumType, appendElement) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); + mmb::Document doc; - mongo::mutablebson::Element t0 = doc.makeObjElement("e0"); + mmb::Element t0 = doc.makeElementObject("e0"); t0.appendSafeNum("a timestamp field", mongo::SafeNum(1352151971LL)); - mongo::mutablebson::SiblingIterator it = findFirstChildNamed(t0, "a timestamp field"); - ASSERT_EQUALS(it.done(), false); - ASSERT_EQUALS(mongo::SafeNum(1352151971LL), - mongo::mutablebson::Element(&doc, it.getRep()).getSafeNumValue()); + mmb::Element it = findFirstChildNamed(t0, "a timestamp field"); + ASSERT_TRUE(it.ok()); + ASSERT_EQUALS(mongo::SafeNum(1352151971LL), it.getValueSafeNum()); } TEST(OIDType, getOidValue) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); - mongo::mutablebson::Element t0 = doc.makeObjElement("e0"); + mmb::Document doc; + mmb::Element t0 = doc.makeElementObject("e0"); const mongo::OID generated = mongo::OID::gen(); t0.appendOID("myOid", generated); - mongo::mutablebson::SiblingIterator it = findFirstChildNamed(t0, "myOid"); - const mongo::OID recovered = mongo::OID((*it).getOIDValue()); + mmb::Element it = findFirstChildNamed(t0, "myOid"); + const mongo::OID recovered = mongo::OID(it.getValueOID()); ASSERT_EQUALS(generated, recovered); } TEST(OIDType, nullOID) { - mongo::mutablebson::BasicHeap myHeap; - mongo::mutablebson::Document doc(&myHeap); - mongo::mutablebson::Element t0 = doc.makeObjElement("e0"); + mmb::Document doc; + mmb::Element t0 = doc.makeElementObject("e0"); const mongo::OID withNull("50a9c82263e413ad0028faad"); t0.appendOID("myOid", withNull); - mongo::mutablebson::SiblingIterator it = findFirstChildNamed(t0, "myOid"); - const mongo::OID recovered = mongo::OID((*it).getOIDValue()); + mmb::Element it = findFirstChildNamed(t0, "myOid"); + const mongo::OID recovered = mongo::OID(it.getValueOID()); ASSERT_EQUALS(withNull, recovered); } -} // unnamed namespace + static const char jsonSample[] = + "{_id:ObjectId(\"47cc67093475061e3d95369d\")," + "query:\"kate hudson\"," + "owner:1234567887654321," + "date:\"2011-05-13T14:22:46.777Z\"," + "score:123.456," + "field1:Infinity," + "\"field2\":-Infinity," + "\"field3\":NaN," + "users:[" + "{uname:\"@aaaa\",editid:\"123\",date:1303959350,yes_votes:0,no_votes:0}," + "{uname:\"@bbbb\",editid:\"456\",date:1303959350,yes_votes:0,no_votes:0}," + "{uname:\"@cccc\",editid:\"789\",date:1303959350,yes_votes:0,no_votes:0}]," + "pattern:/match.*this/," + "lastfield:\"last\"}"; + + TEST(Serialization, RoundTrip) { + mongo::BSONObj obj = mongo::fromjson(jsonSample); + mmb::Document doc(obj.copy()); + mongo::BSONObj built = doc.getObject(); + ASSERT_EQUALS(obj, built); + } + + TEST(Documentation, Example1) { + + // Create a new document + mmb::Document doc; + ASSERT_EQUALS(mongo::fromjson("{}"), + doc.getObject()); + + // Get the root of the document. + mmb::Element root = doc.root(); + + // Create a new mongo::NumberInt typed Element to represent life, the universe, and + // everything, then push that Element into the root object, making it a child of root. + mmb::Element e0 = doc.makeElementInt("ltuae", 42); + ASSERT_OK(root.pushBack(e0)); + ASSERT_EQUALS(mongo::fromjson("{ ltuae : 42 }"), + doc.getObject()); + + // Create a new empty mongo::Object-typed Element named 'magic', and push it back as a + // child of the root, making it a sibling of e0. + mmb::Element e1 = doc.makeElementObject("magic"); + ASSERT_OK(root.pushBack(e1)); + ASSERT_EQUALS(mongo::fromjson("{ ltuae : 42, magic : {} }"), + doc.getObject()); + + // Create a new mongo::NumberDouble typed Element to represent Pi, and insert it as child + // of the new object we just created. + mmb::Element e3 = doc.makeElementDouble("pi", 3.14); + ASSERT_OK(e1.pushBack(e3)); + ASSERT_EQUALS(mongo::fromjson("{ ltuae : 42, magic : { pi : 3.14 } }"), + doc.getObject()); + + // Create a new mongo::NumberDouble to represent Plancks constant in electrovolt + // micrometers, and add it as a child of the 'magic' object. + mmb::Element e4 = doc.makeElementDouble("hbar", 1.239); + ASSERT_OK(e1.pushBack(e4)); + ASSERT_EQUALS(mongo::fromjson("{ ltuae : 42, magic : { pi : 3.14, hbar : 1.239 } }"), + doc.getObject()); + + // Rename the parent element of 'hbar' to be 'constants'. + ASSERT_OK(e4.parent().rename("constants")); + ASSERT_EQUALS(mongo::fromjson("{ ltuae : 42, constants : { pi : 3.14, hbar : 1.239 } }"), + doc.getObject()); + + // Rename 'ltuae' to 'answer' by accessing it as the root objects left child. + ASSERT_OK(doc.root().leftChild().rename("answer")); + ASSERT_EQUALS(mongo::fromjson("{ answer : 42, constants : { pi : 3.14, hbar : 1.239 } }"), + doc.getObject()); + + // Sort the constants by name. + mmb::sortChildren(doc.root().rightChild(), mmb::FieldNameLessThan()); + ASSERT_EQUALS(mongo::fromjson("{ answer : 42, constants : { hbar : 1.239, pi : 3.14 } }"), + doc.getObject()); + } + + TEST(Documentation, Example2) { + + static const char inJson[] = + "{" + " 'whale': { 'alive': true, 'dv': -9.8, 'height': 50.0, attrs : [ 'big' ] }," + " 'petunias': { 'alive': true, 'dv': -9.8, 'height': 50.0 } " + "}"; + mongo::BSONObj obj = mongo::fromjson(inJson); + + // Create a new document representing BSONObj with the above contents. + mmb::Document doc(obj); + + // The whale hits the planet and dies. + mmb::Element whale = mmb::findFirstChildNamed(doc.root(), "whale"); + ASSERT_TRUE(whale.ok()); + // Find the 'dv' field in the whale. + mmb::Element whale_deltav = mmb::findFirstChildNamed(whale, "dv"); + ASSERT_TRUE(whale_deltav.ok()); + // Set the dv field to zero. + ASSERT_OK(whale_deltav.setValueDouble(0.0)); + // Find the 'height' field in the whale. + mmb::Element whale_height = mmb::findFirstChildNamed(whale, "height"); + ASSERT_TRUE(whale_height.ok()); + // Set the height field to zero. + ASSERT_OK(whale_height.setValueDouble(0)); + // Find the 'alive' field, and set it to false. + mmb::Element whale_alive = mmb::findFirstChildNamed(whale, "alive"); + ASSERT_TRUE(whale_alive.ok()); + ASSERT_OK(whale_alive.setValueBool(false)); + + // The petunias survive, update its fields much like we did above. + mmb::Element petunias = mmb::findFirstChildNamed(doc.root(), "petunias"); + ASSERT_TRUE(petunias.ok()); + mmb::Element petunias_deltav = mmb::findFirstChildNamed(petunias, "dv"); + ASSERT_TRUE(petunias_deltav.ok()); + ASSERT_OK(petunias_deltav.setValueDouble(0.0)); + mmb::Element petunias_height = mmb::findFirstChildNamed(petunias, "height"); + ASSERT_TRUE(petunias_height.ok()); + ASSERT_OK(petunias_height.setValueDouble(0)); + + // Replace the whale by its wreckage, saving only its attributes: + // Construct a new mongo::Object element for the ex-whale. + mmb::Element ex_whale = doc.makeElementObject("ex-whale"); + ASSERT_OK(doc.root().pushBack(ex_whale)); + // Find the attributes of the old 'whale' element. + mmb::Element whale_attrs = mmb::findFirstChildNamed(whale, "attrs"); + // Remove the attributes from the whale (they remain valid, but detached). + ASSERT_OK(whale_attrs.remove()); + // Insert the attributes into the ex-whale. + ASSERT_OK(ex_whale.pushBack(whale_attrs)); + // Remove the whale object. + ASSERT_OK(whale.remove()); + + static const char outJson[] = + "{" + " 'petunias': { 'alive': true, 'dv': 0.0, 'height': 0 }," + " 'ex-whale': { 'attrs': [ 'big' ] } })" + "}"; + + mongo::BSONObjBuilder builder; + doc.writeTo(&builder); + ASSERT_EQUALS(mongo::fromjson(outJson), doc.getObject()); + } + + namespace { + void apply(mongo::BSONObj* obj, const mmb::DamageVector& damages, const char* source) { + const mmb::DamageVector::const_iterator end = damages.end(); + mmb::DamageVector::const_iterator where = damages.begin(); + char* const target = const_cast(obj->objdata()); + for ( ; where != end; ++where ) { + std::memcpy( + target + where->targetOffset, + source + where->sourceOffset, + where->size); + } + } + } // namespace + + TEST(Documentation, Example2InPlaceWithDamageVector) { + + static const char inJson[] = + "{" + " 'whale': { 'alive': true, 'dv': -9.8, 'height': 50.0, attrs : [ 'big' ] }," + " 'petunias': { 'alive': true, 'dv': -9.8, 'height': 50.0 } " + "}"; + + // Make the object, and make a copy for reference. + mongo::BSONObj obj = mongo::fromjson(inJson); + const mongo::BSONObj copyOfObj = obj.getOwned(); + ASSERT_EQUALS(obj, copyOfObj); + + // Create a new document representing BSONObj with the above contents. + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_EQUALS(obj, doc); + ASSERT_EQUALS(copyOfObj, doc); + + // Enable in-place mutation for this document + ASSERT_EQUALS(mmb::Document::kInPlaceEnabled, doc.getCurrentInPlaceMode()); + + // The whale hits the planet and dies. + mmb::Element whale = mmb::findFirstChildNamed(doc.root(), "whale"); + ASSERT_TRUE(whale.ok()); + // Find the 'dv' field in the whale. + mmb::Element whale_deltav = mmb::findFirstChildNamed(whale, "dv"); + ASSERT_TRUE(whale_deltav.ok()); + // Set the dv field to zero. + ASSERT_OK(whale_deltav.setValueDouble(0.0)); + // Find the 'height' field in the whale. + mmb::Element whale_height = mmb::findFirstChildNamed(whale, "height"); + ASSERT_TRUE(whale_height.ok()); + // Set the height field to zero. + ASSERT_OK(whale_height.setValueDouble(0)); + // Find the 'alive' field, and set it to false. + mmb::Element whale_alive = mmb::findFirstChildNamed(whale, "alive"); + ASSERT_TRUE(whale_alive.ok()); + ASSERT_OK(whale_alive.setValueBool(false)); + + // The petunias survive, update its fields much like we did above. + mmb::Element petunias = mmb::findFirstChildNamed(doc.root(), "petunias"); + ASSERT_TRUE(petunias.ok()); + mmb::Element petunias_deltav = mmb::findFirstChildNamed(petunias, "dv"); + ASSERT_TRUE(petunias_deltav.ok()); + ASSERT_OK(petunias_deltav.setValueDouble(0.0)); + mmb::Element petunias_height = mmb::findFirstChildNamed(petunias, "height"); + ASSERT_TRUE(petunias_height.ok()); + ASSERT_OK(petunias_height.setValueDouble(0)); + + // Demonstrate that while the document has changed, the underlying BSONObj has not yet + // changed. + ASSERT_FALSE(obj == doc); + ASSERT_EQUALS(copyOfObj, obj); + + // Ensure that in-place updates are still enabled. + ASSERT_EQUALS(mmb::Document::kInPlaceEnabled, doc.getCurrentInPlaceMode()); + + // Extract the damage events + mmb::DamageVector damages; + const char* source = NULL; + size_t size = 0; + ASSERT_EQUALS(true, doc.getInPlaceUpdates(&damages, &source, &size)); + ASSERT_NOT_EQUALS(0U, damages.size()); + ASSERT_NOT_EQUALS(static_cast(NULL), source); + ASSERT_NOT_EQUALS(0U, size); + + apply(&obj, damages, source); + + static const char outJson[] = + "{" + " 'whale': { 'alive': false, 'dv': 0, 'height': 0, attrs : [ 'big' ] }," + " 'petunias': { 'alive': true, 'dv': 0, 'height': 0 } " + "}"; + mongo::BSONObj outObj = mongo::fromjson(outJson); + + ASSERT_EQUALS(outObj, doc); + + mongo::BSONObjBuilder builder; + doc.writeTo(&builder); + ASSERT_EQUALS(mongo::fromjson(outJson), doc.getObject()); + } + + TEST(Documentation, Example3) { + static const char inJson[] = + "{" + " 'xs': { 'x' : 'x', 'X' : 'X' }," + " 'ys': { 'y' : 'y' }" + "}"; + mongo::BSONObj inObj = mongo::fromjson(inJson); + + mmb::Document doc(inObj); + mmb::Element xs = doc.root().leftChild(); + ASSERT_TRUE(xs.ok()); + mmb::Element ys = xs.rightSibling(); + ASSERT_TRUE(ys.ok()); + mmb::Element dne = ys.rightSibling(); + ASSERT_FALSE(dne.ok()); + mmb::Element ycaps = doc.makeElementString("Y", "Y"); + ASSERT_OK(ys.pushBack(ycaps)); + mmb::Element pun = doc.makeElementArray("why"); + ASSERT_OK(ys.pushBack(pun)); + pun.appendString("na", "not"); + mongo::BSONObj outObj = doc.getObject(); + + static const char outJson[] = + "{" + " 'xs': { 'x' : 'x', 'X' : 'X' }," + " 'ys': { 'y' : 'y', 'Y' : 'Y', 'why' : ['not'] }" + "}"; + ASSERT_EQUALS(mongo::fromjson(outJson), outObj); + } + + TEST(Document, LifecycleConstructDefault) { + // Verify the state of a newly created empty Document. + mmb::Document doc; + ASSERT_TRUE(doc.root().ok()); + ASSERT_TRUE(const_cast(doc).root().ok()); + ASSERT_TRUE(doc.root().isType(mongo::Object)); + ASSERT_FALSE(doc.root().leftSibling().ok()); + ASSERT_FALSE(doc.root().rightSibling().ok()); + ASSERT_FALSE(doc.root().leftChild().ok()); + ASSERT_FALSE(doc.root().rightChild().ok()); + ASSERT_FALSE(doc.root().parent().ok()); + ASSERT_FALSE(doc.root().hasValue()); + } + + TEST(Document, LifecycleConstructEmptyBSONObj) { + // Verify the state of a newly created empty Document where the construction argument + // is an empty BSONObj. + mongo::BSONObj obj; + mmb::Document doc(obj); + ASSERT_TRUE(doc.root().ok()); + ASSERT_TRUE(const_cast(doc).root().ok()); + ASSERT_TRUE(doc.root().isType(mongo::Object)); + ASSERT_FALSE(doc.root().leftSibling().ok()); + ASSERT_FALSE(doc.root().rightSibling().ok()); + ASSERT_FALSE(doc.root().leftChild().ok()); + ASSERT_FALSE(doc.root().rightChild().ok()); + ASSERT_FALSE(doc.root().parent().ok()); + ASSERT_FALSE(doc.root().hasValue()); + } + + TEST(Document, LifecycleConstructSimpleBSONObj) { + // Verify the state of a newly created Document where the construction argument is a + // simple (flat) BSONObj. + mongo::BSONObj obj = mongo::fromjson("{ e1: 1, e2: 'hello', e3: false }"); + mmb::Document doc(obj); + + // Check the state of the root. + ASSERT_TRUE(doc.root().ok()); + ASSERT_TRUE(const_cast(doc).root().ok()); + ASSERT_TRUE(doc.root().isType(mongo::Object)); + ASSERT_FALSE(doc.root().parent().ok()); + ASSERT_FALSE(doc.root().leftSibling().ok()); + ASSERT_FALSE(doc.root().rightSibling().ok()); + ASSERT_FALSE(doc.root().hasValue()); + + mmb::ConstElement e1Child = doc.root().leftChild(); + // Check the connectivity of 'e1'. + ASSERT_TRUE(e1Child.ok()); + ASSERT_EQUALS(doc.root(), e1Child.parent()); + ASSERT_FALSE(e1Child.leftSibling().ok()); + ASSERT_TRUE(e1Child.rightSibling().ok()); + ASSERT_FALSE(e1Child.leftChild().ok()); + ASSERT_FALSE(e1Child.rightChild().ok()); + + // Check the type, name, and value of 'e1'. + ASSERT_TRUE(e1Child.isType(mongo::NumberInt)); + ASSERT_EQUALS("e1", e1Child.getFieldName()); + ASSERT_TRUE(e1Child.hasValue()); + ASSERT_EQUALS(int32_t(1), e1Child.getValueInt()); + + mmb::ConstElement e2Child = e1Child.rightSibling(); + // Check the connectivity of 'e2'. + ASSERT_TRUE(e2Child.ok()); + ASSERT_EQUALS(doc.root(), e2Child.parent()); + ASSERT_TRUE(e2Child.leftSibling().ok()); + ASSERT_TRUE(e2Child.rightSibling().ok()); + ASSERT_EQUALS(e1Child, e2Child.leftSibling()); + ASSERT_FALSE(e2Child.leftChild().ok()); + ASSERT_FALSE(e2Child.rightChild().ok()); + + // Check the type, name and value of 'e2'. + ASSERT_TRUE(e2Child.isType(mongo::String)); + ASSERT_EQUALS("e2", e2Child.getFieldName()); + ASSERT_TRUE(e2Child.hasValue()); + ASSERT_EQUALS("hello", e2Child.getValueString()); + + mmb::ConstElement e3Child = e2Child.rightSibling(); + // Check the connectivity of 'e3'. + ASSERT_TRUE(e3Child.ok()); + ASSERT_EQUALS(doc.root(), e3Child.parent()); + ASSERT_TRUE(e3Child.leftSibling().ok()); + ASSERT_FALSE(e3Child.rightSibling().ok()); + ASSERT_EQUALS(e2Child, e3Child.leftSibling()); + ASSERT_FALSE(e2Child.leftChild().ok()); + ASSERT_FALSE(e2Child.rightChild().ok()); + + // Check the type, name and value of 'e3'. + ASSERT_TRUE(e3Child.isType(mongo::Bool)); + ASSERT_EQUALS("e3", e3Child.getFieldName()); + ASSERT_TRUE(e3Child.hasValue()); + ASSERT_EQUALS(false, e3Child.getValueBool()); + } + + TEST(Document, RenameDeserialization) { + // Regression test for a bug where certain rename operations failed to deserialize up + // the tree correctly, resulting in a lost rename + static const char inJson[] = + "{" + " 'a' : { 'b' : { 'c' : { 'd' : 4 } } }" + "}"; + mongo::BSONObj inObj = mongo::fromjson(inJson); + + mmb::Document doc(inObj); + mmb::Element a = doc.root().leftChild(); + ASSERT_TRUE(a.ok()); + mmb::Element b = a.leftChild(); + ASSERT_TRUE(b.ok()); + mmb::Element c = b.leftChild(); + ASSERT_TRUE(c.ok()); + c.rename("C"); + mongo::BSONObj outObj = doc.getObject(); + static const char outJson[] = + "{" + " 'a' : { 'b' : { 'C' : { 'd' : 4 } } }" + "}"; + ASSERT_EQUALS(mongo::fromjson(outJson), outObj); + } + + TEST(Document, CantRenameRootElement) { + mmb::Document doc; + ASSERT_NOT_OK(doc.root().rename("foo")); + } + + TEST(Document, RemoveElementWithOpaqueRightSibling) { + // Regression test for a bug where removing an element with an opaque right sibling + // would access an invalidated rep. Note that this test may or may not fail depending + // on the details of memory allocation: failures would be clearly visible with + // valgrind, however. + static const char inJson[] = + "{" + " 'a' : 1, 'b' : 2, 'c' : 3" + "}"; + + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + + mmb::Element a = doc.root().leftChild(); + ASSERT_TRUE(a.ok()); + a.remove(); + + static const char outJson[] = + "{" + " 'b' : 2, 'c' : 3" + "}"; + mongo::BSONObj outObj = doc.getObject(); + ASSERT_EQUALS(mongo::fromjson(outJson), outObj); + } + + TEST(Document, AddRightSiblingToElementWithOpaqueRightSibling) { + // Regression test for a bug where adding a right sibling to a node with an opaque + // right sibling would potentially access an invalidated rep. Like the 'remove' test + // above, this may or may not crash, but would be visible under a memory checking tool. + static const char inJson[] = + "{" + " 'a' : 1, 'b' : 2, 'c' : 3" + "}"; + + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + + mmb::Element a = doc.root().leftChild(); + ASSERT_TRUE(a.ok()); + mmb::Element newElt = doc.makeElementString("X", "X"); + ASSERT_OK(a.addSiblingRight(newElt)); + + static const char outJson[] = + "{" + " 'a' : 1, 'X' : 'X', 'b' : 2, 'c' : 3" + "}"; + mongo::BSONObj outObj = doc.getObject(); + ASSERT_EQUALS(mongo::fromjson(outJson), outObj); + } + + TEST(Document, ArrayIndexedAccessFromJson) { + static const char inJson[] = + "{" + " a : 1, b : [{ c : 1 }]" + "}"; + + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + + mmb::Element a = doc.root().leftChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS("a", a.getFieldName()); + ASSERT_EQUALS(mongo::NumberInt, a.getType()); + + mmb::Element b = a.rightSibling(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS("b", b.getFieldName()); + ASSERT_EQUALS(mongo::Array, b.getType()); + + mmb::Element b0 = b[0]; + ASSERT_TRUE(b0.ok()); + ASSERT_EQUALS("0", b0.getFieldName()); + ASSERT_EQUALS(mongo::Object, b0.getType()); + } + + TEST(Document, ArrayIndexedAccessFromManuallyBuilt) { + mmb::Document doc; + mmb::Element root = doc.root(); + ASSERT_TRUE(root.ok()); + { + ASSERT_OK(root.appendInt("a", 1)); + mmb::Element b = doc.makeElementArray("b"); + ASSERT_TRUE(b.ok()); + ASSERT_OK(root.pushBack(b)); + mmb::Element b0 = doc.makeElementObject("ignored"); + ASSERT_TRUE(b0.ok()); + ASSERT_OK(b.pushBack(b0)); + ASSERT_OK(b0.appendInt("c", 1)); + } + + mmb::Element a = doc.root().leftChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS("a", a.getFieldName()); + ASSERT_EQUALS(mongo::NumberInt, a.getType()); + + mmb::Element b = a.rightSibling(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS("b", b.getFieldName()); + ASSERT_EQUALS(mongo::Array, b.getType()); + + mmb::Element b0 = b[0]; + ASSERT_TRUE(b0.ok()); + ASSERT_EQUALS("ignored", b0.getFieldName()); + ASSERT_EQUALS(mongo::Object, b0.getType()); + } + + TEST(Document, EndElement) { + mmb::Document doc; + mmb::Element end = doc.end(); + ASSERT_FALSE(end.ok()); + mmb::Element missing = doc.root().leftChild(); + ASSERT_EQUALS(end, missing); + missing = doc.root().rightChild(); + ASSERT_EQUALS(end, missing); + missing = doc.root().leftSibling(); + ASSERT_EQUALS(end, missing); + missing = doc.root().rightSibling(); + ASSERT_EQUALS(end, missing); + } + + TEST(Document, ConstEndElement) { + const mmb::Document doc; + mmb::ConstElement end = doc.end(); + ASSERT_FALSE(end.ok()); + mmb::ConstElement missing = doc.root().leftChild(); + ASSERT_EQUALS(end, missing); + missing = doc.root().rightChild(); + ASSERT_EQUALS(end, missing); + missing = doc.root().leftSibling(); + ASSERT_EQUALS(end, missing); + missing = doc.root().rightSibling(); + ASSERT_EQUALS(end, missing); + } + + TEST(Element, EmptyDocHasNoChildren) { + mmb::Document doc; + ASSERT_FALSE(doc.root().hasChildren()); + } + + TEST(Element, PopulatedDocHasChildren) { + mmb::Document doc; + ASSERT_OK(doc.root().appendInt("a", 1)); + ASSERT_TRUE(doc.root().hasChildren()); + mmb::Element lc = doc.root().leftChild(); + ASSERT_FALSE(lc.hasChildren()); + } + + TEST(Element, LazyEmptyDocHasNoChildren) { + static const char inJson[] = "{}"; + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + ASSERT_FALSE(doc.root().hasChildren()); + } + + TEST(Element, LazySingletonDocHasChildren) { + static const char inJson[] = "{ a : 1 }"; + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + ASSERT_TRUE(doc.root().hasChildren()); + ASSERT_FALSE(doc.root().leftChild().hasChildren()); + } + + TEST(Element, LazyConstDoubletonDocHasChildren) { + static const char inJson[] = "{ a : 1, b : 2 }"; + mongo::BSONObj inObj = mongo::fromjson(inJson); + const mmb::Document doc(inObj); + ASSERT_TRUE(doc.root().hasChildren()); + ASSERT_FALSE(doc.root().leftChild().hasChildren()); + ASSERT_FALSE(doc.root().rightChild().hasChildren()); + ASSERT_FALSE(doc.root().leftChild() == doc.root().rightChild()); + } + + TEST(Document, AddChildToEmptyOpaqueSubobject) { + mongo::BSONObj inObj = mongo::fromjson("{a: {}}"); + mmb::Document doc(inObj); + + mmb::Element elem = doc.root()["a"]; + ASSERT_TRUE(elem.ok()); + + mmb::Element newElem = doc.makeElementInt("0", 1); + ASSERT_TRUE(newElem.ok()); + + ASSERT_OK(elem.pushBack(newElem)); + } + + TEST(Element, IsNumeric) { + mmb::Document doc; + + mmb::Element elt = doc.makeElementNull("dummy"); + ASSERT_FALSE(elt.isNumeric()); + + elt = doc.makeElementInt("dummy", 42); + ASSERT_TRUE(elt.isNumeric()); + + elt = doc.makeElementString("dummy", "dummy"); + ASSERT_FALSE(elt.isNumeric()); + + elt = doc.makeElementLong("dummy", 42); + ASSERT_TRUE(elt.isNumeric()); + + elt = doc.makeElementBool("dummy", false); + ASSERT_FALSE(elt.isNumeric()); + + elt = doc.makeElementDouble("dummy", 42.0); + ASSERT_TRUE(elt.isNumeric()); + } + + TEST(Element, IsIntegral) { + mmb::Document doc; + + mmb::Element elt = doc.makeElementNull("dummy"); + ASSERT_FALSE(elt.isIntegral()); + + elt = doc.makeElementInt("dummy", 42); + ASSERT_TRUE(elt.isIntegral()); + + elt = doc.makeElementString("dummy", "dummy"); + ASSERT_FALSE(elt.isIntegral()); + + elt = doc.makeElementLong("dummy", 42); + ASSERT_TRUE(elt.isIntegral()); + + elt = doc.makeElementDouble("dummy", 42.0); + ASSERT_FALSE(elt.isIntegral()); + } + + TEST(Document, ArraySerialization) { + + static const char inJson[] = + "{ " + " 'a' : { 'b' : [ 'c', 'd' ] } " + "}"; + + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + + mmb::Element root = doc.root(); + mmb::Element a = root.leftChild(); + mmb::Element b = a.leftChild(); + mmb::Element new_array = doc.makeElementArray("XXX"); + mmb::Element e = doc.makeElementString("e", "e"); + new_array.pushBack(e); + b.pushBack(new_array); + + static const char outJson[] = + "{ " + " 'a' : { 'b' : [ 'c', 'd', [ 'e' ] ] } " + "}"; + + const mongo::BSONObj outObj = doc.getObject(); + ASSERT_EQUALS(mongo::fromjson(outJson), outObj); + } + + TEST(Document, SetValueBSONElementFieldNameHandling) { + static const char inJson[] = "{ a : 4 }"; + mongo::BSONObj inObj = mongo::fromjson(inJson); + mmb::Document doc(inObj); + + static const char inJson2[] = "{ b : 5 }"; + mongo::BSONObj inObj2 = mongo::fromjson(inJson2); + mongo::BSONObjIterator iterator = inObj2.begin(); + + ASSERT_TRUE(iterator.more()); + const mongo::BSONElement b = iterator.next(); + + mmb::Element a = doc.root().leftChild(); + a.setValueBSONElement(b); + + static const char outJson[] = "{ a : 5 }"; + ASSERT_EQUALS(mongo::fromjson(outJson), doc.getObject()); + } + + TEST(Document, CreateElementWithEmptyFieldName) { + mmb::Document doc; + mmb::Element noname = doc.makeElementObject(mongo::StringData()); + ASSERT_TRUE(noname.ok()); + ASSERT_EQUALS(mongo::StringData(), noname.getFieldName()); + } + + TEST(Document, CreateElementFromBSONElement) { + mongo::BSONObj obj = mongo::fromjson("{a:1}}"); + mmb::Document doc; + ASSERT_OK(doc.root().appendElement(obj["a"])); + + mmb::Element newElem = doc.root()["a"]; + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(newElem.getType(), mongo::NumberInt); + ASSERT_EQUALS(newElem.getValueInt(), 1); + } + + TEST(Document, toStringEmpty) { + mongo::BSONObj obj; + mmb::Document doc; + ASSERT_EQUALS(obj.toString(), doc.toString()); + } + + TEST(Document, toStringComplex) { + mongo::BSONObj obj = mongo::fromjson("{a : 1, b : [1, 2, 3], c : 'c'}"); + mmb::Document doc(obj); + ASSERT_EQUALS(obj.toString(), doc.toString()); + } + + TEST(Document, toStringEphemeralObject) { + mmb::Document doc; + mmb::Element e = doc.makeElementObject("foo"); + ASSERT_OK(doc.root().pushBack(e)); + ASSERT_OK(e.appendDouble("d", 1.0)); + ASSERT_OK(e.appendString("s", "str")); + ASSERT_EQUALS( + mongo::fromjson("{ foo: { d : 1.0, s : 'str' } }").firstElement().toString(), + e.toString()); + } + + TEST(Document, toStringEphemeralArray) { + mmb::Document doc; + mmb::Element e = doc.makeElementArray("foo"); + ASSERT_OK(doc.root().pushBack(e)); + ASSERT_OK(e.appendDouble(mongo::StringData(), 1.0)); + ASSERT_OK(e.appendString(mongo::StringData(), "str")); + ASSERT_EQUALS( + mongo::fromjson("{ foo: [ 1.0, 'str' ] }").firstElement().toString(), + e.toString()); + } + + TEST(Document, ElementCloningToDifferentDocument) { + + const char initial[] = "{ a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6 ] }"; + + mmb::Document source(mongo::fromjson(initial)); + + // Dirty the 'd' node and parents. + source.root()["d"].pushBack(source.makeElementInt(mongo::StringData(), 7)); + + mmb::Document target; + + mmb::Element newElement = target.makeElement(source.root()["d"]); + ASSERT_TRUE(newElement.ok()); + mongo::Status status = target.root().pushBack(newElement); + ASSERT_OK(status); + const char* expected = + "{ d : [ 4, 5, 6, 7 ] }"; + ASSERT_EQUALS(mongo::fromjson(expected), target); + + newElement = target.makeElement(source.root()["b"]); + ASSERT_TRUE(newElement.ok()); + status = target.root().pushBack(newElement); + ASSERT_OK(status); + expected = + "{ d : [ 4, 5, 6, 7 ], b : [ 1, 2, 3 ] }"; + ASSERT_EQUALS(mongo::fromjson(expected), target); + + newElement = target.makeElementWithNewFieldName("C", source.root()["c"]); + ASSERT_TRUE(newElement.ok()); + status = target.root().pushBack(newElement); + ASSERT_OK(status); + expected = + "{ d : [ 4, 5, 6, 7 ], b : [ 1, 2, 3 ], C : { 'c' : 'c' } }"; + ASSERT_EQUALS(mongo::fromjson(expected), target); + } + + TEST(Document, ElementCloningToSameDocument) { + + const char initial[] = "{ a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6 ] }"; + + mmb::Document doc(mongo::fromjson(initial)); + + // Dirty the 'd' node and parents. + doc.root()["d"].pushBack(doc.makeElementInt(mongo::StringData(), 7)); + + mmb::Element newElement = doc.makeElement(doc.root()["d"]); + ASSERT_TRUE(newElement.ok()); + mongo::Status status = doc.root().pushBack(newElement); + ASSERT_OK(status); + const char* expected = + "{ " + " a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6, 7 ], " + " d : [ 4, 5, 6, 7 ] " + "}"; + ASSERT_EQUALS(mongo::fromjson(expected), doc); + + newElement = doc.makeElement(doc.root()["b"]); + ASSERT_TRUE(newElement.ok()); + status = doc.root().pushBack(newElement); + ASSERT_OK(status); + expected = + "{ " + " a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6, 7 ], " + " d : [ 4, 5, 6, 7 ], " + " b : [ 1, 2, 3 ] " + "}"; + ASSERT_EQUALS(mongo::fromjson(expected), doc); + + newElement = doc.makeElementWithNewFieldName("C", doc.root()["c"]); + ASSERT_TRUE(newElement.ok()); + status = doc.root().pushBack(newElement); + ASSERT_OK(status); + expected = + "{ " + " a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6, 7 ], " + " d : [ 4, 5, 6, 7 ], " + " b : [ 1, 2, 3 ], " + " C : { 'c' : 'c' } " + "}"; + ASSERT_EQUALS(mongo::fromjson(expected), doc); + } + + TEST(Document, RootCloningToDifferentDocument) { + + const char initial[] = "{ a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6 ] }"; + + mmb::Document source(mongo::fromjson(initial)); + + // Dirty the 'd' node and parents. + source.root()["d"].pushBack(source.makeElementInt(mongo::StringData(), 7)); + + mmb::Document target; + + mmb::Element newElement = target.makeElementWithNewFieldName("X", source.root()); + mongo::Status status = target.root().pushBack(newElement); + ASSERT_OK(status); + const char expected[] = + "{ X : { a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6, 7 ] } }"; + + ASSERT_EQUALS(mongo::fromjson(expected), target); + } + + TEST(Document, RootCloningToSameDocument) { + + const char initial[] = "{ a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6 ] }"; + + mmb::Document doc(mongo::fromjson(initial)); + + // Dirty the 'd' node and parents. + doc.root()["d"].pushBack(doc.makeElementInt(mongo::StringData(), 7)); + + mmb::Element newElement = doc.makeElementWithNewFieldName("X", doc.root()); + mongo::Status status = doc.root().pushBack(newElement); + ASSERT_OK(status); + const char expected[] = + "{ " + " a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6, 7 ], " + "X : { a : 1, b : [ 1, 2, 3 ], c : { 'c' : 'c' }, d : [ 4, 5, 6, 7 ] }" + "}"; + + ASSERT_EQUALS(mongo::fromjson(expected), doc); + } + + TEST(Element, PopOpsOnEmpty) { + mmb::Document doc; + mmb::Element root = doc.root(); + ASSERT_NOT_OK(root.popFront()); + ASSERT_NOT_OK(root.popBack()); + } + + TEST(Document, NameOfRootElementIsEmpty) { + mmb::Document doc; + // NOTE: You really shouldn't rely on this behavior; this test is mostly for coverage. + ASSERT_EQUALS(mongo::StringData(), doc.root().getFieldName()); + } + + TEST(Document, SetValueOnRootFails) { + mmb::Document doc; + ASSERT_NOT_OK(doc.root().setValueInt(5)); + } + + TEST(Document, ValueOfEphemeralObjectElementIsEmpty) { + mmb::Document doc; + mmb::Element root = doc.root(); + mmb::Element ephemeralObject = doc.makeElementObject("foo"); + ASSERT_OK(root.pushBack(ephemeralObject)); + ASSERT_FALSE(ephemeralObject.hasValue()); + // NOTE: You really shouldn't rely on this behavior; this test is mostly for coverage. + ASSERT_EQUALS(mongo::BSONElement(), ephemeralObject.getValue()); + } + + TEST(Element, RemovingRemovedElementFails) { + // Once an Element is removed, you can't remove it again until you re-attach it + // somewhere. However, its children are still manipulable. + mmb::Document doc(mongo::fromjson("{ a : { b : 'c' } }")); + mmb::Element a = doc.root().leftChild(); + ASSERT_TRUE(a.ok()); + ASSERT_OK(a.remove()); + ASSERT_NOT_OK(a.remove()); + mmb::Element b = a.leftChild(); + ASSERT_OK(b.remove()); + ASSERT_NOT_OK(b.remove()); + ASSERT_OK(a.pushBack(b)); + ASSERT_OK(b.remove()); + } + + namespace { + // Checks that two BSONElements are byte-for-byte identical. + bool identical(const mongo::BSONElement& lhs, const mongo::BSONElement& rhs) { + if (lhs.size() != rhs.size()) + return false; + return std::memcmp(lhs.rawdata(), rhs.rawdata(), lhs.size()) == 0; + } + } // namespace + + TEST(TypeSupport, EncodingEquivalenceDouble) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const double value1 = 3.1415926; + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::NumberDouble); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendDouble(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::NumberDouble); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueDouble()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::NumberDouble); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueDouble(value1); + ASSERT_EQUALS(c.getType(), mongo::NumberDouble); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceString) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const std::string value1 = "value1"; + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::String); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendString(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::String); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueString()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::String); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueString(value1); + ASSERT_EQUALS(c.getType(), mongo::String); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceObject) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const mongo::BSONObj value1 = mongo::fromjson("{ a : 1, b : 2.0, c : 'hello' }"); + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Object); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendObject(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Object); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueObject()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Object); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueObject(value1); + ASSERT_EQUALS(c.getType(), mongo::Object); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceArray) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const mongo::BSONObj dummy = (mongo::fromjson("{ x : [ 1, 2.0, 'hello' ] } ")); + const mongo::BSONArray value1(dummy.firstElement().embeddedObject()); + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Array); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendArray(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Array); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueArray()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Array); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueArray(value1); + ASSERT_EQUALS(c.getType(), mongo::Array); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceBinary) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const mongo::BinDataType value1 = mongo::newUUID; + const unsigned char value2[] = { + 0x00, 0x9D, 0x15, 0xA3, + 0x3B, 0xCC, 0x46, 0x60, + 0x90, 0x45, 0xEF, 0x54, + 0x77, 0x8A, 0x87, 0x0C + }; + builder.appendBinData(name, sizeof(value2), value1, &value2[0]); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::BinData); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendBinary(name, sizeof(value2), value1, &value2[0])); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::BinData); + ASSERT_TRUE(a.hasValue()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::BinData); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueBinary(sizeof(value2), value1, &value2[0]); + ASSERT_EQUALS(c.getType(), mongo::BinData); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceUndefined) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + builder.appendUndefined(name); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Undefined); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendUndefined(name)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Undefined); + ASSERT_TRUE(a.hasValue()); + ASSERT_TRUE(mmb::ConstElement(a).isValueUndefined()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Undefined); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueUndefined(); + ASSERT_EQUALS(c.getType(), mongo::Undefined); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceOID) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const mongo::OID value1 = mongo::OID::gen(); + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::jstOID); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendOID(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::jstOID); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueOID()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::jstOID); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueOID(value1); + ASSERT_EQUALS(c.getType(), mongo::jstOID); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceBoolean) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const bool value1 = true; + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Bool); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendBool(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Bool); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueBool()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Bool); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueBool(value1); + ASSERT_EQUALS(c.getType(), mongo::Bool); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceDate) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const mongo::Date_t value1 = mongo::jsTime(); + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Date); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendDate(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Date); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueDate()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Date); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueDate(value1); + ASSERT_EQUALS(c.getType(), mongo::Date); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceNull) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + builder.appendNull(name); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::jstNULL); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::jstNULL); + ASSERT_TRUE(a.hasValue()); + ASSERT_TRUE(mmb::ConstElement(a).isValueNull()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::jstNULL); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendUndefined(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueNull(); + ASSERT_EQUALS(c.getType(), mongo::jstNULL); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceRegex) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const std::string value1 = "some_regex_data"; + const std::string value2 = "flags"; + builder.appendRegex(name, value1, value2); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::RegEx); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendRegex(name, value1, value2)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::RegEx); + ASSERT_TRUE(a.hasValue()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::RegEx); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueRegex(value1, value2); + ASSERT_EQUALS(c.getType(), mongo::RegEx); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceDBRef) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const std::string value1 = "some_ns"; + const mongo::OID value2 = mongo::OID::gen(); + builder.appendDBRef(name, value1, value2); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::DBRef); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendDBRef(name, value1, value2)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::DBRef); + ASSERT_TRUE(a.hasValue()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::DBRef); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueDBRef(value1, value2); + ASSERT_EQUALS(c.getType(), mongo::DBRef); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceCode) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const std::string value1 = "{ print 4; }"; + builder.appendCode(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Code); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendCode(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Code); + ASSERT_TRUE(a.hasValue()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Code); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueCode(value1); + ASSERT_EQUALS(c.getType(), mongo::Code); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceSymbol) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const std::string value1 = "#symbol"; + builder.appendSymbol(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Symbol); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendSymbol(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Symbol); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueSymbol()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Symbol); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueSymbol(value1); + ASSERT_EQUALS(c.getType(), mongo::Symbol); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceCodeWithScope) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const std::string value1 = "print x;"; + const mongo::BSONObj value2 = mongo::fromjson("{ x : 4 }"); + builder.appendCodeWScope(name, value1, value2); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::CodeWScope); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendCodeWithScope(name, value1, value2)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::CodeWScope); + ASSERT_TRUE(a.hasValue()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::CodeWScope); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueCodeWithScope(value1, value2); + ASSERT_EQUALS(c.getType(), mongo::CodeWScope); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceInt) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const int value1 = true; + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::NumberInt); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendInt(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::NumberInt); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueInt()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::NumberInt); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueInt(value1); + ASSERT_EQUALS(c.getType(), mongo::NumberInt); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceTimestamp) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const mongo::OpTime value1 = mongo::OpTime(mongo::jsTime()); + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::Timestamp); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendTimestamp(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::Timestamp); + ASSERT_TRUE(a.hasValue()); + ASSERT_TRUE(value1 == mmb::ConstElement(a).getValueTimestamp()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::Timestamp); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueTimestamp(value1); + ASSERT_EQUALS(c.getType(), mongo::Timestamp); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceLong) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + const long long value1 = 420000000000000LL; + builder.append(name, value1); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::NumberLong); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendLong(name, value1)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::NumberLong); + ASSERT_TRUE(a.hasValue()); + ASSERT_EQUALS(value1, mmb::ConstElement(a).getValueLong()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::NumberLong); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueLong(value1); + ASSERT_EQUALS(c.getType(), mongo::NumberLong); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceMinKey) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + builder.appendMinKey(name); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::MinKey); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendMinKey(name)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::MinKey); + ASSERT_TRUE(a.hasValue()); + ASSERT_TRUE(mmb::ConstElement(a).isValueMinKey()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::MinKey); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueMinKey(); + ASSERT_EQUALS(c.getType(), mongo::MinKey); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(TypeSupport, EncodingEquivalenceMaxKey) { + mongo::BSONObjBuilder builder; + const char name[] = "thing"; + builder.appendMaxKey(name); + mongo::BSONObj source = builder.done(); + const mongo::BSONElement thing = source.firstElement(); + ASSERT_TRUE(thing.type() == mongo::MaxKey); + + mmb::Document doc; + + // Construct via direct call to append/make + ASSERT_OK(doc.root().appendMaxKey(name)); + mmb::Element a = doc.root().rightChild(); + ASSERT_TRUE(a.ok()); + ASSERT_EQUALS(a.getType(), mongo::MaxKey); + ASSERT_TRUE(a.hasValue()); + ASSERT_TRUE(mmb::ConstElement(a).isValueMaxKey()); + + // Construct via call passing BSON element + ASSERT_OK(doc.root().appendElement(thing)); + mmb::Element b = doc.root().rightChild(); + ASSERT_TRUE(b.ok()); + ASSERT_EQUALS(b.getType(), mongo::MaxKey); + ASSERT_TRUE(b.hasValue()); + + // Construct via setValue call. + ASSERT_OK(doc.root().appendNull(name)); + mmb::Element c = doc.root().rightChild(); + ASSERT_TRUE(c.ok()); + c.setValueMaxKey(); + ASSERT_EQUALS(c.getType(), mongo::MaxKey); + ASSERT_TRUE(c.hasValue()); + + // Ensure identity: + ASSERT_TRUE(identical(thing, mmb::ConstElement(a).getValue())); + ASSERT_TRUE(identical(a.getValue(), b.getValue())); + ASSERT_TRUE(identical(b.getValue(), c.getValue())); + } + + TEST(Document, ManipulateComplexObjInLeafHeap) { + // Test that an object with complex substructure that lives in the leaf builder can be + // manipulated in the same way as an object with complex substructure that lives + // freely. + mmb::Document doc; + static const char inJson[] = "{ a: 1, b: 2, d : ['w', 'x', 'y', 'z'] }"; + mmb::Element embedded = doc.makeElementObject("embedded", mongo::fromjson(inJson)); + ASSERT_OK(doc.root().pushBack(embedded)); + mmb::Element free = doc.makeElementObject("free"); + ASSERT_OK(doc.root().pushBack(free)); + + mmb::Element e_a = embedded.leftChild(); + ASSERT_TRUE(e_a.ok()); + ASSERT_EQUALS("a", e_a.getFieldName()); + mmb::Element e_b = e_a.rightSibling(); + ASSERT_TRUE(e_b.ok()); + ASSERT_EQUALS("b", e_b.getFieldName()); + + mmb::Element new_c = doc.makeElementDouble("c", 2.0); + ASSERT_TRUE(new_c.ok()); + ASSERT_OK(e_b.addSiblingRight(new_c)); + + mmb::Element e_d = new_c.rightSibling(); + ASSERT_TRUE(e_d.ok()); + ASSERT_EQUALS("d", e_d.getFieldName()); + + mmb::Element e_d_0 = e_d.leftChild(); + ASSERT_TRUE(e_d_0.ok()); + + mmb::Element e_d_1 = e_d_0.rightSibling(); + ASSERT_TRUE(e_d_1.ok()); + + mmb::Element e_d_2 = e_d_1.rightSibling(); + ASSERT_TRUE(e_d_2.ok()); + + ASSERT_OK(e_d_1.remove()); + + static const char outJson[] = + "{ embedded: { a: 1, b: 2, c: 2.0, d : ['w', 'y', 'z'] }, free: {} }"; + ASSERT_EQUALS(mongo::fromjson(outJson), doc.getObject()); + } + + TEST(DocumentInPlace, EphemeralDocumentsDoNotUseInPlaceMode) { + mmb::Document doc; + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsHonored1) { + mongo::BSONObj obj; + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + doc.disableInPlaceUpdates(); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsHonored2) { + mongo::BSONObj obj; + mmb::Document doc(obj, mmb::Document::kInPlaceDisabled); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + doc.disableInPlaceUpdates(); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeWorksWithNoMutations) { + mongo::BSONObj obj; + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + const char* source = NULL; + mmb::DamageVector damages; + ASSERT_TRUE(damages.empty()); + doc.getInPlaceUpdates(&damages, &source); + ASSERT_TRUE(damages.empty()); + ASSERT_NOT_EQUALS(static_cast(NULL), source); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsDisabledByAddSiblingLeft) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + mmb::Element newElt = doc.makeElementInt("bar", 42); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().leftChild().addSiblingLeft(newElt)); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsDisabledByAddSiblingRight) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + mmb::Element newElt = doc.makeElementInt("bar", 42); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().leftChild().addSiblingRight(newElt)); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsDisabledByRemove) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().leftChild().remove()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + // NOTE: Someday, we may do in-place renames, but renaming 'foo' to 'foobar' will never + // work because the sizes don't match. Validate that this disables in-place updates. + TEST(DocumentInPlace, InPlaceModeIsDisabledByRename) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().leftChild().rename("foobar")); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsDisabledByPushFront) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + mmb::Element newElt = doc.makeElementInt("bar", 42); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().pushFront(newElt)); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsDisabledByPushBack) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + mmb::Element newElt = doc.makeElementInt("bar", 42); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().pushBack(newElt)); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsDisabledByPopFront) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().popFront()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, InPlaceModeIsDisabledByPopBack) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_OK(doc.root().popBack()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, ReserveDamageEventsIsAlwaysSafeToCall) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + doc.reserveDamageEvents(10); + doc.disableInPlaceUpdates(); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + doc.reserveDamageEvents(10); + } + + TEST(DocumentInPlace, GettingInPlaceUpdatesWhenDisabledClearsArguments) { + mongo::BSONObj obj = mongo::fromjson("{ foo : 'foo' }"); + mmb::Document doc(obj, mmb::Document::kInPlaceDisabled); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + mmb::DamageVector damages; + const mmb::DamageEvent event = { 0 }; + damages.push_back(event); + const char* source = "foo"; + ASSERT_FALSE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_TRUE(damages.empty()); + ASSERT_EQUALS(static_cast(NULL), source); + + damages.push_back(event); + source = "bar"; + size_t size = 1; + ASSERT_FALSE(doc.getInPlaceUpdates(&damages, &source, &size)); + ASSERT_TRUE(damages.empty()); + ASSERT_EQUALS(static_cast(NULL), source); + ASSERT_EQUALS(0U, size); + } + + // This isn't a great test since we aren't testing all possible combinations of compatible + // and incompatible sets, but since all setValueX calls decay to the internal setValue, we + // can be pretty sure that this will at least check the logic somewhat. + TEST(DocumentInPlace, InPlaceModeIsDisabledByIncompatibleSetValue) { + mongo::BSONObj obj(mongo::fromjson("{ foo : false }")); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + mmb::Element foo = doc.root().leftChild(); + ASSERT_TRUE(foo.ok()); + ASSERT_OK(foo.setValueString("foo")); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + } + + TEST(DocumentInPlace, DisablingInPlaceDoesNotDiscardUpdates) { + mongo::BSONObj obj(mongo::fromjson("{ foo : false, bar : true }")); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + mmb::Element foo = doc.root().leftChild(); + ASSERT_TRUE(foo.ok()); + ASSERT_OK(foo.setValueBool(true)); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + mmb::Element bar = doc.root().rightChild(); + ASSERT_TRUE(bar.ok()); + ASSERT_OK(bar.setValueBool(false)); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + ASSERT_OK(doc.root().appendString("baz", "baz")); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + static const char outJson[] = + "{ foo : true, bar : false, baz : 'baz' }"; + ASSERT_EQUALS(mongo::fromjson(outJson), doc.getObject()); + } + + TEST(DocumentInPlace, StringLifecycle) { + mongo::BSONObj obj(mongo::fromjson("{ x : 'foo' }")); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueString("bar"); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::String)); + ASSERT_EQUALS("bar", x.getValueString()); + + // TODO: When in-place updates for leaf elements is implemented, add tests here. + } + + TEST(DocumentInPlace, BinDataLifecycle) { + const char kData1[] = "\x01\x02\x03\x04\x05\x06"; + const char kData2[] = "\x10\x20\x30\x40\x50\x60"; + + const mongo::BSONBinData binData1(kData1, sizeof(kData1) - 1, mongo::BinDataGeneral); + const mongo::BSONBinData binData2(kData2, sizeof(kData2) - 1, mongo::bdtCustom); + + mongo::BSONObj obj(BSON("x" << binData1)); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueBinary(binData2.length, binData2.type, binData2.data); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::BinData)); + + mongo::BSONElement value = x.getValue(); + ASSERT_EQUALS(binData2.type, value.binDataType()); + int len = 0; + const char* const data = value.binDataClean(len); + ASSERT_EQUALS(binData2.length, len); + ASSERT_EQUALS(0, std::memcmp(data, kData2, len)); + + // TODO: When in-place updates for leaf elements is implemented, add tests here. + } + + TEST(DocumentInPlace, OIDLifecycle) { + const mongo::OID oid1 = mongo::OID::gen(); + const mongo::OID oid2 = mongo::OID::gen(); + ASSERT_NOT_EQUALS(oid1, oid2); + + mongo::BSONObj obj(BSON("x" << oid1)); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueOID(oid2); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::jstOID)); + ASSERT_EQUALS(oid2, x.getValueOID()); + + // TODO: When in-place updates for leaf elements is implemented, add tests here. + } + + TEST(DocumentInPlace, BooleanLifecycle) { + mongo::BSONObj obj(mongo::fromjson("{ x : false }")); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueBool(false); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::Bool)); + ASSERT_EQUALS(false, x.getValueBool()); + + // TODO: Re-enable when in-place updates to leaf elements is supported + // x.setValueBool(true); + // ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + // apply(&obj, damages, source); + // ASSERT_TRUE(x.hasValue()); + // ASSERT_TRUE(x.isType(mongo::Bool)); + // ASSERT_EQUALS(true, x.getValueBool()); + } + + TEST(DocumentInPlace, DateLifecycle) { + mongo::BSONObj obj(BSON("x" << mongo::Date_t(1000))); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueDate(mongo::Date_t(20000)); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::Date)); + ASSERT_EQUALS(mongo::Date_t(20000), x.getValueDate()); + + // TODO: When in-place updates for leaf elements is implemented, add tests here. + } + + TEST(DocumentInPlace, NumberIntLifecycle) { + const int value1 = 42; + const int value2 = 3; + mongo::BSONObj obj(BSON("x" << value1)); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueInt(value2); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::NumberInt)); + ASSERT_EQUALS(value2, x.getValueInt()); + + // TODO: Re-enable when in-place updates to leaf elements is supported + // x.setValueInt(value1); + // ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + // apply(&obj, damages, source); + // ASSERT_TRUE(x.hasValue()); + // ASSERT_TRUE(x.isType(mongo::NumberInt)); + // ASSERT_EQUALS(value1, x.getValueInt()); + } + + TEST(DocumentInPlace, TimestampLifecycle) { + mongo::BSONObj obj(BSON("x" << mongo::OpTime(mongo::Date_t(1000)))); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueTimestamp(mongo::OpTime(mongo::Date_t(20000))); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::Timestamp)); + ASSERT_TRUE(mongo::OpTime(mongo::Date_t(20000)) == x.getValueTimestamp()); + + // TODO: When in-place updates for leaf elements is implemented, add tests here. + } + + TEST(DocumentInPlace, NumberLongLifecycle) { + const long long value1 = 42; + const long long value2 = 3; + + mongo::BSONObj obj(BSON("x" << value1)); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueLong(value2); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::NumberLong)); + ASSERT_EQUALS(value2, x.getValueLong()); + + // TODO: Re-enable when in-place updates to leaf elements is supported + // x.setValueLong(value1); + // ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + // apply(&obj, damages, source); + // ASSERT_TRUE(x.hasValue()); + // ASSERT_TRUE(x.isType(mongo::NumberLong)); + // ASSERT_EQUALS(value1, x.getValueLong()); + } + + TEST(DocumentInPlace, NumberDoubleLifecycle) { + const double value1 = 32.0; + const double value2 = 2.0; + + mongo::BSONObj obj(BSON("x" << value1)); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueDouble(value2); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + ASSERT_EQUALS(1U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::NumberDouble)); + ASSERT_EQUALS(value2, x.getValueDouble()); + + // TODO: Re-enable when in-place updates to leaf elements is supported + // x.setValueDouble(value1); + // ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + // apply(&obj, damages, source); + // ASSERT_TRUE(x.hasValue()); + // ASSERT_TRUE(x.isType(mongo::NumberDouble)); + // ASSERT_EQUALS(value1, x.getValueDouble()); + } + + // Doubles and longs are the same size, 8 bytes, so we should be able to do in-place + // updates between them. + TEST(DocumentInPlace, DoubleToLongAndBack) { + const double value1 = 32.0; + const long long value2 = 42; + + mongo::BSONObj obj(BSON("x" << value1)); + mmb::Document doc(obj, mmb::Document::kInPlaceEnabled); + + mmb::Element x = doc.root().leftChild(); + + mmb::DamageVector damages; + const char* source = NULL; + + x.setValueLong(value2); + ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + // We changed the type, so we get an extra damage event. + ASSERT_EQUALS(2U, damages.size()); + apply(&obj, damages, source); + ASSERT_TRUE(x.hasValue()); + ASSERT_TRUE(x.isType(mongo::NumberLong)); + ASSERT_EQUALS(value2, x.getValueLong()); + + // TODO: Re-enable when in-place updates to leaf elements is supported + // x.setValueDouble(value1); + // ASSERT_TRUE(doc.getInPlaceUpdates(&damages, &source)); + // apply(&obj, damages, source); + // ASSERT_TRUE(x.hasValue()); + // ASSERT_TRUE(x.isType(mongo::NumberDouble)); + // ASSERT_EQUALS(value1, x.getValueDouble()); + } + + TEST(DocumentComparison, SimpleComparison) { + const mongo::BSONObj obj = + mongo::fromjson("{ a : 'a', b : ['b', 'b', 'b'], c : { one : 1.0 } }"); + + const mmb::Document doc1(obj.getOwned()); + ASSERT_EQUALS(0, doc1.compareWithBSONObj(obj)); + const mmb::Document doc2(obj.getOwned()); + ASSERT_EQUALS(0, doc1.compareWith(doc2)); + ASSERT_EQUALS(0, doc2.compareWith(doc1)); + } + + TEST(DocumentComparison, SimpleComparisonWithDeserializedElements) { + const mongo::BSONObj obj = + mongo::fromjson("{ a : 'a', b : ['b', 'b', 'b'], c : { one : 1.0 } }"); + + // Perform an operation on 'b' that doesn't change the serialized value, but + // deserializes the node. + mmb::Document doc1(obj.getOwned()); + const mmb::Document doc1Copy(obj.getOwned()); + mmb::Element b = doc1.root()["b"]; + ASSERT_TRUE(b.ok()); + mmb::Element b0 = b[0]; + ASSERT_TRUE(b0.ok()); + ASSERT_OK(b0.remove()); + ASSERT_OK(b.pushBack(b0)); + // Ensure that it compares correctly against the source object. + ASSERT_EQUALS(0, doc1.compareWithBSONObj(obj)); + // Ensure that it compares correctly against a pristine document. + ASSERT_EQUALS(0, doc1.compareWith(doc1Copy)); + ASSERT_EQUALS(0, doc1Copy.compareWith(doc1)); + + // Perform an operation on 'c' that doesn't change the serialized value, but + // deserializeds the node. + mmb::Document doc2(obj.getOwned()); + const mmb::Document doc2Copy(obj.getOwned()); + mmb::Element c = doc2.root()["c"]; + ASSERT_TRUE(c.ok()); + mmb::Element c1 = c.leftChild(); + ASSERT_TRUE(c1.ok()); + ASSERT_OK(c1.remove()); + ASSERT_OK(c.pushBack(c1)); + // Ensure that it compares correctly against the source object + ASSERT_EQUALS(0, doc2.compareWithBSONObj(obj)); + // Ensure that it compares correctly against a pristine document. + ASSERT_EQUALS(0, doc2.compareWith(doc2Copy)); + ASSERT_EQUALS(0, doc2Copy.compareWith(doc2)); + + // Ensure that the two deserialized documents compare with each other correctly. + ASSERT_EQUALS(0, doc1.compareWith(doc2)); + ASSERT_EQUALS(0, doc2.compareWith(doc1)); + } + + TEST(UnorderedEqualityChecker, Identical) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + + const mongo::BSONObj b2 = b1.getOwned(); + + ASSERT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + + TEST(UnorderedEqualityChecker, DifferentValuesAreNotEqual) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + const mongo::BSONObj b2 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 4 } }"); + + ASSERT_NOT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + + TEST(UnorderedEqualityChecker, DifferentTypesAreNotEqual) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + const mongo::BSONObj b2 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : '2', z : 3 } }"); + + ASSERT_NOT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + + TEST(UnorderedEqualityChecker, DifferentFieldNamesAreNotEqual) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + const mongo::BSONObj b2 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, Y : 2, z : 3 } }"); + + ASSERT_NOT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + + TEST(UnorderedEqualityChecker, MissingFieldsInObjectAreNotEqual) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + const mongo::BSONObj b2 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, z : 3 } }"); + + ASSERT_NOT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + + TEST(UnorderedEqualityChecker, ObjectOrderingIsNotConsidered) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + const mongo::BSONObj b2 = mongo::fromjson( + "{ b : { y : 2, z : 3 , x : 1 }, a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ] }"); + + ASSERT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + + TEST(UnorderedEqualityChecker, ArrayOrderingIsConsidered) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + + const mongo::BSONObj b2 = mongo::fromjson( + "{ a : [ 1, { 'a' : 'b', 'x' : 'y' }, 2 ], b : { x : 1, y : 2, z : 3 } }"); + + ASSERT_NOT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + + TEST(UnorderedEqualityChecker, MissingItemsInArrayAreNotEqual) { + const mongo::BSONObj b1 = mongo::fromjson( + "{ a : [ 1, 2, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, y : 2, z : 3 } }"); + const mongo::BSONObj b2 = mongo::fromjson( + "{ a : [ 1, { 'a' : 'b', 'x' : 'y' } ], b : { x : 1, z : 3 } }"); + + ASSERT_NOT_EQUALS(mmb::unordered(b1), mmb::unordered(b2)); + } + +} // namespace diff --git a/src/mongo/bson/mutable/mutable_bson_test_utils.cpp b/src/mongo/bson/mutable/mutable_bson_test_utils.cpp new file mode 100644 index 00000000000..dbec302fc99 --- /dev/null +++ b/src/mongo/bson/mutable/mutable_bson_test_utils.cpp @@ -0,0 +1,206 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/mutable/mutable_bson_test_utils.h" + +#include +#include +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/const_element.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace mutablebson { + + namespace { + + inline void assertSameSign(int lhs, int rhs) { + if (lhs == 0) { + ASSERT_EQUALS(rhs, 0); + } else if (lhs < 0) { + ASSERT_LESS_THAN(rhs, 0); + } else { + ASSERT_GREATER_THAN(rhs, 0); + } + } + + inline void assertOppositeSign(int lhs, int rhs) { + if (lhs == 0) { + ASSERT_EQUALS(rhs, 0); + } else if (lhs < 0) { + ASSERT_GREATER_THAN(rhs, 0); + } else { + ASSERT_LESS_THAN(rhs, 0); + } + } + + void addChildrenToVector(ConstElement elt, std::vector* accumulator) { + ConstElement current = elt.leftChild(); + while (current.ok()) { + accumulator->push_back(current); + current = current.rightSibling(); + } + } + + bool checkDocNoOrderingImpl(ConstElement lhs, ConstElement rhs) { + const BSONType lhsType = lhs.getType(); + const BSONType rhsType = rhs.getType(); + + if (lhsType == mongo::Object) { + + if (rhsType != mongo::Object) + return false; + + // For objects, sort the children by field name, then compare in that order. + + std::vector lhsChildren; + addChildrenToVector(lhs, &lhsChildren); + std::vector rhsChildren; + addChildrenToVector(rhs, &rhsChildren); + if (lhsChildren.size() != rhsChildren.size()) + return false; + + // NOTE: if you have repeated field names, this is not necessarily going to + // work. This is unlikely to be a problem in practice, but we could write a + // more sophisticated comparator if we need to: perhaps one that ordered first + // by field name, then by type, then by woCompare. Performance isn't important + // here. + std::sort(lhsChildren.begin(), lhsChildren.end(), FieldNameLessThan()); + std::sort(rhsChildren.begin(), rhsChildren.end(), FieldNameLessThan()); + + typedef std::vector::const_iterator iter; + iter lhsWhere = lhsChildren.begin(); + iter rhsWhere = rhsChildren.begin(); + const iter lhsEnd = lhsChildren.end(); + + for (; lhsWhere != lhsEnd; ++lhsWhere, ++rhsWhere) { + + if (lhsWhere->getFieldName() != rhsWhere->getFieldName()) + return false; + + if (!checkDocNoOrderingImpl(*lhsWhere, *rhsWhere)) + return false; + } + + return true; + + } else if (lhsType == mongo::Array) { + + if (rhsType != mongo::Array) + return false; + + // For arrays, since they are ordered, we don't need the sorting step. + const size_t lhsChildren = countChildren(lhs); + const size_t rhsChildren = countChildren(rhs); + + if (lhsChildren != rhsChildren) + return false; + + if (lhsChildren == 0) + return true; + + ConstElement lhsChild = lhs.leftChild(); + ConstElement rhsChild = rhs.leftChild(); + + while (lhsChild.ok()) { + if (!checkDocNoOrderingImpl(lhsChild, rhsChild)) + return false; + + lhsChild = lhsChild.rightSibling(); + rhsChild = rhsChild.rightSibling(); + } + + return true; + + } else { + // This is some leaf type. We've already checked or ignored field names, so + // don't recheck it here. + return lhs.compareWithElement(rhs, false) == 0; + } + } + + } // namespace + + // TODO: We should really update this to be an ASSERT_ something, so that we can print out + // the expected and actual documents. + bool checkDoc(const Document& lhs, const BSONObj& rhs) { + + // Get the fundamental result via BSONObj's woCompare path. This is the best starting + // point, because we think that Document::getObject and the serialization mechanism is + // pretty well sorted. + BSONObj fromLhs = lhs.getObject(); + const int primaryResult = fromLhs.woCompare(rhs); + + // Validate primary result via other comparison paths. + const int secondaryResult = lhs.compareWithBSONObj(rhs); + + assertSameSign(primaryResult, secondaryResult); + + // Check that mutables serialized result matches against its origin. + ASSERT_EQUALS(0, lhs.compareWithBSONObj(fromLhs)); + + return (primaryResult == 0); + } + + bool checkDoc(const Document& lhs, const Document& rhs) { + + const int primaryResult = lhs.compareWith(rhs); + + const BSONObj fromLhs = lhs.getObject(); + const BSONObj fromRhs = rhs.getObject(); + + const int result_d_o = lhs.compareWithBSONObj(fromRhs); + const int result_o_d = rhs.compareWithBSONObj(fromLhs); + + assertSameSign(primaryResult, result_d_o); + assertOppositeSign(primaryResult, result_o_d); + + ASSERT_EQUALS(0, lhs.compareWithBSONObj(fromLhs)); + ASSERT_EQUALS(0, rhs.compareWithBSONObj(fromRhs)); + + return (primaryResult == 0); + } + + std::ostream& operator<<(std::ostream& stream, const Document& doc) { + stream << doc.toString(); + return stream; + } + + std::ostream& operator<<(std::ostream& stream, const ConstElement& elt) { + stream << elt.toString(); + return stream; + } + + bool checkEqualNoOrdering(const Document& lhs, const Document& rhs) { + return checkDocNoOrderingImpl(lhs.root(), rhs.root()); + } + + std::ostream& operator<<(std::ostream& stream, const UnorderedWrapper_Doc& uw_d) { + return stream << uw_d.doc; + } + + std::ostream& operator<<(std::ostream& stream, const UnorderedWrapper_Obj& uw_o) { + const Document d(uw_o.obj); + return stream << d; + } + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/mutable/mutable_bson_test_utils.h b/src/mongo/bson/mutable/mutable_bson_test_utils.h new file mode 100644 index 00000000000..164ff81e9f8 --- /dev/null +++ b/src/mongo/bson/mutable/mutable_bson_test_utils.h @@ -0,0 +1,124 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "mongo/bson/mutable/document.h" + +namespace mongo { + + class BSONObj; + +namespace mutablebson { + + // + // Utilities for mutable BSON unit tests. + // + + /** + * Catch all comparator between a mutable 'doc' and the expected BSON 'exp'. It compares + * (a) 'doc's generated object, (b) 'exp', the expected object, and (c) 'doc(exp)', a + * document created from 'exp'. Returns true if all three are equal, otherwise false. + */ + bool checkDoc(const Document& lhs, const BSONObj& rhs); + bool checkDoc(const Document& lhs, const Document& rhs); + + inline bool operator==(const Document& lhs, const Document& rhs) { + return checkDoc(lhs, rhs); + } + + inline bool operator==(const BSONObj& lhs, const Document& rhs) { + return checkDoc(rhs, lhs); + } + + inline bool operator==(const Document& lhs, const BSONObj& rhs) { + return checkDoc(lhs, rhs); + } + + /** Stream out a document; useful within ASSERT calls */ + std::ostream& operator<<(std::ostream& stream, const Document& doc); + + /** Stream out an element; useful within ASSERT calls */ + std::ostream& operator<<(std::ostream& stream, const ConstElement& elt); + + /** Check that the two provided Documents are equivalent modulo field ordering in Object + * Elements. Leaf values are considered equal via woCompare. + */ + bool checkEqualNoOrdering(const Document& lhs, const Document& rhs); + + struct UnorderedWrapper_Obj { + inline explicit UnorderedWrapper_Obj(const BSONObj& o) + : obj(o) {} + const BSONObj& obj; + }; + + struct UnorderedWrapper_Doc { + inline explicit UnorderedWrapper_Doc(const Document& d) + : doc(d) {} + const Document& doc; + }; + + inline UnorderedWrapper_Doc unordered(const Document& d) { + return UnorderedWrapper_Doc(d); + } + + inline UnorderedWrapper_Obj unordered(const BSONObj& o) { + return UnorderedWrapper_Obj(o); + } + + inline bool operator==(const UnorderedWrapper_Doc& lhs, const UnorderedWrapper_Doc& rhs) { + return checkEqualNoOrdering(lhs.doc, rhs.doc); + } + + inline bool operator!=(const UnorderedWrapper_Doc& lhs, const UnorderedWrapper_Doc& rhs) { + return !(lhs == rhs); + } + + + inline bool operator==(const UnorderedWrapper_Obj& lhs, const UnorderedWrapper_Obj& rhs) { + const Document dlhs(lhs.obj); + const Document drhs(rhs.obj); + return checkEqualNoOrdering(dlhs, drhs); + } + + inline bool operator!=(const UnorderedWrapper_Obj& lhs, const UnorderedWrapper_Obj& rhs) { + return !(lhs == rhs); + } + + + inline bool operator==(const UnorderedWrapper_Doc& lhs, const UnorderedWrapper_Obj& rhs) { + const Document drhs(rhs.obj); + return checkEqualNoOrdering(lhs.doc, drhs); + } + + inline bool operator!=(const UnorderedWrapper_Doc& lhs, const UnorderedWrapper_Obj& rhs) { + return !(lhs == rhs); + } + + + inline bool operator==(const UnorderedWrapper_Obj& lhs, const UnorderedWrapper_Doc& rhs) { + const Document dlhs(lhs.obj); + return checkEqualNoOrdering(dlhs, rhs.doc); + } + + inline bool operator!=(const UnorderedWrapper_Obj& lhs, const UnorderedWrapper_Doc& rhs) { + return !(lhs == rhs); + } + + std::ostream& operator<<(std::ostream& stream, const UnorderedWrapper_Doc& uw_d); + std::ostream& operator<<(std::ostream& stream, const UnorderedWrapper_Obj& uw_o); + +} // namespace mutablebson +} // namespace mongo diff --git a/src/mongo/bson/oid.cpp b/src/mongo/bson/oid.cpp index e96b076762a..8847e498159 100644 --- a/src/mongo/bson/oid.cpp +++ b/src/mongo/bson/oid.cpp @@ -20,6 +20,7 @@ #include #include "mongo/platform/atomic_word.h" +#include "mongo/platform/process_id.h" #include "mongo/platform/random.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/oid.h" @@ -46,17 +47,9 @@ namespace mongo { return s; } - unsigned OID::ourPid() { -#ifdef _WIN32 - return static_cast( GetCurrentProcessId() ); -#else - return static_cast( getpid() ); -#endif - } - void OID::foldInPid(OID::MachineAndPid& x) { - unsigned p = ourPid(); - x._pid ^= (unsigned short) p; + unsigned p = ProcessId::getCurrent().asUInt32(); + x._pid ^= static_cast(p); // when the pid is greater than 16 bits, let the high bits modulate the machine id field. unsigned short& rest = (unsigned short &) x._machineNumber[1]; rest ^= p >> 16; diff --git a/src/mongo/bson/oid.h b/src/mongo/bson/oid.h index 8b5f3ba18c8..2c9ee5c1f18 100644 --- a/src/mongo/bson/oid.h +++ b/src/mongo/bson/oid.h @@ -137,7 +137,6 @@ namespace mongo { unsigned char data[kOIDSize]; }; - static unsigned ourPid(); static void foldInPid(MachineAndPid& x); static MachineAndPid genMachineAndPid(); }; diff --git a/src/mongo/bson/optime.cpp b/src/mongo/bson/optime.cpp new file mode 100644 index 00000000000..6cc01969366 --- /dev/null +++ b/src/mongo/bson/optime.cpp @@ -0,0 +1,105 @@ +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/bson/optime.h" + +#include +#include +#include + +#include "mongo/bson/inline_decls.h" +#include "mongo/util/debug_util.h" +#include "mongo/util/log.h" +#include "mongo/util/startup_test.h" + +namespace mongo { + + OpTime OpTime::last(0, 0); + boost::condition OpTime::notifier; + mongo::mutex OpTime::m("optime"); + + NOINLINE_DECL OpTime OpTime::skewed() { + bool toLog = false; + ONCE toLog = true; + RARELY toLog = true; + last.i++; + if ( last.i & 0x80000000 ) + toLog = true; + if ( toLog ) { + log() << "clock skew detected prev: " << last.secs << " now: " << (unsigned) time(0) + << std::endl; + } + if ( last.i & 0x80000000 ) { + log() << "error large clock skew detected, shutting down" << std::endl; + throw ClockSkewException(); + } + return last; + } + + /*static*/ OpTime OpTime::_now() { + OpTime result; + unsigned t = (unsigned) time(0); + if ( last.secs == t ) { + last.i++; + result = last; + } + else if ( t < last.secs ) { + result = skewed(); // separate function to keep out of the hot code path + } + else { + last = OpTime(t, 1); + result = last; + } + notifier.notify_all(); + return last; + } + + OpTime OpTime::now(const mongo::mutex::scoped_lock&) { + return _now(); + } + + OpTime OpTime::getLast(const mongo::mutex::scoped_lock&) { + return last; + } + + OpTime OpTime::max() { + unsigned int t = static_cast(std::numeric_limits::max()); + unsigned int i = std::numeric_limits::max(); + return OpTime(t, i); + } + + void OpTime::waitForDifferent(unsigned millis){ + mutex::scoped_lock lk(m); + while (*this == last) { + if (!notifier.timed_wait(lk.boost(), boost::posix_time::milliseconds(millis))) + return; // timed out + } + } + + struct TestOpTime : public StartupTest { + void run() { + OpTime t; + for ( int i = 0; i < 10; i++ ) { + OpTime s = OpTime::_now(); + verify( s != t ); + t = s; + } + OpTime q = t; + verify( q == t ); + verify( !(q != t) ); + } + } testoptime; + +} diff --git a/src/mongo/bson/optime.h b/src/mongo/bson/optime.h new file mode 100644 index 00000000000..6d549a38824 --- /dev/null +++ b/src/mongo/bson/optime.h @@ -0,0 +1,160 @@ +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/util/assert_util.h" +#include "mongo/util/concurrency/mutex.h" +#include "mongo/util/time_support.h" + +namespace mongo { + + struct ClockSkewException : public DBException { + ClockSkewException() : DBException( "clock skew exception" , 20001 ) {} + }; + + /* replsets used to use RSOpTime. + M/S uses OpTime. + But this is useable from both. + */ + typedef unsigned long long ReplTime; + + /* Operation sequence #. A combination of current second plus an ordinal value. + */ +#pragma pack(4) + class OpTime { + unsigned i; // ordinal comes first so we can do a single 64 bit compare on little endian + unsigned secs; + static OpTime last; + static OpTime skewed(); + public: + static void setLast(const Date_t &date) { + mutex::scoped_lock lk(m); + last = OpTime(date); + notifier.notify_all(); + } + static void setLast(const OpTime &new_last) { + mutex::scoped_lock lk(m); + last = new_last; + notifier.notify_all(); + } + + unsigned getSecs() const { + return secs; + } + unsigned getInc() const { + return i; + } + OpTime(Date_t date) { + reinterpret_cast(*this) = date.millis; + dassert( (int)secs >= 0 ); + } + OpTime(ReplTime x) { + reinterpret_cast(*this) = x; + dassert( (int)secs >= 0 ); + } + OpTime(unsigned a, unsigned b) { + secs = a; + i = b; + dassert( (int)secs >= 0 ); + } + OpTime( const OpTime& other ) { + secs = other.secs; + i = other.i; + dassert( (int)secs >= 0 ); + } + OpTime() { + secs = 0; + i = 0; + } + // it isn't generally safe to not be locked for this. so use now(). some tests use this. + static OpTime _now(); + + static mongo::mutex m; + + static OpTime now(const mongo::mutex::scoped_lock&); + + static OpTime getLast(const mongo::mutex::scoped_lock&); + + // Maximum OpTime value. + static OpTime max(); + + // Waits for global OpTime to be different from *this + void waitForDifferent(unsigned millis); + + /* We store OpTime's in the database as BSON Date datatype -- we needed some sort of + 64 bit "container" for these values. While these are not really "Dates", that seems a + better choice for now than say, Number, which is floating point. Note the BinData type + is perhaps the cleanest choice, lacking a true unsigned64 datatype, but BinData has 5 + bytes of overhead. + */ + unsigned long long asDate() const { + return reinterpret_cast(&i)[0]; + } + long long asLL() const { + return reinterpret_cast(&i)[0]; + } + + bool isNull() const { return secs == 0; } + + string toStringLong() const { + std::stringstream ss; + ss << time_t_to_String_short(secs) << ' '; + ss << std::hex << secs << ':' << i; + return ss.str(); + } + + string toStringPretty() const { + std::stringstream ss; + ss << time_t_to_String_short(secs) << ':' << std::hex << i; + return ss.str(); + } + + string toString() const { + std::stringstream ss; + ss << std::hex << secs << ':' << i; + return ss.str(); + } + + bool operator==(const OpTime& r) const { + return i == r.i && secs == r.secs; + } + bool operator!=(const OpTime& r) const { + return !(*this == r); + } + bool operator<(const OpTime& r) const { + if ( secs != r.secs ) + return secs < r.secs; + return i < r.i; + } + bool operator<=(const OpTime& r) const { + return *this < r || *this == r; + } + bool operator>(const OpTime& r) const { + return !(*this <= r); + } + bool operator>=(const OpTime& r) const { + return !(*this < r); + } + private: + static boost::condition notifier; + }; +#pragma pack() + +} // namespace mongo diff --git a/src/mongo/bson/ordering.h b/src/mongo/bson/ordering.h index 25545b66b3f..8fccde9b7ba 100644 --- a/src/mongo/bson/ordering.h +++ b/src/mongo/bson/ordering.h @@ -17,6 +17,9 @@ #pragma once +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjiterator.h" + namespace mongo { // todo: ideally move to db/ instead of bson/, but elim any dependencies first diff --git a/src/mongo/bson/util/bson_extract.cpp b/src/mongo/bson/util/bson_extract.cpp index cc6405e3dca..0bb2c374348 100644 --- a/src/mongo/bson/util/bson_extract.cpp +++ b/src/mongo/bson/util/bson_extract.cpp @@ -16,6 +16,7 @@ #include "mongo/bson/util/bson_extract.h" #include "mongo/db/jsobj.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { @@ -24,7 +25,9 @@ namespace mongo { BSONElement* outElement) { BSONElement element = object.getField(fieldName); if (element.eoo()) - return Status(ErrorCodes::NoSuchKey, fieldName.toString()); + return Status(ErrorCodes::NoSuchKey, + mongoutils::str::stream() << "Missing expected field \"" << + fieldName.toString() << "\""); *outElement = element; return Status::OK(); } @@ -38,12 +41,24 @@ namespace mongo { return status; if (type != outElement->type()) { return Status(ErrorCodes::TypeMismatch, - std::string("Expected ") + typeName(type) + - " found " + typeName(outElement->type())); + mongoutils::str::stream() << "\"" << fieldName << + "\" had the wrong type. Expected " << typeName(type) << + ", found " << typeName(outElement->type())); } return Status::OK(); } + Status bsonExtractBooleanField(const BSONObj& object, + const StringData& fieldName, + bool* out) { + BSONElement element; + Status status = bsonExtractTypedField(object, fieldName, Bool, &element); + if (!status.isOK()) + return status; + *out = element.boolean(); + return Status::OK(); + } + Status bsonExtractBooleanFieldWithDefault(const BSONObj& object, const StringData& fieldName, bool defaultValue, @@ -52,17 +67,20 @@ namespace mongo { Status status = bsonExtractField(object, fieldName, &value); if (status == ErrorCodes::NoSuchKey) { *out = defaultValue; + return Status::OK(); } else if (!status.isOK()) { return status; } else if (!value.isNumber() && !value.isBoolean()) { - return Status(ErrorCodes::TypeMismatch, "Expected boolean or number type"); + return Status(ErrorCodes::TypeMismatch, mongoutils::str::stream() << + "Expected boolean or number type for field \"" << fieldName << + "\", found " << typeName(value.type())); } else { *out = value.trueValue(); + return Status::OK(); } - return Status::OK(); } Status bsonExtractStringField(const BSONObj& object, @@ -90,4 +108,39 @@ namespace mongo { return Status::OK(); } + Status bsonExtractIntegerField(const BSONObj& object, + const StringData& fieldName, + long long* out) { + BSONElement value; + Status status = bsonExtractField(object, fieldName, &value); + if (!status.isOK()) + return status; + if (!value.isNumber()) { + return Status(ErrorCodes::TypeMismatch, mongoutils::str::stream() << + "Expected field \"" << fieldName << + "\" to have numeric type, but found " << typeName(value.type())); + } + long long result = value.safeNumberLong(); + if (result != value.numberDouble()) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Expected field \"" << fieldName << "\" to have a value " + "exactly representable as a 64-bit integer, but found " << + value); + } + *out = result; + return Status::OK(); + } + + Status bsonExtractIntegerFieldWithDefault(const BSONObj& object, + const StringData& fieldName, + long long defaultValue, + long long* out) { + Status status = bsonExtractIntegerField(object, fieldName, out); + if (status == ErrorCodes::NoSuchKey) { + *out = defaultValue; + status = Status::OK(); + } + return status; + } + } // namespace mongo diff --git a/src/mongo/bson/util/bson_extract.h b/src/mongo/bson/util/bson_extract.h index 1c1db48d036..fd1c6ca057b 100644 --- a/src/mongo/bson/util/bson_extract.h +++ b/src/mongo/bson/util/bson_extract.h @@ -49,6 +49,32 @@ namespace mongo { BSONType type, BSONElement* outElement); + /** + * Finds a bool-like element named "fieldName" in "object". + * + * Returns Status::OK() and sets *out to the found element's boolean value on success. Returns + * ErrorCodes::NoSuchKey if there are no matches for "fieldName", and ErrorCodes::TypeMismatch + * if the type of the matching element is not Bool or a number type. For return values other + * than Status::OK(), the resulting value of "*out" is undefined. + */ + Status bsonExtractBooleanField(const BSONObj& object, + const StringData& fieldName, + bool* out); + + /** + * Finds an element named "fieldName" in "object" that represents an integral value. + * + * Returns Status::OK() and sets *out to the element's 64-bit integer value representation on + * success. Returns ErrorCodes::NoSuchKey if there are no matches for "fieldName". Returns + * ErrorCodes::TypeMismatch if the value of the matching element is not of a numeric type. + * Returns ErrorCodes::BadValue if the value does not have an exact 64-bit integer + * representation. For return values other than Status::OK(), the resulting value of "*out" is + * undefined. + */ + Status bsonExtractIntegerField(const BSONObj& object, + const StringData& fieldName, + long long* out); + /** * Finds a string-typed element named "fieldName" in "object" and stores its value in "out". * @@ -76,6 +102,21 @@ namespace mongo { bool defaultValue, bool* out); + /** + * Finds an element named "fieldName" in "object" that represents an integral value. + * + * If a field named "fieldName" is present and is a value of numeric type with an exact 64-bit + * integer representation, returns that representation in *out and returns Status::OK(). If + * there is no field named "fieldName", stores defaultValue into *out and returns Status::OK(). + * If the field is found, but has non-numeric type, returns ErrorCodes::TypeMismatch. If the + * value has numeric type, but cannot be represented as a 64-bit integer, returns + * ErrorCodes::BadValue. + */ + Status bsonExtractIntegerFieldWithDefault(const BSONObj& object, + const StringData& fieldName, + long long defaultValue, + long long* out); + /** * Finds a string element named "fieldName" in "object". * diff --git a/src/mongo/bson/util/bson_extract_test.cpp b/src/mongo/bson/util/bson_extract_test.cpp index 38c9e568bd0..186833bb531 100644 --- a/src/mongo/bson/util/bson_extract_test.cpp +++ b/src/mongo/bson/util/bson_extract_test.cpp @@ -13,6 +13,7 @@ * limitations under the License. */ +#include #include #include "mongo/bson/util/bson_extract.h" @@ -96,3 +97,43 @@ TEST(ExtractBSON, ExtractBooleanFieldWithDefault) { ASSERT_EQUALS(ErrorCodes::TypeMismatch, bsonExtractBooleanFieldWithDefault(obj1, "b", true, &b)); } + +TEST(ExtractBSON, ExtractIntegerField) { + long long v; + ASSERT_EQUALS(ErrorCodes::NoSuchKey, bsonExtractIntegerField( + BSON("a" << 1), + "b", + &v)); + ASSERT_OK(bsonExtractIntegerFieldWithDefault( + BSON("a" << 1), + "b", + -1LL, + &v)); + ASSERT_EQUALS(-1LL, v); + ASSERT_EQUALS(ErrorCodes::TypeMismatch, bsonExtractIntegerField( + BSON("a" << false), + "a", + &v)); + ASSERT_EQUALS(ErrorCodes::BadValue, bsonExtractIntegerField( + BSON("a" << std::numeric_limits::quiet_NaN()), + "a", + &v)); + ASSERT_EQUALS(ErrorCodes::BadValue, bsonExtractIntegerField( + BSON("a" << pow(2.0, 64)), + "a", + &v)); + ASSERT_EQUALS(ErrorCodes::BadValue, bsonExtractIntegerField( + BSON("a" << -1.5), + "a", + &v)); + ASSERT_OK(bsonExtractIntegerField( + BSON("a" << -pow(2.0, 55)), + "a", + &v)); + ASSERT_EQUALS(-(1LL << 55), v); + ASSERT_OK(bsonExtractIntegerField( + BSON("a" << 5178), + "a", + &v)); + ASSERT_EQUALS(5178, v); +} diff --git a/src/mongo/bson/util/builder.h b/src/mongo/bson/util/builder.h index a66a736bb5c..711ed0b7d31 100644 --- a/src/mongo/bson/util/builder.h +++ b/src/mongo/bson/util/builder.h @@ -57,8 +57,6 @@ namespace mongo { const int BufferMaxSize = 64 * 1024 * 1024; - void msgasserted(int msgid, const char *msg); - template class StringBuilderImpl; @@ -112,6 +110,7 @@ namespace mongo { data = 0; } l = 0; + reservedBytes = 0; } ~_BufBuilder() { kill(); } @@ -124,9 +123,11 @@ namespace mongo { void reset() { l = 0; + reservedBytes = 0; } void reset( int maxSize ) { l = 0; + reservedBytes = 0; if ( maxSize && size > maxSize ) { al.Free(data); data = (char*)al.Malloc(maxSize); @@ -203,19 +204,43 @@ namespace mongo { inline char* grow(int by) { int oldlen = l; int newLen = l + by; - if ( newLen > size ) { - grow_reallocate(newLen); + int minSize = newLen + reservedBytes; + if ( minSize > size ) { + grow_reallocate(minSize); } l = newLen; return data + oldlen; } + /** + * Reserve room for some number of bytes to be claimed at a later time. + */ + void reserveBytes(int bytes) { + int minSize = l + reservedBytes + bytes; + if (minSize > size) + grow_reallocate(minSize); + + // This must happen *after* any attempt to grow. + reservedBytes += bytes; + } + + /** + * Claim an earlier reservation of some number of bytes. These bytes must already have been + * reserved. Appends of up to this many bytes immediately following a claim are + * guaranteed to succeed without a need to reallocate. + */ + void claimReservedBytes(int bytes) { + invariant(reservedBytes >= bytes); + reservedBytes -= bytes; + } + private: /* "slow" portion of 'grow()' */ - void NOINLINE_DECL grow_reallocate(int newLen) { + void NOINLINE_DECL grow_reallocate(int minSize) { int a = 64; - while( a < newLen ) + while (a < minSize) a = a * 2; + if ( a > BufferMaxSize ) { std::stringstream ss; ss << "BufBuilder attempted to grow() to " << a << " bytes, past the 64MB limit."; @@ -230,6 +255,7 @@ namespace mongo { char *data; int l; int size; + int reservedBytes; // eagerly grow_reallocate to keep this many bytes of spare room. friend class StringBuilderImpl; }; diff --git a/src/mongo/bson/util/misc.h b/src/mongo/bson/util/misc.h index 72623310160..62942506c7c 100644 --- a/src/mongo/bson/util/misc.h +++ b/src/mongo/bson/util/misc.h @@ -25,84 +25,10 @@ #include "mongo/platform/cstdint.h" #include "mongo/util/assert_util.h" +#include "mongo/util/time_support.h" namespace mongo { - inline void time_t_to_String(time_t t, char *buf) { -#if defined(_WIN32) - ctime_s(buf, 32, &t); -#else - ctime_r(&t, buf); -#endif - buf[24] = 0; // don't want the \n - } - - inline std::string time_t_to_String(time_t t = time(0) ) { - char buf[64]; -#if defined(_WIN32) - ctime_s(buf, sizeof(buf), &t); -#else - ctime_r(&t, buf); -#endif - buf[24] = 0; // don't want the \n - return buf; - } - - inline std::string time_t_to_String_no_year(time_t t) { - char buf[64]; -#if defined(_WIN32) - ctime_s(buf, sizeof(buf), &t); -#else - ctime_r(&t, buf); -#endif - buf[19] = 0; - return buf; - } - - inline std::string time_t_to_String_short(time_t t) { - char buf[64]; -#if defined(_WIN32) - ctime_s(buf, sizeof(buf), &t); -#else - ctime_r(&t, buf); -#endif - buf[19] = 0; - if( buf[0] && buf[1] && buf[2] && buf[3] ) - return buf + 4; // skip day of week - return buf; - } - - struct Date_t { - // TODO: make signed (and look for related TODO's) - unsigned long long millis; - Date_t(): millis(0) {} - Date_t(unsigned long long m): millis(m) {} - operator unsigned long long&() { return millis; } - operator const unsigned long long&() const { return millis; } - void toTm (tm *buf) { - time_t dtime = toTimeT(); -#if defined(_WIN32) - gmtime_s(buf, &dtime); -#else - gmtime_r(&dtime, buf); -#endif - } - std::string toString() const { - char buf[64]; - time_t_to_String(toTimeT(), buf); - return buf; - } - time_t toTimeT() const { - // cant use uassert from bson/util - verify((long long)millis >= 0); // TODO when millis is signed, delete - verify(((long long)millis/1000) < (std::numeric_limits::max)()); - return millis / 1000; - } - int64_t asInt64() { - return static_cast(millis); - } - }; - // Like strlen, but only scans up to n bytes. // Returns -1 if no '0' found. inline int strnlen( const char *s, int n ) { diff --git a/src/mongo/client/auth_helpers.cpp b/src/mongo/client/auth_helpers.cpp new file mode 100644 index 00000000000..e339500768e --- /dev/null +++ b/src/mongo/client/auth_helpers.cpp @@ -0,0 +1,56 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/client/auth_helpers.h" + +#include "mongo/db/auth/authorization_manager.h" + +namespace mongo { +namespace auth { + + const std::string schemaVersionServerParameter = "authSchemaVersion"; + + Status getRemoteStoredAuthorizationVersion(DBClientBase* conn, int* outVersion) { + try { + BSONObj cmdResult; + conn->runCommand( + "admin", + BSON("getParameter" << 1 << schemaVersionServerParameter << 1), + cmdResult); + if (!cmdResult["ok"].trueValue()) { + std::string errmsg = cmdResult["errmsg"].str(); + if (errmsg == "no option found to get" || + StringData(errmsg).startsWith("no such cmd")) { + + *outVersion = 1; + return Status::OK(); + } + int code = cmdResult["code"].numberInt(); + if (code == 0) { + code = ErrorCodes::UnknownError; + } + return Status(ErrorCodes::Error(code), errmsg); + } + BSONElement versionElement = cmdResult[schemaVersionServerParameter]; + if (versionElement.eoo()) + return Status(ErrorCodes::UnknownError, "getParameter misbehaved."); + *outVersion = versionElement.numberInt(); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } +} // namespace auth +} // namespace mongo diff --git a/src/mongo/client/auth_helpers.h b/src/mongo/client/auth_helpers.h new file mode 100644 index 00000000000..02413054e91 --- /dev/null +++ b/src/mongo/client/auth_helpers.h @@ -0,0 +1,36 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/client/dbclientinterface.h" + +namespace mongo { +namespace auth { + + /** + * Retrieves the schema version of the persistent data describing users and roles from the + * remote server connected to with conn. + */ + Status getRemoteStoredAuthorizationVersion(DBClientBase* conn, int* outVersion); + + /** + * Name of the server parameter used to report the auth schema version (via getParameter). + */ + extern const std::string schemaVersionServerParameter; + +} // namespace auth +} // namespace mongo diff --git a/src/mongo/client/authlevel.h b/src/mongo/client/authlevel.h deleted file mode 100644 index b37b5b3293e..00000000000 --- a/src/mongo/client/authlevel.h +++ /dev/null @@ -1,42 +0,0 @@ -// mongo/db/authlevel.h - -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -namespace mongo { - - /* - * for a particular db - * levels - * 0 : none - * 1 : read - * 2 : write - */ - struct Auth { - - enum Level { NONE = 0 , - READ = 1 , - WRITE = 2 }; - - Auth() : level( NONE ) {} - - Level level; - std::string user; - }; -} // namespace mongo diff --git a/src/mongo/client/clientAndShell.cpp b/src/mongo/client/clientAndShell.cpp index 92f12e40055..317965de927 100644 --- a/src/mongo/client/clientAndShell.cpp +++ b/src/mongo/client/clientAndShell.cpp @@ -15,18 +15,17 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" + +#include "mongo/client/clientOnly-private.h" +#include "mongo/db/client_basic.h" +#include "mongo/db/server_options.h" +#include "mongo/s/shard.h" #include "mongo/util/assert_util.h" -#include "../db/cmdline.h" -#include "../db/client_basic.h" -#include "../s/shard.h" -#include "../util/timer.h" -#include "clientOnly-private.h" +#include "mongo/util/timer.h" namespace mongo { - CmdLine cmdLine; - const char * curNs = "in client mode"; bool dbexitCalled = false; diff --git a/src/mongo/client/clientOnly-private.h b/src/mongo/client/clientOnly-private.h index 782ddad3297..419f385ba31 100644 --- a/src/mongo/client/clientOnly-private.h +++ b/src/mongo/client/clientOnly-private.h @@ -14,8 +14,13 @@ */ #pragma once + +#include "mongo/client/export_macros.h" + namespace mongo { + class mutex; + namespace shell_utils { - extern mongo::mutex &mongoProgramOutputMutex; + extern MONGO_CLIENT_API mongo::mutex &mongoProgramOutputMutex; } } diff --git a/src/mongo/client/connection_factory.cpp b/src/mongo/client/connection_factory.cpp deleted file mode 100644 index eba48a1d173..00000000000 --- a/src/mongo/client/connection_factory.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// connection_factory.cpp - -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/client/connpool.h" - -// This file contains the client-only implementation of the factory functions for getting -// ScopedDbConnections. -namespace mongo { - - ScopedDbConnection* ScopedDbConnection::getScopedDbConnection() { - return new ScopedDbConnection(); - } - - ScopedDbConnection* ScopedDbConnection::getScopedDbConnection(const string& host, - double socketTimeout) { - return new ScopedDbConnection(host, socketTimeout); - } - - ScopedDbConnection* ScopedDbConnection::getScopedDbConnection(const ConnectionString& host, - double socketTimeout) { - return new ScopedDbConnection(host, socketTimeout); - } - - // In the client code, these functions are the same as the ones above, since we don't have to - // do special handling of authentication for commands in the client. - ScopedDbConnection* ScopedDbConnection::getInternalScopedDbConnection() { - return getScopedDbConnection(); - } - - ScopedDbConnection* ScopedDbConnection::getInternalScopedDbConnection(const string& host, - double socketTimeout) { - return getScopedDbConnection( host, socketTimeout ); - } - - ScopedDbConnection* ScopedDbConnection::getInternalScopedDbConnection(const ConnectionString& host, - double socketTimeout) { - return getScopedDbConnection( host, socketTimeout ); - } - -} diff --git a/src/mongo/client/connpool.cpp b/src/mongo/client/connpool.cpp index 740249d87bb..d620f6d5c75 100644 --- a/src/mongo/client/connpool.cpp +++ b/src/mongo/client/connpool.cpp @@ -18,11 +18,12 @@ // _ todo: reconnect? -#include "pch.h" -#include "connpool.h" -#include "syncclusterconnection.h" -#include "../s/shard.h" -#include "mongo/client/dbclient_rs.h" +#include "mongo/pch.h" + +#include "mongo/client/connpool.h" +#include "mongo/client/replica_set_monitor.h" +#include "mongo/client/syncclusterconnection.h" +#include "mongo/s/shard.h" namespace mongo { @@ -40,18 +41,23 @@ namespace mongo { } } - void PoolForHost::done( DBConnectionPool * pool, DBClientBase * c ) { - if (c->isFailed()) { - reportBadConnectionAt(c->getSockCreationMicroSec()); - pool->onDestroy(c); - delete c; - } - else if (_pool.size() >= _maxPerHost || - c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) { + void PoolForHost::done(DBConnectionPool* pool, DBClientBase* c) { + + bool isFailed = c->isFailed(); + + // Remember that this host had a broken connection for later + if (isFailed) reportBadConnectionAt(c->getSockCreationMicroSec()); + + if (isFailed || + // Another (later) connection was reported as broken to this host + (c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) || + // We have a pool size that we need to enforce + (_maxPoolSize >= 0 && static_cast(_pool.size()) >= _maxPoolSize)) { pool->onDestroy(c); delete c; } else { + // The connection is probably fine, save for later _pool.push(c); } } @@ -61,7 +67,8 @@ namespace mongo { microSec > _minValidCreationTimeMicroSec) { _minValidCreationTimeMicroSec = microSec; log() << "Detected bad connection created at " << _minValidCreationTimeMicroSec - << " microSec, clearing pool for " << _hostName << endl; + << " microSec, clearing pool for " << _hostName + << " of " << _pool.size() << " connections" << endl; clear(); } } @@ -95,30 +102,10 @@ namespace mongo { } void PoolForHost::flush() { - vector all; - while ( ! _pool.empty() ) { + while (!_pool.empty()) { StoredConnection c = _pool.top(); _pool.pop(); - bool res; - bool alive = false; - try { - c.conn->isMaster( res ); - alive = true; - } catch ( const DBException e ) { - // There's something wrong with this connection, swallow the exception and do not - // put the connection back in the pool. - LOG(1) << "Exception thrown when checking pooled connection to " << - c.conn->getServerAddress() << ": " << causedBy(e) << endl; - delete c.conn; - c.conn = NULL; - } - if ( alive ) { - all.push_back( c ); - } - } - - for ( vector::iterator i=all.begin(); i != all.end(); ++i ) { - _pool.push( *i ); + delete c.conn; } } @@ -148,8 +135,8 @@ namespace mongo { } bool PoolForHost::StoredConnection::ok( time_t now ) { - // if connection has been idle for 30 minutes, kill it - return ( now - when ) < 1800; + // Poke the connection to see if we're still ok + return conn->isStillConnected(); } void PoolForHost::createdOne( DBClientBase * base) { @@ -164,22 +151,25 @@ namespace mongo { } } - unsigned PoolForHost::_maxPerHost = 50; - // ------ DBConnectionPool ------ DBConnectionPool pool; - DBConnectionPool::DBConnectionPool() + const int PoolForHost::kPoolSizeUnlimited(-1); + + DBConnectionPool::DBConnectionPool() : _mutex("DBConnectionPool") , _name( "dbconnectionpool" ) , - _hooks( new list() ) { + _maxPoolSize(PoolForHost::kPoolSizeUnlimited) , + _hooks( new list() ) { } DBClientBase* DBConnectionPool::_get(const string& ident , double socketTimeout ) { - verify( ! inShutdown() ); + uassert(17382, "Can't use connection pool during shutdown", + !inShutdown()); scoped_lock L(_mutex); PoolForHost& p = _pools[PoolKey(ident,socketTimeout)]; + p.setMaxPoolSize(_maxPoolSize); p.initializeHostName(ident); return p.get( this , socketTimeout ); } @@ -188,6 +178,7 @@ namespace mongo { { scoped_lock L(_mutex); PoolForHost& p = _pools[PoolKey(host,socketTimeout)]; + p.setMaxPoolSize(_maxPoolSize); p.initializeHostName(host); p.createdOne( conn ); } @@ -323,8 +314,6 @@ namespace mongo { map createdByType; - - set replicaSets; BSONObjBuilder bb( b.subobjStart( "hosts" ) ); { @@ -350,7 +339,7 @@ namespace mongo { bb.done(); // Always report all replica sets being tracked - ReplicaSetMonitor::getAllTrackedSets(&replicaSets); + set replicaSets = ReplicaSetMonitor::getAllTrackedSets(); BSONObjBuilder setBuilder( b.subobjStart( "replicaSets" ) ); for ( set::iterator i=replicaSets.begin(); i!=replicaSets.end(); ++i ) { diff --git a/src/mongo/client/connpool.h b/src/mongo/client/connpool.h index 1b9fe2761aa..9bf4583065f 100644 --- a/src/mongo/client/connpool.h +++ b/src/mongo/client/connpool.h @@ -19,9 +19,10 @@ #include -#include "mongo/util/background.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/client/export_macros.h" #include "mongo/platform/cstdint.h" +#include "mongo/util/background.h" namespace mongo { @@ -32,20 +33,40 @@ namespace mongo { * not thread safe * thread safety is handled by DBConnectionPool */ - class PoolForHost { + class MONGO_CLIENT_API PoolForHost { public: - PoolForHost() - : _created(0), _minValidCreationTimeMicroSec(0) {} - PoolForHost( const PoolForHost& other ) { + // Sentinel value indicating pool has no cleanup limit + static const int kPoolSizeUnlimited; + + PoolForHost() : + _created(0), + _minValidCreationTimeMicroSec(0), + _type(ConnectionString::INVALID), + _maxPoolSize(kPoolSizeUnlimited) { + } + + PoolForHost(const PoolForHost& other) : + _created(other._created), + _minValidCreationTimeMicroSec(other._minValidCreationTimeMicroSec), + _type(other._type), + _maxPoolSize(other._maxPoolSize) { + verify(_created == 0); verify(other._pool.size() == 0); - _created = other._created; - _minValidCreationTimeMicroSec = other._minValidCreationTimeMicroSec; - verify( _created == 0 ); } ~PoolForHost(); + /** + * Returns the maximum number of connections stored in the pool + */ + int getMaxPoolSize() { return _maxPoolSize; } + + /** + * Sets the maximum number of connections stored in the pool + */ + void setMaxPoolSize( int maxPoolSize ) { _maxPoolSize = maxPoolSize; } + int numAvailable() const { return (int)_pool.size(); } void createdOne( DBClientBase * base ); @@ -64,7 +85,7 @@ namespace mongo { void done( DBConnectionPool * pool , DBClientBase * c ); void flush(); - + void getStaleConnections( vector& stale ); /** @@ -84,8 +105,6 @@ namespace mongo { */ void initializeHostName(const std::string& hostName); - static void setMaxPerHost( unsigned max ) { _maxPerHost = max; } - static unsigned getMaxPerHost() { return _maxPerHost; } private: struct StoredConnection { @@ -99,12 +118,13 @@ namespace mongo { std::string _hostName; std::stack _pool; - + int64_t _created; uint64_t _minValidCreationTimeMicroSec; ConnectionString::ConnectionType _type; - static unsigned _maxPerHost; + // The maximum number of connections we'll save in the pool + int _maxPoolSize; }; class DBConnectionHook { @@ -130,8 +150,8 @@ namespace mongo { c.conn()... } */ - class DBConnectionPool : public PeriodicTask { - + class MONGO_CLIENT_API DBConnectionPool : public PeriodicTask { + public: DBConnectionPool(); @@ -140,6 +160,22 @@ namespace mongo { /** right now just controls some asserts. defaults to "dbconnectionpool" */ void setName( const string& name ) { _name = name; } + /** + * Returns the maximum number of connections pooled per-host + * + * This setting only applies to new host connection pools, previously-pooled host pools are + * unaffected. + */ + int getMaxPoolSize() { return _maxPoolSize; } + + /** + * Sets the maximum number of connections pooled per-host. + * + * This setting only applies to new host connection pools, previously-pooled host pools are + * unaffected. + */ + void setMaxPoolSize( int maxPoolSize ) { _maxPoolSize = maxPoolSize; } + void onCreate( DBClientBase * conn ); void onHandedOut( DBClientBase * conn ); void onDestroy( DBClientBase * conn ); @@ -179,15 +215,15 @@ namespace mongo { }; virtual string taskName() const { return "DBConnectionPool-cleaner"; } - virtual void taskDoWork(); + virtual void taskDoWork(); private: DBConnectionPool( DBConnectionPool& p ); - + DBClientBase* _get( const string& ident , double socketTimeout ); DBClientBase* _finishCreate( const string& ident , double socketTimeout, DBClientBase* conn ); - + struct PoolKey { PoolKey( const std::string& i , double t ) : ident( i ) , timeout( t ) {} string ident; @@ -202,27 +238,32 @@ namespace mongo { mongo::mutex _mutex; string _name; - + + // The maximum number of connections we'll save in the pool per-host + // PoolForHost::kPoolSizeUnlimited is a sentinel value meaning "no limit" + // 0 effectively disables the pool + int _maxPoolSize; + PoolMap _pools; // pointers owned by me, right now they leak on shutdown // _hooks itself also leaks because it creates a shutdown race condition - list * _hooks; + list * _hooks; }; - extern DBConnectionPool pool; + extern MONGO_CLIENT_API DBConnectionPool pool; - class AScopedConnection : boost::noncopyable { + class MONGO_CLIENT_API AScopedConnection : boost::noncopyable { public: AScopedConnection() { _numConnections++; } virtual ~AScopedConnection() { _numConnections--; } - + virtual DBClientBase* get() = 0; virtual void done() = 0; virtual string getHost() const = 0; - - /** + + /** * @return true iff this has a connection to the db */ virtual bool ok() const = 0; @@ -240,8 +281,8 @@ namespace mongo { clean up nicely (i.e. the socket gets closed automatically when the scopeddbconnection goes out of scope). */ - class ScopedDbConnection : public AScopedConnection { - private: + class MONGO_CLIENT_API ScopedDbConnection : public AScopedConnection { + public: /** the main constructor you want to use throws UserException if can't connect */ @@ -259,25 +300,6 @@ namespace mongo { ScopedDbConnection(const string& host, DBClientBase* conn, double socketTimeout = 0 ) : _host( host ) , _conn( conn ), _socketTimeout( socketTimeout ) { _setSocketTimeout(); } - public: - - // Factory functions for getting ScopedDbConnections. The caller owns the resulting object - // and is responsible for deleting it when finished. This should be used when running a - // command on a shard from the mongos and the command should run with the client's - // authentication. If the command should be run with full permissions regardless - // of whether or not the user is authorized, then use getInternalScopedDbConnection(). - static ScopedDbConnection* getScopedDbConnection(const string& host, - double socketTimeout = 0); - static ScopedDbConnection* getScopedDbConnection(const ConnectionString& host, - double socketTimeout = 0); - static ScopedDbConnection* getScopedDbConnection(); - - // DEPRECATED. This is now just a synonym for getScopedDbConnection. - static ScopedDbConnection* getInternalScopedDbConnection(const string& host, - double socketTimeout = 0); - static ScopedDbConnection* getInternalScopedDbConnection(const ConnectionString& host, - double socketTimeout = 0); - static ScopedDbConnection* getInternalScopedDbConnection(); static void clearPool(); diff --git a/src/mongo/client/dbclient.cpp b/src/mongo/client/dbclient.cpp index badd292124b..c57fcd67da8 100644 --- a/src/mongo/client/dbclient.cpp +++ b/src/mongo/client/dbclient.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/bson/util/bson_extract.h" #include "mongo/bson/util/builder.h" @@ -26,21 +26,19 @@ #include "mongo/client/syncclusterconnection.h" #include "mongo/db/jsobj.h" #include "mongo/db/json.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" #include "mongo/s/stale_exception.h" // for RecvStaleConfigException #include "mongo/util/assert_util.h" -#include "mongo/util/md5.hpp" - -#ifdef MONGO_SSL -// TODO: Remove references to cmdline from the client. -#include "mongo/db/cmdline.h" -#endif // defined MONGO_SSL +#include "mongo/util/net/ssl_manager.h" +#include "mongo/util/net/ssl_options.h" +#include "mongo/util/password_digest.h" namespace mongo { AtomicInt64 DBClientBase::ConnectionIdSequence; + const char* const saslCommandUserSourceFieldName = "userSource"; + void ConnectionString::_fillServers( string s ) { // @@ -413,12 +411,32 @@ namespace mongo { return QueryOptions(0); } - inline bool DBClientWithCommands::runCommand(const string &dbname, - const BSONObj& cmd, - BSONObj &info, - int options) { + void DBClientWithCommands::setRunCommandHook(RunCommandHookFunc func) { + _runCommandHook = func; + } + + void DBClientWithCommands::setPostRunCommandHook(PostRunCommandHookFunc func) { + _postRunCommandHook = func; + } + + bool DBClientWithCommands::runCommand(const string &dbname, + const BSONObj& cmd, + BSONObj &info, + int options) { string ns = dbname + ".$cmd"; - info = findOne(ns, cmd, 0 , options); + if (_runCommandHook) { + BSONObjBuilder cmdObj; + cmdObj.appendElements(cmd); + _runCommandHook(&cmdObj); + + info = findOne(ns, cmdObj.done(), 0 , options); + } + else { + info = findOne(ns, cmd, 0 , options); + } + if (_postRunCommandHook) { + _postRunCommandHook(info, getServerAddress()); + } return isOk(info); } @@ -435,10 +453,9 @@ namespace mongo { } unsigned long long DBClientWithCommands::count(const string &myns, const BSONObj& query, int options, int limit, int skip ) { - NamespaceString ns(myns); BSONObj cmd = _countCmd( myns , query , options , limit , skip ); BSONObj res; - if( !runCommand(ns.db.c_str(), cmd, res, options) ) + if( !runCommand(nsToDatabase(myns), cmd, res, options) ) uasserted(11010,string("count fails:") + res.toString()); return res["n"].numberLong(); } @@ -446,7 +463,7 @@ namespace mongo { BSONObj DBClientWithCommands::_countCmd(const string &myns, const BSONObj& query, int options, int limit, int skip ) { NamespaceString ns(myns); BSONObjBuilder b; - b.append( "count" , ns.coll ); + b.append( "count" , ns.coll() ); b.append( "query" , query ); if ( limit ) b.append( "limit" , limit ); @@ -526,32 +543,53 @@ namespace mongo { BSONObj getnoncecmdobj = fromjson("{getnonce:1}"); string DBClientWithCommands::createPasswordDigest( const string & username , const string & clearTextPassword ) { - md5digest d; - { - md5_state_t st; - md5_init(&st); - md5_append(&st, (const md5_byte_t *) username.data(), username.length()); - md5_append(&st, (const md5_byte_t *) ":mongo:", 7 ); - md5_append(&st, (const md5_byte_t *) clearTextPassword.data(), clearTextPassword.length()); - md5_finish(&st, d); - } - return digestToString( d ); + return mongo::createPasswordDigest(username, clearTextPassword); } + namespace { + class RunCommandHookOverrideGuard { + MONGO_DISALLOW_COPYING(RunCommandHookOverrideGuard); + public: + RunCommandHookOverrideGuard(DBClientWithCommands* cli, + const DBClientWithCommands::RunCommandHookFunc& hookFunc) + : _cli(cli), _oldHookFunc(cli->getRunCommandHook()) { + cli->setRunCommandHook(hookFunc); + } + ~RunCommandHookOverrideGuard() { + _cli->setRunCommandHook(_oldHookFunc); + } + private: + DBClientWithCommands* const _cli; + DBClientWithCommands::RunCommandHookFunc const _oldHookFunc; + }; + } // namespace void DBClientWithCommands::_auth(const BSONObj& params) { + RunCommandHookOverrideGuard hookGuard(this, RunCommandHookFunc()); std::string mechanism; + uassertStatusOK(bsonExtractStringField(params, saslCommandMechanismFieldName, &mechanism)); + uassert(17232, "You cannot specify both 'db' and 'userSource'. Please use only 'db'.", + !(params.hasField(saslCommandUserDBFieldName) + && params.hasField(saslCommandUserSourceFieldName))); + if (mechanism == StringData("MONGODB-CR", StringData::LiteralTag())) { - std::string userSource; - uassertStatusOK(bsonExtractStringField(params, - saslCommandPrincipalSourceFieldName, - &userSource)); + std::string db; + if (params.hasField(saslCommandUserSourceFieldName)) { + uassertStatusOK(bsonExtractStringField(params, + saslCommandUserSourceFieldName, + &db)); + } + else { + uassertStatusOK(bsonExtractStringField(params, + saslCommandUserDBFieldName, + &db)); + } std::string user; uassertStatusOK(bsonExtractStringField(params, - saslCommandPrincipalFieldName, + saslCommandUserFieldName, &user)); std::string password; uassertStatusOK(bsonExtractStringField(params, @@ -562,17 +600,52 @@ namespace mongo { saslCommandDigestPasswordFieldName, true, &digestPassword)); - std::string errmsg; + BSONObj result; + uassert(result["code"].Int(), + result.toString(), + _authMongoCR(db, user, password, &result, digestPassword)); + } +#ifdef MONGO_SSL + else if (mechanism == StringData("MONGODB-X509", StringData::LiteralTag())){ + std::string db; + if (params.hasField(saslCommandUserSourceFieldName)) { + uassertStatusOK(bsonExtractStringField(params, + saslCommandUserSourceFieldName, + &db)); + } + else { + uassertStatusOK(bsonExtractStringField(params, + saslCommandUserDBFieldName, + &db)); + } + std::string user; + uassertStatusOK(bsonExtractStringField(params, + saslCommandUserFieldName, + &user)); + + uassert(ErrorCodes::AuthenticationFailed, + "Please enable SSL on the client-side to use the MONGODB-X509 " + "authentication mechanism.", + getSSLManager() != NULL); + uassert(ErrorCodes::AuthenticationFailed, - errmsg, - _authMongoCR(userSource, user, password, errmsg, digestPassword)); + "Username \"" + user + + "\" does not match the provided client certificate user \"" + + getSSLManager()->getClientSubjectName() + "\"", + user == getSSLManager()->getClientSubjectName()); + + BSONObj result; + uassert(result["code"].Int(), + result.toString(), + _authX509(db, user, &result)); } +#endif else if (saslClientAuthenticate != NULL) { uassertStatusOK(saslClientAuthenticate(this, params)); } else { uasserted(ErrorCodes::BadValue, - "SASL authentication support not compiled into client library."); + mechanism + " mechanism support not compiled into client library."); } }; @@ -587,8 +660,8 @@ namespace mongo { bool digestPassword) { try { _auth(BSON(saslCommandMechanismFieldName << "MONGODB-CR" << - saslCommandPrincipalSourceFieldName << dbname << - saslCommandPrincipalFieldName << username << + saslCommandUserDBFieldName << dbname << + saslCommandUserFieldName << username << saslCommandPasswordFieldName << password_text << saslCommandDigestPasswordFieldName << digestPassword)); return true; @@ -603,21 +676,19 @@ namespace mongo { bool DBClientWithCommands::_authMongoCR(const string &dbname, const string &username, const string &password_text, - string& errmsg, + BSONObj *info, bool digestPassword) { string password = password_text; if( digestPassword ) password = createPasswordDigest( username , password_text ); - BSONObj info; string nonce; - if( !runCommand(dbname, getnoncecmdobj, info) ) { - errmsg = "getnonce failed: " + info.toString(); + if( !runCommand(dbname, getnoncecmdobj, *info) ) { return false; } { - BSONElement e = info.getField("nonce"); + BSONElement e = info->getField("nonce"); verify( e.type() == String ); nonce = e.valuestr(); } @@ -640,11 +711,25 @@ namespace mongo { authCmd = b.done(); } - if( runCommand(dbname, authCmd, info) ) { + if( runCommand(dbname, authCmd, *info) ) { + return true; + } + + return false; + } + + bool DBClientWithCommands::_authX509(const string&dbname, + const string &username, + BSONObj *info){ + BSONObj authCmd; + BSONObjBuilder cmdBuilder; + cmdBuilder << "authenticate" << 1 << "mechanism" << "MONGODB-X509" << "user" << username; + authCmd = cmdBuilder.done(); + + if( runCommand(dbname, authCmd, *info) ) { return true; } - errmsg = info.toString(); return false; } @@ -750,7 +835,10 @@ namespace mongo { list DBClientWithCommands::getDatabaseNames() { BSONObj info; - uassert( 10005 , "listdatabases failed" , runCommand( "admin" , BSON( "listDatabases" << 1 ) , info ) ); + uassert(10005, "listdatabases failed", runCommand("admin", + BSON("listDatabases" << 1), + info, + QueryOption_SlaveOk)); uassert( 10006 , "listDatabases.databases not array" , info["databases"].type() == Array ); list names; @@ -762,29 +850,97 @@ namespace mongo { return names; } - list DBClientWithCommands::getCollectionNames( const string& db ) { + list infos = getCollectionInfos( db ); list names; + for ( list::iterator it = infos.begin(); it != infos.end(); ++it ) { + names.push_back( db + "." + (*it)["name"].valuestr() ); + } + return names; + } + + list DBClientWithCommands::getCollectionInfos( const string& db, + const BSONObj& filter ) { + list infos; + + // first we're going to try the command + // it was only added in 3.0, so if we're talking to an older server + // we'll fail back to querying system.namespaces + // TODO(spencer): remove fallback behavior after 3.0 + + { + BSONObj res; + if (runCommand(db, + BSON("listCollections" << 1 << "filter" << filter + << "cursor" << BSONObj()), + res, + QueryOption_SlaveOk)) { + BSONObj cursorObj = res["cursor"].Obj(); + BSONObj collections = cursorObj["firstBatch"].Obj(); + BSONObjIterator it( collections ); + while ( it.more() ) { + BSONElement e = it.next(); + infos.push_back( e.Obj().getOwned() ); + } + + const long long id = cursorObj["id"].Long(); + + if ( id != 0 ) { + const std::string ns = cursorObj["ns"].String(); + auto_ptr cursor = getMore(ns, id, 0, 0); + while ( cursor->more() ) { + infos.push_back(cursor->nextSafe().getOwned()); + } + } + + return infos; + } + + // command failed + + int code = res["code"].numberInt(); + string errmsg = res["errmsg"].valuestrsafe(); + if ( code == ErrorCodes::CommandNotFound || + errmsg.find( "no such cmd" ) != string::npos ) { + // old version of server, ok, fall through to old code + } + else { + uasserted( 18630, str::stream() << "listCollections failed: " << res ); + } + + } + + // SERVER-14951 filter for old version fallback needs to db qualify the 'name' element + BSONObjBuilder fallbackFilter; + if ( filter.hasField( "name" ) && filter["name"].type() == String ) { + fallbackFilter.append( "name", db + "." + filter["name"].str() ); + } + fallbackFilter.appendElementsUnique( filter ); string ns = db + ".system.namespaces"; - auto_ptr c = query( ns.c_str() , BSONObj() ); + auto_ptr c = query( + ns.c_str(), fallbackFilter.obj(), 0, 0, 0, QueryOption_SlaveOk); + uassert(28611, str::stream() << "listCollections failed querying " << ns, c.get()); + while ( c->more() ) { - string name = c->next()["name"].valuestr(); - if ( name.find( "$" ) != string::npos ) + BSONObj obj = c->nextSafe(); + string ns = obj["name"].valuestr(); + if ( ns.find( "$" ) != string::npos ) continue; - names.push_back( name ); + BSONObjBuilder b; + b.append( "name", ns.substr( db.size() + 1 ) ); + b.appendElementsUnique( obj ); + infos.push_back( b.obj() ); } - return names; + + return infos; } bool DBClientWithCommands::exists( const string& ns ) { - list names; - - string db = nsGetDB( ns ) + ".system.namespaces"; - BSONObj q = BSON( "name" << ns ); - return count( db.c_str() , q, QueryOption_SlaveOk ) != 0; + BSONObj filter = BSON( "name" << nsToCollectionSubstring( ns ) ); + list results = getCollectionInfos( nsToDatabase( ns ), filter ); + return !results.empty(); } - /* --- dbclientconnection --- */ void DBClientConnection::_auth(const BSONObj& params) { @@ -793,7 +949,7 @@ namespace mongo { /* note we remember the auth info before we attempt to auth -- if the connection is broken, we will then have it for the next autoreconnect attempt. */ - authCache[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); + authCache[params[saslCommandUserDBFieldName].str()] = params.getOwned(); } DBClientBase::_auth(params); @@ -837,43 +993,57 @@ namespace mongo { bool DBClientConnection::_connect( string& errmsg ) { _serverString = _server.toString(); + _serverAddrString.clear(); // we keep around SockAddr for connection life -- maybe MessagingPort // requires that? server.reset(new SockAddr(_server.host().c_str(), _server.port())); p.reset(new MessagingPort( _so_timeout, _logLevel )); - if (_server.host().empty() || server->getAddr() == "0.0.0.0") { - stringstream s; - errmsg = - str::stream() << "couldn't connect to server " << _server.toString(); + if (_server.host().empty() ) { + errmsg = str::stream() << "couldn't connect to server " << toString() + << ", host is empty"; + return false; + } + + _serverAddrString = server->getAddr(); + + if ( _serverAddrString == "0.0.0.0" ) { + errmsg = str::stream() << "couldn't connect to server " << toString() + << ", address resolved to 0.0.0.0"; return false; } - // if( _so_timeout == 0 ){ - // printStackTrace(); - // log() << "Connecting to server " << _serverString << " timeout " << _so_timeout << endl; - // } if ( !p->connect(*server) ) { - errmsg = str::stream() << "couldn't connect to server " << _server.toString(); + errmsg = str::stream() << "couldn't connect to server " << toString() + << ", connection attempt failed"; _failed = true; return false; } + else { + LOG( 1 ) << "connected to server " << toString() << endl; + } #ifdef MONGO_SSL - if ( cmdLine.sslOnNormalPorts ) { - p->secure( sslManager() ); + int sslModeVal = sslGlobalParams.sslMode.load(); + if (sslModeVal == SSLGlobalParams::SSLMode_preferSSL || + sslModeVal == SSLGlobalParams::SSLMode_requireSSL) { + return p->secure( sslManager(), _server.host() ); } #endif return true; } + void DBClientConnection::logout(const string& dbname, BSONObj& info){ + authCache.erase(dbname); + runCommand(dbname, BSON("logout" << 1), info); + } - inline bool DBClientConnection::runCommand(const string &dbname, - const BSONObj& cmd, - BSONObj &info, - int options) { + bool DBClientConnection::runCommand(const string &dbname, + const BSONObj& cmd, + BSONObj &info, + int options) { if (DBClientWithCommands::runCommand(dbname, cmd, info, options)) return true; @@ -888,34 +1058,32 @@ namespace mongo { void DBClientConnection::_checkConnection() { if ( !_failed ) return; - if ( lastReconnectTry && time(0)-lastReconnectTry < 2 ) { - // we wait a little before reconnect attempt to avoid constant hammering. - // but we throw we don't want to try to use a connection in a bad state - throw SocketException( SocketException::FAILED_STATE , toString() ); - } + if ( !autoReconnect ) throw SocketException( SocketException::FAILED_STATE , toString() ); - lastReconnectTry = time(0); - LOG(_logLevel) << "trying reconnect to " << _serverString << endl; + // Don't hammer reconnects, backoff if needed + autoReconnectBackoff.nextSleepMillis(); + + LOG(_logLevel) << "trying reconnect to " << toString() << endl; string errmsg; _failed = false; if ( ! _connect(errmsg) ) { _failed = true; - LOG(_logLevel) << "reconnect " << _serverString << " failed " << errmsg << endl; + LOG(_logLevel) << "reconnect " << toString() << " failed " << errmsg << endl; throw SocketException( SocketException::CONNECT_ERROR , toString() ); } - LOG(_logLevel) << "reconnect " << _serverString << " ok" << endl; + LOG(_logLevel) << "reconnect " << toString() << " ok" << endl; for( map::const_iterator i = authCache.begin(); i != authCache.end(); i++ ) { try { DBClientConnection::_auth(i->second); } catch (UserException& ex) { if (ex.getCode() != ErrorCodes::AuthenticationFailed) throw; - LOG(_logLevel) << "reconnect: auth failed db:" << - i->second[saslCommandPrincipalSourceFieldName] << - " user:" << i->second[saslCommandPrincipalFieldName] << ' ' << + LOG(_logLevel) << "reconnect: auth failed " << + i->second[saslCommandUserDBFieldName] << + i->second[saslCommandUserFieldName] << ' ' << ex.what() << std::endl; } } @@ -1149,9 +1317,8 @@ namespace mongo { say( toSend ); } - auto_ptr DBClientWithCommands::getIndexes( const string &ns ) { - return query( Namespace( ns.c_str() ).getSisterNS( "system.indexes" ).c_str() , BSON( "ns" << ns ) ); + return query( NamespaceString( ns ).getSystemIndexesCollection() , BSON( "ns" << ns ) ); } void DBClientWithCommands::dropIndex( const string& ns , BSONObj keys ) { @@ -1162,7 +1329,7 @@ namespace mongo { void DBClientWithCommands::dropIndex( const string& ns , const string& indexName ) { BSONObj info; if ( ! runCommand( nsToDatabase( ns ) , - BSON( "deleteIndexes" << NamespaceString( ns ).coll << "index" << indexName ) , + BSON( "deleteIndexes" << nsToCollectionSubstring(ns) << "index" << indexName ) , info ) ) { LOG(_logLevel) << "dropIndex failed: " << info << endl; uassert( 10007 , "dropIndex failed" , 0 ); @@ -1172,9 +1339,12 @@ namespace mongo { void DBClientWithCommands::dropIndexes( const string& ns ) { BSONObj info; - uassert( 10008 , "dropIndexes failed" , runCommand( nsToDatabase( ns ) , - BSON( "deleteIndexes" << NamespaceString( ns ).coll << "index" << "*") , - info ) ); + uassert( 10008, + "dropIndexes failed", + runCommand( nsToDatabase( ns ), + BSON( "deleteIndexes" << nsToCollectionSubstring(ns) << "index" << "*"), + info ) + ); resetIndexCache(); } @@ -1189,7 +1359,7 @@ namespace mongo { for ( list::iterator i=all.begin(); i!=all.end(); i++ ) { BSONObj o = *i; - insert( Namespace( ns.c_str() ).getSisterNS( "system.indexes" ).c_str() , o ); + insert( NamespaceString( ns ).getSystemIndexesCollection() , o ); } } @@ -1259,7 +1429,7 @@ namespace mongo { if ( ttl > 0 ) toSave.append( "expireAfterSeconds", ttl ); - insert( Namespace( ns.c_str() ).getSisterNS( "system.indexes" ).c_str() , toSave.obj() ); + insert( NamespaceString( ns ).getSystemIndexesCollection() , toSave.obj() ); return 1; } @@ -1392,21 +1562,14 @@ namespace mongo { #ifdef MONGO_SSL static SimpleMutex s_mtx("SSLManager"); - static SSLManager* s_sslMgr(NULL); + static SSLManagerInterface* s_sslMgr(NULL); - SSLManager* DBClientConnection::sslManager() { + SSLManagerInterface* DBClientConnection::sslManager() { SimpleMutex::scoped_lock lk(s_mtx); if (s_sslMgr) return s_sslMgr; - const SSLParams params(cmdLine.sslPEMKeyFile, - cmdLine.sslPEMKeyPassword, - cmdLine.sslCAFile, - cmdLine.sslCRLFile, - cmdLine.sslWeakCertificateValidation, - cmdLine.sslFIPSMode); - s_sslMgr = new SSLManager(params); + s_sslMgr = getSSLManager(); - return s_sslMgr; } #endif diff --git a/src/mongo/client/dbclient.h b/src/mongo/client/dbclient.h index fec62d1614a..9930688b81e 100644 --- a/src/mongo/client/dbclient.h +++ b/src/mongo/client/dbclient.h @@ -25,6 +25,8 @@ #error dbclient.h is for C++ driver consumer use only #endif +#define LIBMONGOCLIENT_CONSUMER + #include "mongo/client/redef_macros.h" #include "mongo/pch.h" @@ -34,8 +36,9 @@ #include "mongo/client/dbclientcursor.h" #include "mongo/client/dbclientinterface.h" #include "mongo/client/gridfs.h" -#include "mongo/client/model.h" +#include "mongo/client/init.h" #include "mongo/client/sasl_client_authenticate.h" #include "mongo/client/syncclusterconnection.h" +#include "mongo/util/net/ssl_options.h" #include "mongo/client/undef_macros.h" diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp index 21a5b0810f4..a298cdfc0ab 100644 --- a/src/mongo/client/dbclient_rs.cpp +++ b/src/mongo/client/dbclient_rs.cpp @@ -1,5 +1,3 @@ -// dbclient.cpp - connect to a Mongo database as a database, from C++ - /* Copyright 2009 10gen Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,95 +13,22 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/client/dbclient_rs.h" -#include #include -#include "mongo/base/init.h" #include "mongo/bson/util/builder.h" #include "mongo/client/connpool.h" #include "mongo/client/dbclientcursor.h" +#include "mongo/client/replica_set_monitor.h" #include "mongo/client/sasl_client_authenticate.h" #include "mongo/db/dbmessage.h" #include "mongo/db/jsobj.h" -#include "mongo/db/json.h" -#include "mongo/util/background.h" -#include "mongo/util/concurrency/mutex.h" // for StaticObserver -#include "mongo/util/scopeguard.h" -#include "mongo/util/timer.h" namespace mongo { - - /* Replica Set statics: - * If a program (such as one built with the C++ driver) exits (by either calling exit() - * or by returning from main()), static objects will be destroyed in the reverse order - * of their creation (within each translation unit (source code file)). This makes it - * vital that the order be explicitly controlled within the source file so that destroyed - * objects never reference objects that have been destroyed earlier. - * - * The order chosen below is intended to allow safe destruction in reverse order from - * construction order: - * _setsLock -- mutex protecting _seedServers and _sets, destroyed last - * _seedServers -- list (map) of servers - * _sets -- list (map) of ReplicaSetMonitors - * replicaSetMonitorWatcher -- background job to check Replica Set members - * staticObserver -- sentinel to detect process termination - * - * Related to: - * SERVER-8891 -- Simple client fail with segmentation fault in mongoclient library - */ - mongo::mutex ReplicaSetMonitor::_setsLock( "ReplicaSetMonitor" ); - map > ReplicaSetMonitor::_seedServers; - map ReplicaSetMonitor::_sets; - - // global background job responsible for checking every X amount of time - class ReplicaSetMonitorWatcher : public BackgroundJob { - public: - ReplicaSetMonitorWatcher() : _safego("ReplicaSetMonitorWatcher::_safego") , _started(false) {} - - virtual string name() const { return "ReplicaSetMonitorWatcher"; } - - void safeGo() { - // check outside of lock for speed - if ( _started ) - return; - - scoped_lock lk( _safego ); - if ( _started ) - return; - _started = true; - - go(); - } - - protected: - void run() { - log() << "starting" << endl; - sleepsecs( 10 ); - while ( !inShutdown() && !StaticObserver::_destroyingStatics ) { - try { - ReplicaSetMonitor::checkAll( true ); - } - catch ( std::exception& e ) { - error() << "check failed: " << e.what() << endl; - } - catch ( ... ) { - error() << "unknown error" << endl; - } - sleepsecs( 10 ); - } - } - - mongo::mutex _safego; - bool _started; - - } replicaSetMonitorWatcher; - - static StaticObserver staticObserver; - +namespace { /* * Set of commands that can be used with $readPreference @@ -125,142 +50,6 @@ namespace mongo { } } _populateReadPrefSecOkCmdList; - /** - * @param ns the namespace of the query. - * @param queryOptionFlags the flags for the query. - * @param queryObj the query object to check. - * - * @return true if the given query can be sent to a secondary node without taking the - * slaveOk flag into account. - */ - bool _isQueryOkToSecondary(const string& ns, int queryOptionFlags, const BSONObj& queryObj) { - if (queryOptionFlags & QueryOption_SlaveOk) { - return true; - } - - if (!Query::hasReadPreference(queryObj)) { - return false; - } - - if (ns.find(".$cmd") == string::npos) { - return true; - } - - BSONObj actualQueryObj; - if (strcmp(queryObj.firstElement().fieldName(), "query") == 0) { - actualQueryObj = queryObj["query"].embeddedObject(); - } - else { - actualQueryObj = queryObj; - } - - const string cmdName = actualQueryObj.firstElementFieldName(); - if (_secOkCmdList.count(cmdName) == 1) { - return true; - } - - if (cmdName == "mapReduce" || cmdName == "mapreduce") { - if (!actualQueryObj.hasField("out")) { - return false; - } - - BSONElement outElem(actualQueryObj["out"]); - if (outElem.isABSONObj() && outElem["inline"].trueValue()) { - return true; - } - } - - return false; - } - - /** - * Selects the right node given the nodes to pick from and the preference. - * This method does strict tag matching, and will not implicitly fallback - * to matching anything. - * - * @param nodes the nodes to select from - * @param readPreferenceTag the tags to use for choosing the right node - * @param secOnly never select a primary if true - * @param localThresholdMillis the exclusive upper bound of ping time to be - * considered as a local node. Local nodes are favored over non-local - * nodes if multiple nodes matches the other criteria. - * @param lastHost the last host returned (mainly used for doing round-robin). - * Will be overwritten with the newly returned host if not empty. Should - * never be NULL. - * @param isPrimarySelected out parameter that is set to true if the returned host - * is a primary. - * - * @return the host object of the node selected. If none of the nodes are - * eligible, returns an empty host. Cannot be NULL and valid only if returned - * host is not empty. - */ - HostAndPort _selectNode(const vector& nodes, - const BSONObj& readPreferenceTag, - bool secOnly, - int localThresholdMillis, - HostAndPort* lastHost /* in/out */, - bool* isPrimarySelected) { - HostAndPort fallbackHost; - - // Implicit: start from index 0 if lastHost doesn't exist anymore - size_t nextNodeIndex = 0; - - if (!lastHost->empty()) { - for (size_t x = 0; x < nodes.size(); x++) { - if (*lastHost == nodes[x].addr) { - nextNodeIndex = x; - break; - } - } - } - - for (size_t itNode = 0; itNode < nodes.size(); ++itNode) { - nextNodeIndex = (nextNodeIndex + 1) % nodes.size(); - const ReplicaSetMonitor::Node& node = nodes[nextNodeIndex]; - - if (!node.ok) { - LOG(2) << "dbclient_rs not selecting " << node << ", not currently ok" << endl; - continue; - } - - if (secOnly && !node.okForSecondaryQueries()) { - LOG(3) << "dbclient_rs not selecting " << node - << ", not ok for secondary queries (" - << ( !node.secondary ? "not secondary" : "hidden" ) << ")" - << endl; - continue; - } - - if (node.matchesTag(readPreferenceTag)) { - // found an ok candidate; may not be local. - fallbackHost = node.addr; - *isPrimarySelected = node.ismaster; - - if (node.isLocalSecondary(localThresholdMillis)) { - // found a local node. return early. - LOG(2) << "dbclient_rs selecting local secondary " << fallbackHost - << ", ping time: " << node.pingTimeMillis << endl; - *lastHost = fallbackHost; - return fallbackHost; - } - } - } - - if (!fallbackHost.empty()) { - *lastHost = fallbackHost; - } - - if ( fallbackHost.empty() ) { - LOG(3) << "dbclient_rs no node selected for tag " << readPreferenceTag << endl; - } - else { - LOG(3) << "dbclient_rs node " << fallbackHost << " selected for tag " - << readPreferenceTag << endl; - } - - return fallbackHost; - } - /** * Extracts the read preference settings from the query document. Note that this method * assumes that the query is ok for secondaries so it defaults to @@ -271,1176 +60,213 @@ namespace mongo { * { query: , $readPreference: } * * Format B (unofficial internal format from mongos): - * { , $queryOptions: { $readPreference: }} - * - * @param query the raw query document - * - * @return the read preference setting. If the tags field was not present, it will contain one - * empty tag document {} which matches any tag. - * - * @throws AssertionException if the read preference object is malformed - */ - ReadPreferenceSetting* _extractReadPref(const BSONObj& query) { - ReadPreference pref = mongo::ReadPreference_SecondaryPreferred; - - if (Query::hasReadPreference(query)) { - BSONElement readPrefElement; - - if (query.hasField(Query::ReadPrefField.name())) { - readPrefElement = query[Query::ReadPrefField.name()]; - } - else { - readPrefElement = query["$queryOptions"][Query::ReadPrefField.name()]; - } - - uassert(16381, "$readPreference should be an object", - readPrefElement.isABSONObj()); - const BSONObj& prefDoc = readPrefElement.Obj(); - - uassert(16382, "mode not specified for read preference", - prefDoc.hasField(Query::ReadPrefModeField.name())); - - const string mode = prefDoc[Query::ReadPrefModeField.name()].String(); - - if (mode == "primary") { - pref = mongo::ReadPreference_PrimaryOnly; - } - else if (mode == "primaryPreferred") { - pref = mongo::ReadPreference_PrimaryPreferred; - } - else if (mode == "secondary") { - pref = mongo::ReadPreference_SecondaryOnly; - } - else if (mode == "secondaryPreferred") { - pref = mongo::ReadPreference_SecondaryPreferred; - } - else if (mode == "nearest") { - pref = mongo::ReadPreference_Nearest; - } - else { - uasserted(16383, str::stream() << "Unknown read preference mode: " << mode); - } - - if (prefDoc.hasField(Query::ReadPrefTagsField.name())) { - const BSONElement& tagsElem = prefDoc[Query::ReadPrefTagsField.name()]; - uassert(16385, "tags for read preference should be an array", - tagsElem.type() == mongo::Array); - - TagSet tags(BSONArray(tagsElem.Obj().getOwned())); - if (pref == mongo::ReadPreference_PrimaryOnly && !tags.isExhausted()) { - uassert(16384, "Only empty tags are allowed with primary read preference", - tags.getCurrentTag().isEmpty()); - } - - return new ReadPreferenceSetting(pref, tags); - } - } - - TagSet tags(BSON_ARRAY(BSONObj())); - return new ReadPreferenceSetting(pref, tags); - } - - /** - * @return the connection associated with the monitor node. Will also attempt - * to establish connection if NULL. Can still return NULL if reconnect failed. - */ - shared_ptr _getConnWithRefresh(ReplicaSetMonitor::Node& node) { - if (node.conn.get() == NULL) { - ConnectionString connStr(node.addr); - string errmsg; - - try { - node.conn.reset(dynamic_cast( - connStr.connect(errmsg, ReplicaSetMonitor::SOCKET_TIMEOUT_SECS))); - } - catch (const AssertionException&) { - node.ok = false; - } - } - - return node.conn; - } - - // -------------------------------- - // ----- ReplicaSetMonitor --------- - // -------------------------------- - - string seedString( const vector& servers ){ - string seedStr; - for ( unsigned i = 0; i < servers.size(); i++ ){ - seedStr += servers[i].toString(); - if( i < servers.size() - 1 ) seedStr += ","; - } - - return seedStr; - } - - const double ReplicaSetMonitor::SOCKET_TIMEOUT_SECS = 5; - - // Must already be in _setsLock when constructing a new ReplicaSetMonitor. This is why you - // should only create ReplicaSetMonitors from ReplicaSetMonitor::get and - // ReplicaSetMonitor::createIfNeeded. - ReplicaSetMonitor::ReplicaSetMonitor( const string& name , const vector& servers ) - : _lock( "ReplicaSetMonitor instance" ), - _checkConnectionLock( "ReplicaSetMonitor check connection lock" ), - _name( name ), _master(-1), - _nextSlave(0), _failedChecks(0), _localThresholdMillis(cmdLine.defaultLocalThresholdMillis) { - - uassert( 13642 , "need at least 1 node for a replica set" , servers.size() > 0 ); - - if ( _name.size() == 0 ) { - warning() << "replica set name empty, first node: " << servers[0] << endl; - } - - log() << "starting new replica set monitor for replica set " << _name << " with seed of " << seedString( servers ) << endl; - _populateHosts_inSetsLock(servers); - - _seedServers.insert( pair >(name, servers) ); - - log() << "replica set monitor for replica set " << _name << " started, address is " << getServerAddress() << endl; - - } - - // Must already be in _setsLock when destroying a ReplicaSetMonitor. This is why you should only - // delete ReplicaSetMonitors from ReplicaSetMonitor::remove. - ReplicaSetMonitor::~ReplicaSetMonitor() { - scoped_lock lk ( _lock ); - log() << "deleting replica set monitor for: " << _getServerAddress_inlock() << endl; - _cacheServerAddresses_inlock(); - pool.removeHost( _getServerAddress_inlock() ); - _nodes.clear(); - _master = -1; - } - - void ReplicaSetMonitor::_cacheServerAddresses_inlock() { - // Save list of current set members so that the monitor can be rebuilt if needed. - vector& servers = _seedServers[_name]; - servers.clear(); - for ( vector::iterator it = _nodes.begin(); it < _nodes.end(); ++it ) { - servers.push_back( it->addr ); - } - } - - void ReplicaSetMonitor::createIfNeeded( const string& name , const vector& servers ) { - scoped_lock lk( _setsLock ); - ReplicaSetMonitorPtr& m = _sets[name]; - if ( ! m ) - m.reset( new ReplicaSetMonitor( name , servers ) ); - - replicaSetMonitorWatcher.safeGo(); - } - - ReplicaSetMonitorPtr ReplicaSetMonitor::get( const string& name , const bool createFromSeed ) { - scoped_lock lk( _setsLock ); - map::const_iterator i = _sets.find( name ); - if ( i != _sets.end() ) { - return i->second; - } - if ( createFromSeed ) { - map >::const_iterator j = _seedServers.find( name ); - if ( j != _seedServers.end() ) { - LOG(4) << "Creating ReplicaSetMonitor from cached address" << endl; - ReplicaSetMonitorPtr& m = _sets[name]; - verify( !m ); - m.reset( new ReplicaSetMonitor( name, j->second ) ); - replicaSetMonitorWatcher.safeGo(); - return m; - } - } - return ReplicaSetMonitorPtr(); - } - - void ReplicaSetMonitor::getAllTrackedSets(set* activeSets) { - scoped_lock lk( _setsLock ); - for (map::const_iterator it = _sets.begin(); - it != _sets.end(); ++it) - { - activeSets->insert(it->first); - } - } - - void ReplicaSetMonitor::checkAll( bool checkAllSecondaries ) { - set seen; - - while ( true ) { - ReplicaSetMonitorPtr m; - { - scoped_lock lk( _setsLock ); - for ( map::iterator i=_sets.begin(); i!=_sets.end(); ++i ) { - string name = i->first; - if ( seen.count( name ) ) - continue; - LOG(1) << "checking replica set: " << name << endl; - seen.insert( name ); - m = i->second; - break; - } - } - - if ( ! m ) - break; - - m->check( checkAllSecondaries ); - { - scoped_lock lk( _setsLock ); - if ( m->_failedChecks >= _maxFailedChecks ) { - log() << "Replica set " << m->getName() << " was down for " << m->_failedChecks - << " checks in a row. Stopping polled monitoring of the set." << endl; - _remove_inlock( m->getName() ); - } - } - } - - - } - - void ReplicaSetMonitor::remove( const string& name, bool clearSeedCache ) { - scoped_lock lk( _setsLock ); - _remove_inlock( name, clearSeedCache ); - } - - void ReplicaSetMonitor::_remove_inlock( const string& name, bool clearSeedCache ) { - LOG(2) << "Removing ReplicaSetMonitor for " << name << " from replica set table" << endl; - _sets.erase( name ); - if ( clearSeedCache ) { - _seedServers.erase( name ); - } - } - - void ReplicaSetMonitor::setConfigChangeHook( ConfigChangeHook hook ) { - massert( 13610 , "ConfigChangeHook already specified" , _hook == 0 ); - _hook = hook; - } - - void ReplicaSetMonitor::setLocalThresholdMillis( const int millis ) { - scoped_lock lk( _lock ); - _localThresholdMillis = millis; - } - - string ReplicaSetMonitor::getServerAddress() const { - scoped_lock lk( _lock ); - return _getServerAddress_inlock(); - } - - string ReplicaSetMonitor::_getServerAddress_inlock() const { - StringBuilder ss; - if ( _name.size() ) - ss << _name << "/"; - - for ( unsigned i=0; i<_nodes.size(); i++ ) { - if ( i > 0 ) - ss << ","; - _nodes[i].addr.append( ss ); - } - - return ss.str(); - } - - bool ReplicaSetMonitor::contains( const string& server ) const { - scoped_lock lk( _lock ); - for ( unsigned i=0; i<_nodes.size(); i++ ) { - if ( _nodes[i].addr == server ) - return true; - } - return false; - } - - - void ReplicaSetMonitor::notifyFailure( const HostAndPort& server ) { - scoped_lock lk( _lock ); - - if ( _master >= 0 && _master < (int)_nodes.size() ) { - if ( server == _nodes[_master].addr ) { - _nodes[_master].ok = false; - _master = -1; - } - } - } - - - - HostAndPort ReplicaSetMonitor::getMaster() { - { - scoped_lock lk( _lock ); - verify(_master < static_cast(_nodes.size())); - if ( _master >= 0 && _nodes[_master].ok ) - return _nodes[_master].addr; - } - - _check( false ); - - scoped_lock lk( _lock ); - uassert( 10009 , str::stream() << "ReplicaSetMonitor no master found for set: " << _name , _master >= 0 ); - verify(_master < static_cast(_nodes.size())); - return _nodes[_master].addr; - } - - HostAndPort ReplicaSetMonitor::getSlave( const HostAndPort& prev ) { - // make sure its valid - - bool wasFound = false; - bool wasMaster = false; - - // This is always true, since checked in port() - verify( prev.port() >= 0 ); - if( prev.host().size() ){ - scoped_lock lk( _lock ); - for ( unsigned i=0; i<_nodes.size(); i++ ) { - if ( prev != _nodes[i].addr ) - continue; - - wasFound = true; - - if ( _nodes[i].okForSecondaryQueries() ) - return prev; - - wasMaster = _nodes[i].ok && !_nodes[i].secondary; - break; - } - } - - if( prev.host().size() ){ - if( wasFound ){ LOG(1) << "slave '" << prev << ( wasMaster ? "' is master node, trying to find another node" : - "' is no longer ok to use" ) << endl; } - else{ LOG(1) << "slave '" << prev << "' was not found in the replica set" << endl; } - } - else LOG(1) << "slave '" << prev << "' is not initialized or invalid" << endl; - - return getSlave(); - } - - HostAndPort ReplicaSetMonitor::getSlave( bool preferLocal ) { - LOG(2) << "dbclient_rs getSlave " << getServerAddress() << endl; - - HostAndPort fallbackNode; - scoped_lock lk( _lock ); - - for ( size_t itNode = 0; itNode < _nodes.size(); ++itNode ) { - _nextSlave = ( _nextSlave + 1 ) % _nodes.size(); - if ( _nextSlave != _master ) { - if ( _nodes[ _nextSlave ].okForSecondaryQueries() ) { - // found an ok slave; may not be local. - fallbackNode = _nodes[ _nextSlave ].addr; - if ( ! preferLocal ) - return fallbackNode; - else if ( _nodes[ _nextSlave ].isLocalSecondary( _localThresholdMillis ) ) { - // found a local slave. return early. - LOG(2) << "dbclient_rs getSlave found local secondary for queries: " - << _nextSlave << ", ping time: " - << _nodes[ _nextSlave ].pingTimeMillis << endl; - return fallbackNode; - } - } - else - LOG(2) << "dbclient_rs getSlave not selecting " << _nodes[_nextSlave] - << ", not currently okForSecondaryQueries" << endl; - } - } - - if ( ! fallbackNode.empty() ) { - // use a non-local secondary, even if local was preferred - LOG(1) << "dbclient_rs getSlave falling back to a non-local secondary node" << endl; - return fallbackNode; - } - - massert(15899, str::stream() << "No suitable secondary found for slaveOk query" - "in replica set: " << _name, _master >= 0 && - _master < static_cast(_nodes.size()) && _nodes[_master].ok); - - // Fall back to primary - LOG(1) << "dbclient_rs getSlave no member in secondary state found, " - "returning primary " << _nodes[ _master ] << endl; - return _nodes[_master].addr; - } - - /** - * notify the monitor that server has failed - */ - void ReplicaSetMonitor::notifySlaveFailure( const HostAndPort& server ) { - scoped_lock lk( _lock ); - int x = _find_inlock( server ); - if ( x >= 0 ) { - _nodes[x].ok = false; - } - } - - void ReplicaSetMonitor::_checkStatus( const string& hostAddr ) { - BSONObj status; - - /* replSetGetStatus requires admin auth so use a connection from the pool, - * and tell it to use the internal credentials. - */ - scoped_ptr authenticatedConn( - ScopedDbConnection::getInternalScopedDbConnection( hostAddr, 5.0 ) ); - - if ( !authenticatedConn->get()->runCommand( "admin", - BSON( "replSetGetStatus" << 1 ), - status )) { - LOG(1) << "dbclient_rs replSetGetStatus failed" << status << endl; - authenticatedConn->done(); // connection worked properly, but we got an error from server - return; - } - - // Make sure we return when finished - authenticatedConn->done(); - - if( !status.hasField("members") ) { - log() << "dbclient_rs error expected members field in replSetGetStatus result" << endl; - return; - } - if( status["members"].type() != Array) { - log() << "dbclient_rs error expected members field in replSetGetStatus result to be an array" << endl; - return; - } - - BSONObjIterator hi(status["members"].Obj()); - while (hi.more()) { - BSONObj member = hi.next().Obj(); - string host = member["name"].String(); - - int m = -1; - if ((m = _find(host)) < 0) { - LOG(1) << "dbclient_rs _checkStatus couldn't _find(" << host << ')' << endl; - continue; - } - - double state = member["state"].Number(); - if (member["health"].Number() == 1 && (state == 1 || state == 2)) { - LOG(1) << "dbclient_rs nodes["< nodesFound; - - int index = 0; - BSONObjIterator hi( hostList ); - while( hi.more() ){ - - string toCheck = hi.next().String(); - int nodeIndex = _find_inlock( toCheck ); - - // Node-to-add - if( nodeIndex < 0 ) diff.first.insert( toCheck ); - else nodesFound.insert( nodeIndex ); - - index++; - } - - for( size_t i = 0; i < _nodes.size(); i++ ){ - if( nodesFound.find( static_cast(i) ) == nodesFound.end() ) diff.second.insert( static_cast(i) ); - } - - return diff; - } - - bool ReplicaSetMonitor::_shouldChangeHosts( const BSONObj& hostList, bool inlock ){ - - int origHosts = 0; - if( ! inlock ){ - scoped_lock lk( _lock ); - origHosts = _nodes.size(); - } - else origHosts = _nodes.size(); - int numHosts = 0; - bool changed = false; - - BSONObjIterator hi(hostList); - while ( hi.more() ) { - string toCheck = hi.next().String(); - - numHosts++; - int index = 0; - if( ! inlock ) index = _find( toCheck ); - else index = _find_inlock( toCheck ); - - if ( index >= 0 ) continue; - - changed = true; - break; - } - - return (changed || origHosts != numHosts) && numHosts > 0; - - } - - void ReplicaSetMonitor::_checkHosts( const BSONObj& hostList, bool& changed ) { - - // Fast path, still requires intermittent locking - if( ! _shouldChangeHosts( hostList, false ) ){ - changed = false; - return; - } - - // Slow path, double-checked though - scoped_lock lk( _lock ); - - // Our host list may have changed while waiting for another thread in the meantime, - // so double-check here - // TODO: Do we really need this much protection, this should be pretty rare and not - // triggered from lots of threads, duping old behavior for safety - if( ! _shouldChangeHosts( hostList, true ) ){ - changed = false; - return; - } - - // LogLevel can be pretty low, since replica set reconfiguration should be pretty rare and - // we want to record our changes - log() << "changing hosts to " << hostList << " from " << _getServerAddress_inlock() << endl; - - NodeDiff diff = _getHostDiff_inlock( hostList ); - set added = diff.first; - set removed = diff.second; - - verify( added.size() > 0 || removed.size() > 0 ); - changed = true; - - // Delete from the end so we don't invalidate as we delete, delete indices are ascending - for( set::reverse_iterator i = removed.rbegin(), end = removed.rend(); i != end; ++i ){ - - log() << "erasing host " << _nodes[ *i ] << " from replica set " << this->_name << endl; - _nodes.erase( _nodes.begin() + *i ); - } - - // Add new nodes - for( set::iterator i = added.begin(), end = added.end(); i != end; ++i ){ - - log() << "trying to add new host " << *i << " to replica set " << this->_name << endl; - - // Connect to new node - HostAndPort host(*i); - ConnectionString connStr(host); - - uassert(16530, str::stream() << "cannot create a replSet node connection that " - "is not single: " << host.toString(true), - connStr.type() == ConnectionString::MASTER || - connStr.type() == ConnectionString::CUSTOM); - - DBClientConnection* newConn = NULL; - string errmsg; - try { - // Needs to perform a dynamic_cast because we need to set the replSet - // callback. We should eventually not need this after we remove the - // callback. - newConn = dynamic_cast( - connStr.connect(errmsg, SOCKET_TIMEOUT_SECS)); - } - catch (const AssertionException& ex) { - errmsg = ex.toString(); - } - - if (errmsg.empty()) { - log() << "successfully connected to new host " << *i - << " in replica set " << this->_name << endl; - } - else { - warning() << "cannot connect to new host " << *i - << " to replica set " << this->_name - << ", err: " << errmsg << endl; - } - - _nodes.push_back(Node(host, newConn)); - } - - // Invalidate the cached _master index since the _nodes structure has - // already been modified. - _master = -1; - } - - - bool ReplicaSetMonitor::_checkConnection( DBClientConnection* conn, - string& maybePrimary, bool verbose, int nodesOffset ) { - - verify( conn ); - scoped_lock lk( _checkConnectionLock ); - bool isMaster = false; - bool changed = false; - bool errorOccured = false; - - if ( nodesOffset >= 0 ){ - scoped_lock lk( _lock ); - if ( !_checkConnMatch_inlock( conn, nodesOffset )) { - /* Another thread modified _nodes -> invariant broken. - * This also implies that another thread just passed - * through here and refreshed _nodes. So no need to do - * duplicate work. - */ - return false; - } - } - - try { - Timer t; - BSONObj o; - conn->isMaster( isMaster, &o ); - - if ( o["setName"].type() != String || o["setName"].String() != _name ) { - warning() << "node: " << conn->getServerAddress() - << " isn't a part of set: " << _name - << " ismaster: " << o << endl; - - if ( nodesOffset >= 0 ) { - scoped_lock lk( _lock ); - _nodes[nodesOffset].ok = false; - } - - return false; - } - int commandTime = t.millis(); - - if ( nodesOffset >= 0 ) { - scoped_lock lk( _lock ); - Node& node = _nodes[nodesOffset]; - - if (node.pingTimeMillis == 0) { - node.pingTimeMillis = commandTime; - } - else { - // update ping time with smoothed moving averaged (1/4th the delta) - node.pingTimeMillis += (commandTime - node.pingTimeMillis) / 4; - } - - node.hidden = o["hidden"].trueValue(); - node.secondary = o["secondary"].trueValue(); - node.ismaster = o["ismaster"].trueValue(); - - node.lastIsMaster = o.copy(); - } - - LOG( verbose ? 0 : 1 ) << "ReplicaSetMonitor::_checkConnection: " << conn->toString() - << ' ' << o << endl; - - // add other nodes - BSONArrayBuilder b; - if ( o["hosts"].type() == Array ) { - if ( o["primary"].type() == String ) - maybePrimary = o["primary"].String(); - - BSONObjIterator it( o["hosts"].Obj() ); - while( it.more() ) b.append( it.next() ); - } - - if (o.hasField("passives") && o["passives"].type() == Array) { - BSONObjIterator it( o["passives"].Obj() ); - while( it.more() ) b.append( it.next() ); - } - - _checkHosts( b.arr(), changed); - _checkStatus( conn->getServerAddress() ); - - } - catch ( std::exception& e ) { - LOG( verbose ? 0 : 1 ) << "ReplicaSetMonitor::_checkConnection: caught exception " - << conn->toString() << ' ' << e.what() << endl; - - errorOccured = true; - } - - if ( errorOccured && nodesOffset >= 0 ) { - scoped_lock lk( _lock ); - - if (_checkConnMatch_inlock(conn, nodesOffset)) { - // Make sure _checkHosts didn't modify the _nodes structure - _nodes[nodesOffset].ok = false; - } - } - - if ( changed && _hook ) - _hook( this ); - - return isMaster; - } - - void ReplicaSetMonitor::_check( bool checkAllSecondaries ) { - LOG(1) << "_check : " << getServerAddress() << endl; - - int newMaster = -1; - shared_ptr nodeConn; - - for ( int retry = 0; retry < 2; retry++ ) { - bool triedQuickCheck = false; - - if ( !checkAllSecondaries ) { - scoped_lock lk( _lock ); - verify(_master < static_cast(_nodes.size())); - if ( _master >= 0 && _nodes[_master].ok ) { - /* Nothing else to do since another thread already - * found a usable _master - */ - return; - } - } - - for ( unsigned i = 0; /* should not check while outside of lock! */ ; i++ ) { - { - scoped_lock lk( _lock ); - if ( i >= _nodes.size() ) break; - nodeConn = _getConnWithRefresh(_nodes[i]); - if (nodeConn.get() == NULL) continue; - } - - string maybePrimary; - if ( _checkConnection( nodeConn.get(), maybePrimary, retry, i ) ) { - scoped_lock lk( _lock ); - if ( _checkConnMatch_inlock( nodeConn.get(), i )) { - newMaster = i; - if ( newMaster != _master ) { - log() << "Primary for replica set " << _name - << " changed to " << _nodes[newMaster].addr << endl; - } - _master = i; - - if ( !checkAllSecondaries ) - return; - } - else { - /* - * Somebody modified _nodes and most likely set the new - * _master, so try again. - */ - break; - } - } - - - if ( ! triedQuickCheck && ! maybePrimary.empty() ) { - int probablePrimaryIdx = -1; - shared_ptr probablePrimaryConn; - - { - scoped_lock lk( _lock ); - probablePrimaryIdx = _find_inlock( maybePrimary ); - - if (probablePrimaryIdx >= 0) { - probablePrimaryConn = _getConnWithRefresh( - _nodes[probablePrimaryIdx]); - if (probablePrimaryConn.get() == NULL) continue; - } - } - - if ( probablePrimaryIdx >= 0 ) { - triedQuickCheck = true; - - string dummy; - if ( _checkConnection( probablePrimaryConn.get(), dummy, - false, probablePrimaryIdx ) ) { - - scoped_lock lk( _lock ); - - if ( _checkConnMatch_inlock( probablePrimaryConn.get(), - probablePrimaryIdx )) { - - - newMaster = probablePrimaryIdx; - if ( newMaster != _master ) { - log() << "Primary for replica set " << _name << " changed to " << _nodes[newMaster].addr << endl; - } - _master = probablePrimaryIdx; - - if ( ! checkAllSecondaries ) - return; - } - else { - /* - * Somebody modified _nodes and most likely set the - * new _master, so try again. - */ - break; - } - } - } - } - } - - if ( newMaster >= 0 ) - return; - - sleepsecs( 1 ); - } - - { - warning() << "No primary detected for set " << _name << endl; - scoped_lock lk( _lock ); - _master = -1; - - for (vector::iterator iter = _nodes.begin(); iter < _nodes.end(); ++iter) { - iter->ismaster = false; - } - - if (checkAllSecondaries) { - for ( unsigned i = 0; i < _nodes.size(); i++ ) { - if ( _nodes[i].ok ) { - _failedChecks = 0; - return; - } - } - // None of the nodes are ok. - _failedChecks++; - log() << "All nodes for set " << _name << " are down. This has happened for " << _failedChecks << " checks in a row. Polling will stop after " << _maxFailedChecks - _failedChecks << " more failed checks" << endl; - } - } - } - - void ReplicaSetMonitor::check( bool checkAllSecondaries ) { - bool isNodeEmpty = false; - - { - scoped_lock lk( _lock ); - isNodeEmpty = _nodes.empty(); - } - - if (isNodeEmpty) { - scoped_lock lk(_setsLock); - _populateHosts_inSetsLock(_seedServers[_name]); - /* _populateHosts_inlock already refreshes _nodes so no more work - * needs to be done. If it was unsuccessful, the succeeding lines - * will also fail, so no point in trying. - */ - return; - } - - shared_ptr masterConn; - - { - scoped_lock lk( _lock ); - - // first see if the current master is fine - if ( _master >= 0 ) { - verify(_master < static_cast(_nodes.size())); - masterConn = _getConnWithRefresh(_nodes[_master]); - } - } - - if ( masterConn.get() != NULL ) { - string temp; - - if ( _checkConnection( masterConn.get(), temp, false, _master )) { - if ( ! checkAllSecondaries ) { - // current master is fine, so we're done - return; - } - } - } - - // we either have no master, or the current is dead - _check( checkAllSecondaries ); - } - - int ReplicaSetMonitor::_find( const string& server ) const { - scoped_lock lk( _lock ); - return _find_inlock( server ); - } - - int ReplicaSetMonitor::_find_inlock( const string& server ) const { - const size_t size = _nodes.size(); - - for ( unsigned i = 0; i < size; i++ ) { - if ( _nodes[i].addr == server ) { - return i; - } - } - - return -1; - } - - void ReplicaSetMonitor::appendInfo(BSONObjBuilder& bsonObjBuilder) const { - scoped_lock lk(_lock); - BSONArrayBuilder hosts(bsonObjBuilder.subarrayStart("hosts")); - for (unsigned i = 0; i < _nodes.size(); i++) { - const Node& node = _nodes[i]; - - /* Note: cannot use toBSON helper method due to backwards compatibility. - * In particular, toBSON method uses "isMaster" while this method - * uses "ismaster" - */ - BSONObjBuilder builder; - builder.append("addr", node.addr.toString()); - builder.append("ok", node.ok); - builder.append("ismaster", node.ismaster); - builder.append("hidden", node.hidden); - builder.append("secondary", node.secondary); - builder.append("pingTimeMillis", node.pingTimeMillis); - - const BSONElement& tagElem = node.lastIsMaster["tags"]; - if (tagElem.ok() && tagElem.isABSONObj()) { - builder.append("tags", tagElem.Obj()); - } - - hosts.append(builder.obj()); - } - hosts.done(); - - bsonObjBuilder.append("master", _master); - bsonObjBuilder.append("nextSlave", _nextSlave); - } - - bool ReplicaSetMonitor::_checkConnMatch_inlock( DBClientConnection* conn, - size_t nodeOffset ) const { - return (nodeOffset < _nodes.size() && - // Assumption: value for getServerAddress was extracted from - // HostAndPort::toString() - conn->getServerAddress() == _nodes[nodeOffset].addr.toString()); - } - - HostAndPort ReplicaSetMonitor::selectAndCheckNode(ReadPreference preference, - TagSet* tags, - bool* isPrimarySelected) { - - HostAndPort candidate; - - { - scoped_lock lk(_lock); - candidate = ReplicaSetMonitor::selectNode(_nodes, preference, tags, - _localThresholdMillis, &_lastReadPrefHost, isPrimarySelected); - } - - if (candidate.empty()) { - - LOG( 3 ) << "dbclient_rs no compatible nodes found, refreshing view of replica set " - << _name << endl; - - // mimic checkMaster behavior, which refreshes the local view of the replica set - _check(false); + * { , $queryOptions: { $readPreference: }} + * + * @param query the raw query document + * + * @return the read preference setting if a read preference exists, otherwise the default read + * preference of Primary_Only. If the tags field was not present, it will contain one + * empty tag document {} which matches any tag. + * + * @throws AssertionException if the read preference object is malformed + */ + ReadPreferenceSetting* _extractReadPref(const BSONObj& query, int queryOptions) { - scoped_lock lk(_lock); - return ReplicaSetMonitor::selectNode(_nodes, preference, tags, _localThresholdMillis, - &_lastReadPrefHost, isPrimarySelected); - } + if (Query::hasReadPreference(query)) { - return candidate; - } + ReadPreference pref = mongo::ReadPreference_SecondaryPreferred; - // static - HostAndPort ReplicaSetMonitor::selectNode(const std::vector& nodes, - ReadPreference preference, - TagSet* tags, - int localThresholdMillis, - HostAndPort* lastHost, - bool* isPrimarySelected) { - *isPrimarySelected = false; + BSONElement readPrefElement; - switch (preference) { - case ReadPreference_PrimaryOnly: - for (vector::const_iterator iter = nodes.begin(); iter != nodes.end(); ++iter) { - if (iter->ismaster && iter->ok) { - *isPrimarySelected = true; - return iter->addr; - } + if (query.hasField(Query::ReadPrefField.name())) { + readPrefElement = query[Query::ReadPrefField.name()]; } - - return HostAndPort(); - - case ReadPreference_PrimaryPreferred: - { - HostAndPort candidatePri = selectNode(nodes, ReadPreference_PrimaryOnly, tags, - localThresholdMillis, lastHost, isPrimarySelected); - - if (!candidatePri.empty()) { - return candidatePri; + else { + readPrefElement = query["$queryOptions"][Query::ReadPrefField.name()]; } - return selectNode(nodes, ReadPreference_SecondaryOnly, tags, - localThresholdMillis, lastHost, isPrimarySelected); - } + uassert(16381, "$readPreference should be an object", + readPrefElement.isABSONObj()); + const BSONObj& prefDoc = readPrefElement.Obj(); - case ReadPreference_SecondaryOnly: - { - HostAndPort candidate; + uassert(16382, "mode not specified for read preference", + prefDoc.hasField(Query::ReadPrefModeField.name())); - while (!tags->isExhausted()) { - candidate = _selectNode(nodes, tags->getCurrentTag(), true, localThresholdMillis, - lastHost, isPrimarySelected); + const string mode = prefDoc[Query::ReadPrefModeField.name()].String(); - if (candidate.empty()) { - tags->next(); - } - else { - return candidate; - } + if (mode == "primary") { + pref = mongo::ReadPreference_PrimaryOnly; } - - return candidate; - } - - case ReadPreference_SecondaryPreferred: - { - HostAndPort candidateSec = selectNode(nodes, ReadPreference_SecondaryOnly, tags, - localThresholdMillis, lastHost, isPrimarySelected); - - if (!candidateSec.empty()) { - return candidateSec; + else if (mode == "primaryPreferred") { + pref = mongo::ReadPreference_PrimaryPreferred; } - - return selectNode(nodes, ReadPreference_PrimaryOnly, tags, - localThresholdMillis, lastHost, isPrimarySelected); - } - - case ReadPreference_Nearest: - { - HostAndPort candidate; - - while (!tags->isExhausted()) { - candidate = _selectNode(nodes, tags->getCurrentTag(), false, localThresholdMillis, - lastHost, isPrimarySelected); - - if (candidate.empty()) { - tags->next(); - } - else { - return candidate; - } + else if (mode == "secondary") { + pref = mongo::ReadPreference_SecondaryOnly; + } + else if (mode == "secondaryPreferred") { + pref = mongo::ReadPreference_SecondaryPreferred; + } + else if (mode == "nearest") { + pref = mongo::ReadPreference_Nearest; + } + else { + uasserted(16383, str::stream() << "Unknown read preference mode: " << mode); } - return candidate; - } - - default: - uassert( 16337, "Unknown read preference", false ); - break; - } + if (prefDoc.hasField(Query::ReadPrefTagsField.name())) { + const BSONElement& tagsElem = prefDoc[Query::ReadPrefTagsField.name()]; + uassert(16385, "tags for read preference should be an array", + tagsElem.type() == mongo::Array); - return HostAndPort(); - } + TagSet tags(BSONArray(tagsElem.Obj().getOwned())); + if (pref == mongo::ReadPreference_PrimaryOnly && !tags.getTagBSON().isEmpty()) { + uassert(16384, "Only empty tags are allowed with primary read preference", + tags.getTagBSON().firstElement().Obj().isEmpty()); + } - bool ReplicaSetMonitor::isHostCompatible(const HostAndPort& host, - ReadPreference readPreference, - const TagSet* tagSet) const { - scoped_lock lk(_lock); - for (vector::const_iterator iter = _nodes.begin(); iter != _nodes.end(); ++iter) { - if (iter->addr == host) { - return iter->isCompatible(readPreference, tagSet); + return new ReadPreferenceSetting(pref, tags); + } + else { + return new ReadPreferenceSetting(pref, TagSet()); } } - // host is not part of the set anymore! - return false; + // Default read pref is primary only or secondary preferred with slaveOK + ReadPreference pref = + queryOptions & QueryOption_SlaveOk ? + mongo::ReadPreference_SecondaryPreferred : mongo::ReadPreference_PrimaryOnly; + return new ReadPreferenceSetting(pref, TagSet()); } +} // namespace - void ReplicaSetMonitor::_populateHosts_inSetsLock(const vector& seedList){ - verify(_nodes.empty()); - - for (vector::const_iterator iter = seedList.begin(); - iter != seedList.end(); ++iter) { - // Don't check servers we have already - if (_find(*iter) >= 0) continue; + // -------------------------------- + // ----- DBClientReplicaSet --------- + // -------------------------------- - ConnectionString connStr(*iter); - scoped_ptr conn; + const size_t DBClientReplicaSet::MAX_RETRY = 3; + int DBClientReplicaSet::reevaluatePercentage = 0; - uassert(16531, str::stream() << "cannot create a replSet node connection that " - "is not single: " << iter->toString(true), - connStr.type() == ConnectionString::MASTER || - connStr.type() == ConnectionString::CUSTOM); + DBClientReplicaSet::DBClientReplicaSet( + const string& name, + const vector& servers, + double so_timeout): + _setName(name), + _so_timeout(so_timeout), + _nodeSelectOracle(static_cast(time(0))) { + ReplicaSetMonitor::createIfNeeded( name, set(servers.begin(), servers.end()) ); + } - string errmsg; - try { - // Needs to perform a dynamic_cast because we need to set the replSet - // callback. We should eventually not need this after we remove the - // callback. - conn.reset(dynamic_cast( - connStr.connect(errmsg, SOCKET_TIMEOUT_SECS))); - } - catch (const AssertionException& ex) { - errmsg = ex.toString(); - } + DBClientReplicaSet::~DBClientReplicaSet() { + } - if (conn.get() != NULL && errmsg.empty()) { - log() << "successfully connected to seed " << *iter - << " for replica set " << _name << endl; + ReplicaSetMonitorPtr DBClientReplicaSet::_getMonitor() const { + ReplicaSetMonitorPtr rsm = ReplicaSetMonitor::get( _setName, true ); + // If you can't get a ReplicaSetMonitor then this connection isn't valid + uassert( 16340, str::stream() << "No replica set monitor active and no cached seed " + "found for set: " << _setName, rsm ); + return rsm; + } - string maybePrimary; - _checkConnection(conn.get(), maybePrimary, false, -1); - } - else { - log() << "error connecting to seed " << *iter - << ", err: " << errmsg << endl; - } + // This can't throw an exception because it is called in the destructor of ScopedDbConnection + string DBClientReplicaSet::getServerAddress() const { + ReplicaSetMonitorPtr rsm = ReplicaSetMonitor::get( _setName, true ); + if ( !rsm ) { + warning() << "Trying to get server address for DBClientReplicaSet, but no " + "ReplicaSetMonitor exists for " << _setName << endl; + return str::stream() << _setName << "/" ; } - - // Check everything to get the first data - _check(true); + return rsm->getServerAddress(); } - bool ReplicaSetMonitor::isAnyNodeOk() const { - scoped_lock lock(_lock); - - for (vector::const_iterator iter = _nodes.begin(); - iter != _nodes.end(); ++iter) { - if (iter->ok) { - return true; - } + HostAndPort DBClientReplicaSet::getSuspectedPrimaryHostAndPort() const { + if (!_master) { + return HostAndPort(); } - - return false; + return _master->getServerHostAndPort(); } - bool ReplicaSetMonitor::Node::matchesTag(const BSONObj& tag) const { - if (tag.isEmpty()) { - return true; + void DBClientReplicaSet::setRunCommandHook(DBClientWithCommands::RunCommandHookFunc func) { + // Set the hooks in both our sub-connections and in ourselves. + if (_master) { + _master->setRunCommandHook(func); + } + if (_lastSlaveOkConn) { + _lastSlaveOkConn->setRunCommandHook(func); } + _runCommandHook = func; + } - const BSONElement& myTagElem = lastIsMaster["tags"]; - if (!myTagElem.isABSONObj()) { - return false; + void DBClientReplicaSet::setPostRunCommandHook + (DBClientWithCommands::PostRunCommandHookFunc func) { + // Set the hooks in both our sub-connections and in ourselves. + if (_master) { + _master->setPostRunCommandHook(func); } + if (_lastSlaveOkConn) { + _lastSlaveOkConn->setPostRunCommandHook(func); + } + _postRunCommandHook = func; + } - const BSONObj& myTagObj = myTagElem.Obj(); - for (BSONObjIterator iter(tag); iter.more();) { - const BSONElement& tagCriteria(iter.next()); - const char* field = tagCriteria.fieldName(); + // A replica set connection is never disconnected, since it controls its own reconnection + // logic. + // + // Has the side effect of proactively clearing any cached connections which have been + // disconnected in the background. + bool DBClientReplicaSet::isStillConnected() { - if (!myTagObj.hasField(field) || - !tagCriteria.valuesEqual(myTagObj[field])) { - return false; - } + if ( _master && !_master->isStillConnected() ) { + _master.reset(); + _masterHost = HostAndPort(); + // Don't notify monitor of bg failure, since it's not clear how long ago it happened + } + + if ( _lastSlaveOkConn && !_lastSlaveOkConn->isStillConnected() ) { + _lastSlaveOkConn.reset(); + _lastSlaveOkHost = HostAndPort(); + // Reset read pref too, since we're re-selecting the slaveOk host anyway + _lastReadPref.reset(); + // Don't notify monitor of bg failure, since it's not clear how long ago it happened } return true; } - bool ReplicaSetMonitor::Node::isCompatible(ReadPreference readPreference, - const TagSet* tags) const { - if (!ok) { - return false; - } + // Internal implementation of isSecondaryQuery, takes previously-parsed read preference + static bool _isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + const ReadPreferenceSetting& readPref ) { - if ((readPreference == ReadPreference_SecondaryOnly || - /* This is the original behavior for slaveOk. This can result to reading - * data back in time, but the main idea here is to avoid overloading the - * primary when secondary is available. - */ - readPreference == ReadPreference_SecondaryPreferred) && - !okForSecondaryQueries()) { - return false; + // If the read pref is primary only, this is not a secondary query + if (readPref.pref == ReadPreference_PrimaryOnly) return false; + + if (ns.find(".$cmd") == string::npos) { + return true; } - if ((readPreference == ReadPreference_PrimaryOnly || - readPreference == ReadPreference_PrimaryPreferred) && - secondary) { - return false; + // This is a command with secondary-possible read pref + // Only certain commands are supported for secondary operation. + + BSONObj actualQueryObj; + if (strcmp(queryObj.firstElement().fieldName(), "query") == 0) { + actualQueryObj = queryObj["query"].embeddedObject(); + } + else { + actualQueryObj = queryObj; } - scoped_ptr bsonIter(tags->getIterator()); - if (!bsonIter->more()) { - // Empty tag set + const string cmdName = actualQueryObj.firstElementFieldName(); + if (_secOkCmdList.count(cmdName) == 1) { return true; } - while (bsonIter->more()) { - const BSONElement& nextTag = bsonIter->next(); - uassert(16358, "Tags should be a BSON object", nextTag.isABSONObj()); + if (cmdName == "mapReduce" || cmdName == "mapreduce") { + if (!actualQueryObj.hasField("out")) { + return false; + } - if (matchesTag(nextTag.Obj())) { + BSONElement outElem(actualQueryObj["out"]); + if (outElem.isABSONObj() && outElem["inline"].trueValue()) { return true; } } @@ -1448,71 +274,27 @@ namespace mongo { return false; } - BSONObj ReplicaSetMonitor::Node::toBSON() const { - BSONObjBuilder builder; - builder.append( "addr", addr.toString() ); - builder.append( "isMaster", ismaster ); - builder.append( "secondary", secondary ); - builder.append( "hidden", hidden ); - - const BSONElement& tagElem = lastIsMaster["tags"]; - if ( tagElem.ok() && tagElem.isABSONObj() ){ - builder.append( "tags", tagElem.Obj() ); - } - - builder.append( "ok", ok ); - - return builder.obj(); - } - - ReplicaSetMonitor::ConfigChangeHook ReplicaSetMonitor::_hook; - int ReplicaSetMonitor::_maxFailedChecks = 30; // At 1 check every 10 seconds, 30 checks takes 5 minutes - - // -------------------------------- - // ----- DBClientReplicaSet --------- - // -------------------------------- - - const size_t DBClientReplicaSet::MAX_RETRY = 3; - - DBClientReplicaSet::DBClientReplicaSet( const string& name , const vector& servers, double so_timeout ) - : _setName( name ), _so_timeout( so_timeout ) { - ReplicaSetMonitor::createIfNeeded( name, servers ); - } - - DBClientReplicaSet::~DBClientReplicaSet() { - } - - ReplicaSetMonitorPtr DBClientReplicaSet::_getMonitor() const { - ReplicaSetMonitorPtr rsm = ReplicaSetMonitor::get( _setName, true ); - // If you can't get a ReplicaSetMonitor then this connection isn't valid - uassert( 16340, str::stream() << "No replica set monitor active and no cached seed " - "found for set: " << _setName, rsm ); - return rsm; - } - - // This can't throw an exception because it is called in the destructor of ScopedDbConnection - string DBClientReplicaSet::getServerAddress() const { - ReplicaSetMonitorPtr rsm = ReplicaSetMonitor::get( _setName, true ); - if ( !rsm ) { - warning() << "Trying to get server address for DBClientReplicaSet, but no " - "ReplicaSetMonitor exists for " << _setName << endl; - return str::stream() << _setName << "/" ; - } - return rsm->getServerAddress(); + bool DBClientReplicaSet::isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + int queryOptions ) { + auto_ptr readPref( _extractReadPref( queryObj, queryOptions ) ); + return _isSecondaryQuery( ns, queryObj, *readPref ); } DBClientConnection * DBClientReplicaSet::checkMaster() { ReplicaSetMonitorPtr monitor = _getMonitor(); - HostAndPort h = monitor->getMaster(); + HostAndPort h = monitor->getMasterOrUassert(); if ( h == _masterHost && _master ) { // a master is selected. let's just make sure connection didn't die if ( ! _master->isFailed() ) return _master.get(); - monitor->notifyFailure( _masterHost ); + + monitor->failedHost( _masterHost ); + h = monitor->getMasterOrUassert(); // old master failed, try again. } - _masterHost = monitor->getMaster(); + _masterHost = h; ConnectionString connStr(_masterHost); @@ -1531,7 +313,7 @@ namespace mongo { } if (newConn == NULL || !errmsg.empty()) { - monitor->notifyFailure(_masterHost); + monitor->failedHost(_masterHost); uasserted(13639, str::stream() << "can't connect to new replica set master [" << _masterHost.toString() << "]" << (errmsg.empty()? "" : ", err: ") << errmsg); @@ -1539,24 +321,41 @@ namespace mongo { _master.reset(newConn); _master->setReplSetClientCallback(this); + _master->setRunCommandHook(_runCommandHook); + _master->setPostRunCommandHook(_postRunCommandHook); _auth( _master.get() ); return _master.get(); } bool DBClientReplicaSet::checkLastHost(const ReadPreferenceSetting* readPref) { - if (_lastSlaveOkHost.empty()) { + // Can't use a cached host if we don't have one. + if (!_lastSlaveOkConn || _lastSlaveOkHost.empty()) { return false; } - ReplicaSetMonitorPtr monitor = _getMonitor(); + // Don't pin if the readPrefs differ. + if (!_lastReadPref || !_lastReadPref->equals(*readPref)) { + return false; + } + + // If _lastSlaveOkConn is pointing to a connection to primary, make sure that it is + // the same as _master, as this is the connection that has the version set. + if (_lastSlaveOkHost == _masterHost) { + if (_master.get() == NULL) { + // _master conn has been invalidated, need to reset connections. + return false; + } + _lastSlaveOkConn = _master; + } - if (_lastSlaveOkConn && _lastSlaveOkConn->isFailed()) { + // Make sure we don't think the host is down. + if (_lastSlaveOkConn->isFailed() || !_getMonitor()->isHostUp(_lastSlaveOkHost)) { invalidateLastSlaveOkCache(); return false; } - return _lastSlaveOkConn && _lastReadPref && _lastReadPref->equals(*readPref); + return true; } void DBClientReplicaSet::_auth( DBClientConnection * conn ) { @@ -1566,8 +365,8 @@ namespace mongo { } catch (const UserException&) { warning() << "cached auth failed for set: " << _setName << - " db: " << i->second[saslCommandPrincipalSourceFieldName].str() << - " user: " << i->second[saslCommandPrincipalFieldName].str() << endl; + " db: " << i->second[saslCommandUserDBFieldName].str() << + " user: " << i->second[saslCommandUserFieldName].str() << endl; } } } @@ -1577,10 +376,8 @@ namespace mongo { } DBClientConnection& DBClientReplicaSet::slaveConn() { - BSONArray emptyArray(BSON_ARRAY(BSONObj())); - TagSet tags(emptyArray); shared_ptr readPref( - new ReadPreferenceSetting(ReadPreference_SecondaryPreferred, tags)); + new ReadPreferenceSetting(ReadPreference_SecondaryPreferred, TagSet())); DBClientConnection* conn = selectNodeUsingTags(readPref); uassert( 16369, str::stream() << "No good nodes available for set: " @@ -1590,34 +387,80 @@ namespace mongo { } bool DBClientReplicaSet::connect() { - return _getMonitor()->isAnyNodeOk(); + // Returns true if there are any up hosts. + const ReadPreferenceSetting anyUpHost(ReadPreference_Nearest, TagSet()); + return !_getMonitor()->getHostOrRefresh(anyUpHost).empty(); + } + + static bool isAuthenticationException( const DBException& ex ) { + return ex.getCode() == ErrorCodes::AuthenticationFailed; } - void DBClientReplicaSet::_auth(const BSONObj& params) { - DBClientConnection * m = checkMaster(); + void DBClientReplicaSet::_auth( const BSONObj& params ) { - // first make sure it actually works - m->auth(params); + // We prefer to authenticate against a primary, but otherwise a secondary is ok too + // Empty tag matches every secondary + shared_ptr readPref( + new ReadPreferenceSetting( ReadPreference_PrimaryPreferred, TagSet() ) ); - /* Also authenticate the cached secondary connection. Note that this is only - * needed when we actually have something cached and is last known to be - * working. - */ - if (_lastSlaveOkConn.get() != NULL && !_lastSlaveOkConn->isFailed()) { + LOG(3) << "dbclient_rs authentication of " << _getMonitor()->getName() << endl; + + // NOTE that we retry MAX_RETRY + 1 times, since we're always primary preferred we don't + // fallback to the primary. + Status lastNodeStatus = Status::OK(); + for ( size_t retry = 0; retry < MAX_RETRY + 1; retry++ ) { try { - _lastSlaveOkConn->auth(params); + DBClientConnection* conn = selectNodeUsingTags( readPref ); + + if ( conn == NULL ) { + break; + } + + conn->auth( params ); + + // Cache the new auth information since we now validated it's good + _auths[params[saslCommandUserDBFieldName].str()] = params.getOwned(); + + // Ensure the only child connection open is the one we authenticated against - other + // child connections may not have full authentication information. + // NOTE: _lastSlaveOkConn may or may not be the same as _master + dassert(_lastSlaveOkConn.get() == conn || _master.get() == conn); + if ( conn != _lastSlaveOkConn.get() ) { + _lastSlaveOkHost = HostAndPort(); + _lastSlaveOkConn.reset(); + } + if ( conn != _master.get() ) { + _masterHost = HostAndPort(); + _master.reset(); + } + + return; } - catch (const DBException&) { - /* Swallow exception. _lastSlaveOkConn is now in failed state. - * The next time we create a new secondary connection it will - * be authenticated with the credentials from _auths. - */ - verify(_lastSlaveOkConn->isFailed()); + catch ( const DBException &ex ) { + + // We care if we can't authenticate (i.e. bad password) in credential params. + if ( isAuthenticationException( ex ) ) { + throw; + } + + StringBuilder errMsgB; + errMsgB << "can't authenticate against replica set node " + << _lastSlaveOkHost.toString(); + lastNodeStatus = ex.toStatus( errMsgB.str() ); + + LOG(1) << lastNodeStatus.reason() << endl; + invalidateLastSlaveOkCache(); } } - // now that it does, we should save so that for a new node we can auth - _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); + if ( lastNodeStatus.isOK() ) { + StringBuilder assertMsgB; + assertMsgB << "Failed to authenticate, no good nodes in " << _getMonitor()->getName(); + uasserted( ErrorCodes::NodeNotFound, assertMsgB.str() ); + } + else { + uasserted( lastNodeStatus.code(), lastNodeStatus.reason() ); + } } void DBClientReplicaSet::logout(const string &dbname, BSONObj& info) { @@ -1668,9 +511,8 @@ namespace mongo { int queryOptions, int batchSize) { - if ( _isQueryOkToSecondary( ns, queryOptions, query.obj ) ) { - - shared_ptr readPref(_extractReadPref(query.obj)); + shared_ptr readPref( _extractReadPref( query.obj, queryOptions ) ); + if ( _isSecondaryQuery( ns, query.obj, *readPref ) ) { LOG( 3 ) << "dbclient_rs query using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1682,6 +524,7 @@ namespace mongo { _lastSlaveOkConn->getServerAddress() : "[not cached]" ) << ")" << endl; + string lastNodeErrMsg; for (size_t retry = 0; retry < MAX_RETRY; retry++) { try { DBClientConnection* conn = selectNodeUsingTags(readPref); @@ -1697,15 +540,23 @@ namespace mongo { return checkSlaveQueryResult(cursor); } catch (const DBException &dbExcep) { - LOG(1) << "can't query replica set node " << _lastSlaveOkHost - << ": " << causedBy(dbExcep) << endl; + StringBuilder errMsgBuilder; + errMsgBuilder << "can't query replica set node " + << _lastSlaveOkHost.toString() << ": " << causedBy( dbExcep ); + lastNodeErrMsg = errMsgBuilder.str(); + + LOG(1) << lastNodeErrMsg << endl; invalidateLastSlaveOkCache(); } } - uasserted( 16370, - str::stream() << "Failed to do query, no good nodes in " - << _getMonitor()->getName() ); + StringBuilder assertMsg; + assertMsg << "Failed to do query, no good nodes in " << _getMonitor()->getName(); + if ( !lastNodeErrMsg.empty() ) { + assertMsg << ", last error: " << lastNodeErrMsg; + } + + uasserted( 16370, assertMsg.str() ); } LOG( 3 ) << "dbclient_rs query to primary node in " << _getMonitor()->getName() << endl; @@ -1718,9 +569,9 @@ namespace mongo { const Query& query, const BSONObj *fieldsToReturn, int queryOptions) { - if (_isQueryOkToSecondary(ns, queryOptions, query.obj)) { - shared_ptr readPref(_extractReadPref(query.obj)); + shared_ptr readPref( _extractReadPref( query.obj, queryOptions ) ); + if ( _isSecondaryQuery( ns, query.obj, *readPref ) ) { LOG( 3 ) << "dbclient_rs findOne using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1732,6 +583,8 @@ namespace mongo { _lastSlaveOkConn->getServerAddress() : "[not cached]" ) << ")" << endl; + string lastNodeErrMsg; + for (size_t retry = 0; retry < MAX_RETRY; retry++) { try { DBClientConnection* conn = selectNodeUsingTags(readPref); @@ -1743,14 +596,23 @@ namespace mongo { return conn->findOne(ns,query,fieldsToReturn,queryOptions); } catch ( const DBException &dbExcep ) { - LOG(1) << "can't findone replica set node " << _lastSlaveOkHost << ": " - << causedBy( dbExcep ) << endl; + StringBuilder errMsgBuilder; + errMsgBuilder << "can't findone replica set node " + << _lastSlaveOkHost.toString() << ": " << causedBy( dbExcep ); + lastNodeErrMsg = errMsgBuilder.str(); + + LOG(1) << lastNodeErrMsg << endl; invalidateLastSlaveOkCache(); } } - uasserted(16379, str::stream() << "Failed to call findOne, no good nodes in " - << _getMonitor()->getName()); + StringBuilder assertMsg; + assertMsg << "Failed to call findOne, no good nodes in " << _getMonitor()->getName(); + if ( !lastNodeErrMsg.empty() ) { + assertMsg << ", last error: " << lastNodeErrMsg; + } + + uasserted(16379, assertMsg.str()); } LOG( 3 ) << "dbclient_rs findOne to primary node in " << _getMonitor()->getName() @@ -1773,9 +635,11 @@ namespace mongo { // the monitor doesn't exist. ReplicaSetMonitorPtr monitor = ReplicaSetMonitor::get( _setName ); if ( monitor ) { - monitor->notifyFailure( _masterHost ); + monitor->failedHost( _masterHost ); } - _master.reset(); + + _masterHost = HostAndPort(); + _master.reset(); } auto_ptr DBClientReplicaSet::checkSlaveQueryResult( auto_ptr result ){ @@ -1789,7 +653,8 @@ namespace mongo { // If the error code here ever changes, we need to change this code also BSONElement code = error["code"]; - if( code.isNumber() && code.Int() == 13436 /* not master or secondary */ ){ + if( code.isNumber() && + code.Int() == NotMasterOrSecondaryCode /* not master or secondary */ ) { isntSecondary(); throw DBException( str::stream() << "slave " << _lastSlaveOkHost.toString() << " is no longer secondary", 14812 ); @@ -1801,13 +666,13 @@ namespace mongo { void DBClientReplicaSet::isntSecondary() { log() << "slave no longer has secondary status: " << _lastSlaveOkHost << endl; // Failover to next slave - _getMonitor()->notifySlaveFailure( _lastSlaveOkHost ); + _getMonitor()->failedHost( _lastSlaveOkHost ); _lastSlaveOkConn.reset(); } DBClientConnection* DBClientReplicaSet::selectNodeUsingTags( shared_ptr readPref) { - if (checkLastHost(readPref.get())) { + if (!shouldReevaluate() && checkLastHost(readPref.get())) { LOG( 3 ) << "dbclient_rs selecting compatible last used node " << _lastSlaveOkHost << endl; @@ -1816,9 +681,7 @@ namespace mongo { } ReplicaSetMonitorPtr monitor = _getMonitor(); - bool isPrimarySelected = false; - _lastSlaveOkHost = monitor->selectAndCheckNode(readPref->pref, &readPref->tags, - &isPrimarySelected); + _lastSlaveOkHost = monitor->getHostOrRefresh(*readPref); if ( _lastSlaveOkHost.empty() ){ @@ -1833,7 +696,7 @@ namespace mongo { // versioned in mongos. Therefore, we have to make sure that this object // maintains only one connection to the primary and use that connection // every time we need to talk to the primary. - if (isPrimarySelected) { + if (monitor->isPrimary(_lastSlaveOkHost)) { checkMaster(); _lastSlaveOkConn = _master; _lastSlaveOkHost = _masterHost; // implied, but still assign just to be safe @@ -1858,6 +721,8 @@ namespace mongo { _lastSlaveOkConn.reset(newConn); _lastSlaveOkConn->setReplSetClientCallback(this); + _lastSlaveOkConn->setRunCommandHook(_runCommandHook); + _lastSlaveOkConn->setPostRunCommandHook(_postRunCommandHook); _auth(_lastSlaveOkConn.get()); @@ -1872,17 +737,15 @@ namespace mongo { _lazyState = LazyState(); const int lastOp = toSend.operation(); - bool slaveOk = false; if (lastOp == dbQuery) { // TODO: might be possible to do this faster by changing api DbMessage dm(toSend); QueryMessage qm(dm); - const bool slaveOk = qm.queryOptions & QueryOption_SlaveOk; - if (_isQueryOkToSecondary(qm.ns, qm.queryOptions, qm.query)) { - - shared_ptr readPref(_extractReadPref(qm.query)); + shared_ptr readPref( _extractReadPref( qm.query, + qm.queryOptions ) ); + if ( _isSecondaryQuery( qm.ns, qm.query, *readPref ) ) { LOG( 3 ) << "dbclient_rs say using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1894,6 +757,8 @@ namespace mongo { _lastSlaveOkConn->getServerAddress() : "[not cached]" ) << ")" << endl; + string lastNodeErrMsg; + for (size_t retry = 0; retry < MAX_RETRY; retry++) { _lazyState._retries = retry; try { @@ -1910,12 +775,16 @@ namespace mongo { conn->say(toSend); _lazyState._lastOp = lastOp; - _lazyState._slaveOk = slaveOk; + _lazyState._secondaryQueryOk = true; _lazyState._lastClient = conn; } catch ( const DBException& DBExcep ) { - LOG(1) << "can't callLazy replica set node " << _lastSlaveOkHost << ": " - << causedBy( DBExcep ) << endl; + StringBuilder errMsgBuilder; + errMsgBuilder << "can't callLazy replica set node " + << _lastSlaveOkHost.toString() << ": " << causedBy( DBExcep ); + lastNodeErrMsg = errMsgBuilder.str(); + + LOG(1) << lastNodeErrMsg << endl; invalidateLastSlaveOkCache(); continue; } @@ -1923,8 +792,13 @@ namespace mongo { return; } - uasserted(16380, str::stream() << "Failed to call say, no good nodes in " - << _getMonitor()->getName()); + StringBuilder assertMsg; + assertMsg << "Failed to call say, no good nodes in " << _getMonitor()->getName(); + if ( !lastNodeErrMsg.empty() ) { + assertMsg << ", last error: " << lastNodeErrMsg; + } + + uasserted(16380, assertMsg.str()); } } @@ -1936,7 +810,7 @@ namespace mongo { *actualServer = master->getServerAddress(); _lazyState._lastOp = lastOp; - _lazyState._slaveOk = slaveOk; + _lazyState._secondaryQueryOk = false; // Don't retry requests to primary since there is only one host to try _lazyState._retries = MAX_RETRY; _lazyState._lastClient = master; @@ -1954,7 +828,7 @@ namespace mongo { return _lazyState._lastClient->recv( m ); } catch( DBException& e ){ - log() << "could not receive data from " << _lazyState._lastClient << causedBy( e ) << endl; + log() << "could not receive data from " << _lazyState._lastClient->toString() << causedBy( e ) << endl; return false; } } @@ -1975,46 +849,54 @@ namespace mongo { else if (targetHost) *targetHost = ""; if( ! _lazyState._lastClient ) return; + + // nReturned == 1 means that we got one result back, which might be an error + // nReturned == -1 is a sentinel value for "no data returned" aka (usually) network problem + // If neither, this must be a query result so our response is ok wrt the replica set if( nReturned != 1 && nReturned != -1 ) return; BSONObj dataObj; if( nReturned == 1 ) dataObj = BSONObj( data ); // Check if we should retry here - if( _lazyState._lastOp == dbQuery && _lazyState._slaveOk ){ + if( _lazyState._lastOp == dbQuery && _lazyState._secondaryQueryOk ){ + + // query could potentially go to a secondary, so see if this is an error (or empty) and + // retry if we're not past our retry limit. - // Check the error code for a slave not secondary error - if( nReturned == -1 || - ( hasErrField( dataObj ) && ! dataObj["code"].eoo() && dataObj["code"].Int() == 13436 ) ){ + if( nReturned == -1 /* no result, maybe network problem */ || + ( hasErrField( dataObj ) && ! dataObj["code"].eoo() + && dataObj["code"].Int() == NotMasterOrSecondaryCode ) ){ - bool wasMaster = false; if( _lazyState._lastClient == _lastSlaveOkConn.get() ){ isntSecondary(); } else if( _lazyState._lastClient == _master.get() ){ - wasMaster = true; isntMaster(); } - else - warning() << "passed " << dataObj << " but last rs client " << _lazyState._lastClient->toString() << " is not master or secondary" << endl; + else { + warning() << "passed " << dataObj << " but last rs client " + << _lazyState._lastClient->toString() << " is not master or secondary" + << endl; + } - if( _lazyState._retries < 3 ){ + if ( _lazyState._retries < static_cast( MAX_RETRY ) ) { _lazyState._retries++; *retry = true; } else{ - (void)wasMaster; // silence set-but-not-used warning - // verify( wasMaster ); - // printStackTrace(); - log() << "too many retries (" << _lazyState._retries << "), could not get data from replica set" << endl; + log() << "too many retries (" << _lazyState._retries + << "), could not get data from replica set" << endl; } } } else if( _lazyState._lastOp == dbQuery ){ - // slaveOk is not set, just mark the master as bad - if( nReturned == -1 || - ( hasErrField( dataObj ) && ! dataObj["code"].eoo() && dataObj["code"].Int() == 13435 ) ) + // if query could not potentially go to a secondary, just mark the master as bad + + if( nReturned == -1 /* no result, maybe network problem */ || + ( hasErrField( dataObj ) && ! dataObj["code"].eoo() + && dataObj["code"].Int() == NotMasterNoSlaveOkCode ) ) { if( _lazyState._lastClient == _master.get() ){ isntMaster(); @@ -2036,9 +918,9 @@ namespace mongo { QueryMessage qm(dm); ns = qm.ns; - if (_isQueryOkToSecondary(ns, qm.queryOptions, qm.query)) { - - shared_ptr readPref(_extractReadPref(qm.query)); + shared_ptr readPref( _extractReadPref( qm.query, + qm.queryOptions ) ); + if ( _isSecondaryQuery( ns, qm.query, *readPref ) ) { LOG( 3 ) << "dbclient_rs call using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -2111,59 +993,22 @@ namespace mongo { * because there are certain exceptions that will not make the connection be labeled * as failed. For example, asserts 13079, 13080, 16386 */ - _getMonitor()->notifySlaveFailure(_lastSlaveOkHost); + _getMonitor()->failedHost(_lastSlaveOkHost); _lastSlaveOkHost = HostAndPort(); _lastSlaveOkConn.reset(); } - TagSet::TagSet() : _isExhausted(true), _tagIterator(_tags) { - } - - TagSet::TagSet(const TagSet& other) : - _isExhausted(false), - _tags(other._tags.getOwned()), - _tagIterator(_tags) { - next(); - } - - TagSet::TagSet(const BSONArray& tags) : - _isExhausted(false), - _tags(tags.getOwned()), - _tagIterator(_tags) { - next(); - } - - void TagSet::next() { - if (_tagIterator.more()) { - const BSONElement& nextTag = _tagIterator.next(); - uassert(16357, "Tags should be a BSON object", nextTag.isABSONObj()); - _currentTag = nextTag.Obj(); - } - else { - _isExhausted = true; + bool DBClientReplicaSet::shouldReevaluate() { + if (_nodeSelectOracle.nextInt32(100) < reevaluatePercentage) { + return true; } - } - const BSONObj& TagSet::getCurrentTag() const { - verify(!_isExhausted); - return _currentTag; - } - - bool TagSet::isExhausted() const { - return _isExhausted; - } - - BSONObjIterator* TagSet::getIterator() const { - return new BSONObjIterator(_tags); - } - - bool TagSet::equals(const TagSet& other) const { - return _tags.equal(other._tags); + return false; } - const BSONArray& TagSet::getTagBSON() const { - return _tags; - } + // trying to optimize for the common dont-care-about-tags case. + static const BSONArray tagsMatchesAll = BSON_ARRAY(BSONObj()); + TagSet::TagSet() : _tags(tagsMatchesAll) {} string readPrefToString( ReadPreference pref ) { switch ( pref ) { diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h index 05f7faa9b2b..f77dcd8fc28 100644 --- a/src/mongo/client/dbclient_rs.h +++ b/src/mongo/client/dbclient_rs.h @@ -17,14 +17,12 @@ #pragma once -#include "mongo/pch.h" - -#include #include -#include #include #include "mongo/client/dbclientinterface.h" +#include "mongo/client/export_macros.h" +#include "mongo/platform/random.h" #include "mongo/util/net/hostandport.h" namespace mongo { @@ -33,382 +31,6 @@ namespace mongo { class TagSet; struct ReadPreferenceSetting; typedef shared_ptr ReplicaSetMonitorPtr; - typedef pair,set > NodeDiff; - - /** - * manages state about a replica set for client - * keeps tabs on whose master and what slaves are up - * can hand a slave to someone for SLAVE_OK - * one instance per process per replica set - * TODO: we might be able to use a regular Node * to avoid _lock - */ - class ReplicaSetMonitor { - public: - typedef boost::function1 ConfigChangeHook; - - /** - * Data structure for keeping track of the states of individual replica - * members. This class is not thread-safe so proper measures should be taken - * when sharing this object across multiple threads. - * - * Note: these get copied around in the nodes vector so be sure to maintain - * copyable semantics here - */ - struct Node { - Node( const HostAndPort& a , DBClientConnection* c ) : - addr( a ), - conn(c), - ok( c != NULL ), - ismaster(false), - secondary( false ), - hidden( false ), - pingTimeMillis( 0 ) { - } - - bool okForSecondaryQueries() const { - return ok && secondary && ! hidden; - } - - /** - * Checks if the given tag matches the tag attached to this node. - * - * Example: - * - * Tag of this node: { "dc": "nyc", "region": "na", "rack": "4" } - * - * match: {} - * match: { "dc": "nyc", "rack": 4 } - * match: { "region": "na", "dc": "nyc" } - * not match: { "dc: "nyc", "rack": 2 } - * not match: { "dc": "sf" } - * - * @param tag the tag to use to compare. Should not contain any - * embedded documents. - * - * @return true if the given tag matches the this node's tag - * specification - */ - bool matchesTag(const BSONObj& tag) const; - - /** - * @param threshold max ping time (in ms) to be considered local - * @return true if node is a local secondary, and can handle queries - **/ - bool isLocalSecondary( const int threshold ) const { - return pingTimeMillis < threshold; - } - - /** - * Checks whether this nodes is compatible with the given readPreference and - * tag. Compatibility check is strict in the sense that secondary preferred - * is treated like secondary only and primary preferred is treated like - * primary only. - * - * @return true if this node is compatible with the read preference and tags. - */ - bool isCompatible(ReadPreference readPreference, const TagSet* tag) const; - - BSONObj toBSON() const; - - string toString() const { - return toBSON().toString(); - } - - HostAndPort addr; - boost::shared_ptr conn; - - // if this node is in a failure state - // used for slave routing - // this is too simple, should make it better - bool ok; - - // as reported by ismaster - BSONObj lastIsMaster; - - bool ismaster; - bool secondary; - bool hidden; - - int pingTimeMillis; - - }; - - static const double SOCKET_TIMEOUT_SECS; - - /** - * Selects the right node given the nodes to pick from and the preference. - * - * @param nodes the nodes to select from - * @param preference the read mode to use - * @param tags the tags used for filtering nodes - * @param localThresholdMillis the exclusive upper bound of ping time to be - * considered as a local node. Local nodes are favored over non-local - * nodes if multiple nodes matches the other criteria. - * @param lastHost the host used in the last successful request. This is used for - * selecting a different node as much as possible, by doing a simple round - * robin, starting from the node next to this lastHost. This will be overwritten - * with the newly chosen host if not empty, not primary and when preference - * is not Nearest. - * @param isPrimarySelected out parameter that is set to true if the returned host - * is a primary. Cannot be NULL and valid only if returned host is not empty. - * - * @return the host object of the node selected. If none of the nodes are - * eligible, returns an empty host. - */ - static HostAndPort selectNode(const std::vector& nodes, - ReadPreference preference, - TagSet* tags, - int localThresholdMillis, - HostAndPort* lastHost, - bool* isPrimarySelected); - - /** - * Selects the right node given the nodes to pick from and the preference. This - * will also attempt to refresh the local view of the replica set configuration - * if the primary node needs to be returned but is not currently available (except - * for ReadPrefrence_Nearest). - * - * @param preference the read mode to use. - * @param tags the tags used for filtering nodes. - * @param isPrimarySelected out parameter that is set to true if the returned host - * is a primary. Cannot be NULL and valid only if returned host is not empty. - * - * @return the host object of the node selected. If none of the nodes are - * eligible, returns an empty host. - */ - HostAndPort selectAndCheckNode(ReadPreference preference, - TagSet* tags, - bool* isPrimarySelected); - - /** - * Creates a new ReplicaSetMonitor, if it doesn't already exist. - */ - static void createIfNeeded( const string& name , const vector& servers ); - - /** - * gets a cached Monitor per name. If the monitor is not found and createFromSeed is false, - * it will return none. If createFromSeed is true, it will try to look up the last known - * servers list for this set and will create a new monitor using that as the seed list. - */ - static ReplicaSetMonitorPtr get( const string& name, const bool createFromSeed = false ); - - /** - * Populates activeSets with all the currently tracked replica set names. - */ - static void getAllTrackedSets(set* activeSets); - - /** - * checks all sets for current master and new secondaries - * usually only called from a BackgroundJob - */ - static void checkAll( bool checkAllSecondaries ); - - /** - * Removes the ReplicaSetMonitor for the given set name from _sets, which will delete it. - * If clearSeedCache is true, then the cached seed string for this Replica Set will be removed - * from _seedServers. - */ - static void remove( const string& name, bool clearSeedCache = false ); - - static int getMaxFailedChecks() { return _maxFailedChecks; }; - static void setMaxFailedChecks(int numChecks) { _maxFailedChecks = numChecks; }; - - /** - * this is called whenever the config of any replica set changes - * currently only 1 globally - * asserts if one already exists - * ownership passes to ReplicaSetMonitor and the hook will actually never be deleted - */ - static void setConfigChangeHook( ConfigChangeHook hook ); - - ~ReplicaSetMonitor(); - - /** @return HostAndPort or throws an exception */ - HostAndPort getMaster(); - - /** - * notify the monitor that server has faild - */ - void notifyFailure( const HostAndPort& server ); - - /** - * @deprecated use #getCandidateNode instead - * @return prev if its still ok, and if not returns a random slave that is ok for reads - */ - HostAndPort getSlave( const HostAndPort& prev ); - - /** - * @param preferLocal Prefer a local secondary, otherwise pick any - * secondary, or fall back to primary - * @return a random slave that is ok for reads - */ - HostAndPort getSlave( bool preferLocal = true ); - - /** - * notify the monitor that server has faild - */ - void notifySlaveFailure( const HostAndPort& server ); - - /** - * checks for current master and new secondaries - */ - void check( bool checkAllSecondaries ); - - string getName() const { return _name; } - - string getServerAddress() const; - - bool contains( const string& server ) const; - - void appendInfo( BSONObjBuilder& b ) const; - - /** - * Set the threshold value (in ms) for a node to be considered local. - * NOTE: This function acquires the _lock mutex. - **/ - void setLocalThresholdMillis( const int millis ); - - /** - * @return true if the host is compatible with the given readPreference and tag set. - */ - bool isHostCompatible(const HostAndPort& host, ReadPreference readPreference, - const TagSet* tagSet) const; - - /** - * Performs a quick check if at least one node is up based on the cached - * view of the set. - * - * @return true if any node is ok - */ - bool isAnyNodeOk() const; - - private: - /** - * This populates a list of hosts from the list of seeds (discarding the - * seed list). Should only be called from within _setsLock. - * @param name set name - * @param servers seeds - */ - ReplicaSetMonitor( const string& name , const vector& servers ); - - static void _remove_inlock( const string& name, bool clearSeedCache = false ); - - /** - * Checks all connections from the host list and sets the current - * master. - * - * @param checkAllSecondaries if set to false, stop immediately when - * the master is found or when _master is not -1. - */ - void _check( bool checkAllSecondaries ); - - /** - * Use replSetGetStatus command to make sure hosts in host list are up - * and readable. Sets Node::ok appropriately. - */ - void _checkStatus( const string& hostAddr ); - - /** - * Add array of hosts to host list. Doesn't do anything if hosts are - * already in host list. - * @param hostList the list of hosts to add - * @param changed if new hosts were added - */ - void _checkHosts(const BSONObj& hostList, bool& changed); - - /** - * Updates host list. - * Invariant: if nodesOffset is >= 0, _nodes[nodesOffset].conn should be - * equal to conn. - * - * @param conn the connection to check - * @param maybePrimary OUT - * @param verbose - * @param nodesOffset - offset into _nodes array, -1 for not in it - * - * @return true if the connection is good or false if invariant - * is broken - */ - bool _checkConnection( DBClientConnection* conn, string& maybePrimary, - bool verbose, int nodesOffset ); - - /** - * Save the seed list for the current set into the _seedServers map - * Should only be called if you're already holding _setsLock and this - * monitor's _lock. - */ - void _cacheServerAddresses_inlock(); - - string _getServerAddress_inlock() const; - - NodeDiff _getHostDiff_inlock( const BSONObj& hostList ); - bool _shouldChangeHosts( const BSONObj& hostList, bool inlock ); - - /** - * @return the index to _nodes corresponding to the server address. - */ - int _find( const string& server ) const ; - int _find_inlock( const string& server ) const ; - - /** - * Checks whether the given connection matches the connection stored in _nodes. - * Mainly used for sanity checking to confirm that nodeOffset still - * refers to the right connection after releasing and reacquiring - * a mutex. - */ - bool _checkConnMatch_inlock( DBClientConnection* conn, size_t nodeOffset ) const; - - /** - * Populates the local view of the set using the list of servers. - * - * Invariants: - * 1. Should be called while holding _setsLock and while not holding _lock since - * this calls #_checkConnection, which locks _checkConnectionLock - * 2. _nodes should be empty before this is called - */ - void _populateHosts_inSetsLock(const std::vector& seedList); - - // protects _localThresholdMillis, _nodes and refs to _nodes - // (eg. _master & _lastReadPrefHost) - mutable mongo::mutex _lock; - - /** - * "Synchronizes" the _checkConnection method. Should ideally be one mutex per - * connection object being used. The purpose of this lock is to make sure that - * the reply from the connection the lock holder got is the actual response - * to what it sent. - * - * Deadlock WARNING: never acquire this while holding _lock - */ - mutable mongo::mutex _checkConnectionLock; - - string _name; - - /** - * Host list. - */ - std::vector _nodes; - int _master; // which node is the current master. -1 means no master is known - int _nextSlave; // which node is the current slave, only used by the deprecated getSlave - - // last host returned by _selectNode, used for round robin selection - HostAndPort _lastReadPrefHost; - - // The number of consecutive times the set has been checked and every member in the set was down. - int _failedChecks; - - static mongo::mutex _setsLock; // protects _seedServers and _sets - - // set name to seed list. - // Used to rebuild the monitor if it is cleaned up but then the set is accessed again. - static map > _seedServers; - static map _sets; // set name to Monitor - - static ConfigChangeHook _hook; - int _localThresholdMillis; // local ping latency threshold (protected by _lock) - - static int _maxFailedChecks; - }; /** Use this class to connect to a replica set of servers. The class will manage checking for which server in a replica set is master, and do failover automatically. @@ -418,12 +40,18 @@ namespace mongo { On a failover situation, expect at least one operation to return an error (throw an exception) before the failover is complete. Operations are not retried. */ - class DBClientReplicaSet : public DBClientBase { + class MONGO_CLIENT_API DBClientReplicaSet : public DBClientBase { public: using DBClientBase::query; using DBClientBase::update; using DBClientBase::remove; + // For internal use only. The desired probability for reevaluating node + // selection given a read preference, expressed in percentage. Any value + // less than 0 will be treated as 0% and any value greater than 100 will + // be treated as 100%. + static int reevaluatePercentage; + /** Call connect() after constructing. autoReconnect is always on for DBClientReplicaSet connections. */ DBClientReplicaSet( const string& name , const vector& servers, double so_timeout=0 ); virtual ~DBClientReplicaSet(); @@ -501,12 +129,24 @@ namespace mongo { // ----- status ------ virtual bool isFailed() const { return ! _master || _master->isFailed(); } + bool isStillConnected(); // ----- informational ---- + /** + * Gets the replica set name of the set we are connected to. + */ + const std::string& getSetName() const { return _setName; } + + /** + * Returns the HostAndPort of the server this connection believes belongs to the primary, + * or returns an empty HostAndPort if it doesn't know about a current primary. + */ + HostAndPort getSuspectedPrimaryHostAndPort() const; + double getSoTimeout() const { return _so_timeout; } - string toString() { return getServerAddress(); } + string toString() const { return getServerAddress(); } string getServerAddress() const; @@ -518,6 +158,22 @@ namespace mongo { virtual bool call( Message &toSend, Message &response, bool assertOk=true , string * actualServer = 0 ); virtual bool callRead( Message& toSend , Message& response ) { return checkMaster()->callRead( toSend , response ); } + /** + * Returns whether a query or command can be sent to secondaries based on the query object + * and options. + * + * @param ns the namespace of the query. + * @param queryObj the query object to check. + * @param queryOptions the query options + * + * @return true if the query/cmd could potentially be sent to a secondary, false otherwise + */ + static bool isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + int queryOptions ); + + virtual void setRunCommandHook(DBClientWithCommands::RunCommandHookFunc func); + virtual void setPostRunCommandHook(DBClientWithCommands::PostRunCommandHookFunc func); protected: /** Authorize. Authorizes all nodes as needed @@ -574,6 +230,13 @@ namespace mongo { // Throws a DBException if the monitor doesn't exist and there isn't a cached seed to use. ReplicaSetMonitorPtr _getMonitor() const; + /** + * Returns true if this connection should re-evaluate the node selection for + * a given read preference, regardless of it's compatibility with the currently + * cached connection. + */ + bool shouldReevaluate(); + string _setName; HostAndPort _masterHost; @@ -590,7 +253,7 @@ namespace mongo { // Last used connection in a slaveOk query (can be a primary) boost::shared_ptr _lastSlaveOkConn; boost::shared_ptr _lastReadPref; - + double _so_timeout; // we need to store so that when we connect to a new node on failure @@ -599,6 +262,11 @@ namespace mongo { // not sure if/how we should handle std::map _auths; // dbName -> auth parameters + // For v2.6 only. + // Random generator to be used to determine whether this connection object + // should re-evaluate node selection from scratch. + PseudoRandom _nodeSelectOracle; + protected: /** @@ -606,10 +274,11 @@ namespace mongo { */ class LazyState { public: - LazyState() : _lastClient( NULL ), _lastOp( -1 ), _slaveOk( false ), _retries( 0 ) {} + LazyState() : + _lastClient( NULL ), _lastOp( -1 ), _secondaryQueryOk( false ), _retries( 0 ) {} DBClientConnection* _lastClient; int _lastOp; - bool _slaveOk; + bool _secondaryQueryOk; int _retries; } _lazyState; @@ -617,82 +286,38 @@ namespace mongo { }; /** - * A simple object for representing the list of tags. The initial state will - * have a valid current tag as long as the list is not empty. + * A simple object for representing the list of tags requested by a $readPreference. */ - class TagSet { + class MONGO_CLIENT_API TagSet { public: /** - * Creates an empty tag list that is initially exhausted. + * Creates a TagSet that matches any nodes. + * + * Do not call during static init. */ TagSet(); /** - * Creates a copy of the given TagSet. The new copy will have the - * iterator pointing at the initial position. - */ - explicit TagSet(const TagSet& other); - - /** - * Creates a tag set object that lazily iterates over the tag list. + * Creates a TagSet from a BSONArray of tags. * * @param tags the list of tags associated with this option. This object * will get a shared copy of the list. Therefore, it is important * for the the given tag to live longer than the created tag set. */ - explicit TagSet(const BSONArray& tags); - - /** - * Advance to the next tag. - * - * @throws AssertionException if iterator is exhausted before this is called. - */ - void next(); - - // - // Getters - // + explicit TagSet(const BSONArray& tags) : _tags(tags) {} /** - * @return the current tag. Returned tag is invalid if isExhausted is true. + * Returns the BSONArray listing all tags that should be accepted. */ - const BSONObj& getCurrentTag() const; + const BSONArray& getTagBSON() const { return _tags; } - /** - * @return true if the iterator has been exhausted. - */ - bool isExhausted() const; - - /** - * @return an unordered iterator to the tag list. The caller is responsible for - * destroying the returned iterator. - */ - BSONObjIterator* getIterator() const; - - /** - * @returns true if the other TagSet has the same tag set specification with - * this tag set, disregarding where the current iterator is pointing to. - */ - bool equals(const TagSet& other) const; - - const BSONArray& getTagBSON() const; + bool operator==(const TagSet& other) const { return _tags == other._tags; } private: - /** - * This is purposely undefined as the semantics for assignment can be - * confusing. This is because BSONArrayIteratorSorted shouldn't be - * copied (because of how it manages internal buffer). - */ - TagSet& operator=(const TagSet& other); - BSONObj _currentTag; - bool _isExhausted; - - // Important: do not re-order _tags & _tagIterator BSONArray _tags; - BSONArrayIteratorSorted _tagIterator; }; - struct ReadPreferenceSetting { + struct MONGO_CLIENT_API ReadPreferenceSetting { /** * @parm pref the read preference mode. * @param tag the tag set. Note that this object will have the @@ -705,7 +330,7 @@ namespace mongo { } inline bool equals(const ReadPreferenceSetting& other) const { - return pref == other.pref && tags.equals(other.tags); + return pref == other.pref && tags == other.tags; } BSONObj toBSON() const; diff --git a/src/mongo/client/dbclient_rs_test.cpp b/src/mongo/client/dbclient_rs_test.cpp index ccb7b71063e..402ba5d2a8e 100644 --- a/src/mongo/client/dbclient_rs_test.cpp +++ b/src/mongo/client/dbclient_rs_test.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ /** @@ -23,6 +35,7 @@ #include "mongo/client/connpool.h" #include "mongo/client/dbclientinterface.h" #include "mongo/client/dbclient_rs.h" +#include "mongo/client/replica_set_monitor.h" #include "mongo/dbtests/mock/mock_conn_registry.h" #include "mongo/dbtests/mock/mock_replica_set.h" #include "mongo/unittest/unittest.h" @@ -33,25 +46,6 @@ #include #include -namespace mongo { - // Symbols defined to build the binary correctly. - CmdLine cmdLine; - - bool inShutdown() { - return false; - } - - DBClientBase *createDirectClient() { return NULL; } - - void dbexit(ExitCode rc, const char *why){ - ::_exit(rc); - } - - bool haveLocalShardingInfo(const string& ns) { - return false; - } -} - namespace { using boost::scoped_ptr; using std::auto_ptr; @@ -85,13 +79,14 @@ namespace { class BasicRS: public mongo::unittest::Test { protected: void setUp() { + ReplicaSetMonitor::cleanup(); _replSet.reset(new MockReplicaSet("test", 2)); ConnectionString::setConnectionHook( mongo::MockConnRegistry::get()->getConnStrHook()); } void tearDown() { - ReplicaSetMonitor::remove(_replSet->getSetName(), true); + ReplicaSetMonitor::cleanup(); _replSet.reset(); // TODO: remove this after we remove replSetGetStatus from ReplicaSetMonitor. @@ -136,6 +131,9 @@ namespace { MockReplicaSet* replSet = getReplSet(); DBClientReplicaSet replConn(replSet->getSetName(), replSet->getHosts()); + // Need up-to-date view, since either host is valid if view is stale. + ReplicaSetMonitor::get(replSet->getSetName())->startOrContinueRefresh().refreshAll(); + Query query; query.readPref(mongo::ReadPreference_PrimaryPreferred, BSONArray()); @@ -149,6 +147,9 @@ namespace { MockReplicaSet* replSet = getReplSet(); DBClientReplicaSet replConn(replSet->getSetName(), replSet->getHosts()); + // Need up-to-date view, since either host is valid if view is stale. + ReplicaSetMonitor::get(replSet->getSetName())->startOrContinueRefresh().refreshAll(); + Query query; query.readPref(mongo::ReadPreference_SecondaryPreferred, BSONArray()); @@ -164,6 +165,7 @@ namespace { class AllNodesDown: public mongo::unittest::Test { protected: void setUp() { + ReplicaSetMonitor::cleanup(); _replSet.reset(new MockReplicaSet("test", 2)); ConnectionString::setConnectionHook( mongo::MockConnRegistry::get()->getConnStrHook()); @@ -176,7 +178,7 @@ namespace { } void tearDown() { - ReplicaSetMonitor::remove(_replSet->getSetName(), true); + ReplicaSetMonitor::cleanup(); _replSet.reset(); // TODO: remove this after we remove replSetGetStatus from ReplicaSetMonitor. @@ -242,6 +244,7 @@ namespace { class PrimaryDown: public mongo::unittest::Test { protected: void setUp() { + ReplicaSetMonitor::cleanup(); _replSet.reset(new MockReplicaSet("test", 2)); ConnectionString::setConnectionHook( mongo::MockConnRegistry::get()->getConnStrHook()); @@ -249,7 +252,7 @@ namespace { } void tearDown() { - ReplicaSetMonitor::remove(_replSet->getSetName(), true); + ReplicaSetMonitor::cleanup(); _replSet.reset(); // TODO: remove this after we remove replSetGetStatus from ReplicaSetMonitor. @@ -329,6 +332,7 @@ namespace { class SecondaryDown: public mongo::unittest::Test { protected: void setUp() { + ReplicaSetMonitor::cleanup(); _replSet.reset(new MockReplicaSet("test", 2)); ConnectionString::setConnectionHook( mongo::MockConnRegistry::get()->getConnStrHook()); @@ -337,7 +341,7 @@ namespace { } void tearDown() { - ReplicaSetMonitor::remove(_replSet->getSetName(), true); + ReplicaSetMonitor::cleanup(); _replSet.reset(); // TODO: remove this after we remove replSetGetStatus from ReplicaSetMonitor. @@ -420,6 +424,13 @@ namespace { class TaggedFiveMemberRS: public mongo::unittest::Test { protected: void setUp() { + // Tests for pinning behavior require this. + ReplicaSetMonitor::useDeterministicHostSelection = true; + + // This shuts down the background RSMWatcher thread and prevents it from running. These + // tests depend on controlling when the RSMs are updated. + ReplicaSetMonitor::cleanup(); + _replSet.reset(new MockReplicaSet("test", 5)); _originalConnectionHook = ConnectionString::getConnectionHook(); ConnectionString::setConnectionHook( @@ -486,8 +497,10 @@ namespace { } void tearDown() { + ReplicaSetMonitor::useDeterministicHostSelection = false; + ConnectionString::setConnectionHook(_originalConnectionHook); - ReplicaSetMonitor::remove(_replSet->getSetName(), true); + ReplicaSetMonitor::cleanup(); _replSet.reset(); // TODO: remove this after we remove replSetGetStatus from ReplicaSetMonitor. @@ -531,6 +544,39 @@ namespace { } } + TEST_F(TaggedFiveMemberRS, ConnShouldNotPinIfHostMarkedAsFailed) { + MockReplicaSet* replSet = getReplSet(); + vector seedList; + seedList.push_back(HostAndPort(replSet->getPrimary())); + + DBClientReplicaSet replConn(replSet->getSetName(), seedList); + + string dest; + { + Query query; + query.readPref(mongo::ReadPreference_PrimaryPreferred, BSONArray()); + + // Note: IdentityNS contains the name of the server. + auto_ptr cursor = replConn.query(IdentityNS, query); + BSONObj doc = cursor->next(); + dest = doc[HostField.name()].str(); + } + + // This is the only difference from ConnShouldPinIfSameSettings which tests that we *do* pin + // in if the host is still marked as up. Note that this only notifies the RSM, and does not + // directly effect the DBClientRS. + ReplicaSetMonitor::get(replSet->getSetName())->failedHost(dest); + + { + Query query; + query.readPref(mongo::ReadPreference_PrimaryPreferred, BSONArray()); + auto_ptr cursor = replConn.query(IdentityNS, query); + BSONObj doc = cursor->next(); + const string newDest = doc[HostField.name()].str(); + ASSERT_NOT_EQUALS(dest, newDest); + } + } + TEST_F(TaggedFiveMemberRS, ConnShouldNotPinIfDiffMode) { MockReplicaSet* replSet = getReplSet(); vector seedList; @@ -538,6 +584,9 @@ namespace { DBClientReplicaSet replConn(replSet->getSetName(), seedList); + // Need up-to-date view to ensure there are multiple valid choices. + ReplicaSetMonitor::get(replSet->getSetName())->startOrContinueRefresh().refreshAll(); + string dest; { Query query; @@ -567,6 +616,9 @@ namespace { DBClientReplicaSet replConn(replSet->getSetName(), seedList); + // Need up-to-date view to ensure there are multiple valid choices. + ReplicaSetMonitor::get(replSet->getSetName())->startOrContinueRefresh().refreshAll(); + string dest; { Query query; @@ -600,6 +652,10 @@ namespace { DBClientReplicaSet replConn(replSet->getSetName(), seedList); + // Need up-to-date view since slaveConn() uses SecondaryPreferred, and this test assumes it + // knows about at least one secondary. + ReplicaSetMonitor::get(replSet->getSetName())->startOrContinueRefresh().refreshAll(); + string dest; mongo::DBClientConnection& secConn = replConn.slaveConn(); diff --git a/src/mongo/client/dbclientcursor.cpp b/src/mongo/client/dbclientcursor.cpp index 095b3a7a7cf..2e0b0a0fa4d 100644 --- a/src/mongo/client/dbclientcursor.cpp +++ b/src/mongo/client/dbclientcursor.cpp @@ -15,14 +15,13 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/client/dbclientcursor.h" #include "mongo/client/connpool.h" -#include "mongo/db/cmdline.h" #include "mongo/db/dbmessage.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" #include "mongo/s/shard.h" #include "mongo/s/stale_exception.h" // for RecvStaleConfigException @@ -31,7 +30,7 @@ namespace mongo { void assembleRequest( const string &ns, BSONObj query, int nToReturn, int nToSkip, const BSONObj *fieldsToReturn, int queryOptions, Message &toSend ); void DBClientCursor::_finishConsInit() { - _originalHost = _client->toString(); + _originalHost = _client->getServerAddress(); } int DBClientCursor::nextBatchSize() { @@ -79,6 +78,15 @@ namespace mongo { void DBClientCursor::initLazy( bool isRetry ) { massert( 15875 , "DBClientCursor::initLazy called on a client that doesn't support lazy" , _client->lazySupported() ); + if (DBClientWithCommands::RunCommandHookFunc hook = _client->getRunCommandHook()) { + if (NamespaceString(ns).isCommand()) { + BSONObjBuilder bob; + bob.appendElements(query); + hook(&bob); + query = bob.obj(); + } + } + Message toSend; _assembleInit( toSend ); _client->say( toSend, isRetry, &_originalHost ); @@ -103,6 +111,14 @@ namespace mongo { } dataReceived( retry, _lazyHost ); + + if (DBClientWithCommands::PostRunCommandHookFunc hook = _client->getPostRunCommandHook()) { + if (NamespaceString(ns).isCommand()) { + BSONObj cmdResponse = peekFirst(); + hook(cmdResponse, _lazyHost); + } + } + return ! retry; } @@ -140,14 +156,13 @@ namespace mongo { } else { verify( _scopedHost.size() ); - scoped_ptr conn( - ScopedDbConnection::getScopedDbConnection( _scopedHost ) ); - conn->get()->call( toSend , *response ); - _client = conn->get(); + ScopedDbConnection conn(_scopedHost); + conn->call( toSend , *response ); + _client = conn.get(); this->batch.m = response; dataReceived(); _client = 0; - conn->done(); + conn.done(); } } @@ -310,6 +325,10 @@ namespace mongo { } DBClientCursor::~DBClientCursor() { + kill(); + } + + void DBClientCursor::kill() { if (!this) return; @@ -335,19 +354,21 @@ namespace mongo { } else { verify( _scopedHost.size() ); - scoped_ptr conn( - ScopedDbConnection::getScopedDbConnection( _scopedHost ) ); + ScopedDbConnection conn(_scopedHost); if( DBClientConnection::getLazyKillCursor() ) - conn->get()->sayPiggyBack( m ); + conn->sayPiggyBack( m ); else - conn->get()->say( m ); + conn->say( m ); - conn->done(); + conn.done(); } } ); + + // Mark this cursor as dead since we can't do any getMores. + cursorId = 0; } diff --git a/src/mongo/client/dbclientcursor.h b/src/mongo/client/dbclientcursor.h index 20c28df0a0d..810822f6a91 100644 --- a/src/mongo/client/dbclientcursor.h +++ b/src/mongo/client/dbclientcursor.h @@ -22,6 +22,7 @@ #include #include "mongo/client/dbclientinterface.h" +#include "mongo/client/export_macros.h" #include "mongo/db/jsobj.h" #include "mongo/db/json.h" #include "mongo/util/net/message.h" @@ -30,10 +31,10 @@ namespace mongo { class AScopedConnection; - /** for mock purposes only -- do not create variants of DBClientCursor, nor hang code here + /** for mock purposes only -- do not create variants of DBClientCursor, nor hang code here @see DBClientMockCursor */ - class DBClientCursorInterface : boost::noncopyable { + class MONGO_CLIENT_API DBClientCursorInterface : boost::noncopyable { public: virtual ~DBClientCursorInterface() {} virtual bool more() = 0; @@ -44,7 +45,7 @@ namespace mongo { }; /** Queries return a cursor object */ - class DBClientCursor : public DBClientCursorInterface { + class MONGO_CLIENT_API DBClientCursor : public DBClientCursorInterface { public: /** If true, safe to call next(). Requests more from server if necessary. */ bool more(); @@ -78,8 +79,7 @@ namespace mongo { BSONObj o = next(); if( strcmp(o.firstElementFieldName(), "$err") == 0 ) { string s = "nextSafe(): " + o.toString(); - if( logLevel >= 5 ) - log() << s << endl; + LOG(5) << s; uasserted(13106, s); } return o; @@ -130,6 +130,9 @@ namespace mongo { return (resultFlags & flag) != 0; } + /// Change batchSize after construction. Can change after requesting first batch. + void setBatchSize(int newBatchSize) { batchSize = newBatchSize; } + DBClientCursor( DBClientBase* client, const string &_ns, BSONObj _query, int _nToReturn, int _nToSkip, const BSONObj *_fieldsToReturn, int queryOptions , int bs ) : _client(client), @@ -199,7 +202,19 @@ namespace mongo { void initLazy( bool isRetry = false ); bool initLazyFinish( bool& retry ); - class Batch : boost::noncopyable { + /** + * Marks this object as dead and sends the KillCursors message to the server. + * + * Any errors that result from this are swallowed since this is typically performed as part + * of cleanup and a failure to kill the cursor should not result in a failure of the + * operation using the cursor. + * + * Killing an already killed or exhausted cursor does nothing, so it is safe to always call + * this if you want to ensure that a cursor is killed. + */ + void kill(); + + class Batch : boost::noncopyable { friend class DBClientCursor; auto_ptr m; int nReturned; @@ -215,7 +230,7 @@ namespace mongo { int nextBatchSize(); void _finishConsInit(); - + Batch batch; DBClientBase* _client; string _originalHost; @@ -253,7 +268,7 @@ namespace mongo { /** iterate over objects in current batch only - will not cause a network call */ - class DBClientCursorBatchIterator { + class MONGO_CLIENT_API DBClientCursorBatchIterator { public: DBClientCursorBatchIterator( DBClientCursor &c ) : _c( c ), _n() {} bool moreInCurrentBatch() { return _c.moreInCurrentBatch(); } diff --git a/src/mongo/client/dbclientinterface.h b/src/mongo/client/dbclientinterface.h index b037556ee58..d0555e341f5 100644 --- a/src/mongo/client/dbclientinterface.h +++ b/src/mongo/client/dbclientinterface.h @@ -22,7 +22,12 @@ #include "mongo/pch.h" +#include + +#include "mongo/base/string_data.h" +#include "mongo/client/export_macros.h" #include "mongo/db/jsobj.h" +#include "mongo/logger/log_severity.h" #include "mongo/platform/atomic_word.h" #include "mongo/util/net/message.h" #include "mongo/util/net/message_port.h" @@ -30,7 +35,7 @@ namespace mongo { /** the query field 'options' can have these bits set: */ - enum QueryOptions { + enum MONGO_CLIENT_API QueryOptions { /** Tailable means cursor is not closed when the last data is retrieved. rather, the cursor marks the final object's position. you can resume using the cursor later, from where it was located, if more data were received. Set on dbQuery and dbGetMore. @@ -76,15 +81,27 @@ namespace mongo { /** When sharded, this means its ok to return partial results Usually we will fail a query if all required shards aren't up - If this is set, it'll be a partial result set + If this is set, it'll be a partial result set */ QueryOption_PartialResults = 1 << 7 , - QueryOption_AllSupported = QueryOption_CursorTailable | QueryOption_SlaveOk | QueryOption_OplogReplay | QueryOption_NoCursorTimeout | QueryOption_AwaitData | QueryOption_Exhaust | QueryOption_PartialResults - + QueryOption_AllSupported = QueryOption_CursorTailable | + QueryOption_SlaveOk | + QueryOption_OplogReplay | + QueryOption_NoCursorTimeout | + QueryOption_AwaitData | + QueryOption_Exhaust | + QueryOption_PartialResults, + + QueryOption_AllSupportedForSharding = QueryOption_CursorTailable | + QueryOption_SlaveOk | + QueryOption_OplogReplay | + QueryOption_NoCursorTimeout | + QueryOption_AwaitData | + QueryOption_PartialResults, }; - enum UpdateOptions { + enum MONGO_CLIENT_API UpdateOptions { /** Upsert - that is, insert the item if no matching item is found. */ UpdateOption_Upsert = 1 << 0, @@ -96,7 +113,7 @@ namespace mongo { UpdateOption_Broadcast = 1 << 2 }; - enum RemoveOptions { + enum MONGO_CLIENT_API RemoveOptions { /** only delete one option */ RemoveOption_JustOne = 1 << 0, @@ -104,11 +121,11 @@ namespace mongo { RemoveOption_Broadcast = 1 << 1 }; - - /** + + /** * need to put in DbMesssage::ReservedOptions as well */ - enum InsertOptions { + enum MONGO_CLIENT_API InsertOptions { /** With muli-insert keep processing inserts if one fails */ InsertOption_ContinueOnError = 1 << 0 }; @@ -116,7 +133,7 @@ namespace mongo { /** * Start from *top* of bits, these are generic write options that apply to all */ - enum WriteOptions { + enum MONGO_CLIENT_API WriteOptions { /** logical writeback option */ WriteOption_FromWriteback = 1 << 31 }; @@ -127,12 +144,12 @@ namespace mongo { // the api user, but we need these constants to disassemble/reassemble the messages correctly. // - enum ReservedOptions { + enum MONGO_CLIENT_API ReservedOptions { Reserved_InsertOption_ContinueOnError = 1 << 0 , Reserved_FromWriteback = 1 << 1 }; - enum ReadPreference { + enum MONGO_CLIENT_API ReadPreference { /** * Read from primary only. All operations produce an error (throw an * exception where applicable) if primary is unavailable. Cannot be @@ -164,8 +181,8 @@ namespace mongo { ReadPreference_Nearest, }; - class DBClientBase; - class DBClientConnection; + class MONGO_CLIENT_API DBClientBase; + class MONGO_CLIENT_API DBClientConnection; /** * ConnectionString handles parsing different ways to connect to mongo and determining method @@ -174,7 +191,7 @@ namespace mongo { * server:port * foo/server:port,server:port SET * server,server,server SYNC - * Warning - you usually don't want "SYNC", it's used + * Warning - you usually don't want "SYNC", it's used * for some special things such as sharding config servers. * See syncclusterconnection.h for more info. * @@ -184,14 +201,17 @@ namespace mongo { * if ( ! cs.isValid() ) throw "bad: " + errmsg; * DBClientBase * conn = cs.connect( errmsg ); */ - class ConnectionString { + class MONGO_CLIENT_API ConnectionString { public: + enum ConnectionType { INVALID , MASTER , PAIR , SET , SYNC, CUSTOM }; ConnectionString() { _type = INVALID; } + // Note: This should only be used for direct connections to a single server. For replica + // set and SyncClusterConnections, use ConnectionString::parse. ConnectionString( const HostAndPort& server ) { _type = MASTER; _servers.push_back( server ); @@ -223,7 +243,7 @@ namespace mongo { ConnectionString( const string& s , ConnectionType favoredMultipleType ) { _type = INVALID; - + _fillServers( s ); if ( _type != INVALID ) { // set already @@ -241,13 +261,13 @@ namespace mongo { bool isValid() const { return _type != INVALID; } string toString() const { return _string; } - + DBClientBase* connect( string& errmsg, double socketTimeout = 0 ) const; string getSetName() const { return _setName; } vector getServers() const { return _servers; } - + ConnectionType type() const { return _type; } /** @@ -288,6 +308,23 @@ namespace mongo { return _connectHook; } + // Allows ConnectionStrings to be stored more easily in sets/maps + bool operator<(const ConnectionString& other) const { + return _string < other._string; + } + + // + // FOR TESTING ONLY - useful to be able to directly mock a connection string without + // including the entire client library. + // + + static ConnectionString mock( const HostAndPort& server ) { + ConnectionString connStr; + connStr._servers.push_back( server ); + connStr._string = server.toString( true ); + return connStr; + } + private: void _fillServers( string s ); @@ -306,7 +343,7 @@ namespace mongo { * controls how much a clients cares about writes * default is NORMAL */ - enum WriteConcern { + enum MONGO_CLIENT_API WriteConcern { W_NONE = 0 , // TODO: not every connection type fully supports this W_NORMAL = 1 // TODO SAFE = 2 @@ -322,7 +359,7 @@ namespace mongo { QUERY( "age" << 33 << "school" << "UCLA" ).sort("name") QUERY( "age" << GT << 30 << LT << 50 ) */ - class Query { + class MONGO_CLIENT_API Query { public: static const BSONField ReadPrefField; static const BSONField ReadPrefModeField; @@ -444,7 +481,7 @@ namespace mongo { * Represents a full query description, including all options required for the query to be passed on * to other hosts */ - class QuerySpec { + class MONGO_CLIENT_API QuerySpec { string _ns; int _ntoskip; @@ -455,7 +492,7 @@ namespace mongo { Query _queryObj; public: - + QuerySpec( const string& ns, const BSONObj& query, const BSONObj& fields, int ntoskip, int ntoreturn, int options ) @@ -477,40 +514,40 @@ namespace mongo { BSONObj* fieldsData() { return &_fields; } // don't love this, but needed downstrem - const BSONObj* fieldsPtr() const { return &_fields; } + const BSONObj* fieldsPtr() const { return &_fields; } string ns() const { return _ns; } int ntoskip() const { return _ntoskip; } int ntoreturn() const { return _ntoreturn; } int options() const { return _options; } - + void setFields( BSONObj& o ) { _fields = o.getOwned(); } string toString() const { - return str::stream() << "QSpec " << + return str::stream() << "QSpec " << BSON( "ns" << _ns << "n2skip" << _ntoskip << "n2return" << _ntoreturn << "options" << _options << "query" << _query << "fields" << _fields ); } - + }; /** Typically one uses the QUERY(...) macro to construct a Query object. Example: QUERY( "age" << 33 << "school" << "UCLA" ) */ -#define QUERY(x) mongo::Query( BSON(x) ) +#define QUERY(x) ::mongo::Query( BSON(x) ) // Useful utilities for namespaces /** @return the database name portion of an ns string */ - string nsGetDB( const string &ns ); + MONGO_CLIENT_API string nsGetDB( const string &ns ); /** @return the collection name portion of an ns string */ - string nsGetCollection( const string &ns ); + MONGO_CLIENT_API string nsGetCollection( const string &ns ); /** interface that handles communication with the db */ - class DBConnector { + class MONGO_CLIENT_API DBConnector { public: virtual ~DBConnector() {} /** actualServer is set to the actual server where they call went if there was a choice (SlaveOk) */ @@ -529,7 +566,7 @@ namespace mongo { /** The interface that any db connection should implement */ - class DBClientInterface : boost::noncopyable { + class MONGO_CLIENT_API DBClientInterface : boost::noncopyable { public: virtual auto_ptr query(const string &ns, Query query, int nToReturn = 0, int nToSkip = 0, const BSONObj *fieldsToReturn = 0, int queryOptions = 0 , int batchSize = 0 ) = 0; @@ -557,8 +594,8 @@ namespace mongo { */ virtual BSONObj findOne(const string &ns, const Query& query, const BSONObj *fieldsToReturn = 0, int queryOptions = 0); - /** query N objects from the database into an array. makes sense mostly when you want a small number of results. if a huge number, use - query() and iterate the cursor. + /** query N objects from the database into an array. makes sense mostly when you want a small number of results. if a huge number, use + query() and iterate the cursor. */ void findN(vector& out, const string&ns, Query query, int nToReturn, int nToSkip = 0, const BSONObj *fieldsToReturn = 0, int queryOptions = 0); @@ -572,13 +609,13 @@ namespace mongo { DB "commands" Basically just invocations of connection.$cmd.findOne({...}); */ - class DBClientWithCommands : public DBClientInterface { + class MONGO_CLIENT_API DBClientWithCommands : public DBClientInterface { set _seenIndexes; public: /** controls how chatty the client is about network errors & such. See log.h */ - int _logLevel; + logger::LogSeverity _logLevel; - DBClientWithCommands() : _logLevel(0), + DBClientWithCommands() : _logLevel(logger::LogSeverity::Log()), _cachedAvailableOptions( (enum QueryOptions)0 ), _haveCachedAvailableOptions(false) { } @@ -613,9 +650,9 @@ namespace mongo { * are required depends on the mechanism, which is mandatory. * * "mechanism": The string name of the sasl mechanism to use. Mandatory. - * "user": The string name of the principal to authenticate. Mandatory. - * "userSource": The database target of the auth command, which identifies the location - * of the credential information for the principal. May be "$external" if + * "user": The string name of the user to authenticate. Mandatory. + * "db": The database target of the auth command, which identifies the location + * of the credential information for the user. May be "$external" if * credential information is stored outside of the mongo cluster. Mandatory. * "pwd": The password data. * "digestPassword": Boolean, set to true if the "pwd" is undigested (default). @@ -656,7 +693,7 @@ namespace mongo { */ virtual unsigned long long count(const string &ns, const BSONObj& query = BSONObj(), int options=0, int limit=0, int skip=0 ); - string createPasswordDigest( const string &username , const string &clearTextPassword ); + static string createPasswordDigest(const string &username, const string &clearTextPassword); /** returns true in isMaster parm if this db is the current master of a replica pair. @@ -695,7 +732,9 @@ namespace mongo { bool j = false, int w = 0, int wtimeout = 0); - // Same as above but defaults to using admin DB + /** + * Same as the form of getLastError that takes a dbname, but just uses the admin DB. + */ string getLastError(bool fsync = false, bool j = false, int w = 0, int wtimeout = 0); /** Get error result from the last write operation (insert/update/delete) on this connection. @@ -710,10 +749,12 @@ namespace mongo { bool j = false, int w = 0, int wtimeout = 0); - // Same as above but defaults to using admin DB + /** + * Same as the form of getLastErrorDetailed that takes a dbname, but just uses the admin DB. + */ virtual BSONObj getLastErrorDetailed(bool fsync = false, bool j = false, int w = 0, int wtimeout = 0); - /** Can be called with the returned value from getLastErrorDetailed to extract an error string. + /** Can be called with the returned value from getLastErrorDetailed to extract an error string. If all you need is the string, just call getLastError() instead. */ static string getLastErrorString( const BSONObj& res ); @@ -896,6 +937,14 @@ namespace mongo { */ list getCollectionNames( const string& db ); + /** + * { name : "", + * options : { } + * } + */ + std::list getCollectionInfos( const std::string& db, + const BSONObj& filter = BSONObj() ); + bool exists( const string& ns ); /** Create an index if it does not already exist. @@ -946,7 +995,31 @@ namespace mongo { return ret; } - virtual string toString() = 0; + virtual string toString() const = 0; + + /** + * A function type for runCommand hooking; the function takes a pointer + * to a BSONObjBuilder and returns nothing. The builder contains a + * runCommand BSON object. + * Once such a function is set as the runCommand hook, every time the DBClient + * processes a runCommand, the hook will be called just prior to sending it to the server. + */ + typedef boost::function RunCommandHookFunc; + virtual void setRunCommandHook(RunCommandHookFunc func); + RunCommandHookFunc getRunCommandHook() const { + return _runCommandHook; + } + + /** + * Similar to above, but for running a function on a command response after a command + * has been run. + */ + typedef boost::function PostRunCommandHookFunc; + virtual void setPostRunCommandHook(PostRunCommandHookFunc func); + PostRunCommandHookFunc getPostRunCommandHook() const { + return _postRunCommandHook; + } + protected: /** if the result of a command is ok*/ @@ -975,9 +1048,25 @@ namespace mongo { bool _authMongoCR(const string &dbname, const string &username, const string &pwd, - string& errmsg, + BSONObj *info, bool digestPassword); + /** + * Use the MONGODB-X509 protocol to authenticate as "username. The certificate details + * has already been communicated automatically as part of the connect call. + * Returns false on failure and set "errmsg". + */ + bool _authX509(const string&dbname, + const string &username, + BSONObj *info); + + /** + * These functions will be executed by the driver on runCommand calls. + */ + RunCommandHookFunc _runCommandHook; + PostRunCommandHookFunc _postRunCommandHook; + + private: enum QueryOptions _cachedAvailableOptions; bool _haveCachedAvailableOptions; @@ -986,17 +1075,20 @@ namespace mongo { /** abstract class that implements the core db operations */ - class DBClientBase : public DBClientWithCommands, public DBConnector { + class MONGO_CLIENT_API DBClientBase : public DBClientWithCommands, public DBConnector { protected: static AtomicInt64 ConnectionIdSequence; long long _connectionId; // unique connection id for this connection WriteConcern _writeConcern; + int _minWireVersion; + int _maxWireVersion; public: static const uint64_t INVALID_SOCK_CREATION_TIME; DBClientBase() { _writeConcern = W_NORMAL; _connectionId = ConnectionIdSequence.fetchAndAdd(1); + _minWireVersion = _maxWireVersion = 0; } long long getConnectionId() const { return _connectionId; } @@ -1004,6 +1096,14 @@ namespace mongo { WriteConcern getWriteConcern() const { return _writeConcern; } void setWriteConcern( WriteConcern w ) { _writeConcern = w; } + void setWireVersions( int minWireVersion, int maxWireVersion ){ + _minWireVersion = minWireVersion; + _maxWireVersion = maxWireVersion; + } + + int getMinWireVersion() { return _minWireVersion; } + int getMaxWireVersion() { return _maxWireVersion; } + /** send a query to the database. @param ns namespace to query, format is .[.]* @param query query to perform on the collection. this is a BSONObj (binary JSON) @@ -1080,13 +1180,18 @@ namespace mongo { virtual bool isFailed() const = 0; + /** + * if not checked recently, checks whether the underlying socket/sockets are still valid + */ + virtual bool isStillConnected() = 0; + virtual void killCursor( long long cursorID ) = 0; virtual bool callRead( Message& toSend , Message& response ) = 0; // virtual bool callWrite( Message& toSend , Message& response ) = 0; // TODO: add this if needed - + virtual ConnectionString::ConnectionType type() const = 0; - + virtual double getSoTimeout() const = 0; virtual uint64_t getSockCreationMicroSec() const { @@ -1097,7 +1202,7 @@ namespace mongo { class DBClientReplicaSet; - class ConnectException : public UserException { + class MONGO_CLIENT_API ConnectException : public UserException { public: ConnectException(string msg) : UserException(9000,msg) { } }; @@ -1106,7 +1211,7 @@ namespace mongo { A basic connection to the database. This is the main entry point for talking to a simple Mongo setup */ - class DBClientConnection : public DBClientBase { + class MONGO_CLIENT_API DBClientConnection : public DBClientBase { public: using DBClientBase::query; @@ -1117,7 +1222,7 @@ namespace mongo { Connect timeout is fixed, but short, at 5 seconds. */ DBClientConnection(bool _autoReconnect=false, DBClientReplicaSet* cp=0, double so_timeout=0) : - clientSet(cp), _failed(false), autoReconnect(_autoReconnect), lastReconnectTry(0), _so_timeout(so_timeout) { + clientSet(cp), _failed(false), autoReconnect(_autoReconnect), autoReconnectBackoff(1000, 2000), _so_timeout(so_timeout) { _numConnections++; } @@ -1167,6 +1272,15 @@ namespace mongo { throw ConnectException(string("can't connect ") + errmsg); } + /** + * Logs out the connection for the given database. + * + * @param dbname the database to logout from. + * @param info the result object for the logout command (provided for backwards + * compatibility with mongo shell) + */ + virtual void logout(const string& dbname, BSONObj& info); + virtual auto_ptr query(const string &ns, Query query=Query(), int nToReturn = 0, int nToSkip = 0, const BSONObj *fieldsToReturn = 0, int queryOptions = 0 , int batchSize = 0 ) { checkConnection(); @@ -1190,19 +1304,20 @@ namespace mongo { */ bool isFailed() const { return _failed; } + bool isStillConnected() { return p ? p->isStillConnected() : true; } + MessagingPort& port() { verify(p); return *p; } - string toStringLong() const { + string toString() const { stringstream ss; ss << _serverString; + if ( !_serverAddrString.empty() ) ss << " (" << _serverAddrString << ")"; if ( _failed ) ss << " failed"; return ss.str(); } - /** Returns the address of the server */ - string toString() { return _serverString; } - string getServerAddress() const { return _serverString; } + const HostAndPort& getServerHostAndPort() const { return _server; } virtual void killCursor( long long cursorID ); virtual bool callRead( Message& toSend , Message& response ) { return call( toSend , response ); } @@ -1247,9 +1362,10 @@ namespace mongo { boost::scoped_ptr server; bool _failed; const bool autoReconnect; - time_t lastReconnectTry; + Backoff autoReconnectBackoff; HostAndPort _server; // remember for reconnects - string _serverString; + string _serverString; // server host and port + string _serverAddrString; // resolved ip of server void _checkConnection(); // throws SocketException if in failed state and not reconnecting or if waiting to reconnect @@ -1263,20 +1379,20 @@ namespace mongo { static bool _lazyKillCursor; // lazy means we piggy back kill cursors on next op #ifdef MONGO_SSL - SSLManager* sslManager(); + SSLManagerInterface* sslManager(); #endif }; /** pings server to check if it's up */ - bool serverAlive( const string &uri ); + MONGO_CLIENT_API bool serverAlive( const string &uri ); - DBClientBase * createDirectClient(); + MONGO_CLIENT_API DBClientBase * createDirectClient(); - BSONElement getErrField( const BSONObj& result ); - bool hasErrField( const BSONObj& result ); + MONGO_CLIENT_API BSONElement getErrField( const BSONObj& result ); + MONGO_CLIENT_API bool hasErrField( const BSONObj& result ); - inline std::ostream& operator<<( std::ostream &s, const Query &q ) { + MONGO_CLIENT_API inline std::ostream& operator<<( std::ostream &s, const Query &q ) { return s << q.toString(); } diff --git a/src/mongo/client/dbclientmockcursor.h b/src/mongo/client/dbclientmockcursor.h index 8d85ff5ad2e..0e60f0fc947 100644 --- a/src/mongo/client/dbclientmockcursor.h +++ b/src/mongo/client/dbclientmockcursor.h @@ -17,11 +17,12 @@ #pragma once -#include "dbclientcursor.h" +#include "mongo/client/dbclientcursor.h" +#include "mongo/client/export_macros.h" namespace mongo { - class DBClientMockCursor : public DBClientCursorInterface { + class MONGO_CLIENT_API DBClientMockCursor : public DBClientCursorInterface { public: DBClientMockCursor( const BSONArray& mockCollection ) : _iter( mockCollection ) {} virtual ~DBClientMockCursor() {} diff --git a/src/mongo/client/distlock.cpp b/src/mongo/client/distlock.cpp deleted file mode 100644 index 0e38a64f953..00000000000 --- a/src/mongo/client/distlock.cpp +++ /dev/null @@ -1,1159 +0,0 @@ -// @file distlock.h - -/* Copyright 2009 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "pch.h" - -#include "mongo/client/distlock.h" - -#include - -#include "mongo/client/dbclientcursor.h" -#include "mongo/s/type_locks.h" -#include "mongo/s/type_lockpings.h" -#include "mongo/util/timer.h" - -namespace mongo { - - LabeledLevel DistributedLock::logLvl( 1 ); - DistributedLock::LastPings DistributedLock::lastPings; - - ThreadLocalValue distLockIds(""); - - /* ================== - * Module initialization - */ - - static SimpleMutex _cachedProcessMutex("distlock_initmodule"); - static string* _cachedProcessString = NULL; - - static void initModule() { - SimpleMutex::scoped_lock lk(_cachedProcessMutex); - if (_cachedProcessString) { - // someone got the lock before us - return; - } - - // cache process string - stringstream ss; - ss << getHostName() << ":" << cmdLine.port << ":" << time(0) << ":" << rand(); - _cachedProcessString = new string( ss.str() ); - } - - /* =================== */ - - string getDistLockProcess() { - if (!_cachedProcessString) - initModule(); - verify( _cachedProcessString ); - return *_cachedProcessString; - } - - string getDistLockId() { - string s = distLockIds.get(); - if ( s.empty() ) { - stringstream ss; - ss << getDistLockProcess() << ":" << getThreadName() << ":" << rand(); - s = ss.str(); - distLockIds.set( s ); - } - return s; - } - - // Used for disabling the lock pinger during tests - // Should *always* be true otherwise. - static bool lockPingerEnabled = true; - - bool isLockPingerEnabled() { return lockPingerEnabled; } - void setLockPingerEnabled(bool enabled) { lockPingerEnabled = enabled; } - - class DistributedLockPinger { - public: - - DistributedLockPinger() - : _mutex( "DistributedLockPinger" ) { - } - - void _distLockPingThread( ConnectionString addr, - const std::string& process, - unsigned long long sleepTime ) { - - setThreadName( "LockPinger" ); - - string pingId = pingThreadId( addr, process ); - - LOG( DistributedLock::logLvl - 1 ) << "creating distributed lock ping thread for " << addr - << " and process " << process - << " (sleeping for " << sleepTime << "ms)" << endl; - - static int loops = 0; - while( ! inShutdown() && ! shouldKill( addr, process ) ) { - - LOG( DistributedLock::logLvl + 2 ) << "distributed lock pinger '" << pingId << "' about to ping." << endl; - - Date_t pingTime; - - try { - scoped_ptr connPtr( - ScopedDbConnection::getInternalScopedDbConnection( addr.toString(), - 30.0 ) ); - ScopedDbConnection& conn = *connPtr; - - pingTime = jsTime(); - - // refresh the entry corresponding to this process in the lockpings collection - conn->update( LockpingsType::ConfigNS, - BSON( LockpingsType::process(process) ), - BSON( "$set" << BSON( LockpingsType::ping(pingTime) ) ), - true ); - - string err = conn->getLastError(); - if ( ! err.empty() ) { - warning() << "pinging failed for distributed lock pinger '" << pingId << "'." - << causedBy( err ) << endl; - conn.done(); - - // Sleep for normal ping time - sleepmillis(sleepTime); - continue; - } - - // remove really old entries from the lockpings collection if they're not holding a lock - // (this may happen if an instance of a process was taken down and no new instance came up to - // replace it for a quite a while) - // if the lock is taken, the take-over mechanism should handle the situation - auto_ptr c = conn->query( LocksType::ConfigNS , BSONObj() ); - // TODO: Would be good to make clear whether query throws or returns empty on errors - uassert( 16060, str::stream() << "cannot query locks collection on config server " << conn.getHost(), c.get() ); - - set pids; - while ( c->more() ) { - BSONObj lock = c->next(); - if ( ! lock[LocksType::process()].eoo() ) { - pids.insert( lock[LocksType::process()].valuestrsafe() ); - } - } - - Date_t fourDays = pingTime - ( 4 * 86400 * 1000 ); // 4 days - conn->remove( LockpingsType::ConfigNS, - BSON( LockpingsType::process() << NIN << pids << - LockpingsType::ping() << LT << fourDays ) ); - err = conn->getLastError(); - if ( ! err.empty() ) { - warning() << "ping cleanup for distributed lock pinger '" << pingId << " failed." - << causedBy( err ) << endl; - conn.done(); - - // Sleep for normal ping time - sleepmillis(sleepTime); - continue; - } - - // create index so remove is fast even with a lot of servers - if ( loops++ == 0 ) { - conn->ensureIndex( LockpingsType::ConfigNS, BSON( LockpingsType::ping() << 1 ) ); - } - - LOG( DistributedLock::logLvl - ( loops % 10 == 0 ? 1 : 0 ) ) << "cluster " << addr << " pinged successfully at " << pingTime - << " by distributed lock pinger '" << pingId - << "', sleeping for " << sleepTime << "ms" << endl; - - // Remove old locks, if possible - // Make sure no one else is adding to this list at the same time - scoped_lock lk( _mutex ); - - int numOldLocks = _oldLockOIDs.size(); - if( numOldLocks > 0 ) - LOG( DistributedLock::logLvl - 1 ) << "trying to delete " << _oldLockOIDs.size() << " old lock entries for process " << process << endl; - - bool removed = false; - for( list::iterator i = _oldLockOIDs.begin(); i != _oldLockOIDs.end(); - i = ( removed ? _oldLockOIDs.erase( i ) : ++i ) ) { - removed = false; - try { - // Got OID from lock with id, so we don't need to specify id again - conn->update( LocksType::ConfigNS , - BSON( LocksType::lockID(*i) ), - BSON( "$set" << BSON( LocksType::state(0) ) ) ); - - // Either the update went through or it didn't, either way we're done trying to - // unlock - LOG( DistributedLock::logLvl - 1 ) << "handled late remove of old distributed lock with ts " << *i << endl; - removed = true; - } - catch( UpdateNotTheSame& ) { - LOG( DistributedLock::logLvl - 1 ) << "partially removed old distributed lock with ts " << *i << endl; - removed = true; - } - catch ( std::exception& e) { - warning() << "could not remove old distributed lock with ts " << *i - << causedBy( e ) << endl; - } - - } - - if( numOldLocks > 0 && _oldLockOIDs.size() > 0 ){ - LOG( DistributedLock::logLvl - 1 ) << "not all old lock entries could be removed for process " << process << endl; - } - - conn.done(); - - } - catch ( std::exception& e ) { - warning() << "distributed lock pinger '" << pingId << "' detected an exception while pinging." - << causedBy( e ) << endl; - } - - sleepmillis(sleepTime); - } - - warning() << "removing distributed lock ping thread '" << pingId << "'" << endl; - - - if( shouldKill( addr, process ) ) - finishKill( addr, process ); - - } - - void distLockPingThread( ConnectionString addr, - long long clockSkew, - const std::string& processId, - unsigned long long sleepTime ) { - try { - jsTimeVirtualThreadSkew( clockSkew ); - _distLockPingThread( addr, processId, sleepTime ); - } - catch ( std::exception& e ) { - error() << "unexpected error while running distributed lock pinger for " << addr << ", process " << processId << causedBy( e ) << endl; - } - catch ( ... ) { - error() << "unknown error while running distributed lock pinger for " << addr << ", process " << processId << endl; - } - } - - string pingThreadId( const ConnectionString& conn, const string& processId ) { - return conn.toString() + "/" + processId; - } - - string got( DistributedLock& lock, unsigned long long sleepTime ) { - - if (!lockPingerEnabled) return ""; - - // Make sure we don't start multiple threads for a process id - scoped_lock lk( _mutex ); - - const ConnectionString& conn = lock.getRemoteConnection(); - const string& processId = lock.getProcessId(); - string s = pingThreadId( conn, processId ); - - // Ignore if we already have a pinging thread for this process. - if ( _seen.count( s ) > 0 ) return s; - - // Check our clock skew - try { - if( lock.isRemoteTimeSkewed() ) { - throw LockException( str::stream() << "clock skew of the cluster " << conn.toString() << " is too far out of bounds to allow distributed locking." , 13650 ); - } - } - catch( LockException& e) { - throw LockException( str::stream() << "error checking clock skew of cluster " << conn.toString() << causedBy( e ) , 13651); - } - - boost::thread t( boost::bind( &DistributedLockPinger::distLockPingThread, this, conn, getJSTimeVirtualThreadSkew(), processId, sleepTime) ); - - _seen.insert( s ); - - return s; - } - - void addUnlockOID( const OID& oid ) { - // Modifying the lock from some other thread - scoped_lock lk( _mutex ); - _oldLockOIDs.push_back( oid ); - } - - bool willUnlockOID( const OID& oid ) { - scoped_lock lk( _mutex ); - return find( _oldLockOIDs.begin(), _oldLockOIDs.end(), oid ) != _oldLockOIDs.end(); - } - - void kill( const ConnectionString& conn, const string& processId ) { - // Make sure we're in a consistent state before other threads can see us - scoped_lock lk( _mutex ); - - string pingId = pingThreadId( conn, processId ); - - verify( _seen.count( pingId ) > 0 ); - _kill.insert( pingId ); - - } - - bool shouldKill( const ConnectionString& conn, const string& processId ) { - return _kill.count( pingThreadId( conn, processId ) ) > 0; - } - - void finishKill( const ConnectionString& conn, const string& processId ) { - // Make sure we're in a consistent state before other threads can see us - scoped_lock lk( _mutex ); - - string pingId = pingThreadId( conn, processId ); - - _kill.erase( pingId ); - _seen.erase( pingId ); - - } - - static bool _pingerEnabled; - - set _kill; - set _seen; - mongo::mutex _mutex; - list _oldLockOIDs; - - } distLockPinger; - - /** - * Create a new distributed lock, potentially with a custom sleep and takeover time. If a custom sleep time is - * specified (time between pings) - */ - DistributedLock::DistributedLock( const ConnectionString& conn , const string& name , unsigned long long lockTimeout, bool asProcess ) - : _conn(conn), _name(name), - _processId( asProcess ? getDistLockId() : getDistLockProcess() ), - _lockTimeout( lockTimeout == 0 ? LOCK_TIMEOUT : lockTimeout ), - _maxClockSkew( _lockTimeout / LOCK_SKEW_FACTOR ), _maxNetSkew( _maxClockSkew ), - _lockPing( _maxClockSkew ), _mutex( "DistributedLock" ) - { - LOG( logLvl ) << "created new distributed lock for " << name << " on " << conn - << " ( lock timeout : " << _lockTimeout - << ", ping interval : " << _lockPing << ", process : " << asProcess << " )" << endl; - - - } - - DistributedLock::PingData DistributedLock::LastPings::getLastPing( const ConnectionString& conn, const string& lockName ){ - scoped_lock lock( _mutex ); - return _lastPings[ std::pair< string, string >( conn.toString(), lockName ) ]; - } - - void DistributedLock::LastPings::setLastPing( const ConnectionString& conn, const string& lockName, const PingData& pd ){ - scoped_lock lock( _mutex ); - _lastPings[ std::pair< string, string >( conn.toString(), lockName ) ] = pd; - } - - Date_t DistributedLock::getRemoteTime() { - return DistributedLock::remoteTime( _conn, _maxNetSkew ); - } - - bool DistributedLock::isRemoteTimeSkewed() { - return !DistributedLock::checkSkew( _conn, NUM_LOCK_SKEW_CHECKS, _maxClockSkew, _maxNetSkew ); - } - - const ConnectionString& DistributedLock::getRemoteConnection() { - return _conn; - } - - const string& DistributedLock::getProcessId() { - return _processId; - } - - /** - * Returns the remote time as reported by the cluster or server. The maximum difference between the reported time - * and the actual time on the remote server (at the completion of the function) is the maxNetSkew - */ - Date_t DistributedLock::remoteTime( const ConnectionString& cluster, unsigned long long maxNetSkew ) { - - ConnectionString server( *cluster.getServers().begin() ); - scoped_ptr conn( - ScopedDbConnection::getInternalScopedDbConnection( server.toString() ) ); - - BSONObj result; - long long delay; - - try { - Date_t then = jsTime(); - bool success = conn->get()->runCommand( string("admin"), - BSON( "serverStatus" << 1 ), - result ); - delay = jsTime() - then; - - if( !success ) - throw TimeNotFoundException( str::stream() << "could not get status from server " - << server.toString() << " in cluster " << cluster.toString() - << " to check time", 13647 ); - - // Make sure that our delay is not more than 2x our maximum network skew, since this is the max our remote - // time value can be off by if we assume a response in the middle of the delay. - if( delay > (long long) (maxNetSkew * 2) ) - throw TimeNotFoundException( str::stream() << "server " << server.toString() - << " in cluster " << cluster.toString() - << " did not respond within max network delay of " - << maxNetSkew << "ms", 13648 ); - } - catch(...) { - conn->done(); - throw; - } - - conn->done(); - - return result["localTime"].Date() - (delay / 2); - - } - - bool DistributedLock::checkSkew( const ConnectionString& cluster, unsigned skewChecks, unsigned long long maxClockSkew, unsigned long long maxNetSkew ) { - - vector servers = cluster.getServers(); - - if(servers.size() < 1) return true; - - vector avgSkews; - - for(unsigned i = 0; i < skewChecks; i++) { - - // Find the average skew for each server - unsigned s = 0; - for(vector::iterator si = servers.begin(); si != servers.end(); ++si,s++) { - - if(i == 0) avgSkews.push_back(0); - - // Could check if this is self, but shouldn't matter since local network connection should be fast. - ConnectionString server( *si ); - - vector skew; - - BSONObj result; - - Date_t remote = remoteTime( server, maxNetSkew ); - Date_t local = jsTime(); - - // Remote time can be delayed by at most MAX_NET_SKEW - - // Skew is how much time we'd have to add to local to get to remote - avgSkews[s] += (long long) (remote - local); - - LOG( logLvl + 1 ) << "skew from remote server " << server << " found: " << (long long) (remote - local) << endl; - - } - } - - // Analyze skews - - long long serverMaxSkew = 0; - long long serverMinSkew = 0; - - for(unsigned s = 0; s < avgSkews.size(); s++) { - - long long avgSkew = (avgSkews[s] /= skewChecks); - - // Keep track of max and min skews - if(s == 0) { - serverMaxSkew = avgSkew; - serverMinSkew = avgSkew; - } - else { - if(avgSkew > serverMaxSkew) - serverMaxSkew = avgSkew; - if(avgSkew < serverMinSkew) - serverMinSkew = avgSkew; - } - - } - - long long totalSkew = serverMaxSkew - serverMinSkew; - - // Make sure our max skew is not more than our pre-set limit - if(totalSkew > (long long) maxClockSkew) { - LOG( logLvl + 1 ) << "total clock skew of " << totalSkew << "ms for servers " << cluster << " is out of " << maxClockSkew << "ms bounds." << endl; - return false; - } - - LOG( logLvl + 1 ) << "total clock skew of " << totalSkew << "ms for servers " << cluster << " is in " << maxClockSkew << "ms bounds." << endl; - return true; - } - - // For use in testing, ping thread should run indefinitely in practice. - bool DistributedLock::killPinger( DistributedLock& lock ) { - if( lock._threadId == "") return false; - - distLockPinger.kill( lock._conn, lock._processId ); - return true; - } - - - bool DistributedLock::isLockHeld( double timeout, string* errMsg ) { - scoped_ptr connPtr( - ScopedDbConnection::getInternalScopedDbConnection( _conn.toString(), timeout ) ); - ScopedDbConnection& conn = *connPtr; - - BSONObj lockObj; - try { - lockObj = conn->findOne( LocksType::ConfigNS, - BSON( LocksType::name(_name) ) ).getOwned(); - } - catch ( DBException& e ) { - *errMsg = str::stream() << "error checking whether lock " << _name << " is held " - << causedBy( e ); - return false; - } - conn.done(); - - if ( lockObj.isEmpty() ) { - *errMsg = str::stream() << "no lock for " << _name << " exists in the locks collection"; - return false; - } - - if ( lockObj[LocksType::state()].numberInt() < 2 ) { - *errMsg = str::stream() << "lock " << _name << " current state is not held (" - << lockObj[LocksType::state()].numberInt() << ")"; - return false; - } - - if ( lockObj[LocksType::process()].String() != _processId ) { - *errMsg = str::stream() << "lock " << _name << " is currently being held by " - << "another process (" - << lockObj[LocksType::process()].String() << ")"; - return false; - } - - if ( distLockPinger.willUnlockOID( lockObj[LocksType::lockID()].OID() ) ) { - *errMsg = str::stream() << "lock " << _name << " is not held and is currently being " - << "scheduled for lazy unlock by " - << lockObj[LocksType::lockID()].OID(); - return false; - } - - return true; - } - - // Semantics of this method are basically that if the lock cannot be acquired, returns false, can be retried. - // If the lock should not be tried again (some unexpected error) a LockException is thrown. - // If we are only trying to re-enter a currently held lock, reenter should be true. - // Note: reenter doesn't actually make this lock re-entrant in the normal sense, since it can still only - // be unlocked once, instead it is used to verify that the lock is already held. - bool DistributedLock::lock_try( const string& why , bool reenter, BSONObj * other, double timeout ) { - - // TODO: Start pinging only when we actually get the lock? - // If we don't have a thread pinger, make sure we shouldn't have one - if( _threadId == "" ){ - scoped_lock lk( _mutex ); - _threadId = distLockPinger.got( *this, _lockPing ); - } - - // This should always be true, if not, we are using the lock incorrectly. - verify( _name != "" ); - - LOG( logLvl ) << "trying to acquire new distributed lock for " << _name << " on " << _conn - << " ( lock timeout : " << _lockTimeout - << ", ping interval : " << _lockPing << ", process : " << _processId << " )" - << endl; - - // write to dummy if 'other' is null - BSONObj dummyOther; - if ( other == NULL ) - other = &dummyOther; - - scoped_ptr connPtr( - ScopedDbConnection::getInternalScopedDbConnection( _conn.toString(), timeout ) ); - ScopedDbConnection& conn = *connPtr; - - BSONObjBuilder queryBuilder; - queryBuilder.append( LocksType::name() , _name ); - queryBuilder.append( LocksType::state() , 0 ); - - { - // make sure its there so we can use simple update logic below - BSONObj o = conn->findOne( LocksType::ConfigNS , BSON( LocksType::name(_name) ) ).getOwned(); - - // Case 1: No locks - if ( o.isEmpty() ) { - try { - LOG( logLvl ) << "inserting initial doc in " << LocksType::ConfigNS << " for lock " << _name << endl; - conn->insert( LocksType::ConfigNS , BSON( LocksType::name(_name) << LocksType::state(0) << LocksType::who("") ) ); - } - catch ( UserException& e ) { - warning() << "could not insert initial doc for distributed lock " << _name << causedBy( e ) << endl; - } - } - - // Case 2: A set lock that we might be able to force - else if ( o[LocksType::state()].numberInt() > 0 ) { - - string lockName = o[LocksType::name()].String() + string("/") + o[LocksType::process()].String(); - - bool canReenter = reenter && - o[LocksType::process()].String() == _processId && - !distLockPinger.willUnlockOID( o[LocksType::lockID()].OID() ) && - o[LocksType::state()].numberInt() == 2; - if( reenter && ! canReenter ) { - - LOG( logLvl - 1 ) << "not re-entering distributed lock " << lockName; - if ( o[LocksType::process()].String() != _processId ) { - LOG( logLvl - 1 ) << ", different process " << _processId << endl; - } - else if ( o[LocksType::state()].numberInt() == 2 ) { - LOG( logLvl - 1 ) << ", state not finalized" << endl; - } - else { - LOG( logLvl - 1 ) << ", ts " << o[LocksType::lockID()].OID() << " scheduled for late unlock" << endl; - } - - // reset since we've been bounced by a previous lock not being where we thought it was, - // and should go through full forcing process if required. - // (in theory we should never see a ping here if used correctly) - *other = o; other->getOwned(); conn.done(); resetLastPing(); - return false; - } - - BSONObj lastPing = conn->findOne( LockpingsType::ConfigNS, o[LocksType::process()].wrap( LockpingsType::process() ) ); - if ( lastPing.isEmpty() ) { - LOG( logLvl ) << "empty ping found for process in lock '" << lockName << "'" << endl; - // TODO: Using 0 as a "no time found" value Will fail if dates roll over, but then, so will a lot. - lastPing = BSON( LockpingsType::process(o[LocksType::process()].String()) << - LockpingsType::ping((Date_t) 0) ); - } - - unsigned long long elapsed = 0; - unsigned long long takeover = _lockTimeout; - PingData _lastPingCheck = getLastPing(); - - LOG( logLvl ) << "checking last ping for lock '" << lockName << "'" << " against process " << _lastPingCheck.id << " and ping " << _lastPingCheck.lastPing << endl; - - try { - - Date_t remote = remoteTime( _conn ); - - // Timeout the elapsed time using comparisons of remote clock - // For non-finalized locks, timeout 15 minutes since last seen (ts) - // For finalized locks, timeout 15 minutes since last ping - bool recPingChange = o[LocksType::state()].numberInt() == 2 && - ( _lastPingCheck.id != lastPing[LockpingsType::process()].String() || - _lastPingCheck.lastPing != lastPing[LockpingsType::ping()].Date() ); - bool recTSChange = _lastPingCheck.ts != o[LocksType::lockID()].OID(); - - if( recPingChange || recTSChange ) { - // If the ping has changed since we last checked, mark the current date and time - setLastPing( PingData( lastPing[LockpingsType::process()].String().c_str(), - lastPing[LockpingsType::ping()].Date(), - remote, o[LocksType::lockID()].OID() ) ); - } - else { - - // GOTCHA! Due to network issues, it is possible that the current time - // is less than the remote time. We *have* to check this here, otherwise - // we overflow and our lock breaks. - if(_lastPingCheck.remote >= remote) - elapsed = 0; - else - elapsed = remote - _lastPingCheck.remote; - } - } - catch( LockException& e ) { - - // Remote server cannot be found / is not responsive - warning() << "Could not get remote time from " << _conn << causedBy( e ); - // If our config server is having issues, forget all the pings until we can see it again - resetLastPing(); - - } - - if ( elapsed <= takeover && ! canReenter ) { - LOG( logLvl ) << "could not force lock '" << lockName << "' because elapsed time " << elapsed << " <= takeover time " << takeover << endl; - *other = o; other->getOwned(); conn.done(); - return false; - } - else if( elapsed > takeover && canReenter ) { - LOG( logLvl - 1 ) << "not re-entering distributed lock " << lockName << "' because elapsed time " << elapsed << " > takeover time " << takeover << endl; - *other = o; other->getOwned(); conn.done(); - return false; - } - - LOG( logLvl - 1 ) << ( canReenter ? "re-entering" : "forcing" ) << " lock '" << lockName << "' because " - << ( canReenter ? "re-entering is allowed, " : "" ) - << "elapsed time " << elapsed << " > takeover time " << takeover << endl; - - if( elapsed > takeover ) { - - // Lock may forced, reset our timer if succeeds or fails - // Ensures that another timeout must happen if something borks up here, and resets our pristine - // ping state if acquired. - resetLastPing(); - - try { - - // Check the clock skew again. If we check this before we get a lock - // and after the lock times out, we can be pretty sure the time is - // increasing at the same rate on all servers and therefore our - // timeout is accurate - uassert( 14023, str::stream() << "remote time in cluster " << _conn.toString() << " is now skewed, cannot force lock.", !isRemoteTimeSkewed() ); - - // Make sure we break the lock with the correct "ts" (OID) value, otherwise - // we can overwrite a new lock inserted in the meantime. - conn->update( LocksType::ConfigNS, - BSON( LocksType::name(_name) << - LocksType::state(o[LocksType::state()].numberInt()) << - LocksType::lockID(o[LocksType::lockID()].OID()) ), - BSON( "$set" << BSON( LocksType::state(0) ) ) ); - - BSONObj err = conn->getLastErrorDetailed(); - string errMsg = DBClientWithCommands::getLastErrorString(err); - - // TODO: Clean up all the extra code to exit this method, probably with a refactor - if ( !errMsg.empty() || !err["n"].type() || err["n"].numberInt() < 1 ) { - ( errMsg.empty() ? LOG( logLvl - 1 ) : warning() ) << "Could not force lock '" << lockName << "' " - << ( !errMsg.empty() ? causedBy(errMsg) : string("(another force won)") ) << endl; - *other = o; other->getOwned(); conn.done(); - return false; - } - - } - catch( UpdateNotTheSame& ) { - // Ok to continue since we know we forced at least one lock document, and all lock docs - // are required for a lock to be held. - warning() << "lock forcing " << lockName << " inconsistent" << endl; - } - catch( std::exception& e ) { - conn.done(); - throw LockException( str::stream() << "exception forcing distributed lock " - << lockName << causedBy( e ), 13660); - } - - } - else { - - verify( canReenter ); - - // Lock may be re-entered, reset our timer if succeeds or fails - // Not strictly necessary, but helpful for small timeouts where thread scheduling is significant. - // This ensures that two attempts are still required for a force if not acquired, and resets our - // state if we are acquired. - resetLastPing(); - - // Test that the lock is held by trying to update the finalized state of the lock to the same state - // if it does not update or does not update on all servers, we can't re-enter. - try { - - // Test the lock with the correct "ts" (OID) value - conn->update( LocksType::ConfigNS, - BSON( LocksType::name(_name) << - LocksType::state(2) << - LocksType::lockID(o[LocksType::lockID()].OID()) ), - BSON( "$set" << BSON( LocksType::state(2) ) ) ); - - BSONObj err = conn->getLastErrorDetailed(); - string errMsg = DBClientWithCommands::getLastErrorString(err); - - // TODO: Clean up all the extra code to exit this method, probably with a refactor - if ( ! errMsg.empty() || ! err["n"].type() || err["n"].numberInt() < 1 ) { - ( errMsg.empty() ? LOG( logLvl - 1 ) : warning() ) << "Could not re-enter lock '" << lockName << "' " - << ( !errMsg.empty() ? causedBy(errMsg) : string("(not sure lock is held)") ) - << " gle: " << err - << endl; - *other = o; other->getOwned(); conn.done(); - return false; - } - - } - catch( UpdateNotTheSame& ) { - // NOT ok to continue since our lock isn't held by all servers, so isn't valid. - warning() << "inconsistent state re-entering lock, lock " << lockName << " not held" << endl; - *other = o; other->getOwned(); conn.done(); - return false; - } - catch( std::exception& e ) { - conn.done(); - throw LockException( str::stream() << "exception re-entering distributed lock " - << lockName << causedBy( e ), 13660); - } - - LOG( logLvl - 1 ) << "re-entered distributed lock '" << lockName << "'" << endl; - *other = o.getOwned(); - conn.done(); - return true; - - } - - LOG( logLvl - 1 ) << "lock '" << lockName << "' successfully forced" << endl; - - // We don't need the ts value in the query, since we will only ever replace locks with state=0. - } - // Case 3: We have an expired lock - else if ( o[LocksType::lockID()].type() ) { - queryBuilder.append( o[LocksType::lockID()] ); - } - } - - // Always reset our ping if we're trying to get a lock, since getting a lock implies the lock state is open - // and no locks need to be forced. If anything goes wrong, we don't want to remember an old lock. - resetLastPing(); - - bool gotLock = false; - BSONObj currLock; - - BSONObj lockDetails = BSON( LocksType::state(1) - << LocksType::who(getDistLockId()) - << LocksType::process(_processId) - << "when" << jsTime() - << LocksType::why(why) - << LocksType::lockID(OID::gen()) ); - BSONObj whatIWant = BSON( "$set" << lockDetails ); - - BSONObj query = queryBuilder.obj(); - - string lockName = _name + string("/") + _processId; - - try { - - // Main codepath to acquire lock - - LOG( logLvl ) << "about to acquire distributed lock '" << lockName << ":\n" - << lockDetails.jsonString(Strict, true) << "\n" - << query.jsonString(Strict, true) << endl; - - conn->update( LocksType::ConfigNS , query , whatIWant ); - - BSONObj err = conn->getLastErrorDetailed(); - string errMsg = DBClientWithCommands::getLastErrorString(err); - - currLock = conn->findOne( LocksType::ConfigNS , BSON( LocksType::name(_name) ) ); - - if ( !errMsg.empty() || !err["n"].type() || err["n"].numberInt() < 1 ) { - ( errMsg.empty() ? LOG( logLvl - 1 ) : warning() ) << "could not acquire lock '" << lockName << "' " - << ( !errMsg.empty() ? causedBy( errMsg ) : string("(another update won)") ) << endl; - *other = currLock; - other->getOwned(); - gotLock = false; - } - else { - gotLock = true; - } - - } - catch ( UpdateNotTheSame& up ) { - - // this means our update got through on some, but not others - warning() << "distributed lock '" << lockName << " did not propagate properly." << causedBy( up ) << endl; - - // Overall protection derives from: - // All unlocking updates use the ts value when setting state to 0 - // This ensures that during locking, we can override all smaller ts locks with - // our own safe ts value and not be unlocked afterward. - for ( unsigned i = 0; i < up.size(); i++ ) { - - scoped_ptr indDBPtr( - ScopedDbConnection::getInternalScopedDbConnection( up[i].first ) ); - ScopedDbConnection& indDB = *indDBPtr; - BSONObj indUpdate; - - try { - - indUpdate = indDB->findOne( LocksType::ConfigNS , BSON( LocksType::name(_name) ) ); - - // If we override this lock in any way, grab and protect it. - // We assume/ensure that if a process does not have all lock documents, it is no longer - // holding the lock. - // Note - finalized locks may compete too, but we know they've won already if competing - // in this round. Cleanup of crashes during finalizing may take a few tries. - if( indUpdate[LocksType::lockID()] < lockDetails[LocksType::lockID()] || indUpdate[LocksType::state()].numberInt() == 0 ) { - - BSONObj grabQuery = BSON( LocksType::name(_name) - << LocksType::lockID(indUpdate[LocksType::lockID()].OID()) ); - - // Change ts so we won't be forced, state so we won't be relocked - BSONObj grabChanges = BSON( LocksType::lockID(lockDetails[LocksType::lockID()].OID()) - << LocksType::state(1) ); - - // Either our update will succeed, and we'll grab the lock, or it will fail b/c some other - // process grabbed the lock (which will change the ts), but the lock will be set until forcing - indDB->update( LocksType::ConfigNS, grabQuery, BSON( "$set" << grabChanges ) ); - - indUpdate = indDB->findOne( LocksType::ConfigNS, BSON( LocksType::name(_name) ) ); - - // Our lock should now be set until forcing. - // It's possible another lock has won entirely by now, so state could be 1 or 2 here - verify( indUpdate[LocksType::state()].numberInt() > 0 ); - - } - // else our lock is the same, in which case we're safe, or it's a bigger lock, - // in which case we won't need to protect anything since we won't have the lock. - - } - catch( std::exception& e ) { - conn.done(); - throw LockException( str::stream() << "distributed lock " << lockName - << " had errors communicating with individual server " - << up[1].first << causedBy( e ), 13661 ); - } - - verify( !indUpdate.isEmpty() ); - - // Find max TS value - if ( currLock.isEmpty() || currLock[LocksType::lockID()] < indUpdate[LocksType::lockID()] ) { - currLock = indUpdate.getOwned(); - } - - indDB.done(); - - } - - // Locks on all servers are now set and safe until forcing - - if ( currLock[LocksType::lockID()] == lockDetails[LocksType::lockID()] ) { - LOG( logLvl - 1 ) << "lock update won, completing lock propagation for '" << lockName << "'" << endl; - gotLock = true; - } - else { - LOG( logLvl - 1 ) << "lock update lost, lock '" << lockName << "' not propagated." << endl; - - // Register the lock for deletion, to speed up failover - // Not strictly necessary, but helpful - distLockPinger.addUnlockOID( lockDetails[LocksType::lockID()].OID() ); - - gotLock = false; - } - } - catch( std::exception& e ) { - conn.done(); - throw LockException( str::stream() << "exception creating distributed lock " - << lockName << causedBy( e ), 13663 ); - } - - // Complete lock propagation - if( gotLock ) { - - // This is now safe, since we know that no new locks will be placed on top of the ones we've checked for at - // least 15 minutes. Sets the state = 2, so that future clients can determine that the lock is truly set. - // The invariant for rollbacks is that we will never force locks with state = 2 and active pings, since that - // indicates the lock is active, but this means the process creating/destroying them must explicitly poll - // when something goes wrong. - try { - - BSONObjBuilder finalLockDetails; - BSONObjIterator bi( lockDetails ); - while( bi.more() ) { - BSONElement el = bi.next(); - if( (string) ( el.fieldName() ) == LocksType::state() ) - finalLockDetails.append( LocksType::state(), 2 ); - else finalLockDetails.append( el ); - } - - conn->update( LocksType::ConfigNS , BSON( LocksType::name(_name) ) , BSON( "$set" << finalLockDetails.obj() ) ); - - BSONObj err = conn->getLastErrorDetailed(); - string errMsg = DBClientWithCommands::getLastErrorString(err); - - currLock = conn->findOne( LocksType::ConfigNS , BSON( LocksType::name(_name) ) ); - - if ( !errMsg.empty() || !err["n"].type() || err["n"].numberInt() < 1 ) { - warning() << "could not finalize winning lock " << lockName - << ( !errMsg.empty() ? causedBy( errMsg ) : " (did not update lock) " ) << endl; - gotLock = false; - } - else { - // SUCCESS! - gotLock = true; - } - - } - catch( std::exception& e ) { - conn.done(); - - // Register the bad final lock for deletion, in case it exists - distLockPinger.addUnlockOID( lockDetails[LocksType::lockID()].OID() ); - - throw LockException( str::stream() << "exception finalizing winning lock" - << causedBy( e ), 13662 ); - } - - } - - *other = currLock; - other->getOwned(); - - // Log our lock results - if(gotLock) - LOG( logLvl - 1 ) << "distributed lock '" << lockName << - "' acquired, ts : " << currLock[LocksType::lockID()].OID() << endl; - else - LOG( logLvl - 1 ) << "distributed lock '" << lockName << "' was not acquired." << endl; - - conn.done(); - - return gotLock; - } - - // Unlock now takes an optional pointer to the lock, so you can be specific about which - // particular lock you want to unlock. This is required when the config server is down, - // and so cannot tell you what lock ts you should try later. - void DistributedLock::unlock( BSONObj* oldLockPtr ) { - - verify( _name != "" ); - - string lockName = _name + string("/") + _processId; - - const int maxAttempts = 3; - int attempted = 0; - - BSONObj oldLock; - if( oldLockPtr ) oldLock = *oldLockPtr; - - while ( ++attempted <= maxAttempts ) { - - scoped_ptr connPtr( - ScopedDbConnection::getInternalScopedDbConnection( _conn.toString() ) ); - ScopedDbConnection& conn = *connPtr; - - try { - - if( oldLock.isEmpty() ) - oldLock = conn->findOne( LocksType::ConfigNS, BSON( LocksType::name(_name) ) ); - - if( oldLock[LocksType::state()].eoo() || - oldLock[LocksType::state()].numberInt() != 2 || - oldLock[LocksType::lockID()].eoo() ) { - warning() << "cannot unlock invalid distributed lock " << oldLock << endl; - conn.done(); - break; - } - - // Use ts when updating lock, so that new locks can be sure they won't get trampled. - conn->update( LocksType::ConfigNS , - BSON( LocksType::name(_name) << - LocksType::lockID(oldLock[LocksType::lockID()].OID()) ), - BSON( "$set" << BSON( LocksType::state(0) ) ) ); - - // Check that the lock was actually unlocked... if not, try again - BSONObj err = conn->getLastErrorDetailed(); - string errMsg = DBClientWithCommands::getLastErrorString(err); - - if ( !errMsg.empty() || !err["n"].type() || err["n"].numberInt() < 1 ){ - warning() << "distributed lock unlock update failed, retrying " - << ( errMsg.empty() ? causedBy( "( update not registered )" ) : causedBy( errMsg ) ) << endl; - conn.done(); - continue; - } - - LOG( logLvl - 1 ) << "distributed lock '" << lockName << "' unlocked. " << endl; - conn.done(); - return; - } - catch( UpdateNotTheSame& ) { - LOG( logLvl - 1 ) << "distributed lock '" << lockName << "' unlocked (messily). " << endl; - conn.done(); - break; - } - catch ( std::exception& e) { - warning() << "distributed lock '" << lockName << "' failed unlock attempt." - << causedBy( e ) << endl; - - conn.done(); - // TODO: If our lock timeout is small, sleeping this long may be unsafe. - if( attempted != maxAttempts) sleepsecs(1 << attempted); - } - } - - if( attempted > maxAttempts && ! oldLock.isEmpty() && ! oldLock[LocksType::lockID()].eoo() ) { - - LOG( logLvl - 1 ) << "could not unlock distributed lock with ts " << - oldLock[LocksType::lockID()].OID() << ", will attempt again later" << endl; - - // We couldn't unlock the lock at all, so try again later in the pinging thread... - distLockPinger.addUnlockOID( oldLock[LocksType::lockID()].OID() ); - } - else if( attempted > maxAttempts ) { - warning() << "could not unlock untracked distributed lock, a manual force may be required" << endl; - } - - warning() << "distributed lock '" << lockName << "' couldn't consummate unlock request. " - << "lock may be taken over after " << ( _lockTimeout / (60 * 1000) ) - << " minutes timeout." << endl; - } - - ScopedDistributedLock::ScopedDistributedLock(const ConnectionString& conn, const string& name) : - _lock(conn, name), _why(""), _lockTryIntervalMillis(1000), _acquired(false) - { - } - - ScopedDistributedLock::~ScopedDistributedLock() { - if (_acquired) { - unlock(); - } - } - - bool ScopedDistributedLock::tryAcquire(string* errMsg) { - try { - _acquired = _lock.lock_try(_why, false, &_other); - } - catch (const DBException& e) { - - *errMsg = str::stream() << "error acquiring distributed lock " << _lock._name << " for " - << _why << causedBy(e); - - return false; - } - - return _acquired; - } - - void ScopedDistributedLock::unlock() { - _lock.unlock(&_other); - } - - bool ScopedDistributedLock::acquire(long long waitForMillis, string* errMsg) { - - string dummy; - if (!errMsg) errMsg = &dummy; - - Timer timer; - Timer msgTimer; - - while (!_acquired && (waitForMillis <= 0 || timer.millis() < waitForMillis)) { - - string acquireErrMsg; - _acquired = tryAcquire(&acquireErrMsg); - - if (_acquired) break; - - // Set our error message to the last error, in case we break with !_acquired - *errMsg = acquireErrMsg; - - if (waitForMillis == 0) break; - - // Periodically message for debugging reasons - if (msgTimer.seconds() > 10) { - - log() << "waited " << timer.seconds() << "s for distributed lock " << _lock._name - << " for " << _why << endl; - - msgTimer.reset(); - } - - long long timeRemainingMillis = std::max(0LL, waitForMillis - timer.millis()); - sleepmillis(std::min(_lockTryIntervalMillis, timeRemainingMillis)); - } - - if (_acquired) { - verify(!_other.isEmpty()); - return true; - } - - *errMsg = str::stream() << "could not acquire distributed lock " << _lock._name << " for " - << _why << " after " << timer.seconds() - << "s, other lock may be held: " << _other << causedBy(errMsg); - - return false; - } - -} diff --git a/src/mongo/client/distlock.h b/src/mongo/client/distlock.h deleted file mode 100644 index 183826afcce..00000000000 --- a/src/mongo/client/distlock.h +++ /dev/null @@ -1,350 +0,0 @@ -// distlock.h - -/* Copyright 2009 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "mongo/pch.h" -#include "connpool.h" -#include "syncclusterconnection.h" - -#define LOCK_TIMEOUT (15 * 60 * 1000) -#define LOCK_SKEW_FACTOR (30) -#define LOCK_PING (LOCK_TIMEOUT / LOCK_SKEW_FACTOR) -#define MAX_LOCK_NET_SKEW (LOCK_TIMEOUT / LOCK_SKEW_FACTOR) -#define MAX_LOCK_CLOCK_SKEW (LOCK_TIMEOUT / LOCK_SKEW_FACTOR) -#define NUM_LOCK_SKEW_CHECKS (3) - -// The maximum clock skew we need to handle between config servers is -// 2 * MAX_LOCK_NET_SKEW + MAX_LOCK_CLOCK_SKEW. - -// Net effect of *this* clock being slow is effectively a multiplier on the max net skew -// and a linear increase or decrease of the max clock skew. - -namespace mongo { - - /** - * Exception class to encapsulate exceptions while managing distributed locks - */ - class LockException : public DBException { - public: - LockException( const char * msg , int code ) : DBException( msg, code ) {} - LockException( const string& msg, int code ) : DBException( msg, code ) {} - virtual ~LockException() throw() { } - }; - - /** - * Indicates an error in retrieving time values from remote servers. - */ - class TimeNotFoundException : public LockException { - public: - TimeNotFoundException( const char * msg , int code ) : LockException( msg, code ) {} - TimeNotFoundException( const string& msg, int code ) : LockException( msg, code ) {} - virtual ~TimeNotFoundException() throw() { } - }; - - /** - * The distributed lock is a configdb backed way of synchronizing system-wide tasks. A task must be identified by a - * unique name across the system (e.g., "balancer"). A lock is taken by writing a document in the configdb's locks - * collection with that name. - * - * To be maintained, each taken lock needs to be revalidated ("pinged") within a pre-established amount of time. This - * class does this maintenance automatically once a DistributedLock object was constructed. - */ - class DistributedLock { - public: - - static LabeledLevel logLvl; - - struct PingData { - - PingData( const string& _id , Date_t _lastPing , Date_t _remote , OID _ts ) - : id(_id), lastPing(_lastPing), remote(_remote), ts(_ts){ - } - - PingData() - : id(""), lastPing(0), remote(0), ts(){ - } - - string id; - Date_t lastPing; - Date_t remote; - OID ts; - }; - - class LastPings { - public: - LastPings() : _mutex( "DistributedLock::LastPings" ) {} - ~LastPings(){} - - PingData getLastPing( const ConnectionString& conn, const string& lockName ); - void setLastPing( const ConnectionString& conn, const string& lockName, const PingData& pd ); - - mongo::mutex _mutex; - map< std::pair, PingData > _lastPings; - }; - - static LastPings lastPings; - - /** - * The constructor does not connect to the configdb yet and constructing does not mean the lock was acquired. - * Construction does trigger a lock "pinging" mechanism, though. - * - * @param conn address of config(s) server(s) - * @param name identifier for the lock - * @param lockTimeout how long can the log go "unpinged" before a new attempt to lock steals it (in minutes). - * @param lockPing how long to wait between lock pings - * @param legacy use legacy logic - * - */ - DistributedLock( const ConnectionString& conn , const string& name , unsigned long long lockTimeout = 0, bool asProcess = false ); - ~DistributedLock(){}; - - /** - * Attempts to acquire 'this' lock, checking if it could or should be stolen from the previous holder. Please - * consider using the dist_lock_try construct to acquire this lock in an exception safe way. - * - * @param why human readable description of why the lock is being taken (used to log) - * @param whether this is a lock re-entry or a new lock - * @param other configdb's lock document that is currently holding the lock, if lock is taken, or our own lock - * details if not - * @return true if it managed to grab the lock - */ - bool lock_try( const string& why , bool reenter = false, BSONObj * other = 0, double timeout = 0.0 ); - - /** - * Returns true if we currently believe we hold this lock and it was possible to - * confirm that, within 'timeout' seconds, if provided, with the config servers. If the - * lock is not held or if we failed to contact the config servers within the timeout, - * returns false. - */ - bool isLockHeld( double timeout, string* errMsg ); - - /** - * Releases a previously taken lock. - */ - void unlock( BSONObj* oldLockPtr = NULL ); - - Date_t getRemoteTime(); - - bool isRemoteTimeSkewed(); - - const string& getProcessId(); - - const ConnectionString& getRemoteConnection(); - - /** - * Check the skew between a cluster of servers - */ - static bool checkSkew( const ConnectionString& cluster, unsigned skewChecks = NUM_LOCK_SKEW_CHECKS, unsigned long long maxClockSkew = MAX_LOCK_CLOCK_SKEW, unsigned long long maxNetSkew = MAX_LOCK_NET_SKEW ); - - /** - * Get the remote time from a server or cluster - */ - static Date_t remoteTime( const ConnectionString& cluster, unsigned long long maxNetSkew = MAX_LOCK_NET_SKEW ); - - static bool killPinger( DistributedLock& lock ); - - /** - * Namespace for lock pings - */ - static const string lockPingNS; - - /** - * Namespace for locks - */ - static const string locksNS; - - const ConnectionString _conn; - const string _name; - const string _processId; - - // Timeout for lock, usually LOCK_TIMEOUT - const unsigned long long _lockTimeout; - const unsigned long long _maxClockSkew; - const unsigned long long _maxNetSkew; - const unsigned long long _lockPing; - - private: - - void resetLastPing(){ lastPings.setLastPing( _conn, _name, PingData() ); } - void setLastPing( const PingData& pd ){ lastPings.setLastPing( _conn, _name, pd ); } - PingData getLastPing(){ return lastPings.getLastPing( _conn, _name ); } - - // May or may not exist, depending on startup - mongo::mutex _mutex; - string _threadId; - - }; - - // Helper functions for tests, allows us to turn the creation of a lock pinger on and off. - // *NOT* thread-safe - bool isLockPingerEnabled(); - void setLockPingerEnabled(bool enabled); - - - class dist_lock_try { - public: - - dist_lock_try() : _lock(NULL), _got(false) {} - - dist_lock_try( const dist_lock_try& that ) : _lock(that._lock), _got(that._got), _other(that._other) { - _other.getOwned(); - - // Make sure the lock ownership passes to this object, - // so we only unlock once. - ((dist_lock_try&) that)._got = false; - ((dist_lock_try&) that)._lock = NULL; - ((dist_lock_try&) that)._other = BSONObj(); - } - - // Needed so we can handle lock exceptions in context of lock try. - dist_lock_try& operator=( const dist_lock_try& that ){ - - if( this == &that ) return *this; - - _lock = that._lock; - _got = that._got; - _other = that._other; - _other.getOwned(); - _why = that._why; - - // Make sure the lock ownership passes to this object, - // so we only unlock once. - ((dist_lock_try&) that)._got = false; - ((dist_lock_try&) that)._lock = NULL; - ((dist_lock_try&) that)._other = BSONObj(); - - return *this; - } - - dist_lock_try( DistributedLock * lock , const std::string& why, double timeout = 0.0 ) - : _lock(lock), _why(why) { - _got = _lock->lock_try( why , false , &_other, timeout ); - } - - ~dist_lock_try() { - if ( _got ) { - verify( ! _other.isEmpty() ); - _lock->unlock( &_other ); - } - } - - /** - * Returns false if the lock is known _not_ to be held, otherwise asks the underlying - * lock to issue a 'isLockHeld' call and returns whatever that calls does. - */ - bool isLockHeld( double timeout, string* errMsg) { - if ( !_lock ) { - *errMsg = "Lock is not currently set up"; - return false; - } - - if ( !_got ) { - *errMsg = str::stream() << "Lock " << _lock->_name << " is currently held by " - << _other; - return false; - } - - return _lock->isLockHeld( timeout, errMsg ); - } - - bool got() const { return _got; } - BSONObj other() const { return _other; } - - private: - DistributedLock * _lock; - bool _got; - BSONObj _other; - string _why; - }; - - /** - * Scoped wrapper for a distributed lock acquisition attempt. One or more attempts to acquire - * the distributed lock are managed by this class, and the distributed lock is unlocked if - * successfully acquired on object destruction. - */ - class ScopedDistributedLock { - public: - - ScopedDistributedLock(const ConnectionString& conn, const string& name); - - virtual ~ScopedDistributedLock(); - - /** - * Tries once to obtain a lock, and can fail with an error message. - * - * Subclasses of this lock can override this method (and are also required to call the base - * in the overridden method). - * - * @return if the lock was successfully acquired - */ - virtual bool tryAcquire(string* errMsg); - - /** - * Tries to unlock the lock if acquired. Cannot report an error or block indefinitely - * (though it may log messages or continue retrying in a non-blocking way). - * - * Subclasses should define their own destructor unlockXXX() methods. - */ - void unlock(); - - /** - * Tries multiple times to unlock the lock, using the specified lock try interval, until - * a certain amount of time has passed. An error message is immediately returned if the - * lock acquisition attempt fails with an error message. - * waitForMillis = 0 indicates there should only be one attempt to acquire the lock, and - * no waiting. - * waitForMillis = -1 indicates we should retry indefinitely. - * @return true if the lock was acquired - */ - bool acquire(long long waitForMillis, string* errMsg); - - bool isAcquired() const { - return _acquired; - } - - ConnectionString getConfigConnectionString() const { - return _lock._conn; - } - - void setLockTryIntervalMillis(long long lockTryIntervalMillis) { - _lockTryIntervalMillis = lockTryIntervalMillis; - } - - long long getLockTryIntervalMillis() const { - return _lockTryIntervalMillis; - } - - void setLockMessage(const string& why) { - _why = why; - } - - string getLockMessage() const { - return _why; - } - - private: - DistributedLock _lock; - string _why; - long long _lockTryIntervalMillis; - - bool _acquired; - BSONObj _other; - }; - -} - diff --git a/src/mongo/client/distlock_test.cpp b/src/mongo/client/distlock_test.cpp deleted file mode 100644 index f482934769f..00000000000 --- a/src/mongo/client/distlock_test.cpp +++ /dev/null @@ -1,491 +0,0 @@ -// distlock_test.cpp - -/* Copyright 2009 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "pch.h" - -#include "distlock.h" - -#include -#include -#include - -#include "mongo/base/init.h" -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/commands.h" -#include "mongo/util/bson_util.h" -#include "mongo/util/timer.h" - -// Modify some config options for the RNG, since they cause MSVC to fail -#include - -#if defined(BOOST_MSVC) && defined(BOOST_NO_MEMBER_TEMPLATE_FRIENDS) -#undef BOOST_NO_MEMBER_TEMPLATE_FRIENDS -#define BOOST_RNG_HACK -#endif - -// Well, sort-of cross-platform RNG -#include - -#ifdef BOOST_RNG_HACK -#define BOOST_NO_MEMBER_TEMPLATE_FRIENDS -#undef BOOST_RNG_HACK -#endif - - -#include -#include - - -// TODO: Make a method in BSONObj if useful, don't modify for now -#define string_field(obj, name, def) ( obj.hasField(name) ? obj[name].String() : def ) -#define number_field(obj, name, def) ( obj.hasField(name) ? obj[name].Number() : def ) - -namespace mongo { - - class TestDistLockWithSync: public Command { - public: - TestDistLockWithSync() : - Command("_testDistLockWithSyncCluster") { - } - virtual void help(stringstream& help) const { - help << "should not be calling this directly" << endl; - } - - virtual bool slaveOk() const { - return false; - } - virtual bool adminOnly() const { - return true; - } - virtual LockType locktype() const { - return NONE; - } - // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} - static void runThread() { - while (keepGoing) { - if (current->lock_try( "test" )) { - count++; - int before = count; - sleepmillis(3); - int after = count; - - if (after != before) { - error() << " before: " << before << " after: " << after - << endl; - } - - current->unlock(); - } - } - } - - bool run(const string&, BSONObj& cmdObj, int, string& errmsg, - BSONObjBuilder& result, bool) { - Timer t; - DistributedLock lk(ConnectionString(cmdObj["host"].String(), - ConnectionString::SYNC), "testdistlockwithsync", 0, 0); - current = &lk; - count = 0; - gotit = 0; - errors = 0; - keepGoing = true; - - vector > l; - for (int i = 0; i < 4; i++) { - l.push_back( - shared_ptr (new boost::thread(runThread))); - } - - int secs = 10; - if (cmdObj["secs"].isNumber()) - secs = cmdObj["secs"].numberInt(); - sleepsecs(secs); - keepGoing = false; - - for (unsigned i = 0; i < l.size(); i++) - l[i]->join(); - - current = 0; - - result.append("count", count); - result.append("gotit", gotit); - result.append("errors", errors); - result.append("timeMS", t.millis()); - - return errors == 0; - } - - // variables for test - static DistributedLock * current; - static int gotit; - static int errors; - static AtomicUInt count; - - static bool keepGoing; - - }; - MONGO_INITIALIZER(RegisterDistLockWithSyncCmd)(InitializerContext* context) { - if (Command::testCommandsEnabled) { - // Leaked intentionally: a Command registers itself when constructed. - new TestDistLockWithSync(); - } - return Status::OK(); - } - - DistributedLock * TestDistLockWithSync::current; - AtomicUInt TestDistLockWithSync::count; - int TestDistLockWithSync::gotit; - int TestDistLockWithSync::errors; - bool TestDistLockWithSync::keepGoing; - - - - class TestDistLockWithSkew: public Command { - public: - - static const int logLvl = 1; - - TestDistLockWithSkew() : - Command("_testDistLockWithSkew") { - } - virtual void help(stringstream& help) const { - help << "should not be calling this directly" << endl; - } - - virtual bool slaveOk() const { - return false; - } - virtual bool adminOnly() const { - return true; - } - virtual LockType locktype() const { - return NONE; - } - // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} - - void runThread(ConnectionString& hostConn, unsigned threadId, unsigned seed, - BSONObj& cmdObj, BSONObjBuilder& result) { - - stringstream ss; - ss << "thread-" << threadId; - setThreadName(ss.str().c_str()); - - // Lock name - string lockName = string_field(cmdObj, "lockName", this->name + "_lock"); - - // Range of clock skew in diff threads - int skewRange = (int) number_field(cmdObj, "skewRange", 1); - - // How long to wait with the lock - int threadWait = (int) number_field(cmdObj, "threadWait", 30); - if(threadWait <= 0) threadWait = 1; - - // Max amount of time (ms) a thread waits before checking the lock again - int threadSleep = (int) number_field(cmdObj, "threadSleep", 30); - if(threadSleep <= 0) threadSleep = 1; - - // How long until the lock is forced in ms, only compared locally - unsigned long long takeoverMS = (unsigned long long) number_field(cmdObj, "takeoverMS", 0); - - // Whether or not we should hang some threads - int hangThreads = (int) number_field(cmdObj, "hangThreads", 0); - - - boost::mt19937 gen((boost::mt19937::result_type) seed); - - boost::variate_generator > randomSkew(gen, boost::uniform_int<>(0, skewRange)); - boost::variate_generator > randomWait(gen, boost::uniform_int<>(1, threadWait)); - boost::variate_generator > randomSleep(gen, boost::uniform_int<>(1, threadSleep)); - boost::variate_generator > randomNewLock(gen, boost::uniform_int<>(0, 3)); - - - int skew = 0; - if (!lock.get()) { - - // Pick a skew, but the first two threads skew the whole range - if(threadId == 0) - skew = -skewRange / 2; - else if(threadId == 1) - skew = skewRange / 2; - else skew = randomSkew() - (skewRange / 2); - - // Skew this thread - jsTimeVirtualThreadSkew( skew ); - - log() << "Initializing lock with skew of " << skew << " for thread " << threadId << endl; - - lock.reset(new DistributedLock(hostConn, lockName, takeoverMS, true )); - - log() << "Skewed time " << jsTime() << " for thread " << threadId << endl - << " max wait (with lock: " << threadWait << ", after lock: " << threadSleep << ")" << endl - << " takeover in " << takeoverMS << "(ms remote)" << endl; - - } - - DistributedLock* myLock = lock.get(); - - bool errors = false; - BSONObj lockObj; - while (keepGoing) { - try { - - if (myLock->lock_try("Testing distributed lock with skew.", false, &lockObj )) { - - log() << "**** Locked for thread " << threadId << " with ts " << lockObj["ts"] << endl; - - if( count % 2 == 1 && ! myLock->lock_try( "Testing lock re-entry.", true ) ) { - errors = true; - log() << "**** !Could not re-enter lock already held" << endl; - break; - } - - if( count % 3 == 1 && myLock->lock_try( "Testing lock non-re-entry.", false ) ) { - errors = true; - log() << "**** !Invalid lock re-entry" << endl; - break; - } - - count++; - int before = count; - int sleep = randomWait(); - sleepmillis(sleep); - int after = count; - - if(after != before) { - errors = true; - log() << "**** !Bad increment while sleeping with lock for: " << sleep << "ms" << endl; - break; - } - - // Unlock only half the time... - if(hangThreads == 0 || threadId % hangThreads != 0) { - log() << "**** Unlocking for thread " << threadId << " with ts " << lockObj["ts"] << endl; - myLock->unlock( &lockObj ); - } - else { - log() << "**** Not unlocking for thread " << threadId << endl; - verify( DistributedLock::killPinger( *myLock ) ); - // We're simulating a crashed process... - break; - } - } - - } - catch( LockException& e ) { - log() << "*** !Could not try distributed lock." << causedBy( e ) << endl; - break; - } - - // Create a new lock 1/3 of the time - if( randomNewLock() > 1 ){ - lock.reset(new DistributedLock( hostConn, lockName, takeoverMS, true )); - myLock = lock.get(); - } - - sleepmillis(randomSleep()); - } - - result << "errors" << errors - << "skew" << skew - << "takeover" << (long long) takeoverMS - << "localTimeout" << (takeoverMS > 0); - - } - - void test(ConnectionString& hostConn, string& lockName, unsigned seed) { - return; - } - - bool run(const string&, BSONObj& cmdObj, int, string& errmsg, - BSONObjBuilder& result, bool) { - - Timer t; - - ConnectionString hostConn(cmdObj["host"].String(), - ConnectionString::SYNC); - - unsigned seed = (unsigned) number_field(cmdObj, "seed", 0); - int numThreads = (int) number_field(cmdObj, "numThreads", 4); - int wait = (int) number_field(cmdObj, "wait", 10000); - - log() << "Starting " << this->name << " with -" << endl - << " seed: " << seed << endl - << " numThreads: " << numThreads << endl - << " total wait: " << wait << endl << endl; - - // Skew host clocks if needed - try { - skewClocks( hostConn, cmdObj ); - } - catch( DBException e ) { - errmsg = str::stream() << "Clocks could not be skewed." << causedBy( e ); - return false; - } - - count = 0; - keepGoing = true; - - vector > threads; - vector > results; - for (int i = 0; i < numThreads; i++) { - results.push_back(shared_ptr (new BSONObjBuilder())); - threads.push_back(shared_ptr (new boost::thread( - boost::bind(&TestDistLockWithSkew::runThread, this, - hostConn, (unsigned) i, seed + i, boost::ref(cmdObj), - boost::ref(*(results[i].get())))))); - } - - sleepsecs(wait / 1000); - keepGoing = false; - - bool errors = false; - for (unsigned i = 0; i < threads.size(); i++) { - threads[i]->join(); - errors = errors || results[i].get()->obj()["errors"].Bool(); - } - - result.append("count", count); - result.append("errors", errors); - result.append("timeMS", t.millis()); - - return !errors; - - } - - /** - * Skews the clocks of a remote cluster by a particular amount, specified by - * the "skewHosts" element in a BSONObj. - */ - static void skewClocks( ConnectionString& cluster, BSONObj& cmdObj ) { - - vector skew; - if(cmdObj.hasField("skewHosts")) { - bsonArrToNumVector(cmdObj["skewHosts"], skew); - } - else { - LOG( logLvl ) << "No host clocks to skew." << endl; - return; - } - - LOG( logLvl ) << "Skewing clocks of hosts " << cluster << endl; - - unsigned s = 0; - for(vector::iterator i = skew.begin(); i != skew.end(); ++i,s++) { - - ConnectionString server( cluster.getServers()[s] ); - scoped_ptr conn( - ScopedDbConnection::getInternalScopedDbConnection( server.toString() ) ); - - BSONObj result; - try { - bool success = conn->get()->runCommand( string("admin"), - BSON( "_skewClockCommand" << 1 - << "skew" << *i ), - result ); - - uassert(13678, str::stream() << "Could not communicate with server " << server.toString() << " in cluster " << cluster.toString() << " to change skew by " << *i, success ); - - LOG( logLvl + 1 ) << " Skewed host " << server << " clock by " << *i << endl; - } - catch(...) { - conn->done(); - throw; - } - - conn->done(); - - } - - } - - // variables for test - thread_specific_ptr lock; - AtomicUInt count; - bool keepGoing; - - }; - MONGO_INITIALIZER(RegisterDistLockWithSkewCmd)(InitializerContext* context) { - if (Command::testCommandsEnabled) { - // Leaked intentionally: a Command registers itself when constructed. - new TestDistLockWithSkew(); - } - return Status::OK(); - } - - /** - * Utility command to virtually skew the clock of a mongo server a particular amount. - * This skews the clock globally, per-thread skew is also possible. - */ - class SkewClockCommand: public Command { - public: - SkewClockCommand() : - Command("_skewClockCommand") { - } - virtual void help(stringstream& help) const { - help << "should not be calling this directly" << endl; - } - - virtual bool slaveOk() const { - return false; - } - virtual bool adminOnly() const { - return true; - } - virtual LockType locktype() const { - return NONE; - } - // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} - - bool run(const string&, BSONObj& cmdObj, int, string& errmsg, - BSONObjBuilder& result, bool) { - - long long skew = (long long) number_field(cmdObj, "skew", 0); - - log() << "Adjusting jsTime() clock skew to " << skew << endl; - - jsTimeVirtualSkew( skew ); - - log() << "JSTime adjusted, now is " << jsTime() << endl; - - return true; - - } - - }; - MONGO_INITIALIZER(RegisterSkewClockCmd)(InitializerContext* context) { - if (Command::testCommandsEnabled) { - // Leaked intentionally: a Command registers itself when constructed. - new SkewClockCommand(); - } - return Status::OK(); - } -} - diff --git a/src/mongo/client/examples/authTest.cpp b/src/mongo/client/examples/authTest.cpp index 9ecd764199f..81ce22e4502 100644 --- a/src/mongo/client/examples/authTest.cpp +++ b/src/mongo/client/examples/authTest.cpp @@ -15,6 +15,7 @@ * limitations under the License. */ +#include #include #include #include @@ -33,24 +34,51 @@ int main( int argc, const char **argv ) { port = argv[ 2 ]; } - DBClientConnection conn; + Status status = client::initialize(); + if (!status.isOK()) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + std::string errmsg; - if ( ! conn.connect( string( "127.0.0.1:" ) + port , errmsg ) ) { + ConnectionString cs = ConnectionString::parse(string("127.0.0.1:") + port, errmsg); + if (!cs.isValid()) { + cout << "error parsing url: " << errmsg << endl; + return EXIT_FAILURE; + } + + boost::scoped_ptr conn(cs.connect(errmsg)); + if (!conn) { cout << "couldn't connect: " << errmsg << endl; return EXIT_FAILURE; } + BSONObj ret; // clean up old data from any previous tests - conn.remove( "test.system.users" , BSONObj() ); + conn->runCommand( "test", BSON("removeUsersFromDatabase" << 1), ret ); - conn.insert( "test.system.users" , BSON( "user" << "eliot" << "pwd" << conn.createPasswordDigest( "eliot" , "bar" ) ) ); + conn->runCommand( "test", + BSON( "createUser" << "eliot" << + "pwd" << "bar" << + "roles" << BSON_ARRAY("readWrite")), + ret); errmsg.clear(); - bool ok = conn.auth( "test" , "eliot" , "bar" , errmsg ); - if ( ! ok ) - cout << errmsg << endl; - MONGO_verify( ok ); + conn->auth(BSON("user" << "eliot" << + "db" << "test" << + "pwd" << "bar" << + "mechanism" << "MONGODB-CR")); - MONGO_verify( ! conn.auth( "test" , "eliot" , "bars" , errmsg ) ); + try { + conn->auth(BSON("user" << "eliot" << + "db" << "test" << + "pwd" << "bars" << // incorrect password + "mechanism" << "MONGODB-CR")); + // Shouldn't get here. + cout << "Authentication with invalid password should have failed but didn't" << endl; + return EXIT_FAILURE; + } catch (const DBException& e) { + // expected + } return EXIT_SUCCESS; } diff --git a/src/mongo/client/examples/clientTest.cpp b/src/mongo/client/examples/clientTest.cpp index 8d8bd23a131..ab136b326a8 100644 --- a/src/mongo/client/examples/clientTest.cpp +++ b/src/mongo/client/examples/clientTest.cpp @@ -42,6 +42,12 @@ int main( int argc, const char **argv ) { port = argv[ 2 ]; } + Status status = client::initialize(); + if ( !status.isOK() ) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + DBClientConnection conn; string errmsg; if ( ! conn.connect( string( "127.0.0.1:" ) + port , errmsg ) ) { diff --git a/src/mongo/client/examples/first.cpp b/src/mongo/client/examples/first.cpp index 1e8616a9fb2..d5537dcad25 100644 --- a/src/mongo/client/examples/first.cpp +++ b/src/mongo/client/examples/first.cpp @@ -49,6 +49,12 @@ int main( int argc, const char **argv ) { port = argv[ 2 ]; } + mongo::Status status = mongo::client::initialize(); + if ( !status.isOK() ) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + mongo::DBClientConnection conn; string errmsg; if ( ! conn.connect( string( "127.0.0.1:" ) + port , errmsg ) ) { @@ -67,7 +73,7 @@ int main( int argc, const char **argv ) { { mongo::BSONObjBuilder query; - auto_ptr cursor = conn.query( "test.people" , query.obj() ); + std::auto_ptr cursor = conn.query( "test.people" , query.obj() ); if (!cursor.get()) { cout << "query failure" << endl; return EXIT_FAILURE; @@ -95,4 +101,5 @@ int main( int argc, const char **argv ) { cout << res.isEmpty() << "\t" << res.jsonString() << endl; } + return EXIT_SUCCESS; } diff --git a/src/mongo/client/examples/httpClientTest.cpp b/src/mongo/client/examples/httpClientTest.cpp index 43617b592de..0608f78931a 100644 --- a/src/mongo/client/examples/httpClientTest.cpp +++ b/src/mongo/client/examples/httpClientTest.cpp @@ -17,48 +17,59 @@ #include -#include "mongo/base/init.h" #include "mongo/client/dbclient.h" -#include "util/net/httpclient.h" +#include "mongo/util/net/httpclient.h" -using namespace mongo; +#ifndef verify +# define verify(x) MONGO_verify(x) +#endif + +void play( std::string url ) { + using mongo::HttpClient; -void play( string url ) { - cout << "[" << url << "]" << endl; + std::cout << "[" << url << "]" << std::endl; HttpClient c; HttpClient::Result r; - MONGO_verify( c.get( url , &r ) == 200 ); + verify( c.get( url , &r ) == 200 ); HttpClient::Headers h = r.getHeaders(); - MONGO_verify( h["Content-Type"].find( "text/html" ) == 0 ); + verify( h["Content-Type"].find( "text/html" ) == 0 ); - cout << "\tHeaders" << endl; + std::cout << "\tHeaders" << std::endl; for ( HttpClient::Headers::iterator i = h.begin() ; i != h.end(); ++i ) { - cout << "\t\t" << i->first << "\t" << i->second << endl; + std::cout << "\t\t" << i->first << "\t" << i->second << std::endl; } - + } int main( int argc, const char **argv, char **envp) { #ifdef MONGO_SSL - cmdLine.sslOnNormalPorts = true; - runGlobalInitializersOrDie(argc, argv, envp); + mongo::sslGlobalParams.sslMode.store(mongo::SSLGlobalParams::SSLMode_requireSSL); #endif + mongo::Status status = mongo::client::initialize(); + if (!status.isOK()) { + std::cout << "Failed to initialize mongodb client. " << status << std::endl; + return EXIT_FAILURE; + } + int port = 27017; if ( argc != 1 ) { - if ( argc != 3 ) - throw -12; + if ( argc != 3 ) { + std::cout << "need to pass port as second param" << std::endl; + return EXIT_FAILURE; + } port = atoi( argv[ 2 ] ); } port += 1000; - play( str::stream() << "http://localhost:" << port << "/" ); - + play( mongo::str::stream() << "http://localhost:" << port << "/" ); + #ifdef MONGO_SSL - play( "https://www.10gen.com/" ); + play( "https://www.mongodb.com/" ); #endif - + + return EXIT_SUCCESS; } diff --git a/src/mongo/client/examples/insert_demo.cpp b/src/mongo/client/examples/insert_demo.cpp index 39b463027c7..dc3f15d4b04 100644 --- a/src/mongo/client/examples/insert_demo.cpp +++ b/src/mongo/client/examples/insert_demo.cpp @@ -24,13 +24,20 @@ */ #include -#include "dbclient.h" // the mongo c++ driver +#include "mongo/client/dbclient.h" // the mongo c++ driver using namespace std; using namespace mongo; using namespace bson; int main() { + + Status status = client::initialize(); + if ( !status.isOK() ) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + try { cout << "connecting to localhost..." << endl; DBClientConnection c; @@ -55,8 +62,8 @@ int main() { } catch(DBException& e) { cout << "caught DBException " << e.toString() << endl; - return 1; + return EXIT_FAILURE; } - return 0; + return EXIT_SUCCESS; } diff --git a/src/mongo/client/examples/mongoperf.cpp b/src/mongo/client/examples/mongoperf.cpp index 4ea9eb74684..e535f187d59 100644 --- a/src/mongo/client/examples/mongoperf.cpp +++ b/src/mongo/client/examples/mongoperf.cpp @@ -25,7 +25,7 @@ // so we define the following macro #define MONGO_EXPOSE_MACROS 1 -#include "pch.h" +#include "mongo/pch.h" #include diff --git a/src/mongo/client/examples/rs.cpp b/src/mongo/client/examples/rs.cpp index 4db59f92ac5..db82509a7e8 100644 --- a/src/mongo/client/examples/rs.cpp +++ b/src/mongo/client/examples/rs.cpp @@ -79,6 +79,12 @@ int main( int argc , const char ** argv ) { } + Status status = client::initialize(); + if ( !status.isOK() ) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + string errmsg; ConnectionString cs = ConnectionString::parse( "foo/127.0.0.1" , errmsg ); if ( ! cs.isValid() ) { @@ -122,4 +128,5 @@ int main( int argc , const char ** argv ) { threads[i]->join(); } + return EXIT_SUCCESS; } diff --git a/src/mongo/client/examples/second.cpp b/src/mongo/client/examples/second.cpp index e4503957afd..87b34a482e7 100644 --- a/src/mongo/client/examples/second.cpp +++ b/src/mongo/client/examples/second.cpp @@ -33,9 +33,13 @@ int main( int argc, const char **argv ) { port = argv[ 2 ]; } - scoped_ptr connPtr( ScopedDbConnection::getScopedDbConnection( - string( "127.0.0.1:" ) + port ) ); - ScopedDbConnection& conn = *connPtr; + Status status = client::initialize(); + if ( !status.isOK() ) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + + ScopedDbConnection conn(string( "127.0.0.1:" ) + port); const char * ns = "test.second"; @@ -45,6 +49,12 @@ int main( int argc, const char **argv ) { conn->insert( ns , BSON( "name" << "sara" << "num" << 24 ) ); std::auto_ptr cursor = conn->query( ns , BSONObj() ); + + if (!cursor.get()) { + cout << "query failure" << endl; + return EXIT_FAILURE; + } + cout << "using cursor" << endl; while ( cursor->more() ) { BSONObj obj = cursor->next(); @@ -54,4 +64,6 @@ int main( int argc, const char **argv ) { conn->ensureIndex( ns , BSON( "name" << 1 << "num" << -1 ) ); conn.done(); + + return EXIT_SUCCESS; } diff --git a/src/mongo/client/examples/tail.cpp b/src/mongo/client/examples/tail.cpp index 90e62d279c1..d6d7a4f40ce 100644 --- a/src/mongo/client/examples/tail.cpp +++ b/src/mongo/client/examples/tail.cpp @@ -17,8 +17,8 @@ /* example of using a tailable cursor */ -#include "../../client/dbclient.h" -#include "../../util/goodies.h" +#include "mongo/client/dbclient.h" +#include "mongo/util/goodies.h" using namespace mongo; diff --git a/src/mongo/client/examples/tutorial.cpp b/src/mongo/client/examples/tutorial.cpp index 12c5c7e71a9..7b5e3bf10a9 100644 --- a/src/mongo/client/examples/tutorial.cpp +++ b/src/mongo/client/examples/tutorial.cpp @@ -23,15 +23,28 @@ using namespace mongo; -void printIfAge(DBClientConnection& c, int age) { - auto_ptr cursor = c.query("tutorial.persons", QUERY( "age" << age ).sort("name") ); +int printIfAge(DBClientConnection& c, int age) { + std::auto_ptr cursor = c.query("tutorial.persons", QUERY( "age" << age ).sort("name") ); + if (!cursor.get()) { + cout << "query failure" << endl; + return EXIT_FAILURE; + } + while( cursor->more() ) { BSONObj p = cursor->next(); cout << p.getStringField("name") << endl; } + return EXIT_SUCCESS; } -void run() { +int run() { + + Status status = client::initialize(); + if ( !status.isOK() ) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + DBClientConnection c; c.connect("localhost"); //"192.168.58.1"); cout << "connected ok" << endl; @@ -50,21 +63,28 @@ void run() { cout << "count:" << c.count("tutorial.persons") << endl; - auto_ptr cursor = c.query("tutorial.persons", BSONObj()); + std::auto_ptr cursor = c.query("tutorial.persons", BSONObj()); + if (!cursor.get()) { + cout << "query failure" << endl; + return EXIT_FAILURE; + } + while( cursor->more() ) { cout << cursor->next().toString() << endl; } cout << "\nprintifage:\n"; - printIfAge(c, 33); + return printIfAge(c, 33); } int main() { + int ret = EXIT_SUCCESS; try { - run(); + ret = run(); } catch( DBException &e ) { cout << "caught " << e.what() << endl; + ret = EXIT_FAILURE; } - return 0; + return ret; } diff --git a/src/mongo/client/examples/whereExample.cpp b/src/mongo/client/examples/whereExample.cpp index a2f98d7f71d..23afe8fbe0e 100644 --- a/src/mongo/client/examples/whereExample.cpp +++ b/src/mongo/client/examples/whereExample.cpp @@ -20,6 +20,10 @@ #include "mongo/client/dbclient.h" +#ifndef verify +# define verify(x) MONGO_verify(x) +#endif + using namespace std; using namespace mongo; @@ -27,16 +31,25 @@ int main( int argc, const char **argv ) { const char *port = "27017"; if ( argc != 1 ) { - if ( argc != 3 ) - throw -12; + if ( argc != 3 ) { + cout << "need to pass port as second param" << endl; + return EXIT_FAILURE; + } port = argv[ 2 ]; } + + Status status = client::initialize(); + if ( !status.isOK() ) { + std::cout << "failed to initialize the client driver: " << status.toString() << endl; + return EXIT_FAILURE; + } + DBClientConnection conn; string errmsg; if ( ! conn.connect( string( "127.0.0.1:" ) + port , errmsg ) ) { cout << "couldn't connect : " << errmsg << endl; - throw -11; + return EXIT_FAILURE; } const char * ns = "test.where"; @@ -46,7 +59,11 @@ int main( int argc, const char **argv ) { conn.insert( ns , BSON( "name" << "eliot" << "num" << 17 ) ); conn.insert( ns , BSON( "name" << "sara" << "num" << 24 ) ); - auto_ptr cursor = conn.query( ns , BSONObj() ); + std::auto_ptr cursor = conn.query( ns , BSONObj() ); + if (!cursor.get()) { + cout << "query failure" << endl; + return EXIT_FAILURE; + } while ( cursor->more() ) { BSONObj obj = cursor->next(); @@ -58,6 +75,10 @@ int main( int argc, const char **argv ) { Query q = Query("{}").where("this.name == name" , BSON( "name" << "sara" )); cursor = conn.query( ns , q ); + if (!cursor.get()) { + cout << "query failure" << endl; + return EXIT_FAILURE; + } int num = 0; while ( cursor->more() ) { @@ -65,5 +86,7 @@ int main( int argc, const char **argv ) { cout << "\t" << obj.jsonString() << endl; num++; } - MONGO_verify( num == 1 ); + verify( num == 1 ); + + return EXIT_SUCCESS; } diff --git a/src/mongo/client/export_macros.h b/src/mongo/client/export_macros.h new file mode 100644 index 00000000000..15903d0ac6d --- /dev/null +++ b/src/mongo/client/export_macros.h @@ -0,0 +1,52 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongo/platform/compiler.h" + +/** + * Definition of macros used to label the mongo client api. + * + * If a type, free function or global variable is part of the client api, it must be labeled. + * + * To label a type, place the MONGO_CLIENT_API macro after the struct, class or enum keyword. + * Example: + * class MONGO_CLIENT_API DBClientInterface { ... }; + * + * To label a function, place the label on the declaration before the return type. You + * do NOT need to label the methods of exported classes. + * Example: + * MONGO_CLIENT_API Status myFreeFunction(int arg1); + * + * To label a global variable, place the label on the declaration, before the type and + * after the "extern" keyword. + * Example: + * extern MONGO_CLIENT_API int myGlobalVariable; + * + * dbclient.h sets the LIBMONGOCLIENT_CONSUMER macro, so all clients will convert the + * MONGO_CLIENT_API macro to the the import form, while the library code will convert it to the + * export form. + */ + +#if defined(LIBMONGOCLIENT_CONSUMER) && !defined(LIBMONGOCLIENT_BUILDING) +#define MONGO_CLIENT_API MONGO_COMPILER_API_IMPORT +#elif !defined(LIBMONGOCLIENT_CONSUMER) && defined(LIBMONGOCLIENT_BUILDING) +#define MONGO_CLIENT_API MONGO_COMPILER_API_EXPORT +#elif !defined(LIBMONGOCLIENT_CONSUMER) && !defined(LIBMONGOCLIENT_BUILDING) +#define MONGO_CLIENT_API +#else +#error "Must not define both LIBMONGOCLIENT_BUILDING and LIBMONGOCLIENT_CONSUMER" +#endif diff --git a/src/mongo/client/gridfs.cpp b/src/mongo/client/gridfs.cpp index e2d1038d6ee..ef334a55019 100644 --- a/src/mongo/client/gridfs.cpp +++ b/src/mongo/client/gridfs.cpp @@ -15,29 +15,30 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" +#include "mongo/client/gridfs.h" + +#include #include #include #include #include -#include "mongo/client/gridfs.h" -#include "mongo/client/dbclientcursor.h" - #if defined(_WIN32) #include #endif +#include "mongo/client/dbclientcursor.h" + #ifndef MIN #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #endif -#include namespace mongo { - const unsigned DEFAULT_CHUNK_SIZE = 256 * 1024; + const unsigned DEFAULT_CHUNK_SIZE = 255 * 1024; GridFSChunk::GridFSChunk( BSONObj o ) { _data = o; diff --git a/src/mongo/client/gridfs.h b/src/mongo/client/gridfs.h index d6a4d418c3d..003207ccb3f 100644 --- a/src/mongo/client/gridfs.h +++ b/src/mongo/client/gridfs.h @@ -20,6 +20,7 @@ #include "mongo/bson/bsonelement.h" #include "mongo/bson/bsonobj.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -28,7 +29,7 @@ namespace mongo { class GridFS; class GridFile; - class GridFSChunk { + class MONGO_CLIENT_API GridFSChunk { public: GridFSChunk( BSONObj data ); GridFSChunk( BSONObj fileId , int chunkNumber , const char * data , int len ); @@ -53,7 +54,7 @@ namespace mongo { GridFS is for storing large file-style objects in MongoDB. @see http://dochub.mongodb.org/core/gridfsspec */ - class GridFS { + class MONGO_CLIENT_API GridFS { public: /** * @param client - db connection @@ -85,8 +86,7 @@ namespace mongo { * puts the file represented by data into the db * @param data pointer to buffer to store in GridFS * @param length length of buffer - * @param remoteName optional filename to use for file stored in GridFS - * (default is to use fileName parameter) + * @param remoteName filename to use for file stored in GridFS * @param contentType optional MIME type for this object. * (default is to omit) * @return the file object @@ -137,7 +137,7 @@ namespace mongo { /** wrapper for a file stored in the Mongo database */ - class GridFile { + class MONGO_CLIENT_API GridFile { public: /** * @return whether or not this file exists diff --git a/src/mongo/client/include_bsonh_test.cpp b/src/mongo/client/include_bsonh_test.cpp new file mode 100644 index 00000000000..f31f901b54e --- /dev/null +++ b/src/mongo/client/include_bsonh_test.cpp @@ -0,0 +1,20 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +int main() { + return 0; +} diff --git a/src/mongo/client/include_dbclienth_test.cpp b/src/mongo/client/include_dbclienth_test.cpp new file mode 100644 index 00000000000..e3bd28a600a --- /dev/null +++ b/src/mongo/client/include_dbclienth_test.cpp @@ -0,0 +1,20 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +int main() { + return 0; +} diff --git a/src/mongo/client/init.cpp b/src/mongo/client/init.cpp new file mode 100644 index 00000000000..ffdd3d5797c --- /dev/null +++ b/src/mongo/client/init.cpp @@ -0,0 +1,66 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/client/init.h" + +#include + +#include "mongo/base/initializer.h" +#include "mongo/client/connpool.h" +#include "mongo/client/replica_set_monitor.h" +#include "mongo/util/background.h" + +namespace mongo { +namespace client { + + namespace { + void callShutdownAtExit() { + // We can't really do anything of value if this returns a non-OK status. + mongo::client::shutdown(kDefaultShutdownGracePeriodMillis); + } + } // namespace + + Status initialize(bool atexit) { + + if (atexit) { + if (std::atexit(&callShutdownAtExit) != 0) { + return Status( + ErrorCodes::InternalError, + "Failed setting client driver atexit shutdown handler"); + } + } + + Status result = runGlobalInitializers(0, NULL, NULL); + if (!result.isOK()) + return result; + + // Setup default pool parameters + mongo::pool.setName("connection pool"); + mongo::pool.setMaxPoolSize(50); + + PeriodicTask::startRunningPeriodicTasks(); + + return Status::OK(); + } + + Status shutdown(int gracePeriodMillis) { + ReplicaSetMonitor::cleanup(); + return PeriodicTask::stopRunningPeriodicTasks(gracePeriodMillis); + } + +} // namespace client +} // namespace mongo diff --git a/src/mongo/client/init.h b/src/mongo/client/init.h new file mode 100644 index 00000000000..63614037511 --- /dev/null +++ b/src/mongo/client/init.h @@ -0,0 +1,55 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/client/export_macros.h" + +// NOTE: These functions are only intended to be used when linking against the libmongoclient +// library. The below functions are not defined in servers like mongos or mongod, which have +// their own initialization strategy. + +namespace mongo { +namespace client { + + const int kDefaultShutdownGracePeriodMillis = 250; + + /** + * Initializes the client driver. If the 'callShutdownAtExit' parameter is true, then + * 'initialize' schedules a call to 'client::shutdown', with a grace period of + * 'kDefaultShutdownGracePeriodMillis', via std::atexit. Failure to shutdown within the + * grace period in the 'atexit' callback leads to a call to _exit. If the + * 'callShutDownAtExit' parameter is false, then it is the responsibility of the user of + * the client driver to appropriately sequence a call to 'mongo::client::shutdown' and + * respond to any failure to terminate within the grace period. Note that 'initialize' + * invokes 'runGlobalInitializers', so it is not permitted to explicitly call + * 'runGlobalInitializers' if calling 'initialize'. If a non-OK status is returned by this + * function, the error should be reported and the client driver API must not be used. + */ + MONGO_CLIENT_API Status initialize(bool callShutdownAtExit = true); + + /** + * Terminates the client driver. If the driver does not terminate within the provided + * grace period (which defaults to kDefaultShutdownGracePeriodMillis), an + * 'ExceededTimeLimit' Status will be returned, in which case it is legal to retry + * 'shutdown'. Other non-OK status values do not admit retrying the operation, and the + * failure to terminate the driver should be reported, and it may be unsafe to exit the + * process by any mechanism which causes normal destruction of static objects. + */ + MONGO_CLIENT_API Status shutdown(int gracePeriodMillis = kDefaultShutdownGracePeriodMillis); + +} // namespace client +} // namespace mongo diff --git a/src/mongo/client/model.cpp b/src/mongo/client/model.cpp deleted file mode 100644 index 4a66625da39..00000000000 --- a/src/mongo/client/model.cpp +++ /dev/null @@ -1,141 +0,0 @@ -// model.cpp - -/* Copyright 2009 10gen - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "pch.h" -#include "model.h" -#include "connpool.h" - -namespace mongo { - - bool Model::load(BSONObj& query) { - scoped_ptr conn( - ScopedDbConnection::getScopedDbConnection (modelServer() ) ); - - BSONObj b = conn->get()->findOne(getNS(), query); - conn->done(); - - if ( b.isEmpty() ) - return false; - - unserialize(b); - _id = b["_id"].wrap().getOwned(); - return true; - } - - void Model::remove( bool safe ) { - uassert( 10016 , "_id isn't set - needed for remove()" , _id["_id"].type() ); - - scoped_ptr conn( - ScopedDbConnection::getScopedDbConnection (modelServer() ) ); - conn->get()->remove( getNS() , _id ); - - string errmsg = ""; - if ( safe ) - errmsg = conn->get()->getLastError(); - - conn->done(); - - if ( safe && errmsg.size() ) - throw UserException( 9002 , (string)"error on Model::remove: " + errmsg ); - } - - void Model::save( bool safe ) { - scoped_ptr conn( - ScopedDbConnection::getScopedDbConnection (modelServer() ) ); - - BSONObjBuilder b; - serialize( b ); - - BSONElement myId; - { - BSONObjIterator i = b.iterator(); - while ( i.more() ) { - BSONElement e = i.next(); - if ( strcmp( e.fieldName() , "_id" ) == 0 ) { - myId = e; - break; - } - } - } - - if ( myId.type() ) { - if ( _id.isEmpty() ) { - _id = myId.wrap(); - } - else if ( myId.woCompare( _id.firstElement() ) ) { - stringstream ss; - ss << "_id from serialize and stored differ: "; - ss << '[' << myId << "] != "; - ss << '[' << _id.firstElement() << ']'; - throw UserException( 13121 , ss.str() ); - } - } - - if ( _id.isEmpty() ) { - OID oid; - oid.init(); - b.appendOID( "_id" , &oid ); - - BSONObj o = b.obj(); - conn->get()->insert( getNS() , o ); - _id = o["_id"].wrap().getOwned(); - - LOG(4) << "inserted new model " << getNS() << " " << o << endl; - } - else { - if ( myId.eoo() ) { - myId = _id["_id"]; - b.append( myId ); - } - - verify( ! myId.eoo() ); - - BSONObjBuilder qb; - qb.append( myId ); - - BSONObj q = qb.obj(); - BSONObj o = b.obj(); - - LOG(4) << "updated model" << getNS() << " " << q << " " << o << endl; - - conn->get()->update( getNS() , q , o , true ); - - } - - string errmsg = ""; - if ( safe ) - errmsg = conn->get()->getLastError(); - - conn->done(); - - if ( safe && errmsg.size() ) - throw UserException( 9003 , (string)"error on Model::save: " + errmsg ); - } - - BSONObj Model::toObject() { - BSONObjBuilder b; - serialize( b ); - return b.obj(); - } - - void Model::append( const char * name , BSONObjBuilder& b ) { - BSONObjBuilder bb( b.subobjStart( name ) ); - serialize( bb ); - bb.done(); - } - -} // namespace mongo diff --git a/src/mongo/client/model.h b/src/mongo/client/model.h deleted file mode 100644 index 249a78c3640..00000000000 --- a/src/mongo/client/model.h +++ /dev/null @@ -1,61 +0,0 @@ -/** @file model.h */ - -/* Copyright 2009 10gen - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "mongo/bson/bsonelement.h" -#include "mongo/bson/bsonobj.h" - -namespace mongo { - - /** Model is a base class for defining objects which are serializable to the Mongo - database via the database driver. - - Definition - Your serializable class should inherit from Model and implement the abstract methods - below. - - Loading - To load, first construct an (empty) object. Then call load(). Do not load an object - more than once. - */ - class Model { - public: - Model() { } - virtual ~Model() { } - - virtual const char * getNS() = 0; - virtual void serialize(BSONObjBuilder& to) = 0; - virtual void unserialize(const BSONObj& from) = 0; - virtual BSONObj toObject(); - virtual void append( const char * name , BSONObjBuilder& b ); - - virtual string modelServer() = 0; - - /** Load a single object. - @return true if successful. - */ - virtual bool load(BSONObj& query); - virtual void save( bool safe=false ); - virtual void remove( bool safe=false ); - - protected: - BSONObj _id; - }; - -} // namespace mongo - diff --git a/src/mongo/client/mongo_client_lib.cpp b/src/mongo/client/mongo_client_lib.cpp deleted file mode 100644 index 5fa996b8016..00000000000 --- a/src/mongo/client/mongo_client_lib.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* @file client_lib.cpp */ - -/* Copyright 2009 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - - MongoDB C++ Driver - - Normally one includes dbclient.h, and links against libmongoclient.a, when connecting to MongoDB - from C++. However, if you have a situation where the pre-built library does not work, you can use - this file instead to build all the necessary symbols. To do so, include mongo_client_lib.cpp in your - project. - - GCC - --- - For example, to build and run simple_client_demo.cpp with GCC and run it: - - g++ -I .. simple_client_demo.cpp mongo_client_lib.cpp -lboost_thread-mt -lboost_filesystem - ./a.out - - Visual Studio (2010 tested) - --------------------------- - First, see client/examples/simple_client_demo.vcxproj. - - Be sure to include your boost include directory in your project as an Additional Include Directory. - - Define _CRT_SECURE_NO_WARNINGS to avoid warnings on use of strncpy and such by the MongoDB client code. - - Include the boost libraries directory. - - Linker.Input.Additional Dependencies - add ws2_32.lib for the Winsock library. -*/ - -#if defined(_WIN32) -// C4800 forcing value to bool 'true' or 'false' (performance warning) -#pragma warning( disable : 4800 ) -#endif - -#include "../util/md5main.cpp" - -#define MONGO_EXPOSE_MACROS - -#include "../pch.h" - -#include "../util/assert_util.cpp" -#include "../util/net/message.cpp" -#include "../util/util.cpp" -#include "../util/background.cpp" -#include "../util/base64.cpp" -#include "../util/net/sock.cpp" -#include "../util/log.cpp" -#include "../util/password.cpp" -#include "../util/net/message_port.cpp" -#include "../util/concurrency/thread_pool.cpp" -#include "../util/concurrency/task.cpp" -#include "../util/concurrency/spin_lock.cpp" - -// in client/ directory: -#include "authentication_table.cpp" -#include "connpool.cpp" -#include "syncclusterconnection.cpp" -#include "dbclient.cpp" -#include "clientOnly.cpp" -#include "gridfs.cpp" -#include "dbclientcursor.cpp" -#include "dbclient_rs.cpp" - -#include "../util/text.cpp" -#include "../bson/oid.cpp" -#include "../db/lasterror.cpp" -#include "../db/json.cpp" -#include "../db/jsobj.cpp" -#include "../db/dbmessage.cpp" - -#include "../pch.cpp" - -extern "C" { -#include "../util/md5.cpp" -} - -// in client/ directory: -#include "clientAndShell.cpp" -#include "connection_factory.cpp" - -#include "../util/time_support.cpp" -#include "../util/timer.cpp" -#include "../util/concurrency/mutexdebugger.cpp" -#include "../util/stringutils.cpp" - -/* these should probably be in clientOnly.cpp. however as a first step putting here so that there - is no risk we break the LIB build of cpp client. so this is interim. - */ -namespace mongo { - void printStackTrace(class std::basic_ostream > &) { } - void mongo_breakpoint() { } - void printStackAndExit( int signalNum ) { - ::_exit( EXIT_ABRUPT ); - } -} diff --git a/src/mongo/client/parallel.cpp b/src/mongo/client/parallel.cpp index 18649d9c6b9..8ef983abc53 100644 --- a/src/mongo/client/parallel.cpp +++ b/src/mongo/client/parallel.cpp @@ -16,12 +16,16 @@ */ -#include "pch.h" +#include "mongo/pch.h" + +#include "mongo/client/parallel.h" #include "mongo/client/connpool.h" #include "mongo/client/dbclientcursor.h" -#include "mongo/client/parallel.h" +#include "mongo/client/dbclient_rs.h" +#include "mongo/client/replica_set_monitor.h" #include "mongo/db/dbmessage.h" +#include "mongo/db/query/lite_parsed_query.h" #include "mongo/s/chunk.h" #include "mongo/s/chunk_version.h" #include "mongo/s/config.h" @@ -33,65 +37,27 @@ namespace mongo { LabeledLevel pc( "pcursor", 2 ); - // -------- ClusteredCursor ----------- - - ClusteredCursor::ClusteredCursor( const QuerySpec& q ) { - _ns = q.ns(); - _query = q.filter().copy(); - _hint = q.hint(); - _sort = q.sort(); - _options = q.options(); - _fields = q.fields().copy(); - _batchSize = q.ntoreturn(); - if ( _batchSize == 1 ) - _batchSize = 2; - - _done = false; - _didInit = false; - } - - ClusteredCursor::ClusteredCursor( QueryMessage& q ) { - _ns = q.ns; - _query = q.query.copy(); - _options = q.queryOptions; - _fields = q.fields.copy(); - _batchSize = q.ntoreturn; - if ( _batchSize == 1 ) - _batchSize = 2; - - _done = false; - _didInit = false; - } - - ClusteredCursor::ClusteredCursor( const string& ns , const BSONObj& q , int options , const BSONObj& fields ) { - _ns = ns; - _query = q.getOwned(); - _options = options; - _fields = fields.getOwned(); - _batchSize = 0; - - _done = false; - _didInit = false; - } - - ClusteredCursor::~ClusteredCursor() { - _done = true; // just in case - } - - void ClusteredCursor::init() { + void ParallelSortClusteredCursor::init() { if ( _didInit ) return; _didInit = true; - _init(); + + if( ! _qSpec.isEmpty() ) fullInit(); + else _oldInit(); } - void ClusteredCursor::_checkCursor( DBClientCursor * cursor ) { + string ParallelSortClusteredCursor::getNS() { + if( ! _qSpec.isEmpty() ) return _qSpec.ns(); + return _ns; + } + + static void _checkCursor( DBClientCursor * cursor ) { verify( cursor ); if ( cursor->hasResultFlag( ResultFlag_ShardConfigStale ) ) { BSONObj error; cursor->peekError( &error ); - throw RecvStaleConfigException( "ClusteredCursor::_checkCursor", error ); + throw RecvStaleConfigException( "_checkCursor", error ); } if ( cursor->hasResultFlag( ResultFlag_ErrSet ) ) { @@ -107,105 +73,9 @@ namespace mongo { // running with a 2.0 mongod. BSONObj res = cursor->peekFirst(); if ( res.hasField( "code" ) && res["code"].Number() == SendStaleConfigCode ) { - throw RecvStaleConfigException( "ClusteredCursor::_checkCursor", res ); - } - } - } - - auto_ptr ClusteredCursor::query( const string& server , int num , BSONObj extra , int skipLeft , bool lazy ) { - uassert( 10017 , "cursor already done" , ! _done ); - verify( _didInit ); - - BSONObj q = _query; - if ( ! extra.isEmpty() ) { - q = concatQuery( q , extra ); - } - - try { - ShardConnection conn( server , _ns ); - - if ( conn.setVersion() ) { - conn.done(); - // Deprecated, so we don't care about versions here - throw RecvStaleConfigException( _ns , "ClusteredCursor::query" , ChunkVersion( 0, OID() ), ChunkVersion( 0, OID() ), true ); - } - - LOG(5) << "ClusteredCursor::query (" << type() << ") server:" << server - << " ns:" << _ns << " query:" << q << " num:" << num - << " _fields:" << _fields << " options: " << _options << endl; - - auto_ptr cursor = - conn->query( _ns , q , num , 0 , ( _fields.isEmpty() ? 0 : &_fields ) , _options , _batchSize == 0 ? 0 : _batchSize + skipLeft ); - - if ( ! cursor.get() && _options & QueryOption_PartialResults ) { - _done = true; - conn.done(); - return cursor; - } - - massert( 13633 , str::stream() << "error querying server: " << server , cursor.get() ); - - cursor->attach( &conn ); // this calls done on conn - verify( ! conn.ok() ); - _checkCursor( cursor.get() ); - return cursor; - } - catch ( SocketException& e ) { - if ( ! ( _options & QueryOption_PartialResults ) ) - throw e; - _done = true; - return auto_ptr(); - } - } - - BSONObj ClusteredCursor::explain( const string& server , BSONObj extra ) { - BSONObj q = _query; - if ( ! extra.isEmpty() ) { - q = concatQuery( q , extra ); - } - - Query qu( q ); - qu.explain(); - if ( ! _hint.isEmpty() ) - qu.hint( _hint ); - if ( ! _sort.isEmpty() ) - qu.sort( _sort ); - - BSONObj o; - - ShardConnection conn( server , _ns ); - auto_ptr cursor = conn->query( _ns , qu , abs( _batchSize ) * -1 , 0 , _fields.isEmpty() ? 0 : &_fields ); - if ( cursor.get() && cursor->more() ) - o = cursor->next().getOwned(); - conn.done(); - return o; - } - - BSONObj ClusteredCursor::concatQuery( const BSONObj& query , const BSONObj& extraFilter ) { - if ( ! query.hasField( "query" ) ) - return _concatFilter( query , extraFilter ); - - BSONObjBuilder b; - BSONObjIterator i( query ); - while ( i.more() ) { - BSONElement e = i.next(); - - if ( strcmp( e.fieldName() , "query" ) ) { - b.append( e ); - continue; + throw RecvStaleConfigException( "_checkCursor", res ); } - - b.append( "query" , _concatFilter( e.embeddedObjectUserCheck() , extraFilter ) ); } - return b.obj(); - } - - BSONObj ClusteredCursor::_concatFilter( const BSONObj& filter , const BSONObj& extra ) { - BSONObjBuilder b; - b.appendElements( filter ); - b.appendElements( extra ); - return b.obj(); - // TODO: should do some simplification here if possibl ideally } void ParallelSortClusteredCursor::explain(BSONObjBuilder& b) { @@ -237,7 +107,7 @@ namespace mongo { double numExplains = 0; map counters; - + map > out; { _explain( out ); @@ -249,6 +119,17 @@ namespace mongo { BSONArrayBuilder y( x.subarrayStart( shard ) ); for ( list::iterator j=l.begin(); j!=l.end(); ++j ) { BSONObj temp = *j; + + // If appending the next output from the shard is going to make the BSON + // too large, then don't add it. We make sure the BSON doesn't get bigger + // than the allowable "user size" for a BSONObj. This leaves a little bit + // of extra space which mongos can use to add extra data. + if ((x.len() + temp.objsize()) > BSONObjMaxUserSize) { + y.append(BSON("warning" << + "shard output omitted due to nearing 16 MB limit")); + break; + } + y.append( temp ); BSONObjIterator k( temp ); @@ -358,7 +239,6 @@ namespace mongo { BSONObj ret = _next; _next = BSONObj(); - _advance(); return ret; } @@ -385,90 +265,35 @@ namespace mongo { _done = true; } - // -------- SerialServerClusteredCursor ----------- - - SerialServerClusteredCursor::SerialServerClusteredCursor( const set& servers , QueryMessage& q , int sortOrder) : ClusteredCursor( q ) { - for ( set::const_iterator i = servers.begin(); i!=servers.end(); i++ ) - _servers.push_back( *i ); - - if ( sortOrder > 0 ) - sort( _servers.begin() , _servers.end() ); - else if ( sortOrder < 0 ) - sort( _servers.rbegin() , _servers.rend() ); - - _serverIndex = 0; - - _needToSkip = q.ntoskip; - } - - bool SerialServerClusteredCursor::more() { - - // TODO: optimize this by sending on first query and then back counting - // tricky in case where 1st server doesn't have any after - // need it to send n skipped - while ( _needToSkip > 0 && _current.more() ) { - _current.next(); - _needToSkip--; - } - - if ( _current.more() ) - return true; - - if ( _serverIndex >= _servers.size() ) { - return false; - } - - ServerAndQuery& sq = _servers[_serverIndex++]; - - _current.reset( query( sq._server , 0 , sq._extra ) ); - return more(); - } - - BSONObj SerialServerClusteredCursor::next() { - uassert( 10018 , "no more items" , more() ); - return _current.next(); - } - - void SerialServerClusteredCursor::_explain( map< string,list >& out ) { - for ( unsigned i=0; i<_servers.size(); i++ ) { - ServerAndQuery& sq = _servers[i]; - list & l = out[sq._server]; - l.push_back( explain( sq._server , sq._extra ) ); - } - } - // -------- ParallelSortClusteredCursor ----------- - ParallelSortClusteredCursor::ParallelSortClusteredCursor( const set& servers , QueryMessage& q , - const BSONObj& sortKey ) - : ClusteredCursor( q ) , _servers( servers ) { - _sortKey = sortKey.getOwned(); - _needToSkip = q.ntoskip; + ParallelSortClusteredCursor::ParallelSortClusteredCursor( const QuerySpec& qSpec, const CommandInfo& cInfo ) + : _qSpec( qSpec ), _cInfo( cInfo ), _totalTries( 0 ) + { + _done = false; + _didInit = false; + _finishCons(); } + // LEGACY Constructor ParallelSortClusteredCursor::ParallelSortClusteredCursor( const set& servers , const string& ns , const Query& q , int options , const BSONObj& fields ) - : ClusteredCursor( ns , q.obj , options , fields ) , _servers( servers ) { + : _servers( servers ) { + _sortKey = q.getSort().copy(); _needToSkip = 0; - _finishCons(); - } - ParallelSortClusteredCursor::ParallelSortClusteredCursor( const QuerySpec& qSpec, const CommandInfo& cInfo ) - : ClusteredCursor( qSpec ), - _qSpec( qSpec ), _cInfo( cInfo ), _totalTries( 0 ) - { - _finishCons(); - } + _done = false; + _didInit = false; - ParallelSortClusteredCursor::ParallelSortClusteredCursor( const set& qShards, const QuerySpec& qSpec ) - : ClusteredCursor( qSpec ), - _qSpec( qSpec ), _totalTries( 0 ) - { - for( set::const_iterator i = qShards.begin(), end = qShards.end(); i != end; ++i ) - _qShards.insert( *i ); + // Populate legacy fields + _ns = ns; + _query = q.obj.getOwned(); + _options = options; + _fields = fields.getOwned(); + _batchSize = 0; _finishCons(); } @@ -488,11 +313,32 @@ namespace mongo { if( ! isVersioned() ) verify( _cInfo.isEmpty() ); } - if ( ! _sortKey.isEmpty() && ! _fields.isEmpty() ) { - // we need to make sure the sort key is in the projection + // Partition sort key fields into (a) text meta fields and (b) all other fields. + set textMetaSortKeyFields; + set normalSortKeyFields; + + // Transform _sortKey fields {a:{$meta:"textScore"}} into {a:-1}, in order to apply the + // merge sort for text metadata in the correct direction. + BSONObjBuilder transformedSortKeyBuilder; + + BSONObjIterator sortKeyIt( _sortKey ); + while ( sortKeyIt.more() ) { + BSONElement e = sortKeyIt.next(); + if ( LiteParsedQuery::isTextScoreMeta( e ) ) { + textMetaSortKeyFields.insert( e.fieldName() ); + transformedSortKeyBuilder.append( e.fieldName(), -1 ); + } + else { + normalSortKeyFields.insert( e.fieldName() ); + transformedSortKeyBuilder.append( e ); + } + } + _sortKey = transformedSortKeyBuilder.obj(); - set sortKeyFields; - _sortKey.getFieldNames(sortKeyFields); + // Verify that that all text metadata sort fields are in the projection. For all other sort + // fields, copy them into the projection if they are missing (and if projection is + // negative). + if ( ! _sortKey.isEmpty() && ! _fields.isEmpty() ) { BSONObjBuilder b; bool isNegative = false; @@ -504,26 +350,38 @@ namespace mongo { string fieldName = e.fieldName(); - // exact field - bool found = sortKeyFields.erase(fieldName); - - // subfields - set::const_iterator begin = sortKeyFields.lower_bound(fieldName + ".\x00"); - set::const_iterator end = sortKeyFields.lower_bound(fieldName + ".\xFF"); - sortKeyFields.erase(begin, end); - - if ( ! e.trueValue() ) { - uassert( 13431 , "have to have sort key in projection and removing it" , !found && begin == end ); + if ( LiteParsedQuery::isTextScoreMeta( e ) ) { + textMetaSortKeyFields.erase( fieldName ); } - else if (!e.isABSONObj()) { - isNegative = true; + else { + // exact field + bool found = normalSortKeyFields.erase( fieldName ); + + // subfields + set::const_iterator begin = + normalSortKeyFields.lower_bound( fieldName + ".\x00" ); + set::const_iterator end = + normalSortKeyFields.lower_bound( fieldName + ".\xFF" ); + normalSortKeyFields.erase( begin, end ); + + if ( ! e.trueValue() ) { + uassert( 13431, + "have to have sort key in projection and removing it", + !found && begin == end ); + } + else if ( !e.isABSONObj() ) { + isNegative = true; + } } } } - if (isNegative) { - for (set::const_iterator it(sortKeyFields.begin()), end(sortKeyFields.end()); it != end; ++it) { - b.append(*it, 1); + if ( isNegative ) { + for ( set::const_iterator it( normalSortKeyFields.begin() ), + end( normalSortKeyFields.end() ); + it != end; + ++it ) { + b.append( *it, 1 ); } } @@ -533,6 +391,10 @@ namespace mongo { if( ! _qSpec.isEmpty() ){ _qSpec.setFields( _fields ); } + + uassert( 17306, + "have to have all text meta sort keys in projection", + textMetaSortKeyFields.empty() ); } void ParallelConnectionMetadata::cleanup( bool full ){ @@ -540,14 +402,7 @@ namespace mongo { if( full || errored ) retryNext = false; if( ! retryNext && pcState ){ - - if( errored && pcState->conn ){ - // Don't return this conn to the pool if it's bad - pcState->conn->kill(); - pcState->conn.reset(); - } - else if( initialized ){ - + if (initialized && !errored) { verify( pcState->cursor ); verify( pcState->conn ); @@ -673,7 +528,7 @@ namespace mongo { void ParallelSortClusteredCursor::_handleStaleNS( const NamespaceString& staleNS, bool forceReload, bool fullReload ){ - DBConfigPtr config = grid.getDBConfig( staleNS.db ); + DBConfigPtr config = grid.getDBConfig( staleNS.db() ); // Reload db if needed, make sure it works if( config && fullReload && ! config->reload() ){ @@ -715,26 +570,40 @@ namespace mongo { } const DBClientBase* rawConn = state->conn->getRawConn(); - if (( _options & QueryOption_SlaveOk ) && - rawConn->type() == ConnectionString::SET && - rawConn->isFailed() ) { - /* A side effect of this short circuiting is this will not be - * able figure out that the primary is now up on it's own and - * has to rely on other threads to refresh the node states. - */ + bool allowShardVersionFailure = + rawConn->type() == ConnectionString::SET && + DBClientReplicaSet::isSecondaryQuery( _qSpec.ns(), _qSpec.query(), _qSpec.options() ); + bool connIsDown = rawConn->isFailed(); + if (allowShardVersionFailure && !connIsDown) { + // If the replica set connection believes that it has a valid primary that is up, + // confirm that the replica set monitor agrees that the suspected primary is indeed up. + const DBClientReplicaSet* replConn = dynamic_cast(rawConn); + ReplicaSetMonitorPtr rsMonitor = ReplicaSetMonitor::get(replConn->getSetName()); + if (!rsMonitor->isHostUp(replConn->getSuspectedPrimaryHostAndPort())) { + connIsDown = true; + } + } + + if (allowShardVersionFailure && connIsDown) { + // If we're doing a secondary-allowed query and the primary is down, don't attempt to + // set the shard version. + + state->conn->donotCheckVersion(); + + // A side effect of this short circuiting is the mongos will not be able figure out that + // the primary is now up on it's own and has to rely on other threads to refresh node + // states. OCCASIONALLY { - const DBClientReplicaSet* repl = - dynamic_cast( rawConn ); + const DBClientReplicaSet* repl = dynamic_cast( rawConn ); + dassert(repl); warning() << "Primary for " << repl->getServerAddress() << " was down before, bypassing setShardVersion." - << " Local config view can be stale." << endl; + << " The local replica set view and targeting may be stale." << endl; } - } else { + } + else { try { - /* TODO: Undo SERVER-5797. This try-catch is a temporary hack until - * secondaries can properly handle shard versioning - */ if ( state->conn->setVersion() ) { // It's actually okay if we set the version here, since either the // manager will be verified as compatible, or if the manager doesn't @@ -742,19 +611,20 @@ namespace mongo { LOG( pc ) << "needed to set remote version on connection to value " << "compatible with " << vinfo << endl; } - } catch ( const DBException& dbEx ) { - if ( (dbEx.getCode() == 10009 /* no master */ && - ( _options & QueryOption_SlaveOk )) ) { + } + catch ( const DBException& ) { + if ( allowShardVersionFailure ) { + + // It's okay if we don't set the version when talking to a secondary, we can + // be stale in any case. OCCASIONALLY { const DBClientReplicaSet* repl = - dynamic_cast( - state->conn->getRawConn() ); - - warning() << "Cannot contact primary for " - << repl->getServerAddress() - << " to check shard version. " - << "SlaveOk query can be sent to the wrong shard." + dynamic_cast( state->conn->getRawConn() ); + dassert(repl); + warning() << "Cannot contact primary for " << repl->getServerAddress() + << " to check shard version." + << " The local replica set view and targeting may be stale." << endl; } } @@ -767,16 +637,14 @@ namespace mongo { void ParallelSortClusteredCursor::startInit() { - bool returnPartial = ( _qSpec.options() & QueryOption_PartialResults ); - bool specialVersion = _cInfo.versionedNS.size() > 0; - bool specialFilter = ! _cInfo.cmdFilter.isEmpty(); - NamespaceString ns = specialVersion ? _cInfo.versionedNS : _qSpec.ns(); + const bool returnPartial = ( _qSpec.options() & QueryOption_PartialResults ); + NamespaceString ns( !_cInfo.isEmpty() ? _cInfo.versionedNS : _qSpec.ns() ); ChunkManagerPtr manager; ShardPtr primary; string prefix; - if (MONGO_unlikely(logLevel >= pc)) { + if (MONGO_unlikely(logger::globalLogDomain()->shouldLog(pc))) { if( _totalTries > 0 ) { prefix = str::stream() << "retrying (" << _totalTries << " tries)"; } @@ -792,13 +660,13 @@ namespace mongo { if( isVersioned() ){ - DBConfigPtr config = grid.getDBConfig( ns.db ); // Gets or loads the config + DBConfigPtr config = grid.getDBConfig( ns.db() ); // Gets or loads the config uassert( 15989, "database not found for parallel cursor request", config ); // Try to get either the chunk manager or the primary shard config->getChunkManagerOrPrimary( ns, manager, primary ); - if (MONGO_unlikely(logLevel >= pc)) { + if (MONGO_unlikely(logger::globalLogDomain()->shouldLog(pc))) { if (manager) { vinfo = str::stream() << "[" << manager->getns() << " @ " << manager->getVersion().toString() << "]"; @@ -809,7 +677,7 @@ namespace mongo { } } - if( manager ) manager->getShardsForQuery( todo, specialFilter ? _cInfo.cmdFilter : _qSpec.filter() ); + if( manager ) manager->getShardsForQuery( todo, !_cInfo.isEmpty() ? _cInfo.cmdFilter : _qSpec.filter() ); else if( primary ) todo.insert( *primary ); // Close all cursors on extra shards first, as these will be invalid @@ -826,7 +694,7 @@ namespace mongo { // Don't use version to get shards here todo = _qShards; - if (MONGO_unlikely(logLevel >= pc)) { + if (MONGO_unlikely(logger::globalLogDomain()->shouldLog(pc))) { vinfo = str::stream() << "[" << _qShards.size() << " shards specified]"; } } @@ -978,7 +846,7 @@ namespace mongo { catch( StaleConfigException& e ){ // Our version isn't compatible with the current version anymore on at least one shard, need to retry immediately - NamespaceString staleNS = e.getns(); + NamespaceString staleNS( e.getns() ); // For legacy reasons, this may not be set in the exception :-( if( staleNS.size() == 0 ) staleNS = ns; // ns is the *versioned* namespace, be careful of this @@ -1195,7 +1063,7 @@ namespace mongo { if( staleNSExceptions.size() ){ for( map::iterator i = staleNSExceptions.begin(), end = staleNSExceptions.end(); i != end; ++i ){ - const string& staleNS = i->first; + NamespaceString staleNS( i->first ); const StaleConfigException& exception = i->second; bool forceReload, fullReload; @@ -1306,16 +1174,34 @@ namespace mongo { else return i->second.pcState->cursor; } - void ParallelSortClusteredCursor::_init() { - if( ! _qSpec.isEmpty() ) fullInit(); - else _oldInit(); + static BSONObj _concatFilter( const BSONObj& filter , const BSONObj& extra ) { + BSONObjBuilder b; + b.appendElements( filter ); + b.appendElements( extra ); + return b.obj(); + // TODO: should do some simplification here if possibl ideally } + static BSONObj concatQuery( const BSONObj& query , const BSONObj& extraFilter ) { + if ( ! query.hasField( "query" ) ) + return _concatFilter( query , extraFilter ); + + BSONObjBuilder b; + BSONObjIterator i( query ); + while ( i.more() ) { + BSONElement e = i.next(); - // DEPRECATED + if ( strcmp( e.fieldName() , "query" ) ) { + b.append( e ); + continue; + } + b.append( "query" , _concatFilter( e.embeddedObjectUserCheck() , extraFilter ) ); + } + return b.obj(); + } - // TODO: Merge with futures API? We do a lot of error checking here that would be useful elsewhere. + // DEPRECATED void ParallelSortClusteredCursor::_oldInit() { // log() << "Starting parallel search..." << endl; @@ -1579,6 +1465,15 @@ namespace mongo { // Clear out our metadata after removing legacy cursor data _cursorMap.clear(); + + // Just to be sure + _done = true; + } + + void ParallelSortClusteredCursor::setBatchSize(int newBatchSize) { + for ( int i=0; i<_numServers; i++ ) { + _cursors[i].setBatchSize(newBatchSize); + } } bool ParallelSortClusteredCursor::more() { @@ -1673,7 +1568,7 @@ namespace mongo { void Future::CommandResult::init(){ try { if ( ! _conn ){ - _connHolder.reset( ScopedDbConnection::getScopedDbConnection( _server ) ); + _connHolder.reset( new ScopedDbConnection( _server ) ); _conn = _connHolder->get(); } diff --git a/src/mongo/client/parallel.h b/src/mongo/client/parallel.h index bae89c36cc6..a596bddcf7e 100644 --- a/src/mongo/client/parallel.h +++ b/src/mongo/client/parallel.h @@ -21,9 +21,10 @@ #pragma once +#include "mongo/client/export_macros.h" #include "mongo/db/dbmessage.h" #include "mongo/db/matcher.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" #include "mongo/s/shard.h" #include "mongo/s/stale_exception.h" // for StaleConfigException #include "mongo/util/concurrency/mvar.h" @@ -33,7 +34,7 @@ namespace mongo { /** * holder for a server address and a query to run */ - class ServerAndQuery { + class MONGO_CLIENT_API ServerAndQuery { public: ServerAndQuery( const string& server , BSONObj extra = BSONObj() , BSONObj orderObject = BSONObj() ) : _server( server ) , _extra( extra.getOwned() ) , _orderObject( orderObject.getOwned() ) { @@ -65,182 +66,10 @@ namespace mongo { BSONObj _orderObject; }; - /** - * this is a cursor that works over a set of servers - * can be used in serial/parallel as controlled by sub classes - */ - class ClusteredCursor { - public: - ClusteredCursor( const QuerySpec& q ); - ClusteredCursor( QueryMessage& q ); - ClusteredCursor( const string& ns , const BSONObj& q , int options=0 , const BSONObj& fields=BSONObj() ); - virtual ~ClusteredCursor(); - - /** call before using */ - void init(); - - virtual std::string getNS() { return _ns; } - - virtual bool more() = 0; - virtual BSONObj next() = 0; - - static BSONObj concatQuery( const BSONObj& query , const BSONObj& extraFilter ); - - virtual string type() const = 0; - - virtual void explain(BSONObjBuilder& b) = 0; - - protected: - - virtual void _init() = 0; - - auto_ptr query( const string& server , int num = 0 , BSONObj extraFilter = BSONObj() , int skipLeft = 0 , bool lazy=false ); - BSONObj explain( const string& server , BSONObj extraFilter = BSONObj() ); - - /** - * checks the cursor for any errors - * will throw an exceptionif an error is encountered - */ - void _checkCursor( DBClientCursor * cursor ); - - static BSONObj _concatFilter( const BSONObj& filter , const BSONObj& extraFilter ); - - virtual void _explain( map< string,list >& out ) = 0; - - string _ns; - BSONObj _query; - BSONObj _hint; - BSONObj _sort; - - int _options; - BSONObj _fields; - int _batchSize; - - bool _didInit; - - bool _done; - }; - class ParallelConnectionMetadata; + class FilteringClientCursor; - // TODO: We probably don't really need this as a separate class. - class FilteringClientCursor { - public: - FilteringClientCursor( const BSONObj filter = BSONObj() ); - FilteringClientCursor( DBClientCursor* cursor , const BSONObj filter = BSONObj() ); - FilteringClientCursor( auto_ptr cursor , const BSONObj filter = BSONObj() ); - ~FilteringClientCursor(); - - void reset( auto_ptr cursor ); - void reset( DBClientCursor* cursor, ParallelConnectionMetadata* _pcmData = NULL ); - - bool more(); - BSONObj next(); - - BSONObj peek(); - - DBClientCursor* raw() { return _cursor.get(); } - ParallelConnectionMetadata* rawMData(){ return _pcmData; } - - // Required for new PCursor - void release(){ - _cursor.release(); - _pcmData = NULL; - } - - private: - void _advance(); - - Matcher _matcher; - auto_ptr _cursor; - ParallelConnectionMetadata* _pcmData; - - BSONObj _next; - bool _done; - }; - - - class Servers { - public: - Servers() { - } - - void add( const ServerAndQuery& s ) { - add( s._server , s._extra ); - } - - void add( const string& server , const BSONObj& filter ) { - vector& mine = _filters[server]; - mine.push_back( filter.getOwned() ); - } - - // TOOO: pick a less horrible name - class View { - View( const Servers* s ) { - for ( map >::const_iterator i=s->_filters.begin(); i!=s->_filters.end(); ++i ) { - _servers.push_back( i->first ); - _filters.push_back( i->second ); - } - } - public: - int size() const { - return _servers.size(); - } - - string getServer( int n ) const { - return _servers[n]; - } - - vector getFilter( int n ) const { - return _filters[ n ]; - } - - private: - vector _servers; - vector< vector > _filters; - - friend class Servers; - }; - - View view() const { - return View( this ); - } - - - private: - map > _filters; - - friend class View; - }; - - - /** - * runs a query in serial across any number of servers - * returns all results from 1 server, then the next, etc... - */ - class SerialServerClusteredCursor : public ClusteredCursor { - public: - SerialServerClusteredCursor( const set& servers , QueryMessage& q , int sortOrder=0); - virtual bool more(); - virtual BSONObj next(); - virtual string type() const { return "SerialServer"; } - - protected: - virtual void _explain( map< string,list >& out ); - - void _init() {} - - vector _servers; - unsigned _serverIndex; - - FilteringClientCursor _current; - - int _needToSkip; - }; - - - - class CommandInfo { + class MONGO_CLIENT_API CommandInfo { public: string versionedNS; BSONObj cmdFilter; @@ -262,12 +91,14 @@ namespace mongo { class DBClientCursor; typedef shared_ptr DBClientCursorPtr; - class ParallelConnectionState { + class MONGO_CLIENT_API ParallelConnectionState { public: ParallelConnectionState() : count( 0 ), done( false ) { } + // Please do not reorder. cursor destructor can use conn. + // On a related note, never attempt to cleanup these pointers manually. ShardConnectionPtr conn; DBClientCursorPtr cursor; @@ -289,7 +120,7 @@ namespace mongo { typedef ParallelConnectionState PCState; typedef shared_ptr PCStatePtr; - class ParallelConnectionMetadata { + class MONGO_CLIENT_API ParallelConnectionMetadata { public: ParallelConnectionMetadata() : @@ -322,27 +153,36 @@ namespace mongo { typedef shared_ptr PCMDataPtr; /** - * Runs a query in parallel across N servers. New logic has several modes - - * 1) Standard query, enforces compatible chunk versions for queries across all results - * 2) Standard query, sent to particular servers with no compatible chunk version enforced, but handling - * stale configuration exceptions - * 3) Command query, either enforcing compatible chunk versions or sent to particular shards. + * Runs a query in parallel across N servers, enforcing compatible chunk versions for queries + * across all shards. + * + * If CommandInfo is provided, the ParallelCursor does not use the direct .$cmd namespace in the + * query spec, but instead enforces versions across another namespace specified by CommandInfo. + * This is to support commands like: + * db.runCommand({ fileMD5 : "" }) + * + * There is a deprecated legacy mode as well which effectively does a merge-sort across a number + * of servers, but does not correctly enforce versioning (used only in mapreduce). */ - class ParallelSortClusteredCursor : public ClusteredCursor { + class MONGO_CLIENT_API ParallelSortClusteredCursor { public: ParallelSortClusteredCursor( const QuerySpec& qSpec, const CommandInfo& cInfo = CommandInfo() ); - ParallelSortClusteredCursor( const set& servers, const QuerySpec& qSpec ); - // LEGACY Constructors - ParallelSortClusteredCursor( const set& servers , QueryMessage& q , const BSONObj& sortKey ); + // DEPRECATED legacy constructor for pure mergesort functionality - do not use ParallelSortClusteredCursor( const set& servers , const string& ns , const Query& q , int options=0, const BSONObj& fields=BSONObj() ); - virtual ~ParallelSortClusteredCursor(); - virtual bool more(); - virtual BSONObj next(); - virtual string type() const { return "ParallelSort"; } + ~ParallelSortClusteredCursor(); + + std::string getNS(); + + /** call before using */ + void init(); + + bool more(); + BSONObj next(); + string type() const { return "ParallelSort"; } void fullInit(); void startInit(); @@ -352,6 +192,11 @@ namespace mongo { bool isExplain(){ return _qSpec.isExplain(); } bool isVersioned(){ return _qShards.size() == 0; } + /** + * Sets the batch size on all underlying cursors to 'newBatchSize'. + */ + void setBatchSize(int newBatchSize); + bool isSharded(); ShardPtr getPrimary(); void getQueryShards( set& shards ); @@ -361,18 +206,19 @@ namespace mongo { BSONObj toBSON() const; string toString() const; - virtual void explain(BSONObjBuilder& b); + void explain(BSONObjBuilder& b); - protected: + private: void _finishCons(); - void _init(); - void _oldInit(); - virtual void _explain( map< string,list >& out ); + void _explain( map< string,list >& out ); void _markStaleNS( const NamespaceString& staleNS, const StaleConfigException& e, bool& forceReload, bool& fullReload ); void _handleStaleNS( const NamespaceString& staleNS, bool forceReload, bool fullReload ); + bool _didInit; + bool _done; + set _qShards; QuerySpec _qSpec; CommandInfo _cInfo; @@ -392,7 +238,6 @@ namespace mongo { FilteringClientCursor * _cursors; int _needToSkip; - private: /** * Setups the shard version of the connection. When using a replica * set connection and the primary cannot be reached, the version @@ -404,14 +249,66 @@ namespace mongo { const NamespaceString& ns, const std::string& vinfo, ChunkManagerPtr manager /* in */ ); + + // LEGACY init - Needed for map reduce + void _oldInit(); + + // LEGACY - Needed ONLY for _oldInit + string _ns; + BSONObj _query; + int _options; + BSONObj _fields; + int _batchSize; + }; + + + // TODO: We probably don't really need this as a separate class. + class MONGO_CLIENT_API FilteringClientCursor { + public: + FilteringClientCursor( const BSONObj filter = BSONObj() ); + FilteringClientCursor( DBClientCursor* cursor , const BSONObj filter = BSONObj() ); + FilteringClientCursor( auto_ptr cursor , const BSONObj filter = BSONObj() ); + ~FilteringClientCursor(); + + void reset( auto_ptr cursor ); + void reset( DBClientCursor* cursor, ParallelConnectionMetadata* _pcmData = NULL ); + + bool more(); + BSONObj next(); + + BSONObj peek(); + + void setBatchSize(int newBatchSize) { _cursor->setBatchSize(newBatchSize); } + + DBClientCursor* raw() { return _cursor.get(); } + ParallelConnectionMetadata* rawMData(){ return _pcmData; } + + // Required for new PCursor + void release(){ + _cursor.release(); + _pcmData = NULL; + } + + private: + void _advance(); + + Matcher _matcher; + auto_ptr _cursor; + ParallelConnectionMetadata* _pcmData; + + BSONObj _next; + bool _done; }; /** + * Generally clients should be using Strategy::commandOp() wherever possible - the Future API + * does not handle versioning. + * * tools for doing asynchronous operations * right now uses underlying sync network ops and uses another thread * should be changed to use non-blocking io */ - class Future { + class MONGO_CLIENT_API Future { public: class CommandResult { public: @@ -457,7 +354,7 @@ namespace mongo { friend class Future; }; - + /** * @param server server name * @param db db name diff --git a/src/mongo/client/redef_macros.h b/src/mongo/client/redef_macros.h index 90a97b3933b..90e58f9a5f1 100644 --- a/src/mongo/client/redef_macros.h +++ b/src/mongo/client/redef_macros.h @@ -36,6 +36,9 @@ #pragma push_macro("verify") #undef verify #define verify MONGO_verify +#pragma push_macro("invariant") +#undef invariant +#define invariant MONGO_invariant #pragma push_macro("dassert") #undef dassert #define dassert MONGO_dassert @@ -48,9 +51,6 @@ #pragma push_macro("uassert") #undef uassert #define uassert MONGO_uassert -#pragma push_macro("uassertStatusOK") -#undef uassertStatusOK -#define uassertStatusOK MONGO_uassertStatusOK #pragma push_macro("DESTRUCTOR_GUARD") #undef DESTRUCTOR_GUARD #define DESTRUCTOR_GUARD MONGO_DESTRUCTOR_GUARD diff --git a/src/mongo/client/replica_set_monitor.cpp b/src/mongo/client/replica_set_monitor.cpp new file mode 100644 index 00000000000..73e39cebd0f --- /dev/null +++ b/src/mongo/client/replica_set_monitor.cpp @@ -0,0 +1,1092 @@ +/* Copyright 2014 MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/client/replica_set_monitor.h" + +#include +#include +#include + +#include "mongo/client/connpool.h" +#include "mongo/client/replica_set_monitor_internal.h" +#include "mongo/util/concurrency/mutex.h" // for StaticObserver +#include "mongo/util/background.h" +#include "mongo/util/string_map.h" +#include "mongo/util/timer.h" + +#if 0 // enable this to ease debugging of this file. +#undef DEV +#define DEV if (true) + +#undef LOG +#define LOG(x) log() +#endif + +namespace mongo { +namespace { + // Pull nested types to top-level scope + typedef ReplicaSetMonitor::IsMasterReply IsMasterReply; + typedef ReplicaSetMonitor::ScanState ScanState; + typedef ReplicaSetMonitor::ScanStatePtr ScanStatePtr; + typedef ReplicaSetMonitor::SetState SetState; + typedef ReplicaSetMonitor::SetStatePtr SetStatePtr; + typedef ReplicaSetMonitor::Refresher Refresher; + typedef Refresher::NextStep NextStep; + typedef ScanState::UnconfirmedReplies UnconfirmedReplies; + typedef SetState::Node Node; + typedef SetState::Nodes Nodes; + + const double socketTimeoutSecs = 5; + + /* Replica Set Monitor shared state: + * If a program (such as one built with the C++ driver) exits (by either calling exit() + * or by returning from main()), static objects will be destroyed in the reverse order + * of their creation (within each translation unit (source code file)). This makes it + * vital that the order be explicitly controlled within the source file so that destroyed + * objects never reference objects that have been destroyed earlier. + * + * The order chosen below is intended to allow safe destruction in reverse order from + * construction order: + * setsLock -- mutex protecting _seedServers and _sets, destroyed last + * seedServers -- list (map) of servers + * sets -- list (map) of ReplicaSetMonitors + * replicaSetMonitorWatcher -- background job to check Replica Set members + * staticObserver -- sentinel to detect process termination + * + * Related to: + * SERVER-8891 -- Simple client fail with segmentation fault in mongoclient library + * + * Mutex locking order: + * Don't lock setsLock while holding any SetState::mutex. It is however safe to grab a + * SetState::mutex without holder setsLock, but then you can't grab setsLock until you + * release the SetState::mutex. + */ + mongo::mutex setsLock("ReplicaSetMonitor"); + StringMap > seedServers; + StringMap sets; + + // global background job responsible for checking every X amount of time + class ReplicaSetMonitorWatcher : public BackgroundJob { + public: + ReplicaSetMonitorWatcher(): + _monitorMutex("ReplicaSetMonitorWatcher::_safego"), + _started(false), + _stopRequested(false) { + } + + ~ReplicaSetMonitorWatcher() { + stop(); + + // We relying on the fact that if the monitor was rerun again, wait will not hang + // because _destroyingStatics will make the run method exit immediately. + dassert(StaticObserver::_destroyingStatics); + if (running()) { + wait(); + } + } + + virtual string name() const { return "ReplicaSetMonitorWatcher"; } + + void safeGo() { + scoped_lock lk( _monitorMutex ); + if ( _started ) + return; + + _started = true; + _stopRequested = false; + + go(); + } + + /** + * Stops monitoring the sets and wait for the monitoring thread to terminate. + */ + void stop() { + scoped_lock sl( _monitorMutex ); + _stopRequested = true; + _stopRequestedCV.notify_one(); + } + + protected: + void run() { + log() << "starting"; // includes thread name in output + + // Added only for patching timing problems in test. Remove after tests + // are fixed - see 392b933598668768bf12b1e41ad444aa3548d970. + // Should not be needed after SERVER-7533 gets implemented and tests start + // using it. + if (!inShutdown() && !StaticObserver::_destroyingStatics) { + scoped_lock sl( _monitorMutex ); + _stopRequestedCV.timed_wait(sl.boost(), boost::posix_time::seconds(10)); + } + + while ( !inShutdown() && + !StaticObserver::_destroyingStatics ) { + { + scoped_lock sl( _monitorMutex ); + if (_stopRequested) { + break; + } + } + + try { + checkAllSets(); + } + catch ( std::exception& e ) { + error() << "check failed: " << e.what(); + } + catch ( ... ) { + error() << "unknown error"; + } + + scoped_lock sl( _monitorMutex ); + if (_stopRequested) { + break; + } + + _stopRequestedCV.timed_wait(sl.boost(), boost::posix_time::seconds(10)); + } + } + + void checkAllSets() { + // make a copy so we can quickly unlock setsLock + StringMap setsCopy; + { + scoped_lock lk( setsLock ); + setsCopy = sets; + } + + for (StringMap::const_iterator it = setsCopy.begin(); + it != setsCopy.end(); ++it) { + LOG(1) << "checking replica set: " << it->first; + ReplicaSetMonitorPtr m = it->second; + + m->startOrContinueRefresh().refreshAll(); + + const int numFails = m->getConsecutiveFailedScans(); + if (numFails >= ReplicaSetMonitor::maxConsecutiveFailedChecks) { + log() << "Replica set " << m->getName() << " was down for " << numFails + << " checks in a row. Stopping polled monitoring of the set."; + + // locks setsLock + ReplicaSetMonitor::remove(m->getName(), false); + } + } + } + + // protects _started, _stopRequested + mongo::mutex _monitorMutex; + bool _started; + + boost::condition _stopRequestedCV; + bool _stopRequested; + } replicaSetMonitorWatcher; + + StaticObserver staticObserver; + + // + // Helpers for stl algorithms + // + + bool isMaster(const Node& node) { return node.isMaster; } + bool compareLatencies(const Node* lhs, const Node* rhs) { + // NOTE: this automatically compares Node::unknownLatency worse than all others. + return lhs->latencyMicros < rhs->latencyMicros; + } + + bool hostsEqual(const Node& lhs, const HostAndPort& rhs) { return lhs.host == rhs; } + + // Allows comparing two Nodes, or a HostAndPort and a Node. + // NOTE: the two HostAndPort overload is only needed to support extra checks in some STL + // implementations. For simplicity, no comparator should be used with collections of just + // HostAndPort. + struct CompareHosts { + bool operator() (const Node& lhs, const Node& rhs) { return lhs.host < rhs.host; } + bool operator() (const Node& lhs, const HostAndPort& rhs) { return lhs.host < rhs; } + bool operator() (const HostAndPort& lhs, const Node& rhs) { return lhs < rhs.host; } + bool operator() (const HostAndPort& lhs, const HostAndPort& rhs) { return lhs < rhs; } + } compareHosts; // like an overloaded function, but able to pass to stl algorithms + + // The following structs should be treated as functions returning a UnaryPredicate. + // Usage example: std::find_if(nodes.begin(), nodes.end(), hostIs(someHost)); + // They all hold their constructor argument by reference. + + struct hostIs { + explicit hostIs(const HostAndPort& host) :_host(host) {} + bool operator() (const HostAndPort& host) { return host == _host; } + bool operator() (const Node& node) { return node.host == _host; } + const HostAndPort& _host; + }; + + struct hostNotIn { + explicit hostNotIn(const std::set& hosts) :_hosts(hosts) {} + bool operator() (const HostAndPort& host) { return !_hosts.count(host); } + bool operator() (const Node& node) { return !_hosts.count(node.host); } + const std::set& _hosts; + }; +} // namespace + + // At 1 check every 10 seconds, 30 checks takes 5 minutes + int ReplicaSetMonitor::maxConsecutiveFailedChecks = 30; + + // Defaults to random selection as required by the spec + bool ReplicaSetMonitor::useDeterministicHostSelection = false; + + ReplicaSetMonitor::ReplicaSetMonitor(StringData name, const std::set& seeds) + : _state(boost::make_shared(name, seeds)) { + LogstreamBuilder lsb = log(); + lsb << "starting new replica set monitor for replica set " << name << " with seeds "; + for (std::set::const_iterator it = seeds.begin(); + it != seeds.end(); ++it) { + if (it != seeds.begin()) + lsb << ','; + lsb << *it; + } + } + + HostAndPort ReplicaSetMonitor::getHostOrRefresh(const ReadPreferenceSetting& criteria) { + { + boost::mutex::scoped_lock lk(_state->mutex); + HostAndPort out = _state->getMatchingHost(criteria); + if (!out.empty()) + return out; + } + + { + Refresher refresher = startOrContinueRefresh(); + HostAndPort out = refresher.refreshUntilMatches(criteria); + if (!out.empty() || refresher.startedNewScan()) + return out; + } + + // We didn't find any matching hosts and the scan we just finished may have stale data from + // before we joined. Therefore we should participate in a new scan to make sure all hosts + // are contacted at least once (possibly by other threads) before this function gives up. + + return startOrContinueRefresh().refreshUntilMatches(criteria); + } + + HostAndPort ReplicaSetMonitor::getMasterOrUassert() { + const ReadPreferenceSetting masterOnly(ReadPreference_PrimaryOnly, TagSet()); + HostAndPort master = getHostOrRefresh(masterOnly); + uassert(10009, str::stream() << "ReplicaSetMonitor no master found for set: " << getName(), + !master.empty()); + return master; + } + + Refresher ReplicaSetMonitor::startOrContinueRefresh() { + boost::mutex::scoped_lock lk(_state->mutex); + + Refresher out(_state); + DEV _state->checkInvariants(); + return out; + } + + void ReplicaSetMonitor::failedHost(const HostAndPort& host) { + boost::mutex::scoped_lock lk(_state->mutex); + Node* node = _state->findNode(host); + if (node) + node->markFailed(); + DEV _state->checkInvariants(); + } + + bool ReplicaSetMonitor::isPrimary(const HostAndPort& host) const { + boost::mutex::scoped_lock lk(_state->mutex); + Node* node = _state->findNode(host); + return node ? node->isMaster : false; + } + + bool ReplicaSetMonitor::isHostUp(const HostAndPort& host) const { + boost::mutex::scoped_lock lk(_state->mutex); + Node* node = _state->findNode(host); + return node ? node->isUp : false; + } + + int ReplicaSetMonitor::getConsecutiveFailedScans() const { + boost::mutex::scoped_lock lk(_state->mutex); + return _state->consecutiveFailedScans; + } + + std::string ReplicaSetMonitor::getName() const { + // name is const so don't need to lock + return _state->name; + } + + std::string ReplicaSetMonitor::getServerAddress() const { + boost::mutex::scoped_lock lk(_state->mutex); + return _state->getServerAddress(); + } + + bool ReplicaSetMonitor::contains(const HostAndPort& host) const { + boost::mutex::scoped_lock lk(_state->mutex); + return _state->seedNodes.count(host); + } + + void ReplicaSetMonitor::createIfNeeded(const string& name, const set& servers) { + LOG(3) << "ReplicaSetMonitor::createIfNeeded " << name; + scoped_lock lk(setsLock); + ReplicaSetMonitorPtr& m = sets[name]; + if ( ! m ) + m = boost::make_shared( name , servers ); + + replicaSetMonitorWatcher.safeGo(); + } + + ReplicaSetMonitorPtr ReplicaSetMonitor::get(const string& name, const bool createFromSeed) { + LOG(3) << "ReplicaSetMonitor::get " << name; + scoped_lock lk( setsLock ); + StringMap::const_iterator i = sets.find( name ); + if ( i != sets.end() ) { + return i->second; + } + if ( createFromSeed ) { + StringMap >::const_iterator j = seedServers.find( name ); + if ( j != seedServers.end() ) { + LOG(4) << "Creating ReplicaSetMonitor from cached address"; + ReplicaSetMonitorPtr& m = sets[name]; + invariant( !m ); + m.reset( new ReplicaSetMonitor( name, j->second ) ); + replicaSetMonitorWatcher.safeGo(); + return m; + } + } + return ReplicaSetMonitorPtr(); + } + + set ReplicaSetMonitor::getAllTrackedSets() { + set activeSets; + scoped_lock lk( setsLock ); + for (StringMap::const_iterator it = sets.begin(); + it != sets.end(); ++it) + { + activeSets.insert(it->first); + } + return activeSets; + } + + void ReplicaSetMonitor::remove(const string& name, bool clearSeedCache) { + LOG(2) << "Removing ReplicaSetMonitor for " << name << " from replica set table" + << (clearSeedCache ? " and the seed cache" : ""); + + scoped_lock lk( setsLock ); + const StringMap::const_iterator setIt = sets.find(name); + if (setIt != sets.end()) { + if (!clearSeedCache) { + // Save list of current set members so that the monitor can be rebuilt if needed. + const ReplicaSetMonitorPtr& rsm = setIt->second; + boost::mutex::scoped_lock lk(rsm->_state->mutex); + seedServers[name] = rsm->_state->seedNodes; + } + sets.erase(setIt); + } + + if ( clearSeedCache ) { + seedServers.erase( name ); + } + + // Kill all pooled ReplicaSetConnections for this set. They will not function correctly + // after we kill the ReplicaSetMonitor. + // TODO we may only need to do this if clearSeedCache is true. + pool.removeHost(name); + } + + void ReplicaSetMonitor::setConfigChangeHook(ConfigChangeHook hook) { + massert(13610, "ConfigChangeHook already specified", !SetState::configChangeHook); + SetState::configChangeHook = hook; + } + + // TODO move to correct order with non-statics before pushing + void ReplicaSetMonitor::appendInfo(BSONObjBuilder& bsonObjBuilder) const { + boost::mutex::scoped_lock lk(_state->mutex); + + // NOTE: the format here must be consistent for backwards compatibility + BSONArrayBuilder hosts(bsonObjBuilder.subarrayStart("hosts")); + for (unsigned i = 0; i < _state->nodes.size(); i++) { + const Node& node = _state->nodes[i]; + + BSONObjBuilder builder; + builder.append("addr", node.host.toString()); + builder.append("ok", node.isUp); + builder.append("ismaster", node.isMaster); // intentionally not camelCase + builder.append("hidden", false); // we don't keep hidden nodes in the set + builder.append("secondary", node.isUp && !node.isMaster); + builder.append("pingTimeMillis", int(node.latencyMicros / 1000)); + + if (!node.tags.isEmpty()) { + builder.append("tags", node.tags); + } + + hosts.append(builder.obj()); + } + hosts.done(); + } + + void ReplicaSetMonitor::cleanup() { + // Call cancel first, in case the RSMW was never started. + replicaSetMonitorWatcher.cancel(); + replicaSetMonitorWatcher.stop(); + replicaSetMonitorWatcher.wait(); + scoped_lock lock(setsLock); + sets = StringMap(); + seedServers = StringMap >(); + } + + Refresher::Refresher(const SetStatePtr& setState) + : _set(setState) + , _scan(setState->currentScan) + , _startedNewScan(false) { + if (_scan) + return; // participate in in-progress scan + + LOG(2) << "Starting new refresh of replica set " << _set->name; + _scan = startNewScan(_set.get()); + _set->currentScan = _scan; + _startedNewScan = true; + } + + Refresher::NextStep Refresher::getNextStep() { + if (_scan != _set->currentScan) + return NextStep(NextStep::DONE); // No longer the current scan. + + // Wait for all dispatched hosts to return before trying any fallback hosts. + if (_scan->hostsToScan.empty() && !_scan->waitingFor.empty()) + return NextStep(NextStep::WAIT); + + // If we haven't yet found a master, try contacting unconfirmed hosts + if (_scan->hostsToScan.empty() && !_scan->foundUpMaster) { + _scan->enqueAllUntriedHosts(_scan->possibleNodes, _set->rand); + _scan->possibleNodes.clear(); + } + + if (_scan->hostsToScan.empty()) { + // We've tried all hosts we can, so nothing more to do in this round. + if (!_scan->foundUpMaster) { + warning() << "No primary detected for set " << _set->name; + + // Since we've talked to everyone we could but still didn't find a primary, we + // do the best we can, and assume all unconfirmedReplies are actually from nodes + // in the set (we've already confirmed that they think they are). This is + // important since it allows us to bootstrap to a usable state even if we are + // unable to talk to a master while starting up. As soon as we are able to + // contact a master, we will remove any nodes that it doesn't think are part of + // the set, undoing the damage we cause here. + + // NOTE: we don't modify seedNodes or notify about set membership change in this + // case since it hasn't been confirmed by a master. + for (UnconfirmedReplies::iterator it = _scan->unconfirmedReplies.begin(); + it != _scan->unconfirmedReplies.end(); ++it) { + _set->findOrCreateNode(it->host)->update(*it); + } + } + + if (_scan->foundAnyUpNodes) { + _set->consecutiveFailedScans = 0; + } + else { + _set->consecutiveFailedScans++; + log() << "All nodes for set " << _set->name << " are down. " + << "This has happened for " << _set->consecutiveFailedScans + << " checks in a row. Polling will stop after " + << maxConsecutiveFailedChecks - _set->consecutiveFailedScans + << " more failed checks"; + } + + _set->currentScan.reset(); // Makes sure all other Refreshers in this round return DONE + return NextStep(NextStep::DONE); + } + + // Pop and return the next hostToScan. + HostAndPort host = _scan->hostsToScan.front(); + _scan->hostsToScan.pop_front(); + _scan->waitingFor.insert(host); + _scan->triedHosts.insert(host); + return NextStep(NextStep::CONTACT_HOST, host); + } + + void Refresher::receivedIsMaster(const HostAndPort& from, + int64_t latencyMicros, + const BSONObj& replyObj) { + // Be careful: all return paths must call either failedHost or cv.notify_all! + _scan->waitingFor.erase(from); + + const IsMasterReply reply(from, latencyMicros, replyObj); + // Handle various failure cases + if (!reply.ok) { + failedHost(from); + return; + } + + if (reply.setName != _set->name) { + warning() << "node: " << from + << " isn't a part of set: " << _set->name + << " ismaster: " << replyObj; + failedHost(from); + return; + } + + if (reply.isMaster) { + const bool stalePrimary = !receivedIsMasterFromMaster(reply); + if (stalePrimary) { + log() << "node " << from << " believes it is primary, but its election id of " + << reply.electionId << " is older than the most recent election id" + << " for this set, " << _set->maxElectionId; + failedHost(from); + return; + } + } + + if (_scan->foundUpMaster) { + // We only update a Node if a master has confirmed it is in the set. + _set->updateNodeIfInNodes(reply); + } + else { + receivedIsMasterBeforeFoundMaster(reply); + _scan->unconfirmedReplies.push_back(reply); + } + + // _set->nodes may still not have any nodes with isUp==true, but we have at least found a + // connectible host that is that claims to be in the set. + _scan->foundAnyUpNodes = true; + + // TODO consider only notifying if we've updated a node or we've emptied waitingFor. + _set->cv.notify_all(); + + DEV _set->checkInvariants(); + } + + void Refresher::failedHost(const HostAndPort& host) { + _scan->waitingFor.erase(host); + + // Failed hosts can't pass criteria, so the only way they'd effect the _refreshUntilMatches + // loop is if it was the last host we were waitingFor. + if (_scan->waitingFor.empty()) + _set->cv.notify_all(); + + Node* node = _set->findNode(host); + if (node) + node->markFailed(); + } + + ScanStatePtr Refresher::startNewScan(const SetState* set) { + const ScanStatePtr scan = boost::make_shared(); + + // The heuristics we use in deciding the order to contact hosts are designed to find a + // master as quickly as possible. This is because we can't use any hosts we find until + // we either get the latest set of members from a master or talk to all possible hosts + // without finding a master. + + // TODO It might make sense to check down nodes first if the last seen master is still + // marked as up. + + int upNodes = 0; + for (Nodes::const_iterator it(set->nodes.begin()), end(set->nodes.end()); + it != end; ++it) { + if (it->isUp) { + // scan the nodes we think are up first + scan->hostsToScan.push_front(it->host); + upNodes++; + } + else { + scan->hostsToScan.push_back(it->host); + } + } + + // shuffle the queue, but keep "up" nodes at the front + std::random_shuffle(scan->hostsToScan.begin(), + scan->hostsToScan.begin() + upNodes, + set->rand); + std::random_shuffle(scan->hostsToScan.begin() + upNodes, + scan->hostsToScan.end(), + set->rand); + + if (!set->lastSeenMaster.empty()) { + // move lastSeenMaster to front of queue + std::stable_partition(scan->hostsToScan.begin(), + scan->hostsToScan.end(), + hostIs(set->lastSeenMaster)); + } + + return scan; + } + + bool Refresher::receivedIsMasterFromMaster(const IsMasterReply& reply) { + invariant(reply.isMaster); + + if (reply.electionId.isSet()) { + if (_set->maxElectionId.isSet() && _set->maxElectionId.compare(reply.electionId) > 0) { + return false; + } + _set->maxElectionId = reply.electionId; + } + + // Mark all nodes as not master. We will mark ourself as master before releasing the lock. + // NOTE: we use a "last-wins" policy if multiple hosts claim to be master. + for (size_t i = 0; i < _set->nodes.size(); i++) { + _set->nodes[i].isMaster = false; + } + + // Check if the master agrees with our current list of nodes. + // REMINDER: both _set->nodes and reply.normalHosts are sorted. + if (_set->nodes.size() != reply.normalHosts.size() || + !std::equal(_set->nodes.begin(), _set->nodes.end(), + reply.normalHosts.begin(), + hostsEqual)) { + + LOG(2) << "Adjusting nodes in our view of replica set " << _set->name + << " based on master reply: " << reply.raw; + + // remove non-members from _set->nodes + _set->nodes.erase(std::remove_if(_set->nodes.begin(), + _set->nodes.end(), + hostNotIn(reply.normalHosts)), + _set->nodes.end()); + + // add new members to _set->nodes + for (std::set::const_iterator it = reply.normalHosts.begin(); + it != reply.normalHosts.end(); ++it) { + _set->findOrCreateNode(*it); + } + + // replace hostToScan queue with untried normal hosts. can both add and remove + // hosts from the queue. + _scan->hostsToScan.clear(); + _scan->enqueAllUntriedHosts(reply.normalHosts, _set->rand); + + if (!_scan->waitingFor.empty()) { + // make sure we don't wait for any hosts that aren't considered members + std::set newWaitingFor; + std::set_intersection(reply.normalHosts.begin(), reply.normalHosts.end(), + _scan->waitingFor.begin(), _scan->waitingFor.end(), + std::inserter(newWaitingFor, newWaitingFor.end())); + _scan->waitingFor.swap(newWaitingFor); + } + } + + if (reply.normalHosts != _set->seedNodes) { + const string oldAddr = _set->getServerAddress(); + _set->seedNodes = reply.normalHosts; + + // LogLevel can be pretty low, since replica set reconfiguration should be pretty rare + // and we want to record our changes + log() << "changing hosts to " << _set->getServerAddress() << " from " << oldAddr; + + if (SetState::configChangeHook) { + // call from a separate thread to avoid blocking and holding lock while potentially + // going over the network + boost::thread bg(SetState::configChangeHook, _set->name, _set->getServerAddress()); + bg.detach(); + } + } + + // Update other nodes's information based on replies we've already seen + for (UnconfirmedReplies::iterator it = _scan->unconfirmedReplies.begin(); + it != _scan->unconfirmedReplies.end(); ++it) { + // this ignores replies from hosts not in _set->nodes (as modified above) + _set->updateNodeIfInNodes(*it); + } + _scan->unconfirmedReplies.clear(); + + _scan->foundUpMaster = true; + _set->lastSeenMaster = reply.host; + + return true; + } + + void Refresher::receivedIsMasterBeforeFoundMaster(const IsMasterReply& reply) { + invariant(!reply.isMaster); + // This function doesn't alter _set at all. It only modifies the work queue in _scan. + + // Add everyone this host claims is in the set to possibleNodes. + _scan->possibleNodes.insert(reply.normalHosts.begin(), reply.normalHosts.end()); + + // If this node thinks the primary is someone we haven't tried, make that the next + // hostToScan. + if (!reply.primary.empty() && !_scan->triedHosts.count(reply.primary)) { + std::deque::iterator it = + std::stable_partition(_scan->hostsToScan.begin(), + _scan->hostsToScan.end(), + hostIs(reply.primary)); + + if (it == _scan->hostsToScan.begin()) { + // reply.primary wasn't in hostsToScan + _scan->hostsToScan.push_front(reply.primary); + } + } + } + + HostAndPort Refresher::_refreshUntilMatches(const ReadPreferenceSetting* criteria) { + boost::mutex::scoped_lock lk(_set->mutex); + while (true) { + if (criteria) { + HostAndPort out = _set->getMatchingHost(*criteria); + if (!out.empty()) + return out; + } + + const NextStep ns = getNextStep(); + switch(ns.step) { + case NextStep::DONE: + DEV _set->checkInvariants(); + // getNextStep may have updated nodes if no master was found. + return criteria ? _set->getMatchingHost(*criteria) : HostAndPort(); + + case NextStep::WAIT: // TODO consider treating as DONE for refreshAll + DEV _set->checkInvariants(); + _set->cv.wait(lk); + continue; + + case NextStep::CONTACT_HOST: { + BSONObj reply; // empty on error + int64_t pingMicros = 0; + + DEV _set->checkInvariants(); + lk.unlock(); // relocked after attempting to call isMaster + try { + ScopedDbConnection conn(ConnectionString(ns.host), socketTimeoutSecs); + bool ignoredOutParam = false; + Timer timer; + conn->isMaster(ignoredOutParam, &reply); + pingMicros = timer.micros(); + conn.done(); // return to pool on success. + } + catch (...) { + reply = BSONObj(); // should be a no-op but want to be sure + } + lk.lock(); + + // Ignore the reply and return if we are no longer the current scan. This might + // happen if it was decided that the host we were contacting isn't part of the set. + if (_scan != _set->currentScan) + return criteria ? _set->getMatchingHost(*criteria) : HostAndPort(); + + if (reply.isEmpty()) + failedHost(ns.host); + else + receivedIsMaster(ns.host, pingMicros, reply); + } + } + } + } + + void IsMasterReply::parse(const BSONObj& obj) { + try { + raw = obj.getOwned(); // don't use obj again after this line + + ok = raw["ok"].trueValue(); + if (!ok) + return; + + setName = raw["setName"].str(); + hidden = raw["hidden"].trueValue(); + secondary = raw["secondary"].trueValue(); + + // hidden nodes can't be master, even if they claim to be. + isMaster = !hidden && raw["ismaster"].trueValue(); + + if (isMaster && raw.hasField("electionId")) { + electionId = raw["electionId"].OID(); + } + + const string primaryString = raw["primary"].str(); + primary = primaryString.empty() ? HostAndPort() : HostAndPort(primaryString); + + // both hosts and passives, but not arbiters, are considered "normal hosts" + normalHosts.clear(); + BSONForEach(host, raw.getObjectField("hosts")) { + normalHosts.insert(HostAndPort(host.String())); + } + BSONForEach(host, raw.getObjectField("passives")) { + normalHosts.insert(HostAndPort(host.String())); + } + + tags = raw.getObjectField("tags"); + } catch (const std::exception& e) { + ok = false; + log() << "exception while parsing isMaster reply: " << e.what() << " " << obj; + } + } + + const int64_t Node::unknownLatency = numeric_limits::max(); + + bool Node::matches(const ReadPreference& pref) const { + if (!isUp) + return false; + + if (pref == ReadPreference_PrimaryOnly) { + return isMaster; + } + + if (pref == ReadPreference_SecondaryOnly) { + if (isMaster) + return false; + } + + return true; + } + + bool Node::matches(const BSONObj& tag) const { + BSONForEach(tagCriteria, tag) { + if (this->tags[tagCriteria.fieldNameStringData()] != tagCriteria) + return false; + } + + return true; + } + + void Node::update(const IsMasterReply& reply) { + invariant(host == reply.host); + invariant(reply.ok); + + LOG(3) << "Updating host " << host << " based on ismaster reply: " << reply.raw; + + // Nodes that are hidden or neither master or secondary are considered down since we can't + // send any operations to them. + isUp = !reply.hidden && (reply.isMaster || reply.secondary); + isMaster = reply.isMaster; + + // save a copy if unchanged + if (!tags.binaryEqual(reply.tags)) + tags = reply.tags.getOwned(); + + if (reply.latencyMicros >= 0) { // TODO upper bound? + if (latencyMicros == unknownLatency) { + latencyMicros = reply.latencyMicros; + } + else { + // update latency with smoothed moving average (1/4th the delta) + latencyMicros += (reply.latencyMicros - latencyMicros) / 4; + } + } + } + + ReplicaSetMonitor::ConfigChangeHook SetState::configChangeHook; + + SetState::SetState(StringData name, const std::set& seedNodes) + : name(name.toString()) + , consecutiveFailedScans(0) + , seedNodes(seedNodes) + , latencyThresholdMicros(serverGlobalParams.defaultLocalThresholdMillis * 1000) + , rand(int64_t(time(0))) + , roundRobin(0) + { + uassert(13642, "Replica set seed list can't be empty", !seedNodes.empty()); + + if (name.empty()) + warning() << "Replica set name empty, first node: " << *(seedNodes.begin()); + + // This adds the seed hosts to nodes, but they aren't usable for anything except seeding a + // scan until we start a scan and either find a master or contact all hosts without finding + // one. + // WARNING: if seedNodes is ever changed to not imply sorted iteration, you will need to + // sort nodes after this loop. + for (std::set::const_iterator it = seedNodes.begin(); + it != seedNodes.end(); ++it) { + nodes.push_back(Node(*it)); + } + + DEV checkInvariants(); + } + + HostAndPort SetState::getMatchingHost(const ReadPreferenceSetting& criteria) const { + switch (criteria.pref) { + // "Prefered" read preferences are defined in terms of other preferences + case ReadPreference_PrimaryPreferred: { + HostAndPort out = getMatchingHost(ReadPreferenceSetting(ReadPreference_PrimaryOnly, + criteria.tags)); + // NOTE: the spec says we should use the primary even if tags don't match + if (!out.empty()) + return out; + return getMatchingHost(ReadPreferenceSetting(ReadPreference_SecondaryOnly, + criteria.tags)); + } + + case ReadPreference_SecondaryPreferred: { + HostAndPort out = + getMatchingHost(ReadPreferenceSetting(ReadPreference_SecondaryOnly, + criteria.tags)); + if (!out.empty()) + return out; + // NOTE: the spec says we should use the primary even if tags don't match + return getMatchingHost(ReadPreferenceSetting(ReadPreference_PrimaryOnly, + criteria.tags)); + } + + case ReadPreference_PrimaryOnly: { + // NOTE: isMaster implies isUp + Nodes::const_iterator it = std::find_if(nodes.begin(), nodes.end(), isMaster); + if (it == nodes.end()) + return HostAndPort(); + return it->host; + } + + // The difference between these is handled by Node::matches + case ReadPreference_SecondaryOnly: + case ReadPreference_Nearest: { + BSONForEach(tagElem, criteria.tags.getTagBSON()) { + uassert(16358, "Tags should be a BSON object", tagElem.isABSONObj()); + BSONObj tag = tagElem.Obj(); + + std::vector matchingNodes; + for (size_t i = 0; i < nodes.size(); i++ ) { + if (nodes[i].matches(criteria.pref) && nodes[i].matches(tag)) { + matchingNodes.push_back(&nodes[i]); + } + } + + // don't do more complicated selection if not needed + if (matchingNodes.empty()) continue; + if (matchingNodes.size() == 1) return matchingNodes.front()->host; + + // order by latency and don't consider hosts further than a threshold from the + // closest. + std::sort(matchingNodes.begin(), matchingNodes.end(), compareLatencies); + for (size_t i = 1; i < matchingNodes.size(); i++ ) { + int64_t distance = matchingNodes[i]->latencyMicros + - matchingNodes[0]->latencyMicros; + if (distance >= latencyThresholdMicros) { + // this node and all remaining ones are too far away + matchingNodes.erase(matchingNodes.begin() + i, matchingNodes.end()); + break; + } + } + + // of the remaining nodes, pick one at random (or use round-robin) + if (ReplicaSetMonitor::useDeterministicHostSelection) { + // only in tests + return matchingNodes[roundRobin++ % matchingNodes.size()]->host; + } + else { + // normal case + return matchingNodes[rand.nextInt32(matchingNodes.size())]->host; + }; + } + + return HostAndPort(); + } + default: + uassert( 16337, "Unknown read preference", false ); + break; + } + } + + Node* SetState::findNode(const HostAndPort& host) { + const Nodes::iterator it = std::lower_bound(nodes.begin(), nodes.end(), host, compareHosts); + if (it == nodes.end() || it->host != host) + return NULL; + + return &(*it); + } + + Node* SetState::findOrCreateNode(const HostAndPort& host) { + // This is insertion sort, but N is currently guaranteed to be <= 12 (although this class + // must function correctly even with more nodes). If we lift that restriction, we may need + // to consider alternate algorithms. + Nodes::iterator it = std::lower_bound(nodes.begin(), nodes.end(), host, compareHosts); + if (it == nodes.end() || it->host != host) { + LOG(2) << "Adding node " << host << " to our view of replica set " << name; + it = nodes.insert(it, Node(host)); + } + return &(*it); + } + + void SetState::updateNodeIfInNodes(const IsMasterReply& reply) { + Node* node = findNode(reply.host); + if (!node) { + LOG(2) << "Skipping application of ismaster reply from " << reply.host + << " since it isn't a confirmed member of set " << name; + return; + } + + node->update(reply); + } + + std::string SetState::getServerAddress() const { + StringBuilder ss; + if (!name.empty()) + ss << name << "/"; + + for (std::set::const_iterator it = seedNodes.begin(); + it != seedNodes.end(); ++it) { + if (it != seedNodes.begin()) + ss << ","; + it->append(ss); + } + + return ss.str(); + } + + void SetState::checkInvariants() const { + bool foundMaster = false; + for (size_t i = 0; i < nodes.size(); i++) { + // no empty hosts + invariant(!nodes[i].host.empty()); + + if (nodes[i].isMaster) { + // masters must be up + invariant(nodes[i].isUp); + + // at most one master + invariant(!foundMaster); + foundMaster = true; + + // if we have a master it should be the same as lastSeenMaster + invariant(nodes[i].host == lastSeenMaster); + } + + // should never end up with negative latencies + invariant(nodes[i].latencyMicros >= 0); + + // nodes must be sorted by host with no-dupes + invariant(i == 0 || (nodes[i-1].host < nodes[i].host)); + } + + // nodes should be a (non-strict) superset of the seedNodes + invariant(std::includes(nodes.begin(), nodes.end(), + seedNodes.begin(), seedNodes.end(), + compareHosts)); + + if (currentScan) { + // hostsToScan can't have dups or hosts already in triedHosts. + std::set cantSee = currentScan->triedHosts; + for (std::deque::const_iterator it = currentScan->hostsToScan.begin(); + it != currentScan->hostsToScan.end(); ++it) { + invariant(!cantSee.count(*it)); + cantSee.insert(*it); // make sure we don't see this again + } + + // We should only be waitingFor hosts that are in triedHosts + invariant( + std::includes(currentScan->triedHosts.begin(), currentScan->triedHosts.end(), + currentScan->waitingFor.begin(), currentScan->waitingFor.end())); + + // We should only have unconfirmedReplies if we haven't found a master yet + invariant(!currentScan->foundUpMaster || currentScan->unconfirmedReplies.empty()); + } + } + + template + void ScanState::enqueAllUntriedHosts(const Container& container, PseudoRandom& rand) { + invariant(hostsToScan.empty()); // because we don't try to dedup hosts already in the queue. + + // no std::copy_if before c++11 + for (typename Container::const_iterator it(container.begin()), end(container.end()); + it != end; ++it) { + if (!triedHosts.count(*it)) { + hostsToScan.push_back(*it); + } + } + std::random_shuffle(hostsToScan.begin(), hostsToScan.end(), rand); + } +} diff --git a/src/mongo/client/replica_set_monitor.h b/src/mongo/client/replica_set_monitor.h new file mode 100644 index 00000000000..194a4710349 --- /dev/null +++ b/src/mongo/client/replica_set_monitor.h @@ -0,0 +1,326 @@ +/* Copyright 2014 MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" +#include "mongo/util/net/hostandport.h" + +namespace mongo { + class ReplicaSetMonitor; + class TagSet; + struct ReadPreferenceSetting; + typedef shared_ptr ReplicaSetMonitorPtr; + + /** + * Holds state about a replica set and provides a means to refresh the local view. + * All methods perform the required synchronization to allow callers from multiple threads. + */ + class MONGO_CLIENT_API ReplicaSetMonitor { + MONGO_DISALLOW_COPYING(ReplicaSetMonitor); + public: + class Refresher; + + typedef boost::function + ConfigChangeHook; + + /** + * Initializes local state. + * + * seeds must not be empty. + */ + ReplicaSetMonitor(StringData name, const std::set& seeds); + + /** + * Returns a host matching criteria or an empty HostAndPort if no host matches. + * + * If no host matches initially, will then attempt to refresh our view of the set by + * contacting other hosts. May still return no result if no host matches following a + * refresh. + */ + HostAndPort getHostOrRefresh(const ReadPreferenceSetting& criteria); + + /** + * Returns the host we think is the current master or uasserts. + * + * This is a thin wrapper around getHostOrRefresh so this will also refresh our view if we + * don't think there is a master at first. The main difference is that this will uassert + * rather than returning an empty HostAndPort. + */ + HostAndPort getMasterOrUassert(); + + /** + * Returns a refresher object that can be used to update our view of the set. + * If a refresh is currently in-progress, the returned Refresher will participate in the + * current refresh round. + */ + Refresher startOrContinueRefresh(); + + /** + * Notifies this Monitor that a host has failed and should be considered down. + * + * Call this when you get a connection error. If you get an error while trying to refresh + * our view of a host, call Refresher::hostFailed() instead. + */ + void failedHost(const HostAndPort& host); + + /** + * Returns true if this node is the master based ONLY on local data. Be careful, return may + * be stale. + */ + bool isPrimary(const HostAndPort& host) const; + + /** + * Returns true if host is part of this set and is considered up (meaning it can accept + * queries). + */ + bool isHostUp(const HostAndPort& host) const; + + /** + * How may times in a row have we tried to refresh without successfully contacting any hosts + * who claim to be members of this set? + */ + int getConsecutiveFailedScans() const; + + /** + * The name of the set. + */ + std::string getName() const; + + /** + * Returns a string with the format name/server1,server2. + * If name is empty, returns just comma-separated list of servers. + */ + std::string getServerAddress() const; + + /** + * Is server part of this set? Uses only cached information. + */ + bool contains(const HostAndPort& server) const; + + /** + * Writes information about our cached view of the set to a BSONObjBuilder. + */ + void appendInfo(BSONObjBuilder& b) const; + + /** + * Creates a new ReplicaSetMonitor, if it doesn't already exist. + */ + static void createIfNeeded(const std::string& name , const std::set& servers); + + /** + * gets a cached Monitor per name. If the monitor is not found and createFromSeed is false, + * it will return none. If createFromSeed is true, it will try to look up the last known + * servers list for this set and will create a new monitor using that as the seed list. + */ + static ReplicaSetMonitorPtr get(const std::string& name, bool createFromSeed = false); + + /** + * Returns all the currently tracked replica set names. + */ + static std::set getAllTrackedSets(); + + /** + * Removes the ReplicaSetMonitor for the given set name from _sets, which will delete it. + * If clearSeedCache is true, then the cached seed string for this Replica Set will be + * removed from _seedServers. + */ + static void remove(const std::string& name, bool clearSeedCache = false); + + /** + * Sets the hook to be called whenever the config of any replica set changes. + * Currently only 1 globally, so this asserts if one already exists. + * + * The hook will be called from a fresh thread. It is responsible for initializing any + * thread-local state and ensuring that no exceptions escape. + * + * The hook must not be changed while the program has multiple threads. + */ + static void setConfigChangeHook(ConfigChangeHook hook); + + /** + * Permanently stops all monitoring on replica sets and clears all cached information + * as well. As a consequence, NEVER call this if you have other threads that have a + * DBClientReplicaSet instance. + */ + static void cleanup(); + + /** + * If a ReplicaSetMonitor has been refreshed more than this many times in a row without + * finding any live nodes claiming to be in the set, the ReplicaSetMonitorWatcher will stop + * periodic background refreshes of this set. + */ + static int maxConsecutiveFailedChecks; + + // + // internal types (defined in replica_set_monitor_internal.h) + // + + struct IsMasterReply; + struct ScanState; + struct SetState; + typedef boost::shared_ptr ScanStatePtr; + typedef boost::shared_ptr SetStatePtr; + + // + // FOR TESTING ONLY + // + + /** + * Allows tests to set initial conditions and introspect the current state. + */ + explicit ReplicaSetMonitor(const SetStatePtr& initialState) + : _state(initialState) + {} + + /** + * Defaults to false, meaning that if multiple hosts meet a criteria we pick one at random. + * This is required by the replica set driver spec. Set this to true in tests that need host + * selection to be deterministic. + */ + static bool useDeterministicHostSelection; + + private: + const SetStatePtr _state; // never NULL + }; + + /** + * Refreshes the local view of a replica set. + * + * Use ReplicaSetMonitor::startOrContinueRefresh() to obtain a Refresher. + * + * Multiple threads can refresh a single set without any additional synchronization, however + * they must each use their own Refresher object. + * + * All logic related to choosing the hosts to contact and updating the SetState based on replies + * lives in this class. + */ + class MONGO_CLIENT_API ReplicaSetMonitor::Refresher { + public: + /** + * Contact hosts in the set to refresh our view, but stop once a host matches criteria. + * Returns the matching host or empty if none match after a refresh. + * + * This is called by ReplicaSetMonitor::getHostWithRefresh() + */ + HostAndPort refreshUntilMatches(const ReadPreferenceSetting& criteria) { + return _refreshUntilMatches(&criteria); + }; + + /** + * Refresh all hosts. Equivalent to refreshUntilMatches with a criteria that never + * matches. + * + * This is intended to be called periodically, possibly from a background thread. + */ + void refreshAll() { _refreshUntilMatches(NULL); } + + // + // Remaining methods are only for testing and internal use. + // Callers are responsible for holding SetState::mutex before calling any of these methods. + // + + /** + * Any passed-in pointers are shared with caller. + * + * If no scan is in-progress, this function is responsible for setting up a new scan. + */ + explicit Refresher(const SetStatePtr& setState); + + struct NextStep { + enum StepKind { + CONTACT_HOST, /// Contact the returned host + WAIT, /// Wait on condition variable and try again. + DONE, /// No more hosts to contact in this Refresh round + }; + + explicit NextStep(StepKind step, const HostAndPort& host=HostAndPort()) + : step(step) + , host(host) + {} + + StepKind step; + HostAndPort host; + }; + + /** + * Returns the next step to take. + * + * By calling this, you promise to call receivedIsMaster or failedHost if the NextStep is + * CONTACT_HOST. + */ + NextStep getNextStep(); + + /** + * Call this if a host returned from getNextStep successfully replied to an isMaster call. + * Negative latencyMicros are ignored. + */ + void receivedIsMaster(const HostAndPort& from, int64_t latencyMicros, const BSONObj& reply); + + /** + * Call this if a host returned from getNextStep failed to reply to an isMaster call. + */ + void failedHost(const HostAndPort& host); + + /** + * True if this Refresher started a new full scan rather than joining an existing one. + */ + bool startedNewScan() const { return _startedNewScan; } + + /** + * Starts a new scan over the hosts in set. + */ + static ScanStatePtr startNewScan(const SetState* set); + + private: + + /** + * First, checks that the "reply" is not from a stale primary by + * comparing the electionId of "reply" to the maxElectionId recorded by the SetState. + * Returns true if "reply" belongs to a non-stale primary. + * + * Updates _set and _scan based on set-membership information from a master. + * Applies _scan->unconfirmedReplies to confirmed nodes. + * Does not update this host's node in _set->nodes. + */ + bool receivedIsMasterFromMaster(const IsMasterReply& reply); + + /** + * Adjusts the _scan work queue based on information from this host. + * This should only be called with replies from non-masters. + * Does not update _set at all. + */ + void receivedIsMasterBeforeFoundMaster(const IsMasterReply& reply); + + /** + * Shared implementation of refreshUntilMatches and refreshAll. + * NULL criteria means refresh every host. + * Handles own locking. + */ + HostAndPort _refreshUntilMatches(const ReadPreferenceSetting* criteria); + + // Both pointers are never NULL + SetStatePtr _set; + ScanStatePtr _scan; // May differ from _set->currentScan if a new scan has started. + bool _startedNewScan; + }; +} diff --git a/src/mongo/client/replica_set_monitor_internal.h b/src/mongo/client/replica_set_monitor_internal.h new file mode 100644 index 00000000000..61f3101e47f --- /dev/null +++ b/src/mongo/client/replica_set_monitor_internal.h @@ -0,0 +1,199 @@ +/* Copyright 2014 MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is an internal header. + * This should only be included by replica_set_monitor.cpp and replica_set_monitor_test.cpp. + * This should never be included by any header. + */ + +#pragma once + +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/client/dbclient_rs.h" // for TagSet and ReadPreferenceSettings +#include "mongo/client/replica_set_monitor.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/cstdint.h" +#include "mongo/platform/random.h" +#include "mongo/util/net/hostandport.h" + +namespace mongo { + struct ReplicaSetMonitor::IsMasterReply { + IsMasterReply() : ok(false) {} + IsMasterReply(const HostAndPort& host, int64_t latencyMicros, const BSONObj& reply) + : ok(false) + , host(host) + , latencyMicros(latencyMicros) { + parse(reply); + } + + /** + * Never throws. If parsing fails for any reason, sets ok to false. + */ + void parse(const BSONObj& obj); + + bool ok; // if false, ignore all other fields + BSONObj raw; // Always owned. Other fields are allowed to be a view into this. + std::string setName; + bool isMaster; + bool secondary; + bool hidden; + OID electionId; // Set if this isMaster reply is from the primary + HostAndPort primary; // empty if not present + std::set normalHosts; // both "hosts" and "passives" + BSONObj tags; + + // remaining fields aren't in isMaster reply, but are known to caller. + HostAndPort host; + int64_t latencyMicros; // ignored if negative + }; + + struct ReplicaSetMonitor::SetState { + MONGO_DISALLOW_COPYING(SetState); + public: + + // A single node in the replicaSet + struct Node { + explicit Node(const HostAndPort& host) + : host(host) + , latencyMicros(unknownLatency) { + markFailed(); + } + + void markFailed() { + isUp = false; + isMaster = false; + } + + bool matches(const ReadPreference& pref) const; + + /** + * Checks if the given tag matches the tag attached to this node. + * + * Example: + * + * Tag of this node: { "dc": "nyc", "region": "na", "rack": "4" } + * + * match: {} + * match: { "dc": "nyc", "rack": 4 } + * match: { "region": "na", "dc": "nyc" } + * not match: { "dc": "nyc", "rack": 2 } + * not match: { "dc": "sf" } + */ + bool matches(const BSONObj& tag) const; + + /** + * Updates this Node based on information in reply. The reply must be from this host. + */ + void update(const IsMasterReply& reply); + + // Intentionally chosen to compare worse than all known latencies. + static const int64_t unknownLatency; // = numeric_limits::max() + + HostAndPort host; + bool isUp; + bool isMaster; // implies isUp + int64_t latencyMicros; // unknownLatency if unknown + BSONObj tags; // owned + }; + typedef std::vector Nodes; + + /** + * seedNodes must not be empty + */ + SetState(StringData name, const std::set& seedNodes); + + /** + * Returns a host matching criteria or an empty host if no known host matches. + * + * Note: Uses only local data and does not go over the network. + */ + HostAndPort getMatchingHost(const ReadPreferenceSetting& criteria) const; + + /** + * Returns the Node with the given host, or NULL if no Node has that host. + */ + Node* findNode(const HostAndPort& host); + + /** + * Returns the Node with the given host, or creates one if no Node has that host. + * Maintains the sorted order of nodes. + */ + Node* findOrCreateNode(const HostAndPort& host); + + void updateNodeIfInNodes(const IsMasterReply& reply); + + std::string getServerAddress() const; + + /** + * Before unlocking, do DEV checkInvariants(); + */ + void checkInvariants() const; + + static ConfigChangeHook configChangeHook; + + boost::mutex mutex; // must hold this to access any other member or method (except name). + + // If Refresher::getNextStep returns WAIT, you should wait on the condition_variable, + // releasing mutex. It will be notified when either getNextStep will return something other + // than WAIT, or a new host is available for consideration by getMatchingHost. Essentially, + // this will be hit whenever the _refreshUntilMatches loop has the potential to make + // progress. + // TODO consider splitting cv into two: one for when looking for a master, one for all other + // cases. + boost::condition_variable cv; + + const std::string name; // safe to read outside lock since it is const + int consecutiveFailedScans; + std::set seedNodes; // updated whenever a master reports set membership changes + OID maxElectionId; // largest election id observed by this ReplicaSetMonitor + HostAndPort lastSeenMaster; // empty if we have never seen a master. can be same as current + Nodes nodes; // maintained sorted and unique by host + ScanStatePtr currentScan; // NULL if no scan in progress + int64_t latencyThresholdMicros; + mutable PseudoRandom rand; // only used for host selection to balance load + mutable int roundRobin; // used when useDeterministicHostSelection is true + }; + + struct ReplicaSetMonitor::ScanState { + MONGO_DISALLOW_COPYING(ScanState); + public: + ScanState() : foundUpMaster(false), foundAnyUpNodes(false) {} + + /** + * Adds all hosts in container that aren't in triedHosts to hostsToScan, then shuffles the + * queue. + */ + template + void enqueAllUntriedHosts(const Container& container, PseudoRandom& rand); + + // Access to fields is guarded by associated SetState's mutex. + bool foundUpMaster; + bool foundAnyUpNodes; + std::deque hostsToScan; // Work queue. + std::set possibleNodes; // Nodes reported by non-primary hosts. + std::set waitingFor; // Hosts we have dispatched but haven't replied yet. + std::set triedHosts; // Hosts that have been returned from getNextStep. + + // All responses go here until we find a master. + typedef std::vector UnconfirmedReplies; + UnconfirmedReplies unconfirmedReplies; + }; +} diff --git a/src/mongo/client/replica_set_monitor_test.cpp b/src/mongo/client/replica_set_monitor_test.cpp new file mode 100644 index 00000000000..60775087314 --- /dev/null +++ b/src/mongo/client/replica_set_monitor_test.cpp @@ -0,0 +1,964 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/client/connpool.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/client/dbclient_rs.h" +#include "mongo/client/replica_set_monitor.h" +#include "mongo/client/replica_set_monitor_internal.h" +#include "mongo/dbtests/mock/mock_conn_registry.h" +#include "mongo/dbtests/mock/mock_replica_set.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +// Pull nested types to top-level scope +typedef ReplicaSetMonitor::IsMasterReply IsMasterReply; +typedef ReplicaSetMonitor::ScanState ScanState; +typedef ReplicaSetMonitor::ScanStatePtr ScanStatePtr; +typedef ReplicaSetMonitor::SetState SetState; +typedef ReplicaSetMonitor::SetStatePtr SetStatePtr; +typedef ReplicaSetMonitor::Refresher Refresher; +typedef Refresher::NextStep NextStep; +typedef ScanState::UnconfirmedReplies UnconfirmedReplies; +typedef SetState::Node Node; +typedef SetState::Nodes Nodes; + +std::vector basicSeedsBuilder() { + std::vector out; + out.push_back(HostAndPort("a")); + out.push_back(HostAndPort("b")); + out.push_back(HostAndPort("c")); + return out; +} + +const std::vector basicSeeds = basicSeedsBuilder(); +const std::set basicSeedsSet(basicSeeds.begin(), basicSeeds.end()); + +// NOTE: Unless stated otherwise, all tests assume exclusive access to state belongs to the +// current (only) thread, so they do not lock SetState::mutex before examining state. This is +// NOT something that non-test code should do. + +TEST(ReplicaSetMonitorTests, InitialState) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + ASSERT_EQUALS(state->name, "name"); + ASSERT(state->seedNodes == basicSeedsSet); + ASSERT(state->lastSeenMaster.empty()); + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size()); + for (size_t i = 0; i < basicSeeds.size(); i++) { + Node* node = state->findNode(basicSeeds[i]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[i].toString()); + ASSERT(!node->isUp); + ASSERT(!node->isMaster); + ASSERT(node->tags.isEmpty()); + } +} + +TEST(ReplicaSetMonitorTests, IsMasterBadParse) { + BSONObj ismaster = BSON("hosts" << BSON_ARRAY("mongo.example:badport")); + IsMasterReply imr(HostAndPort("mongo.example:27017"), -1, ismaster); + ASSERT_EQUALS(imr.ok, false); +} + +TEST(ReplicaSetMonitorTests, IsMasterReplyRSNotInitiated) { + BSONObj ismaster = BSON( + "ismaster" << false + << "secondary" << false + << "info" << "can't get local.system.replset config from self or any seed (EMPTYCONFIG)" + << "isreplicaset" << true + << "maxBsonObjectSize" << 16777216 + << "maxMessageSizeBytes" << 48000000 + << "maxWriteBatchSize" << 1000 + << "localTime" << mongo::jsTime() + << "maxWireVersion" << 2 + << "minWireVersion" << 0 + << "ok" << 1 + ); + + IsMasterReply imr(HostAndPort(), -1, ismaster); + + ASSERT_EQUALS(imr.ok, true); + ASSERT_EQUALS(imr.setName, ""); + ASSERT_EQUALS(imr.hidden, false); + ASSERT_EQUALS(imr.secondary, false); + ASSERT_EQUALS(imr.isMaster, false); + ASSERT(imr.primary.empty()); + ASSERT(imr.normalHosts.empty()); + ASSERT(imr.tags.isEmpty()); +} + +TEST(ReplicaSetMonitorTests, IsMasterReplyRSPrimary) { + BSONObj ismaster = BSON( + "setName" << "test" + << "setVersion" << 1 + << "ismaster" << true + << "secondary" << false + << "hosts" << BSON_ARRAY("mongo.example:3000") + << "primary" << "mongo.example:3000" + << "me" << "mongo.example:3000" + << "maxBsonObjectSize" << 16777216 + << "maxMessageSizeBytes" << 48000000 + << "maxWriteBatchSize" << 1000 + << "localTime" << mongo::jsTime() + << "maxWireVersion" << 2 + << "minWireVersion" << 0 + << "ok" << 1 + ); + + IsMasterReply imr(HostAndPort("mongo.example:3000"), -1, ismaster); + + ASSERT_EQUALS(imr.ok, true); + ASSERT_EQUALS(imr.host.toString(), HostAndPort("mongo.example:3000").toString()); + ASSERT_EQUALS(imr.setName, "test"); + ASSERT_EQUALS(imr.hidden, false); + ASSERT_EQUALS(imr.secondary, false); + ASSERT_EQUALS(imr.isMaster, true); + ASSERT_EQUALS(imr.primary.toString(), HostAndPort("mongo.example:3000").toString()); + ASSERT(imr.normalHosts.count(HostAndPort("mongo.example:3000"))); + ASSERT(imr.tags.isEmpty()); +} + +TEST(ReplicaSetMonitorTests, IsMasterReplyPassiveSecondary) { + BSONObj ismaster = BSON( + "setName" << "test" + << "setVersion" << 1 + << "ismaster" << false + << "secondary" << true + << "hosts" << BSON_ARRAY("mongo.example:3000") + << "passives" << BSON_ARRAY("mongo.example:3001") + << "primary" << "mongo.example:3000" + << "passive" << true + << "me" << "mongo.example:3001" + << "maxBsonObjectSize" << 16777216 + << "maxMessageSizeBytes" << 48000000 + << "maxWriteBatchSize" << 1000 + << "localTime" << mongo::jsTime() + << "maxWireVersion" << 2 + << "minWireVersion" << 0 + << "ok" << 1 + ); + + IsMasterReply imr(HostAndPort("mongo.example:3001"), -1, ismaster); + + ASSERT_EQUALS(imr.ok, true); + ASSERT_EQUALS(imr.host.toString(), HostAndPort("mongo.example:3001").toString()); + ASSERT_EQUALS(imr.setName, "test"); + ASSERT_EQUALS(imr.hidden, false); + ASSERT_EQUALS(imr.secondary, true); + ASSERT_EQUALS(imr.isMaster, false); + ASSERT_EQUALS(imr.primary.toString(), HostAndPort("mongo.example:3000").toString()); + ASSERT(imr.normalHosts.count(HostAndPort("mongo.example:3000"))); + ASSERT(imr.normalHosts.count(HostAndPort("mongo.example:3001"))); + ASSERT(imr.tags.isEmpty()); +} + +TEST(ReplicaSetMonitorTests, IsMasterReplyHiddenSecondary) { + BSONObj ismaster = BSON( + "setName" << "test" + << "setVersion" << 1 + << "ismaster" << false + << "secondary" << true + << "hosts" << BSON_ARRAY("mongo.example:3000") + << "primary" << "mongo.example:3000" + << "passive" << true + << "hidden" << true + << "me" << "mongo.example:3001" + << "maxBsonObjectSize" << 16777216 + << "maxMessageSizeBytes" << 48000000 + << "maxWriteBatchSize" << 1000 + << "localTime" << mongo::jsTime() + << "maxWireVersion" << 2 + << "minWireVersion" << 0 + << "ok" << 1 + ); + + IsMasterReply imr(HostAndPort("mongo.example:3001"), -1, ismaster); + + ASSERT_EQUALS(imr.ok, true); + ASSERT_EQUALS(imr.host.toString(), HostAndPort("mongo.example:3001").toString()); + ASSERT_EQUALS(imr.setName, "test"); + ASSERT_EQUALS(imr.hidden, true); + ASSERT_EQUALS(imr.secondary, true); + ASSERT_EQUALS(imr.isMaster, false); + ASSERT_EQUALS(imr.primary.toString(), HostAndPort("mongo.example:3000").toString()); + ASSERT(imr.normalHosts.count(HostAndPort("mongo.example:3000"))); + ASSERT(imr.tags.isEmpty()); +} + +TEST(ReplicaSetMonitorTests, IsMasterSecondaryWithTags) { + BSONObj ismaster = BSON( + "setName" << "test" + << "setVersion" << 1 + << "ismaster" << false + << "secondary" << true + << "hosts" << BSON_ARRAY("mongo.example:3000" << "mongo.example:3001") + << "primary" << "mongo.example:3000" + << "me" << "mongo.example:3001" + << "maxBsonObjectSize" << 16777216 + << "maxMessageSizeBytes" << 48000000 + << "maxWriteBatchSize" << 1000 + << "localTime" << mongo::jsTime() + << "maxWireVersion" << 2 + << "minWireVersion" << 0 + << "tags" << BSON("dc" << "nyc" << "use" << "production") + << "ok" << 1 + ); + + IsMasterReply imr(HostAndPort("mongo.example:3001"), -1, ismaster); + + ASSERT_EQUALS(imr.ok, true); + ASSERT_EQUALS(imr.host.toString(), HostAndPort("mongo.example:3001").toString()); + ASSERT_EQUALS(imr.setName, "test"); + ASSERT_EQUALS(imr.hidden, false); + ASSERT_EQUALS(imr.secondary, true); + ASSERT_EQUALS(imr.isMaster, false); + ASSERT_EQUALS(imr.primary.toString(), HostAndPort("mongo.example:3000").toString()); + ASSERT(imr.normalHosts.count(HostAndPort("mongo.example:3000"))); + ASSERT(imr.normalHosts.count(HostAndPort("mongo.example:3001"))); + ASSERT(imr.tags.hasElement("dc")); + ASSERT(imr.tags.hasElement("use")); + ASSERT_EQUALS(imr.tags["dc"].str(), "nyc"); + ASSERT_EQUALS(imr.tags["use"].str(), "production"); +} + +TEST(ReplicaSetMonitorTests, CheckAllSeedsSerial) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + for (size_t i = 0; i < basicSeeds.size(); i++) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + // mock a reply + bool primary = ns.host.host() == "a"; + refresher.receivedIsMaster(ns.host, -1, BSON( + "setName" << "name" + << "ismaster" << primary + << "secondary" << !primary + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true + )); + } + + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); + + // validate final state + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size()); + for (size_t i = 0; i < basicSeeds.size(); i++) { + Node* node = state->findNode(basicSeeds[i]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[i].toString()); + ASSERT(node->isUp); + ASSERT_EQUALS(node->isMaster, node->host.host() == "a"); + ASSERT(node->tags.isEmpty()); + } +} + +TEST(ReplicaSetMonitorTests, CheckAllSeedsParallel) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + // get all hosts to contact first + for (size_t i = 0; i < basicSeeds.size(); i++) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + } + + + // mock all replies + for (size_t i = 0; i < basicSeeds.size(); i++) { + // All hosts to talk to are already dispatched, but no reply has been received + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::WAIT); + ASSERT(ns.host.empty()); + + bool primary = i == 0; + refresher.receivedIsMaster(basicSeeds[i], -1, BSON( + "setName" << "name" + << "ismaster" << primary + << "secondary" << !primary + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true + )); + } + + // Now all hosts have returned data + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); + + // validate final state + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size()); + for (size_t i = 0; i < basicSeeds.size(); i++) { + Node* node = state->findNode(basicSeeds[i]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[i].toString()); + ASSERT(node->isUp); + ASSERT_EQUALS(node->isMaster, i == 0); + ASSERT(node->tags.isEmpty()); + } +} + +TEST(ReplicaSetMonitorTests, NoMasterInitAllUp) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + for (size_t i = 0; i < basicSeeds.size(); i++) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + // mock a reply + refresher.receivedIsMaster(ns.host, -1, BSON( + "setName" << "name" + << "ismaster" << false + << "secondary" << true + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true + )); + } + + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); + + // validate final state + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size()); + for (size_t i = 0; i < basicSeeds.size(); i++) { + Node* node = state->findNode(basicSeeds[i]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[i].toString()); + ASSERT(node->isUp); + ASSERT_EQUALS(node->isMaster, false); + ASSERT(node->tags.isEmpty()); + } +} + +TEST(ReplicaSetMonitorTests, MasterNotInSeeds_NoPrimaryInIsMaster) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + for (size_t i = 0; i < basicSeeds.size(); i++) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + // mock a reply + refresher.receivedIsMaster(ns.host, -1, BSON( + "setName" << "name" + << "ismaster" << false + << "secondary" << true + << "hosts" << BSON_ARRAY("a" << "b" << "c" << "d") + << "ok" << true + )); + } + + // Only look at "d" after exhausting all other hosts + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT_EQUALS(ns.host.host(), "d"); + refresher.receivedIsMaster(ns.host, -1, BSON( + "setName" << "name" + << "ismaster" << true + << "secondary" << false + << "hosts" << BSON_ARRAY("a" << "b" << "c" << "d") + << "ok" << true + )); + + + ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); + + // validate final state + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size() + 1); + for (size_t i = 0; i < basicSeeds.size(); i++) { + Node* node = state->findNode(basicSeeds[i]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[i].toString()); + ASSERT(node->isUp); + ASSERT_EQUALS(node->isMaster, false); + ASSERT(node->tags.isEmpty()); + } + + Node* node = state->findNode(HostAndPort("d")); + ASSERT(node); + ASSERT_EQUALS(node->host.host(), "d"); + ASSERT(node->isUp); + ASSERT_EQUALS(node->isMaster, true); + ASSERT(node->tags.isEmpty()); +} + +TEST(ReplicaSetMonitorTests, MasterNotInSeeds_PrimaryInIsMaster) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + for (size_t i = 0; i < basicSeeds.size() + 1; i++) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + if (i == 1) // d should be the second host we contact since we are told it is primary + ASSERT_EQUALS(ns.host.host(), "d"); + else + ASSERT(basicSeedsSet.count(ns.host)); + + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + // mock a reply + bool primary = ns.host.host() == "d"; + refresher.receivedIsMaster(ns.host, -1, BSON( + "setName" << "name" + << "ismaster" << primary + << "secondary" << !primary + << "primary" << "d" + << "hosts" << BSON_ARRAY("a" << "b" << "c" << "d") + << "ok" << true + )); + } + + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); + + // validate final state + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size() + 1); + for (size_t i = 0; i < basicSeeds.size(); i++) { + Node* node = state->findNode(basicSeeds[i]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[i].toString()); + ASSERT(node->isUp); + ASSERT_EQUALS(node->isMaster, false); + ASSERT(node->tags.isEmpty()); + } + + Node* node = state->findNode(HostAndPort("d")); + ASSERT(node); + ASSERT_EQUALS(node->host.host(), "d"); + ASSERT(node->isUp); + ASSERT_EQUALS(node->isMaster, true); + ASSERT(node->tags.isEmpty()); +} + +// Make sure we can use slaves we find even if we can't find a primary +TEST(ReplicaSetMonitorTests, SlavesUsableEvenIfNoMaster) { + std::set seeds; + seeds.insert(HostAndPort("a")); + SetStatePtr state = boost::make_shared("name", seeds); + Refresher refresher(state); + + const ReadPreferenceSetting secondary(ReadPreference_SecondaryOnly, TagSet()); + + // Mock a reply from the only host we know about and have it claim to not be master or know + // about any other hosts. This leaves the scan with no more hosts to scan, but all hosts are + // still marked as down since we never contacted a master. The next call to + // Refresher::getNextStep will apply all unconfimedReplies and return DONE. + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT_EQUALS(ns.host.host(), "a"); + refresher.receivedIsMaster(ns.host, -1, BSON( + "setName" << "name" + << "ismaster" << false + << "secondary" << true + << "hosts" << BSON_ARRAY("a") + << "ok" << true + )); + + // Check intended conditions for entry to refreshUntilMatches. + ASSERT(state->currentScan->hostsToScan.empty()); + ASSERT(state->currentScan->waitingFor.empty()); + ASSERT(state->currentScan->possibleNodes == state->currentScan->triedHosts); + ASSERT(state->getMatchingHost(secondary).empty()); + + // This calls getNextStep after not finding a matching host. We want to ensure that it checks + // again after being told that there are no more hosts to contact. + ASSERT(!refresher.refreshUntilMatches(secondary).empty()); + + // Future calls should be able to return directly from the cached data. + ASSERT(!state->getMatchingHost(secondary).empty()); +} + +// Test multiple nodes that claim to be master (we use a last-wins policy) +TEST(ReplicaSetMonitorTests, MultipleMasterLastNodeWins) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + // get all hosts to contact first + for (size_t i = 0; i != basicSeeds.size(); ++i) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + } + + const ReadPreferenceSetting primaryOnly(ReadPreference_PrimaryOnly, TagSet()); + + // mock all replies + for (size_t i = 0; i != basicSeeds.size(); ++i) { + // All hosts to talk to are already dispatched, but no reply has been received + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::WAIT); + ASSERT(ns.host.empty()); + + refresher.receivedIsMaster(basicSeeds[i], -1, BSON( + "setName" << "name" + << "ismaster" << true + << "secondary" << false + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true + )); + + // Ensure the set primary is the host we just got a reply from + HostAndPort currentPrimary = state->getMatchingHost(primaryOnly); + ASSERT_EQUALS(currentPrimary.host(), basicSeeds[i].host()); + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size()); + + // Check the state of each individual node + for (size_t j = 0; j != basicSeeds.size(); ++j) { + Node* node = state->findNode(basicSeeds[j]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[j].toString()); + ASSERT_EQUALS(node->isUp, j <= i); + ASSERT_EQUALS(node->isMaster, j == i); + ASSERT(node->tags.isEmpty()); + } + } + + // Now all hosts have returned data + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); +} + +// Test nodes disagree about who is in the set, master is source of truth +TEST(ReplicaSetMonitorTests, MasterIsSourceOfTruth) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + BSONArray primaryHosts = BSON_ARRAY("a" << "b" << "d"); + BSONArray secondaryHosts = BSON_ARRAY("a" << "b" << "c"); + + // mock all replies + NextStep ns = refresher.getNextStep(); + while (ns.step == NextStep::CONTACT_HOST) { + bool primary = ns.host.host() == "a"; + refresher.receivedIsMaster(ns.host, -1, BSON( + "setName" << "name" + << "ismaster" << primary + << "secondary" << !primary + << "hosts" << (primary ? primaryHosts : secondaryHosts) + << "ok" << true + )); + + ns = refresher.getNextStep(); + } + + // Ensure that we have heard from all hosts and scan is done + ASSERT_EQUALS(ns.step, NextStep::DONE); + + // Ensure that d is in the set but c is not + ASSERT(state->findNode(HostAndPort("d"))); + ASSERT(!state->findNode(HostAndPort("c"))); +} + +// Test multiple master nodes that disagree about set membership +TEST(ReplicaSetMonitorTests, MultipleMastersDisagree) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + BSONArray hostsForSeed[3]; + hostsForSeed[0] = BSON_ARRAY("a" << "b" << "c" << "d"); + hostsForSeed[1] = BSON_ARRAY("a" << "b" << "c" << "e"); + hostsForSeed[2] = hostsForSeed[0]; + + set seen; + + for (size_t i = 0; i != basicSeeds.size(); ++i) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + } + + const ReadPreferenceSetting primaryOnly(ReadPreference_PrimaryOnly, TagSet()); + + // mock all replies + for (size_t i = 0; i != basicSeeds.size(); ++i) { + refresher.receivedIsMaster(basicSeeds[i], -1, BSON( + "setName" << "name" + << "ismaster" << true + << "secondary" << false + << "hosts" << hostsForSeed[i % 2] + << "ok" << true + )); + + // Ensure the primary is the host we just got a reply from + HostAndPort currentPrimary = state->getMatchingHost(primaryOnly); + ASSERT_EQUALS(currentPrimary.host(), basicSeeds[i].host()); + + // Ensure each primary discovered becomes source of truth + if (i == 1) { + // "b" thinks node "e" is a member but "d" is not + ASSERT(state->findNode(HostAndPort("e"))); + ASSERT(!state->findNode(HostAndPort("d"))); + } + else { + // "a" and "c" think node "d" is a member but "e" is not + ASSERT(state->findNode(HostAndPort("d"))); + ASSERT(!state->findNode(HostAndPort("e"))); + } + } + + // next step should be to contact "d" + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT_EQUALS(ns.host.host(), "d"); + seen.insert(ns.host); + + // reply from "d" + refresher.receivedIsMaster(HostAndPort("d"), -1, BSON( + "setName" << "name" + << "ismaster" << false + << "secondary" << true + << "hosts" << hostsForSeed[0] + << "ok" << true + )); + + // scan should be complete + ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); + + // Validate final state (only "c" should be master and "d" was added) + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size() + 1); + + std::vector nodes = state->nodes; + for (std::vector::const_iterator it = nodes.begin(); it != nodes.end(); ++it) { + const Node& node = *it; + ASSERT(node.isUp); + ASSERT_EQUALS(node.isMaster, node.host.host() == "c"); + ASSERT(seen.count(node.host)); + } +} + +// Ensure getMatchingHost returns hosts even if scan is ongoing +TEST(ReplicaSetMonitorTests, GetMatchingDuringScan) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + const ReadPreferenceSetting primaryOnly(ReadPreference_PrimaryOnly, TagSet()); + const ReadPreferenceSetting secondaryOnly(ReadPreference_SecondaryOnly, TagSet()); + + for (std::vector::const_iterator it = basicSeeds.begin(); it != basicSeeds.end(); + ++it) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(state->getMatchingHost(primaryOnly).empty()); + ASSERT(state->getMatchingHost(secondaryOnly).empty()); + } + + // mock replies and validate set state as replies come back + for (size_t i = 0; i != basicSeeds.size(); ++i) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::WAIT); + ASSERT(ns.host.empty()); + + bool primary = (i == 1); + refresher.receivedIsMaster(basicSeeds[i], -1, BSON( + "setName" << "name" + << "ismaster" << primary + << "secondary" << !primary + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true + )); + + bool hasPrimary = !(state->getMatchingHost(primaryOnly).empty()); + bool hasSecondary = !(state->getMatchingHost(secondaryOnly).empty()); + + // secondary node has not been confirmed by primary until i == 1 + if (i >= 1) { + ASSERT(hasPrimary); + ASSERT(hasSecondary); + } + else { + ASSERT(!hasPrimary); + ASSERT(!hasSecondary); + } + } + + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); +} + +// Ensure nothing breaks when out-of-band failedHost is called during scan +TEST(ReplicaSetMonitorTests, OutOfBandFailedHost) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + ReplicaSetMonitorPtr rsm = boost::make_shared(state); + Refresher refresher = rsm->startOrContinueRefresh(); + + for (size_t i = 0; i != basicSeeds.size(); ++i) { + NextStep ns = refresher.getNextStep(); + } + + for (size_t i = 0; i != basicSeeds.size(); ++i) { + bool primary = (i == 0); + + refresher.receivedIsMaster(basicSeeds[i], -1, BSON( + "setName" << "name" + << "ismaster" << primary + << "secondary" << !primary + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true + )); + + if (i >= 1) { + HostAndPort a("a"); + rsm->failedHost(a); + Node* node = state->findNode(a); + ASSERT(node); + ASSERT(!node->isUp); + ASSERT(!node->isMaster); + } + else { + Node* node = state->findNode(HostAndPort("a")); + ASSERT(node); + ASSERT(node->isUp); + ASSERT(node->isMaster); + } + } +} + +// Newly elected primary with electionId >= maximum electionId seen by the Refresher +TEST(ReplicaSetMonitorTests, NewPrimaryWithMaxElectionId) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + // get all hosts to contact first + for (size_t i = 0; i != basicSeeds.size(); ++i) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + } + + const ReadPreferenceSetting primaryOnly(ReadPreference_PrimaryOnly, TagSet()); + + // mock all replies + for (size_t i = 0; i != basicSeeds.size(); ++i) { + // All hosts to talk to are already dispatched, but no reply has been received + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::WAIT); + ASSERT(ns.host.empty()); + + refresher.receivedIsMaster(basicSeeds[i], + -1, + BSON("setName" << "name" + << "ismaster" << true + << "secondary" << false + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "electionId" << OID::gen() + << "ok" << true)); + + // Ensure the set primary is the host we just got a reply from + HostAndPort currentPrimary = state->getMatchingHost(primaryOnly); + ASSERT_EQUALS(currentPrimary.host(), basicSeeds[i].host()); + ASSERT_EQUALS(state->nodes.size(), basicSeeds.size()); + + // Check the state of each individual node + for (size_t j = 0; j != basicSeeds.size(); ++j) { + Node* node = state->findNode(basicSeeds[j]); + ASSERT(node); + ASSERT_EQUALS(node->host.toString(), basicSeeds[j].toString()); + ASSERT_EQUALS(node->isUp, j <= i); + ASSERT_EQUALS(node->isMaster, j == i); + ASSERT(node->tags.isEmpty()); + } + } + + // Now all hosts have returned data + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); +} + +// Ignore electionId of secondaries +TEST(ReplicaSetMonitorTests, IgnoreElectionIdFromSecondaries) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + set seen; + + const OID primaryElectionId = OID::gen(); + + // mock all replies + for (size_t i = 0; i != basicSeeds.size(); ++i) { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + // mock a reply + const bool primary = ns.host.host() == "a"; + refresher.receivedIsMaster(ns.host, + -1, + BSON("setName" << "name" + << "ismaster" << primary + << "secondary" << !primary + << "electionId" << (primary ? + primaryElectionId : OID::gen()) + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true)); + } + + // check that the SetState's maxElectionId == primary's electionId + ASSERT_EQUALS(state->maxElectionId, primaryElectionId); + + // Now all hosts have returned data + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); +} + +// Stale Primary with obsolete electionId +TEST(ReplicaSetMonitorTests, StalePrimaryWithObsoleteElectionId) { + SetStatePtr state = boost::make_shared("name", basicSeedsSet); + Refresher refresher(state); + + const OID firstElectionId = OID::gen(); + const OID secondElectionId = OID::gen(); + + set seen; + + // contact first host claiming to be primary with greater electionId + { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + refresher.receivedIsMaster(ns.host, + -1, + BSON("setName" << "name" + << "ismaster" << true + << "secondary" << false + << "electionId" << secondElectionId + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true)); + + Node* node = state->findNode(ns.host); + ASSERT(node); + ASSERT_TRUE(node->isMaster); + ASSERT_EQUALS(state->maxElectionId, secondElectionId); + } + + // contact second host claiming to be primary with smaller electionId + { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + refresher.receivedIsMaster(ns.host, + -1, + BSON("setName" << "name" + << "ismaster" << true + << "secondary" << false + << "electionId" << firstElectionId + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true)); + + Node* node = state->findNode(ns.host); + ASSERT(node); + // The SetState shouldn't see this host as master + ASSERT_FALSE(node->isMaster); + // the max electionId should remain the same + ASSERT_EQUALS(state->maxElectionId, secondElectionId); + } + + // third host is a secondary + { + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST); + ASSERT(basicSeedsSet.count(ns.host)); + ASSERT(!seen.count(ns.host)); + seen.insert(ns.host); + + refresher.receivedIsMaster(ns.host, + -1, + BSON("setName" << "name" + << "ismaster" << false + << "secondary" << true + << "hosts" << BSON_ARRAY("a" << "b" << "c") + << "ok" << true)); + + Node* node = state->findNode(ns.host); + ASSERT(node); + ASSERT_FALSE(node->isMaster); + // the max electionId should remain the same + ASSERT_EQUALS(state->maxElectionId, secondElectionId); + } + + // Now all hosts have returned data + NextStep ns = refresher.getNextStep(); + ASSERT_EQUALS(ns.step, NextStep::DONE); + ASSERT(ns.host.empty()); +} diff --git a/src/mongo/client/sasl_client_authenticate.cpp b/src/mongo/client/sasl_client_authenticate.cpp index 24cd5000fd7..250e40f746d 100644 --- a/src/mongo/client/sasl_client_authenticate.cpp +++ b/src/mongo/client/sasl_client_authenticate.cpp @@ -40,8 +40,8 @@ namespace mongo { const char* const saslCommandMechanismListFieldName = "supportedMechanisms"; const char* const saslCommandPasswordFieldName = "pwd"; const char* const saslCommandPayloadFieldName = "payload"; - const char* const saslCommandPrincipalFieldName = "user"; - const char* const saslCommandPrincipalSourceFieldName = "userSource"; + const char* const saslCommandUserDBFieldName = "db"; + const char* const saslCommandUserFieldName = "user"; const char* const saslCommandServiceHostnameFieldName = "serviceHostname"; const char* const saslCommandServiceNameFieldName = "serviceName"; const char* const saslCommandDigestPasswordFieldName = "digestPassword"; diff --git a/src/mongo/client/sasl_client_authenticate.h b/src/mongo/client/sasl_client_authenticate.h index c95cdc42660..3bc73f41672 100644 --- a/src/mongo/client/sasl_client_authenticate.h +++ b/src/mongo/client/sasl_client_authenticate.h @@ -18,6 +18,7 @@ #include "mongo/base/status.h" #include "mongo/bson/bsontypes.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/client/export_macros.h" namespace mongo { class BSONObj; @@ -40,9 +41,9 @@ namespace mongo { * "autoAuthorize": Truthy values tell the server to automatically acquire privileges on * all resources after successful authentication, which is the default. Falsey values * instruct the server to await separate privilege-acquisition commands. - * "user": The string name of the principal to authenticate. - * "userSource": The database target of the auth command, which identifies the location - * of the credential information for the principal. May be "$external" if credential + * "user": The string name of the user to authenticate. + * "db": The database target of the auth command, which identifies the location + * of the credential information for the user. May be "$external" if credential * information is stored outside of the mongo cluster. * "pwd": The password. * "serviceName": The GSSAPI service name to use. Defaults to "mongodb". @@ -54,7 +55,7 @@ namespace mongo { * rejected. Other failures, all of which are tantamount to authentication failure, may also be * returned. */ - extern Status (*saslClientAuthenticate)(DBClientWithCommands* client, + extern MONGO_CLIENT_API Status (*saslClientAuthenticate)(DBClientWithCommands* client, const BSONObj& saslParameters); /** @@ -66,69 +67,69 @@ namespace mongo { * stores into "*payload". If the type is BinData, the contents are stored directly * into "*payload". In all other cases, returns */ - Status saslExtractPayload(const BSONObj& cmdObj, std::string* payload, BSONType* type); + Status MONGO_CLIENT_API saslExtractPayload(const BSONObj& cmdObj, std::string* payload, BSONType* type); // Constants /// String name of the saslStart command. - extern const char* const saslStartCommandName; + extern MONGO_CLIENT_API const char* const saslStartCommandName; /// String name of the saslContinue command. - extern const char* const saslContinueCommandName; + extern MONGO_CLIENT_API const char* const saslContinueCommandName; /// Name of the saslStart parameter indicating that the server should automatically grant the - /// connection all privileges associated with the principal after successful authentication. - extern const char* const saslCommandAutoAuthorizeFieldName; + /// connection all privileges associated with the user after successful authentication. + extern MONGO_CLIENT_API const char* const saslCommandAutoAuthorizeFieldName; /// Name of the field contain the status code in responses from the server. - extern const char* const saslCommandCodeFieldName; + extern MONGO_CLIENT_API const char* const saslCommandCodeFieldName; /// Name of the field containing the conversation identifier in server respones and saslContinue /// commands. - extern const char* const saslCommandConversationIdFieldName; + extern MONGO_CLIENT_API const char* const saslCommandConversationIdFieldName; /// Name of the field that indicates whether or not the server believes authentication has /// completed successfully. - extern const char* const saslCommandDoneFieldName; + extern MONGO_CLIENT_API const char* const saslCommandDoneFieldName; /// Field in which to store error messages associated with non-success return codes. - extern const char* const saslCommandErrmsgFieldName; + extern MONGO_CLIENT_API const char* const saslCommandErrmsgFieldName; /// Name of parameter to saslStart command indiciating the client's desired sasl mechanism. - extern const char* const saslCommandMechanismFieldName; + extern MONGO_CLIENT_API const char* const saslCommandMechanismFieldName; /// In the event that saslStart supplies an unsupported mechanism, the server responds with a /// field by this name, with a list of supported mechanisms. - extern const char* const saslCommandMechanismListFieldName; + extern MONGO_CLIENT_API const char* const saslCommandMechanismListFieldName; /// Field containing password information for saslClientAuthenticate(). - extern const char* const saslCommandPasswordFieldName; + extern MONGO_CLIENT_API const char* const saslCommandPasswordFieldName; /// Field containing sasl payloads passed to and from the server. - extern const char* const saslCommandPayloadFieldName; + extern MONGO_CLIENT_API const char* const saslCommandPayloadFieldName; - /// Field containing the string identifier of the principal to authenticate in + /// Field containing the string identifier of the user to authenticate in /// saslClientAuthenticate(). - extern const char* const saslCommandPrincipalFieldName; + extern MONGO_CLIENT_API const char* const saslCommandUserFieldName; /// Field containing the string identifier of the database containing credential information, /// or "$external" if the credential information is stored outside of the mongo cluster. - extern const char* const saslCommandPrincipalSourceFieldName; + extern MONGO_CLIENT_API const char* const saslCommandUserDBFieldName; /// Field overriding the FQDN of the hostname hosting the mongodb srevice in /// saslClientAuthenticate(). - extern const char* const saslCommandServiceHostnameFieldName; + extern MONGO_CLIENT_API const char* const saslCommandServiceHostnameFieldName; /// Field overriding the name of the mongodb service saslClientAuthenticate(). - extern const char* const saslCommandServiceNameFieldName; + extern MONGO_CLIENT_API const char* const saslCommandServiceNameFieldName; /// Default database against which sasl authentication commands should run. - extern const char* const saslDefaultDBName; + extern MONGO_CLIENT_API const char* const saslDefaultDBName; /// Default sasl service name, "mongodb". - extern const char* const saslDefaultServiceName; + extern MONGO_CLIENT_API const char* const saslDefaultServiceName; // Field whose value should be set to true if the field in saslCommandPasswordFieldName needs to // be digested. - extern const char* const saslCommandDigestPasswordFieldName; + extern MONGO_CLIENT_API const char* const saslCommandDigestPasswordFieldName; } diff --git a/src/mongo/client/sasl_client_authenticate_impl.cpp b/src/mongo/client/sasl_client_authenticate_impl.cpp index 30ee4bc9b40..87393688380 100644 --- a/src/mongo/client/sasl_client_authenticate_impl.cpp +++ b/src/mongo/client/sasl_client_authenticate_impl.cpp @@ -34,6 +34,7 @@ #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/net/hostandport.h" +#include "mongo/util/password_digest.h" namespace mongo { namespace { @@ -56,16 +57,16 @@ namespace { /** * Gets the password data from "saslParameters" and stores it to "outPassword". * - * If "saslParameters" indicates that the password needs to be "digested" via - * DBClientWithCommands::createPasswordDigest(), this method takes care of that. + * If "digestPassword" indicates that the password needs to be "digested" via + * mongo::createPasswordDigest(), this method takes care of that. * On success, the value of "*outPassword" is always the correct value to set * as the password on the SaslClientSession. * * Returns Status::OK() on success, and ErrorCodes::NoSuchKey if the password data is not * present in "saslParameters". Other ErrorCodes returned indicate other errors. */ - Status extractPassword(DBClientWithCommands* client, - const BSONObj& saslParameters, + Status extractPassword(const BSONObj& saslParameters, + bool digestPassword, std::string* outPassword) { std::string rawPassword; @@ -75,23 +76,15 @@ namespace { if (!status.isOK()) return status; - bool digest; - status = bsonExtractBooleanFieldWithDefault(saslParameters, - saslCommandDigestPasswordFieldName, - true, - &digest); - if (!status.isOK()) - return status; - - if (digest) { + if (digestPassword) { std::string user; status = bsonExtractStringField(saslParameters, - saslCommandPrincipalFieldName, + saslCommandUserFieldName, &user); if (!status.isOK()) return status; - *outPassword = client->createPasswordDigest(user, rawPassword); + *outPassword = mongo::createPasswordDigest(user, rawPassword); } else { *outPassword = rawPassword; @@ -109,16 +102,18 @@ namespace { */ Status configureSession(SaslClientSession* session, DBClientWithCommands* client, + const std::string& targetDatabase, const BSONObj& saslParameters) { - std::string value; + std::string mechanism; Status status = bsonExtractStringField(saslParameters, saslCommandMechanismFieldName, - &value); + &mechanism); if (!status.isOK()) return status; - session->setParameter(SaslClientSession::parameterMechanism, value); + session->setParameter(SaslClientSession::parameterMechanism, mechanism); + std::string value; status = bsonExtractStringFieldWithDefault(saslParameters, saslCommandServiceNameFieldName, saslDefaultServiceName, @@ -136,13 +131,24 @@ namespace { session->setParameter(SaslClientSession::parameterServiceHostname, value); status = bsonExtractStringField(saslParameters, - saslCommandPrincipalFieldName, + saslCommandUserFieldName, &value); if (!status.isOK()) return status; session->setParameter(SaslClientSession::parameterUser, value); - status = extractPassword(client, saslParameters, &value); + bool digestPasswordDefault = + !(targetDatabase == "$external" && mechanism == "PLAIN") && + !(targetDatabase == "$external" && mechanism == "GSSAPI"); + bool digestPassword; + status = bsonExtractBooleanFieldWithDefault(saslParameters, + saslCommandDigestPasswordFieldName, + digestPasswordDefault, + &digestPassword); + if (!status.isOK()) + return status; + + status = extractPassword(saslParameters, digestPassword, &value); if (status.isOK()) { session->setParameter(SaslClientSession::parameterPassword, value); } @@ -161,20 +167,20 @@ namespace { int saslLogLevel = getSaslClientLogLevel(saslParameters); - SaslClientSession session; - Status status = configureSession(&session, client, saslParameters); - if (!status.isOK()) - return status; - std::string targetDatabase; try { - status = bsonExtractStringFieldWithDefault(saslParameters, - saslCommandPrincipalSourceFieldName, - saslDefaultDBName, - &targetDatabase); + Status status = bsonExtractStringFieldWithDefault(saslParameters, + saslCommandUserDBFieldName, + saslDefaultDBName, + &targetDatabase); + if (!status.isOK()) + return status; } catch (const DBException& ex) { return ex.toStatus(); } + + SaslClientSession session; + Status status = configureSession(&session, client, targetDatabase, saslParameters); if (!status.isOK()) return status; diff --git a/src/mongo/client/sasl_client_session.cpp b/src/mongo/client/sasl_client_session.cpp index 24c1f343ed9..0ec7e34e407 100644 --- a/src/mongo/client/sasl_client_session.cpp +++ b/src/mongo/client/sasl_client_session.cpp @@ -29,17 +29,26 @@ namespace { * doesn't initialize the library for us. */ - void* saslOurMalloc(unsigned long sz) { +// Version 2.1.26 is the first version to use size_t in the allocator signatures +#if (SASL_VERSION_FULL >= ((2 << 16) | (1 << 8) | 26)) + typedef size_t SaslAllocSize; +#else + typedef unsigned long SaslAllocSize; +#endif + + typedef int(*SaslCallbackFn)(); + + void* saslOurMalloc(SaslAllocSize sz) { return ourmalloc(sz); } - void* saslOurCalloc(unsigned long count, unsigned long size) { + void* saslOurCalloc(SaslAllocSize count, SaslAllocSize size) { void* ptr = calloc(count, size); - if (!ptr) printStackAndExit(0); + if (!ptr) abort(); return ptr; } - void* saslOurRealloc(void* ptr, unsigned long sz) { + void* saslOurRealloc(void* ptr, SaslAllocSize sz) { return ourrealloc(ptr, sz); } @@ -83,6 +92,10 @@ namespace { return Status::OK(); } + int saslClientLogSwallow(void *context, int priority, const char *message) { + return SASL_OK; // do nothing + } + /** * Initializes the client half of the SASL library, but is effectively a no-op if the client * application has already done it. @@ -97,7 +110,9 @@ namespace { MONGO_INITIALIZER_WITH_PREREQUISITES(SaslClientContext, ("CyrusSaslAllocatorsAndMutexes"))( InitializerContext* context) { - static sasl_callback_t saslClientGlobalCallbacks[] = { { SASL_CB_LIST_END } }; + static sasl_callback_t saslClientGlobalCallbacks[] = + { { SASL_CB_LOG, SaslCallbackFn(saslClientLogSwallow), NULL /* context */ }, + { SASL_CB_LIST_END } }; // If the client application has previously called sasl_client_init(), the callbacks passed // in here are ignored. @@ -178,8 +193,6 @@ namespace { _step(0), _done(false) { - typedef int(*SaslCallbackFn)(); - const sasl_callback_t callbackTemplate[maxCallbacks] = { { SASL_CB_AUTHNAME, SaslCallbackFn(saslClientGetSimple), this }, { SASL_CB_USER, SaslCallbackFn(saslClientGetSimple), this }, @@ -220,7 +233,7 @@ namespace { bool SaslClientSession::hasParameter(Parameter id) { if (id < 0 || id >= numParameters) return false; - return _parameters[id].data; + return static_cast(_parameters[id].data); } StringData SaslClientSession::getParameter(Parameter id) { diff --git a/src/mongo/client/sasl_client_session.h b/src/mongo/client/sasl_client_session.h index 9316b7c84a1..776a3c02224 100644 --- a/src/mongo/client/sasl_client_session.h +++ b/src/mongo/client/sasl_client_session.h @@ -21,6 +21,7 @@ #include "mongo/base/disallow_copying.h" #include "mongo/base/status.h" #include "mongo/base/string_data.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -37,7 +38,7 @@ namespace mongo { * parameters must be UTF-8 encoded strings with no embedded NUL characters. The * parameterPassword parameter is not constrained. */ - class SaslClientSession { + class MONGO_CLIENT_API SaslClientSession { MONGO_DISALLOW_COPYING(SaslClientSession); public: /** diff --git a/src/mongo/client/sasl_sspi.cpp b/src/mongo/client/sasl_sspi.cpp new file mode 100644 index 00000000000..537a52aebe6 --- /dev/null +++ b/src/mongo/client/sasl_sspi.cpp @@ -0,0 +1,529 @@ +/* + * Copyright (C) 2013 10gen, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef _WIN32 + +#define SECURITY_WIN32 1 // Required for SSPI support. + +#include "mongo/platform/basic.h" + +#include +#include +#include +#include + +#include "mongo/base/init.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/base/status.h" +#include "mongo/util/scopeguard.h" +#include "mongo/util/text.h" + +extern "C" int plain_client_plug_init(const sasl_utils_t *utils, + int maxversion, + int *out_version, + sasl_client_plug_t **pluglist, + int *plugcount); + +extern "C" int crammd5_client_plug_init(const sasl_utils_t *utils, + int maxversion, + int *out_version, + sasl_client_plug_t **pluglist, + int *plugcount); + +namespace mongo { +namespace { + /* + * SSPI client plugin impl + */ + + // The SSPI plugin implements the GSSAPI interface. + char sspiPluginName[] = "GSSAPI"; + + // This structure is passed through each callback to us by the sasl glue code. + struct SspiConnContext { + CredHandle cred; + bool haveCred; + CtxtHandle ctx; + bool haveCtxt; + bool authComplete; + std::wstring nameToken; + std::string userPlusRealm; + + SspiConnContext() : + haveCred(false), + haveCtxt(false), + authComplete(false) + {} + ~SspiConnContext() { + if (haveCtxt) { + DeleteSecurityContext(&ctx); + } + if (haveCred) { + FreeCredentialsHandle(&cred); + } + } + }; + + // Utility function for fetching error text from Windows API function calls. + void HandleLastError(const sasl_utils_t* utils, DWORD errCode, const char* msg) { + char *err; + if (!FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + errCode, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR) &err, + 0, + NULL)) { + return; + } + + std::string buffer(mongoutils::str::stream() << "SSPI: " << msg << ": " << err); + utils->seterror(utils->conn, 0, "%s", buffer.c_str()); + LocalFree(err); + } + + int sspiClientMechNew(void *glob_context, + sasl_client_params_t *cparams, + void **conn_context) throw() { + // Prepare auth identity to pass to AcquireCredentialsHandle + SEC_WINNT_AUTH_IDENTITY authIdentity; + authIdentity.Flags = SEC_WINNT_AUTH_IDENTITY_UNICODE; + + // Fetch username@realm. + sasl_getsimple_t *user_cb; + void *user_context; + int ret = cparams->utils->getcallback(cparams->utils->conn, + SASL_CB_USER, + (sasl_callback_ft *)&user_cb, + &user_context); + if (ret != SASL_OK) { + cparams->utils->seterror(cparams->utils->conn, 0, "getcallback user failed"); + return ret; + } + const char *rawUserPlusRealm; + unsigned rawUserPlusRealmLength = 0; + + ret = user_cb(user_context, SASL_CB_USER, &rawUserPlusRealm, &rawUserPlusRealmLength); + if (ret != SASL_OK) { + cparams->utils->seterror(cparams->utils->conn, 0, "user callback failed"); + return ret; + } + std::string userPlusRealm(rawUserPlusRealm, rawUserPlusRealmLength); + + // Parse out the username and realm. + size_t atSign = userPlusRealm.find('@'); + if (atSign == std::string::npos) { + cparams->utils->seterror(cparams->utils->conn, 0, "no @REALM found in username"); + return SASL_BADPARAM; + } + std::string utf8Username(userPlusRealm, 0, atSign); + std::wstring utf16Username(toWideString(utf8Username.c_str())); + + authIdentity.UserLength = utf16Username.length(); + authIdentity.User = reinterpret_cast(const_cast( + utf16Username.c_str())); + + std::string utf8Domain(userPlusRealm, atSign + 1); + std::wstring utf16Domain(toWideString(utf8Domain.c_str())); + authIdentity.DomainLength = utf16Domain.length(); + authIdentity.Domain = reinterpret_cast(const_cast( + utf16Domain.c_str())); + + // Fetch password, if available. + authIdentity.PasswordLength = 0; + authIdentity.Password = NULL; + std::wstring utf16Password; + + sasl_secret_t* password = NULL; + sasl_getsecret_t *pass_cb; + void *pass_context; + ret = cparams->utils->getcallback(cparams->utils->conn, + SASL_CB_PASS, + (sasl_callback_ft *)&pass_cb, + &pass_context); + + if ((ret == SASL_OK) && pass_cb) { + ret = pass_cb(cparams->utils->conn, pass_context, SASL_CB_PASS, &password); + if ((ret == SASL_OK) && password) { + std::string utf8Password(reinterpret_cast(password->data), password->len); + utf16Password = toWideString(utf8Password.c_str()); + authIdentity.PasswordLength = utf16Password.length(); + authIdentity.Password = reinterpret_cast( + const_cast( + utf16Password.c_str())); + } + } + + // Actually acquire the handle to the client credentials. + std::auto_ptr pcctx(new SspiConnContext()); + pcctx->userPlusRealm = userPlusRealm; + TimeStamp ignored; + SECURITY_STATUS status = AcquireCredentialsHandleW(NULL, // principal + L"kerberos", + SECPKG_CRED_OUTBOUND, + NULL, // LOGON id + &authIdentity, // auth data + NULL, // get key fn + NULL, // get key arg + &pcctx->cred, + &ignored); + if (status != SEC_E_OK) { + HandleLastError(cparams->utils, status, "AcquireCredentialsHandle"); + return SASL_FAIL; + } + + pcctx->haveCred = true; + + // Compose target name token. + if (cparams->serverFQDN == NULL || strlen(cparams->serverFQDN) == 0) { + cparams->utils->seterror(cparams->utils->conn, 0, "SSPI: no serverFQDN"); + return SASL_FAIL; + } + pcctx->nameToken = toWideString((std::string(cparams->service) + + "/" + + cparams->serverFQDN).c_str()); + + *conn_context = pcctx.release(); + + return SASL_OK; + } + + int sspiValidateServerSecurityLayerOffering(SspiConnContext* pcctx, + sasl_client_params_t* cparams, + const char *serverin, + unsigned serverinlen) { + boost::scoped_array message(new char[serverinlen]); + memcpy(message.get(), serverin, serverinlen); + + SecBuffer wrapBufs[2]; + SecBufferDesc wrapBufDesc; + wrapBufDesc.cBuffers = 2; + wrapBufDesc.pBuffers = wrapBufs; + wrapBufDesc.ulVersion = SECBUFFER_VERSION; + + wrapBufs[0].cbBuffer = serverinlen; + wrapBufs[0].BufferType = SECBUFFER_STREAM; + wrapBufs[0].pvBuffer = message.get(); + + wrapBufs[1].cbBuffer = 0; + wrapBufs[1].BufferType = SECBUFFER_DATA; + wrapBufs[1].pvBuffer = NULL; + + SECURITY_STATUS status = DecryptMessage(&pcctx->ctx, + &wrapBufDesc, + 0, + NULL); + if (status != SEC_E_OK) { + HandleLastError(cparams->utils, status, "DecryptMessage"); + return SASL_FAIL; + } + + // Validate the server's plaintext message. + // Length (as per RFC 4752) + if (wrapBufs[1].cbBuffer < 4) { + cparams->utils->seterror(cparams->utils->conn, 0, "SSPI: server message is too short"); + return SASL_FAIL; + } + // First bit of first byte set, indicating that the client may elect to use no + // security layer. As a client we are uninterested in any of the other features the + // server offers and thus we ignore the other bits. + if (!(static_cast(wrapBufs[1].pvBuffer)[0] & 1)) { + cparams->utils->seterror(cparams->utils->conn, 0, + "SSPI: server does not support the required security layer"); + return SASL_BADAUTH; + } + return SASL_OK; + } + + + int sspiSendClientAuthzId(SspiConnContext* pcctx, + sasl_client_params_t* cparams, + const char *serverin, + unsigned serverinlen, + const char **clientout, + unsigned *clientoutlen, + sasl_out_params_t* oparams) { + + // Ensure server response is decryptable. + int decryptStatus = sspiValidateServerSecurityLayerOffering(pcctx, + cparams, + serverin, + serverinlen); + if (decryptStatus != SASL_OK) { + return decryptStatus; + } + + // Fill in AUTHID and AUTHZID fields in oparams. + int ret = cparams->canon_user(cparams->utils->conn, + pcctx->userPlusRealm.c_str(), + 0, + SASL_CU_AUTHID | SASL_CU_AUTHZID, + oparams); + + // Reply to server with security capability and authz name. + SecPkgContext_Sizes sizes; + SECURITY_STATUS status = QueryContextAttributes(&pcctx->ctx, + SECPKG_ATTR_SIZES, + &sizes); + if (status != SEC_E_OK) { + HandleLastError(cparams->utils, status, "QueryContextAttributes(sizes)"); + return SASL_FAIL; + } + + // See RFC4752. + int plaintextMessageSize = 4 + pcctx->userPlusRealm.size(); + boost::scoped_array message(new char[sizes.cbSecurityTrailer + + plaintextMessageSize + + sizes.cbBlockSize]); + char* plaintextMessage = message.get() + sizes.cbSecurityTrailer; + plaintextMessage[0] = 1; // LAYER_NONE + plaintextMessage[1] = 0; + plaintextMessage[2] = 0; + plaintextMessage[3] = 0; + memcpy(&plaintextMessage[4], pcctx->userPlusRealm.c_str(), pcctx->userPlusRealm.size()); + + SecBuffer wrapBufs[3]; + SecBufferDesc wrapBufDesc; + wrapBufDesc.cBuffers = 3; + wrapBufDesc.pBuffers = wrapBufs; + wrapBufDesc.ulVersion = SECBUFFER_VERSION; + + wrapBufs[0].cbBuffer = sizes.cbSecurityTrailer; + wrapBufs[0].BufferType = SECBUFFER_TOKEN; + wrapBufs[0].pvBuffer = message.get(); + + wrapBufs[1].cbBuffer = plaintextMessageSize; + wrapBufs[1].BufferType = SECBUFFER_DATA; + wrapBufs[1].pvBuffer = message.get() + sizes.cbSecurityTrailer; + + wrapBufs[2].cbBuffer = sizes.cbBlockSize; + wrapBufs[2].BufferType = SECBUFFER_PADDING; + wrapBufs[2].pvBuffer = message.get() + sizes.cbSecurityTrailer + plaintextMessageSize; + + status = EncryptMessage(&pcctx->ctx, + SECQOP_WRAP_NO_ENCRYPT, + &wrapBufDesc, + 0); + + if (status != SEC_E_OK) { + HandleLastError(cparams->utils, status, "EncryptMessage"); + return SASL_FAIL; + } + + // Create the message to send to server. + *clientoutlen = wrapBufs[0].cbBuffer + wrapBufs[1].cbBuffer + wrapBufs[2].cbBuffer; + char *newoutbuf = static_cast(cparams->utils->malloc(*clientoutlen)); + memcpy(newoutbuf, + wrapBufs[0].pvBuffer, + wrapBufs[0].cbBuffer); + memcpy(newoutbuf + wrapBufs[0].cbBuffer, + wrapBufs[1].pvBuffer, + wrapBufs[1].cbBuffer); + memcpy(newoutbuf + wrapBufs[0].cbBuffer + wrapBufs[1].cbBuffer, + wrapBufs[2].pvBuffer, + wrapBufs[2].cbBuffer); + *clientout = newoutbuf; + + return SASL_OK; + } + + + int sspiClientMechStep(void *conn_context, + sasl_client_params_t *cparams, + const char *serverin, + unsigned serverinlen, + sasl_interact_t **prompt_need, + const char **clientout, + unsigned *clientoutlen, + sasl_out_params_t *oparams) throw() { + SspiConnContext* pcctx = static_cast(conn_context); + *clientout = NULL; + *clientoutlen = 0; + + if (pcctx->authComplete) { + return sspiSendClientAuthzId(pcctx, cparams, + serverin, serverinlen, + clientout, clientoutlen, + oparams); + } + + SecBufferDesc inbuf; + SecBuffer inBufs[1]; + SecBufferDesc outbuf; + SecBuffer outBufs[1]; + + if (pcctx->haveCtxt) { + // If we already have a context, we now have data to send. + // Put this data in an inbuf. + inbuf.ulVersion = SECBUFFER_VERSION; + inbuf.cBuffers = 1; + inbuf.pBuffers = inBufs; + inBufs[0].pvBuffer = const_cast(serverin); + inBufs[0].cbBuffer = serverinlen; + inBufs[0].BufferType = SECBUFFER_TOKEN; + } + + outbuf.ulVersion = SECBUFFER_VERSION; + outbuf.cBuffers = 1; + outbuf.pBuffers = outBufs; + outBufs[0].pvBuffer = NULL; + outBufs[0].cbBuffer = 0; + outBufs[0].BufferType = SECBUFFER_TOKEN; + + ULONG contextAttr = 0; + SECURITY_STATUS status = InitializeSecurityContextW(&pcctx->cred, + pcctx->haveCtxt ? &pcctx->ctx : NULL, + const_cast( + pcctx->nameToken.c_str()), + ISC_REQ_ALLOCATE_MEMORY | + ISC_REQ_MUTUAL_AUTH, + 0, + SECURITY_NETWORK_DREP, + (pcctx->haveCtxt ? &inbuf : NULL), + 0, + &pcctx->ctx, + &outbuf, + &contextAttr, + NULL); + + if (status != SEC_E_OK && status != SEC_I_CONTINUE_NEEDED) { + HandleLastError(cparams->utils, status, "InitializeSecurityContext"); + return SASL_FAIL; + } + + ON_BLOCK_EXIT(FreeContextBuffer, outbuf.pBuffers[0].pvBuffer); + pcctx->haveCtxt = true; + + if (status == SEC_E_OK) { + // Send back nothing and wait for the server to reply with the security capabilities + *clientout = NULL; + *clientoutlen = 0; + pcctx->authComplete = true; + return SASL_CONTINUE; + } + + char *newoutbuf = static_cast(cparams->utils->malloc(outBufs[0].cbBuffer)); + *clientoutlen = outBufs[0].cbBuffer; + memcpy(newoutbuf, outBufs[0].pvBuffer, *clientoutlen); + *clientout = newoutbuf; + return SASL_CONTINUE; + } + + void sspiClientMechDispose(void *conn_context, const sasl_utils_t *utils) { + SspiConnContext* pcctx = static_cast(conn_context); + delete pcctx; + } + + void sspiClientMechFree(void *glob_context, const sasl_utils_t *utils) { + } + + sasl_client_plug_t sspiClientPlugin[] = { + { + sspiPluginName, /* mechanism name */ + 112, /* TODO: (taken from gssapi) best mech additional security layer strength factor */ + SASL_SEC_NOPLAINTEXT /* eam: copied from gssapi */ + | SASL_SEC_NOACTIVE + | SASL_SEC_NOANONYMOUS + | SASL_SEC_MUTUAL_AUTH + | SASL_SEC_PASS_CREDENTIALS, /* security_flags */ + SASL_FEAT_NEEDSERVERFQDN + | SASL_FEAT_WANT_CLIENT_FIRST + | SASL_FEAT_ALLOWS_PROXY, + NULL, /* required prompt ids, NULL = user/pass only */ + NULL, /* global state for mechanism */ + sspiClientMechNew, + sspiClientMechStep, + sspiClientMechDispose, + sspiClientMechFree, + NULL, + NULL, + NULL + } + }; + + int sspiClientPluginInit(const sasl_utils_t *utils, + int max_version, + int *out_version, + sasl_client_plug_t **pluglist, + int *plugcount) { + if (max_version < SASL_CLIENT_PLUG_VERSION) { + utils->seterror(utils->conn, 0, "Wrong SSPI version"); + return SASL_BADVERS; + } + + *out_version = SASL_CLIENT_PLUG_VERSION; + *pluglist = sspiClientPlugin; + *plugcount = 1; + + return SASL_OK; + } + + /** + * Registers the plugin at process initialization time. + * Must be run after the AllocatorsAndMutexes are registered, but before the ClientContext is + * created. + */ + MONGO_INITIALIZER_WITH_PREREQUISITES(SaslSspiClientPlugin, + ("CyrusSaslAllocatorsAndMutexes", + "SaslClientContext")) + (InitializerContext*) { + + int ret = sasl_client_add_plugin(sspiPluginName, + sspiClientPluginInit); + if (SASL_OK != ret) { + return Status(ErrorCodes::UnknownError, + mongoutils::str::stream() << "could not add SASL Client SSPI plugin " + << sspiPluginName << ": " << sasl_errstring(ret, NULL, NULL)); + } + + return Status::OK(); + } + MONGO_INITIALIZER_WITH_PREREQUISITES(SaslCramClientPlugin, + ("CyrusSaslAllocatorsAndMutexes", + "SaslClientContext")) + (InitializerContext*) { + int ret = sasl_client_add_plugin("CRAMMD5", + crammd5_client_plug_init); + if (SASL_OK != ret) { + return Status(ErrorCodes::UnknownError, + mongoutils::str::stream() << "Could not add SASL Client CRAM-MD5 plugin " + << sspiPluginName << ": " << sasl_errstring(ret, NULL, NULL)); + } + + return Status::OK(); + } + + MONGO_INITIALIZER_WITH_PREREQUISITES(SaslPlainClientPlugin, + ("CyrusSaslAllocatorsAndMutexes", + "SaslClientContext")) + (InitializerContext*) { + int ret = sasl_client_add_plugin("PLAIN", + plain_client_plug_init); + if (SASL_OK != ret) { + return Status(ErrorCodes::UnknownError, + mongoutils::str::stream() << "Could not add SASL Client PLAIN plugin " + << sspiPluginName << ": " << sasl_errstring(ret, NULL, NULL)); + } + + return Status::OK(); + } + +} // namespace +} // namespace mongo + +#endif // ifdef _WIN32 diff --git a/src/mongo/client/scoped_db_conn_test.cpp b/src/mongo/client/scoped_db_conn_test.cpp index 6926588544f..09109fbaa09 100644 --- a/src/mongo/client/scoped_db_conn_test.cpp +++ b/src/mongo/client/scoped_db_conn_test.cpp @@ -51,7 +51,6 @@ namespace { namespace mongo { // Symbols defined to build the binary correctly. - CmdLine cmdLine; bool inShutdown() { scoped_lock sl(shutDownMutex); @@ -178,6 +177,7 @@ namespace mongo_test { * Helper method for running the server on a separate thread. */ static void runServer(mongo::MessageServer* server) { + server->setupSockets(); server->run(); } @@ -193,7 +193,7 @@ namespace mongo_test { class DummyServerFixture: public mongo::unittest::Test { public: void setUp() { - _maxPoolSizePerHost = mongo::PoolForHost::getMaxPerHost(); + _maxPoolSizePerHost = mongo::pool.getMaxPoolSize(); _dummyServer = new DummyServer(TARGET_PORT); _dummyServer->run(&dummyHandler); @@ -217,7 +217,7 @@ namespace mongo_test { ScopedDbConnection::clearPool(); delete _dummyServer; - mongo::PoolForHost::setMaxPerHost(_maxPoolSizePerHost); + mongo::pool.setMaxPoolSize(_maxPoolSizePerHost); } protected: @@ -242,8 +242,7 @@ namespace mongo_test { size_t newConnsToCreate) { vector newConnList; for (size_t x = 0; x < newConnsToCreate; x++) { - ScopedDbConnection* newConn = - ScopedDbConnection::getScopedDbConnection(TARGET_HOST); + ScopedDbConnection* newConn = new ScopedDbConnection(TARGET_HOST); checkFunc(newConn->get()->getSockCreationMicroSec(), arg2); newConnList.push_back(newConn); } @@ -260,8 +259,7 @@ namespace mongo_test { // Check that connections created after the purge was put back to the pool. for (size_t x = 0; x < newConnsToCreate; x++) { - ScopedDbConnection* newConn = - ScopedDbConnection::getScopedDbConnection(TARGET_HOST); + ScopedDbConnection* newConn = new ScopedDbConnection(TARGET_HOST); ASSERT_LESS_THAN(newConn->get()->getSockCreationMicroSec(), oldCreationTime); newConnList.push_back(newConn); } @@ -275,6 +273,7 @@ namespace mongo_test { private: static void runServer(mongo::MessageServer* server) { + server->setupSockets(); server->run(); } @@ -283,32 +282,26 @@ namespace mongo_test { }; TEST_F(DummyServerFixture, BasicScopedDbConnection) { - scoped_ptr conn1( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - scoped_ptr conn2( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); + ScopedDbConnection conn1(TARGET_HOST); + ScopedDbConnection conn2(TARGET_HOST); - DBClientBase* conn1Ptr = conn1->get(); - conn1->done(); + DBClientBase* conn1Ptr = conn1.get(); + conn1.done(); - scoped_ptr conn3( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - ASSERT_EQUALS(conn1Ptr, conn3->get()); + ScopedDbConnection conn3(TARGET_HOST); + ASSERT_EQUALS(conn1Ptr, conn3.get()); - conn2->done(); - conn3->done(); + conn2.done(); + conn3.done(); } TEST_F(DummyServerFixture, InvalidateBadConnInPool) { - scoped_ptr conn1( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - scoped_ptr conn2( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - scoped_ptr conn3( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); + ScopedDbConnection conn1(TARGET_HOST); + ScopedDbConnection conn2(TARGET_HOST); + ScopedDbConnection conn3(TARGET_HOST); - conn1->done(); - conn3->done(); + conn1.done(); + conn3.done(); const uint64_t badCreationTime = mongo::curTimeMicros64(); @@ -316,33 +309,30 @@ namespace mongo_test { setMode(FailPoint::alwaysOn); try { - conn2->get()->query("test.user", mongo::Query()); + conn2->query("test.user", mongo::Query()); } catch (const mongo::SocketException&) { } mongo::getGlobalFailPointRegistry()->getFailPoint("throwSockExcep")-> setMode(FailPoint::off); - conn2->done(); + conn2.done(); checkNewConns(assertGreaterThan, badCreationTime, 10); } TEST_F(DummyServerFixture, DontReturnKnownBadConnToPool) { - scoped_ptr conn1( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - scoped_ptr conn2( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - scoped_ptr conn3( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); + ScopedDbConnection conn1(TARGET_HOST); + ScopedDbConnection conn2(TARGET_HOST); + ScopedDbConnection conn3(TARGET_HOST); - conn1->done(); + conn1.done(); mongo::getGlobalFailPointRegistry()->getFailPoint("throwSockExcep")-> setMode(FailPoint::alwaysOn); try { - conn3->get()->query("test.user", mongo::Query()); + conn3->query("test.user", mongo::Query()); } catch (const mongo::SocketException&) { } @@ -350,26 +340,23 @@ namespace mongo_test { mongo::getGlobalFailPointRegistry()->getFailPoint("throwSockExcep")-> setMode(FailPoint::off); - const uint64_t badCreationTime = conn3->get()->getSockCreationMicroSec(); - conn3->done(); + const uint64_t badCreationTime = conn3->getSockCreationMicroSec(); + conn3.done(); // attempting to put a 'bad' connection back to the pool - conn2->done(); + conn2.done(); checkNewConns(assertGreaterThan, badCreationTime, 10); } TEST_F(DummyServerFixture, InvalidateBadConnEvenWhenPoolIsFull) { - mongo::PoolForHost::setMaxPerHost(2); + mongo::pool.setMaxPoolSize(2); - scoped_ptr conn1( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - scoped_ptr conn2( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - scoped_ptr conn3( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); + ScopedDbConnection conn1(TARGET_HOST); + ScopedDbConnection conn2(TARGET_HOST); + ScopedDbConnection conn3(TARGET_HOST); - conn1->done(); - conn3->done(); + conn1.done(); + conn3.done(); const uint64_t badCreationTime = mongo::curTimeMicros64(); @@ -377,32 +364,30 @@ namespace mongo_test { setMode(FailPoint::alwaysOn); try { - conn2->get()->query("test.user", mongo::Query()); + conn2->query("test.user", mongo::Query()); } catch (const mongo::SocketException&) { } mongo::getGlobalFailPointRegistry()->getFailPoint("throwSockExcep")-> setMode(FailPoint::off); - conn2->done(); + conn2.done(); checkNewConns(assertGreaterThan, badCreationTime, 2); } TEST_F(DummyServerFixture, DontReturnConnGoneBadToPool) { - scoped_ptr conn1( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); + ScopedDbConnection conn1(TARGET_HOST); - const uint64_t conn1CreationTime = conn1->get()->getSockCreationMicroSec(); + const uint64_t conn1CreationTime = conn1->getSockCreationMicroSec(); uint64_t conn2CreationTime = 0; { - scoped_ptr conn2( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - conn2CreationTime = conn2->get()->getSockCreationMicroSec(); + ScopedDbConnection conn2(TARGET_HOST); + conn2CreationTime = conn2->getSockCreationMicroSec(); - conn1->done(); + conn1.done(); // conn2 gets out of scope without calling done() } @@ -410,12 +395,11 @@ namespace mongo_test { // also not invalidate older connections since it didn't encounter // a socket exception. - scoped_ptr conn1Again( - ScopedDbConnection::getScopedDbConnection(TARGET_HOST)); - ASSERT_EQUALS(conn1CreationTime, conn1Again->get()->getSockCreationMicroSec()); + ScopedDbConnection conn1Again(TARGET_HOST); + ASSERT_EQUALS(conn1CreationTime, conn1Again->getSockCreationMicroSec()); checkNewConns(assertNotEqual, conn2CreationTime, 10); - conn1Again->done(); + conn1Again.done(); } } diff --git a/src/mongo/client/simple_client_demo.cpp b/src/mongo/client/simple_client_demo.cpp index 51b370ab2d3..3aea72c2273 100644 --- a/src/mongo/client/simple_client_demo.cpp +++ b/src/mongo/client/simple_client_demo.cpp @@ -19,13 +19,9 @@ How to build and run: - (1) Using the mongoclient: g++ simple_client_demo.cpp -lmongoclient -lboost_thread-mt -lboost_filesystem -lboost_program_options ./a.out - (2) using client_lib.cpp: - g++ -I .. simple_client_demo.cpp mongo_client_lib.cpp -lboost_thread-mt -lboost_filesystem - ./a.out */ #include diff --git a/src/mongo/client/syncclusterconnection.cpp b/src/mongo/client/syncclusterconnection.cpp index 3a927a55777..a4324a3376b 100644 --- a/src/mongo/client/syncclusterconnection.cpp +++ b/src/mongo/client/syncclusterconnection.cpp @@ -16,14 +16,14 @@ */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/client/syncclusterconnection.h" #include "mongo/client/dbclientcursor.h" #include "mongo/client/dbclientinterface.h" #include "mongo/db/dbmessage.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" // error codes 8000-8009 @@ -131,7 +131,9 @@ namespace mongo { for ( size_t i = 0; i<_conns.size(); i++ ) { BSONObj res = _lastErrors[i]; - if ( res["ok"].trueValue() && (res["fsyncFiles"].numberInt() > 0 || res.hasElement("waited"))) + if ( res["ok"].trueValue() && (res["fsyncFiles"].numberInt() > 0 || + res.hasElement("waited") || + res["syncMillis"].numberInt() >= 0 ) ) continue; ok = false; err << _conns[i]->toString() << ": " << res << " " << errors[i]; @@ -159,6 +161,8 @@ namespace mongo { void SyncClusterConnection::_connect( const std::string& host ) { log() << "SyncClusterConnection connecting to [" << host << "]" << endl; DBClientConnection * c = new DBClientConnection( true ); + c->setRunCommandHook(_runCommandHook); + c->setPostRunCommandHook(_postRunCommandHook); c->setSoTimeout( _socketTimeout ); string errmsg; if ( ! c->connect( host , errmsg ) ) @@ -211,39 +215,39 @@ namespace mongo { } void SyncClusterConnection::_auth(const BSONObj& params) { - // A SCC is authorized if any connection has been authorized + // A SCC is authenticated if any connection has been authenticated // Credentials are stored in the auto-reconnect connections. bool authedOnce = false; vector errors; - for( vector::iterator it = _conns.begin(); it < _conns.end(); ++it ){ + for ( vector::iterator it = _conns.begin(); it < _conns.end(); ++it ) { massert( 15848, "sync cluster of sync clusters?", (*it)->type() != ConnectionString::SYNC ); - // Authorize or collect the error message + // Authenticate or collect the error message string lastErrmsg; bool authed; - try{ + try { // Auth errors can manifest either as exceptions or as false results // TODO: Make this better (*it)->auth(params); authed = true; } - catch( const DBException& e ){ + catch ( const DBException& e ) { // auth will be retried on reconnect lastErrmsg = e.what(); authed = false; } - if( ! authed ){ + if ( ! authed ) { // Since we're using auto-reconnect connections, we're sure the auth info has been // stored if needed for later lastErrmsg = str::stream() << "auth error on " << (*it)->getServerAddress() - << causedBy( lastErrmsg ); + << causedBy( lastErrmsg ); LOG(1) << lastErrmsg << endl; errors.push_back( lastErrmsg ); @@ -287,22 +291,47 @@ namespace mongo { return isOk( info ); } + void SyncClusterConnection::attachQueryHandler( QueryHandler* handler ) { + _customQueryHandler.reset( handler ); + } + auto_ptr SyncClusterConnection::_queryOnActive(const string &ns, Query query, int nToReturn, int nToSkip, const BSONObj *fieldsToReturn, int queryOptions, int batchSize ) { + if ( _customQueryHandler && _customQueryHandler->canHandleQuery( ns, query ) ) { + + LOG( 2 ) << "custom query handler used for query on " << ns << ": " + << query.toString() << endl; + + return _customQueryHandler->handleQuery( _connAddresses, + ns, + query, + nToReturn, + nToSkip, + fieldsToReturn, + queryOptions, + batchSize ); + } + for ( size_t i=0; i<_conns.size(); i++ ) { try { auto_ptr cursor = _conns[i]->query( ns , query , nToReturn , nToSkip , fieldsToReturn , queryOptions , batchSize ); if ( cursor.get() ) return cursor; - log() << "query failed to: " << _conns[i]->toString() << " no data" << endl; + + log() << "query on " << ns << ": " << query.toString() << " failed to: " + << _conns[i]->toString() << " no data" << endl; } catch ( std::exception& e ) { - log() << "query failed to: " << _conns[i]->toString() << " exception: " << e.what() << endl; + + log() << "query on " << ns << ": " << query.toString() << " failed to: " + << _conns[i]->toString() << " exception: " << e.what() << endl; } catch ( ... ) { - log() << "query failed to: " << _conns[i]->toString() << " exception" << endl; + + log() << "query on " << ns << ": " << query.toString() << " failed to: " + << _conns[i]->toString() << " exception" << endl; } } throw UserException( 8002 , str::stream() << "all servers down/unreachable when querying: " << _address ); @@ -317,8 +346,8 @@ namespace mongo { void SyncClusterConnection::insert( const string &ns, BSONObj obj , int flags) { uassert(13119, - (string)"SyncClusterConnection::insert obj has to have an _id: " + obj.jsonString(), - NamespaceString(ns).coll == "system.indexes" || obj["_id"].type()); + str::stream() << "SyncClusterConnection::insert obj has to have an _id: " << obj, + nsToCollectionSubstring(ns) == "system.indexes" || obj["_id"].type()); string errmsg; if ( ! prepare( errmsg ) ) @@ -385,7 +414,7 @@ namespace mongo { if ( _writeConcern ) { string errmsg; if ( ! prepare( errmsg ) ) - throw UserException( 8005 , (string)"SyncClusterConnection::udpate prepare failed: " + errmsg ); + throw UserException( 8005 , (string)"SyncClusterConnection::update prepare failed: " + errmsg ); } for ( size_t i = 0; i < _conns.size(); i++ ) { @@ -423,7 +452,18 @@ namespace mongo { string SyncClusterConnection::_toString() const { stringstream ss; - ss << "SyncClusterConnection [" << _address << "]"; + ss << "SyncClusterConnection "; + ss << " ["; + for ( size_t i = 0; i < _conns.size(); i++ ) { + if ( i != 0 ) ss << ","; + if ( _conns[i] ) { + ss << _conns[i]->toString(); + } + else { + ss << "(no conn)"; + } + } + ss << "]"; return ss.str(); } @@ -492,6 +532,18 @@ namespace mongo { verify(0); } + // A SCC should be reused only if all the existing connections haven't been broken in the + // background. + // Note: an SCC may have missing connections if a config server is temporarily offline, + // but reading from the others is still allowed. + bool SyncClusterConnection::isStillConnected() { + for ( size_t i = 0; i < _conns.size(); i++ ) { + if ( _conns[i] && !_conns[i]->isStillConnected() ) return false; + + } + return true; + } + void SyncClusterConnection::setAllSoTimeouts( double socketTimeout ){ _socketTimeout = socketTimeout; for ( size_t i=0; i<_conns.size(); i++ ) @@ -499,4 +551,24 @@ namespace mongo { if( _conns[i] ) _conns[i]->setSoTimeout( socketTimeout ); } + void SyncClusterConnection::setRunCommandHook(DBClientWithCommands::RunCommandHookFunc func) { + // Set the hooks in both our sub-connections and in ourselves. + for (size_t i = 0; i < _conns.size(); ++i) { + if (_conns[i]) { + _conns[i]->setRunCommandHook(func); + } + } + _runCommandHook = func; + } + + void SyncClusterConnection::setPostRunCommandHook + (DBClientWithCommands::PostRunCommandHookFunc func) { + // Set the hooks in both our sub-connections and in ourselves. + for (size_t i = 0; i < _conns.size(); ++i) { + if (_conns[i]) { + _conns[i]->setPostRunCommandHook(func); + } + } + _postRunCommandHook = func; + } } diff --git a/src/mongo/client/syncclusterconnection.h b/src/mongo/client/syncclusterconnection.h index 3511285117c..0efd49fdae1 100644 --- a/src/mongo/client/syncclusterconnection.h +++ b/src/mongo/client/syncclusterconnection.h @@ -22,6 +22,7 @@ #include "mongo/bson/bsonelement.h" #include "mongo/bson/bsonobj.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -39,13 +40,15 @@ namespace mongo { * The class checks if a command is read or write style, and sends to a single * node if a read lock command and to all in two phases with a write style command. */ - class SyncClusterConnection : public DBClientBase { + class MONGO_CLIENT_API SyncClusterConnection : public DBClientBase { public: using DBClientBase::query; using DBClientBase::update; using DBClientBase::remove; + class QueryHandler; + /** * @param commaSeparated should be 3 hosts comma separated */ @@ -92,7 +95,8 @@ namespace mongo { virtual string getServerAddress() const { return _address; } virtual bool isFailed() const { return false; } - virtual string toString() { return _toString(); } + virtual bool isStillConnected(); + virtual string toString() const { return _toString(); } virtual BSONObj getLastErrorDetailed(const std::string& db, bool fsync=false, @@ -111,6 +115,16 @@ namespace mongo { virtual bool lazySupported() const { return false; } + virtual void setRunCommandHook(DBClientWithCommands::RunCommandHookFunc func); + virtual void setPostRunCommandHook(DBClientWithCommands::PostRunCommandHookFunc func); + + /** + * Allow custom query processing through an external (e.g. mongos-only) service. + * + * Takes ownership of attached handler. + */ + void attachQueryHandler( QueryHandler* handler ); + protected: virtual void _auth(const BSONObj& params); @@ -127,15 +141,48 @@ namespace mongo { string _address; vector _connAddresses; vector _conns; - map _lockTypes; - mongo::mutex _mutex; vector _lastErrors; + // Optionally attached by user + scoped_ptr _customQueryHandler; + + mongo::mutex _mutex; + map _lockTypes; + // End mutex + double _socketTimeout; }; - class UpdateNotTheSame : public UserException { + /** + * Interface for custom query processing for the SCC. + * Allows plugging different host query behaviors for different types of queries. + */ + class SyncClusterConnection::QueryHandler { + public: + + virtual ~QueryHandler() {}; + + /** + * Returns true if the query can be processed using this handler. + */ + virtual bool canHandleQuery( const string& ns, Query query ) = 0; + + /** + * Returns a cursor on one of the hosts with the desired results for the query. + * May throw or return an empty auto_ptr on failure. + */ + virtual auto_ptr handleQuery( const vector& hosts, + const string &ns, + Query query, + int nToReturn, + int nToSkip, + const BSONObj *fieldsToReturn, + int queryOptions, + int batchSize ) = 0; + }; + + class MONGO_CLIENT_API UpdateNotTheSame : public UserException { public: UpdateNotTheSame( int code , const string& msg , const vector& addrs , const vector& lastErrors ) : UserException( code , msg ) , _addrs( addrs ) , _lastErrors( lastErrors ) { diff --git a/src/mongo/client/undef_macros.h b/src/mongo/client/undef_macros.h index 6aed7d943b9..b9d12b4d3e6 100644 --- a/src/mongo/client/undef_macros.h +++ b/src/mongo/client/undef_macros.h @@ -38,10 +38,10 @@ #pragma pop_macro("massert") #undef uassert #pragma pop_macro("uassert") -#undef uassertStatusOK -#pragma pop_macro("uassertStatusOK") #undef verify #pragma pop_macro("verify") +#undef invariant +#pragma pop_macro("invariant") #undef DESTRUCTOR_GUARD #pragma pop_macro("DESTRUCTOR_GUARD") diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 2d3a4f65734..04a9febc820 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -3,13 +3,67 @@ Import("env") # -# We'd like 'common' to have the abstractions that are shared by several components of the +# The db/'common' lib has the abstractions that are shared by components of the # server. Ideally, many of the object in 'coredb' should be moved here when their dependencies # get resolved. # -env.StaticLibrary('common', ['field_ref.cpp'], - LIBDEPS=['$BUILD_DIR/mongo/bson', - '$BUILD_DIR/mongo/foundation']) +env.Library( + target= 'common', + source= [ + 'field_ref.cpp', + 'field_ref_set.cpp', + 'field_parser.cpp', + 'write_concern_options.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/bson', + '$BUILD_DIR/mongo/foundation', + ], +) -env.CppUnitTest('field_ref_test', ['field_ref_test.cpp'], LIBDEPS=['common']) +env.CppUnitTest( + target= 'field_ref_test', + source= 'field_ref_test.cpp', + LIBDEPS=[ + 'common', + ], +) + +env.CppUnitTest( + target= 'field_ref_set_test', + source = 'field_ref_set_test.cpp', + LIBDEPS=[ + 'common', + ], +) + +env.CppUnitTest( + target= 'field_parser_test', + source= 'field_parser_test.cpp', + LIBDEPS=[ + 'common', + ], +) + +env.CppUnitTest( + target= 'hasher_test', + source= [ + 'hasher_test.cpp', + ], + LIBDEPS=[ + 'common', + '$BUILD_DIR/mongo/mongohasher', + ], +) + +env.CppUnitTest( + target="dbmessage_test", + source=[ + "dbmessage_test.cpp" + ], + LIBDEPS=[ + "common", + "$BUILD_DIR/mongo/clientdriver", + ], +) \ No newline at end of file diff --git a/src/mongo/db/audit.cpp b/src/mongo/db/audit.cpp new file mode 100644 index 00000000000..a06d0058a8f --- /dev/null +++ b/src/mongo/db/audit.cpp @@ -0,0 +1,228 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/audit.h" + +#if MONGO_ENTERPRISE_VERSION +#define MONGO_AUDIT_STUB ; +#else +#define MONGO_AUDIT_STUB {} +#endif + +namespace mongo { +namespace audit { + + void logAuthentication(ClientBasic* client, + const StringData& mechanism, + const UserName& user, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logCommandAuthzCheck(ClientBasic* client, + const NamespaceString& ns, + const mutablebson::Document& cmdObj, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logDeleteAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& pattern, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logFsyncUnlockAuthzCheck( + ClientBasic* client, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logGetMoreAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + long long cursorId, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logInProgAuthzCheck( + ClientBasic* client, + const BSONObj& filter, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logInsertAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& insertedObj, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logKillCursorsAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + long long cursorId, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logKillOpAuthzCheck( + ClientBasic* client, + const BSONObj& filter, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logQueryAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& query, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logUpdateAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& query, + const BSONObj& updateObj, + bool isUpsert, + bool isMulti, + ErrorCodes::Error result) MONGO_AUDIT_STUB + + void logCreateUser(ClientBasic* client, + const UserName& username, + bool password, + const BSONObj* customData, + const std::vector& roles) MONGO_AUDIT_STUB + + void logDropUser(ClientBasic* client, + const UserName& username) MONGO_AUDIT_STUB + + void logDropAllUsersFromDatabase(ClientBasic* client, + const StringData& dbname) MONGO_AUDIT_STUB + + void logUpdateUser(ClientBasic* client, + const UserName& username, + bool password, + const BSONObj* customData, + const std::vector* roles) MONGO_AUDIT_STUB + + void logGrantRolesToUser(ClientBasic* client, + const UserName& username, + const std::vector& roles) MONGO_AUDIT_STUB + + void logRevokeRolesFromUser(ClientBasic* client, + const UserName& username, + const std::vector& roles) MONGO_AUDIT_STUB + + void logCreateRole(ClientBasic* client, + const RoleName& role, + const std::vector& roles, + const PrivilegeVector& privileges) MONGO_AUDIT_STUB + + void logUpdateRole(ClientBasic* client, + const RoleName& role, + const std::vector* roles, + const PrivilegeVector* privileges) MONGO_AUDIT_STUB + + void logDropRole(ClientBasic* client, + const RoleName& role) MONGO_AUDIT_STUB + + void logDropAllRolesFromDatabase(ClientBasic* client, + const StringData& dbname) MONGO_AUDIT_STUB + + void logGrantRolesToRole(ClientBasic* client, + const RoleName& role, + const std::vector& roles) MONGO_AUDIT_STUB + + void logRevokeRolesFromRole(ClientBasic* client, + const RoleName& role, + const std::vector& roles) MONGO_AUDIT_STUB + + void logGrantPrivilegesToRole(ClientBasic* client, + const RoleName& role, + const PrivilegeVector& privileges) MONGO_AUDIT_STUB + + void logRevokePrivilegesFromRole(ClientBasic* client, + const RoleName& role, + const PrivilegeVector& privileges) MONGO_AUDIT_STUB + + void logReplSetReconfig(ClientBasic* client, + const BSONObj* oldConfig, + const BSONObj* newConfig) MONGO_AUDIT_STUB + + void logApplicationMessage(ClientBasic* client, + const StringData& msg) MONGO_AUDIT_STUB + + void logShutdown(ClientBasic* client) MONGO_AUDIT_STUB + + void logCreateIndex(ClientBasic* client, + const BSONObj* indexSpec, + const StringData& indexname, + const StringData& nsname) MONGO_AUDIT_STUB + + void logCreateCollection(ClientBasic* client, + const StringData& nsname) MONGO_AUDIT_STUB + + void logCreateDatabase(ClientBasic* client, + const StringData& dbname) MONGO_AUDIT_STUB + + + void logDropIndex(ClientBasic* client, + const StringData& indexname, + const StringData& nsname) MONGO_AUDIT_STUB + + void logDropCollection(ClientBasic* client, + const StringData& nsname) MONGO_AUDIT_STUB + + void logDropDatabase(ClientBasic* client, + const StringData& dbname) MONGO_AUDIT_STUB + + void logRenameCollection(ClientBasic* client, + const StringData& source, + const StringData& target) MONGO_AUDIT_STUB + + void logEnableSharding(ClientBasic* client, + const StringData& dbname) MONGO_AUDIT_STUB + + void logAddShard(ClientBasic* client, + const StringData& name, + const std::string& servers, + long long maxSize) MONGO_AUDIT_STUB + + void logRemoveShard(ClientBasic* client, + const StringData& shardname) MONGO_AUDIT_STUB + + void logShardCollection(ClientBasic* client, + const StringData& ns, + const BSONObj& keyPattern, + bool unique) MONGO_AUDIT_STUB + + void appendImpersonatedUsers(BSONObjBuilder* cmd) MONGO_AUDIT_STUB + + void parseAndRemoveImpersonatedUsersField( + BSONObj cmdObj, + AuthorizationSession* authSession, + std::vector* parsedUserNames, + bool* fieldIsPresent) MONGO_AUDIT_STUB + + void parseAndRemoveImpersonatedRolesField( + BSONObj cmdObj, + AuthorizationSession* authSession, + std::vector* parsedRoleNames, + bool* fieldIsPresent) MONGO_AUDIT_STUB + +} // namespace audit +} // namespace mongo + diff --git a/src/mongo/db/audit.h b/src/mongo/db/audit.h new file mode 100644 index 00000000000..0e6d07cf92a --- /dev/null +++ b/src/mongo/db/audit.h @@ -0,0 +1,404 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This module describes free functions for logging various operations of interest to a + * party interested in generating logs of user activity in a MongoDB server instance. + */ + +#pragma once + +#include "mongo/base/error_codes.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/user.h" + +namespace mongo { + + class AuthorizationSession; + class BSONObj; + class ClientBasic; + class NamespaceString; + class ReplSetConfig; + class StringData; + class UserName; + +namespace mutablebson { + class Document; +} // namespace mutablebson + +namespace audit { + + /** + * Logs the result of an authentication attempt. + */ + void logAuthentication(ClientBasic* client, + const StringData& mechanism, + const UserName& user, + ErrorCodes::Error result); + + // + // Authorization (authz) logging functions. + // + // These functions generate log messages describing the disposition of access control + // checks. + // + + /** + * Logs the result of a command authorization check. + */ + void logCommandAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const mutablebson::Document& cmdObj, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for an OP_DELETE wire protocol message. + */ + void logDeleteAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& pattern, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for the "unlock" pseudo-command. + */ + void logFsyncUnlockAuthzCheck( + ClientBasic* client, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for an OP_GET_MORE wire protocol message. + */ + void logGetMoreAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + long long cursorId, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for an "inprog" pseudo-command. + */ + void logInProgAuthzCheck( + ClientBasic* client, + const BSONObj& filter, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for an OP_INSERT wire protocol message. + */ + void logInsertAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& insertedObj, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for an OP_KILL_CURSORS wire protocol message. + */ + void logKillCursorsAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + long long cursorId, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for a "killop" pseudo-command. + */ + void logKillOpAuthzCheck( + ClientBasic* client, + const BSONObj& filter, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for an OP_QUERY wire protocol message. + */ + void logQueryAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& query, + ErrorCodes::Error result); + + /** + * Logs the result of an authorization check for an OP_UPDATE wire protocol message. + */ + void logUpdateAuthzCheck( + ClientBasic* client, + const NamespaceString& ns, + const BSONObj& query, + const BSONObj& updateObj, + bool isUpsert, + bool isMulti, + ErrorCodes::Error result); + + /** + * Logs the result of a createUser command. + */ + void logCreateUser(ClientBasic* client, + const UserName& username, + bool password, + const BSONObj* customData, + const std::vector& roles); + + /** + * Logs the result of a dropUser command. + */ + void logDropUser(ClientBasic* client, + const UserName& username); + + /** + * Logs the result of a dropAllUsersFromDatabase command. + */ + void logDropAllUsersFromDatabase(ClientBasic* client, + const StringData& dbname); + + /** + * Logs the result of a updateUser command. + */ + void logUpdateUser(ClientBasic* client, + const UserName& username, + bool password, + const BSONObj* customData, + const std::vector* roles); + + /** + * Logs the result of a grantRolesToUser command. + */ + void logGrantRolesToUser(ClientBasic* client, + const UserName& username, + const std::vector& roles); + + /** + * Logs the result of a revokeRolesFromUser command. + */ + void logRevokeRolesFromUser(ClientBasic* client, + const UserName& username, + const std::vector& roles); + + /** + * Logs the result of a createRole command. + */ + void logCreateRole(ClientBasic* client, + const RoleName& role, + const std::vector& roles, + const PrivilegeVector& privileges); + + /** + * Logs the result of a updateRole command. + */ + void logUpdateRole(ClientBasic* client, + const RoleName& role, + const std::vector* roles, + const PrivilegeVector* privileges); + + /** + * Logs the result of a dropRole command. + */ + void logDropRole(ClientBasic* client, + const RoleName& role); + + /** + * Logs the result of a dropAllRolesForDatabase command. + */ + void logDropAllRolesFromDatabase(ClientBasic* client, + const StringData& dbname); + + /** + * Logs the result of a grantRolesToRole command. + */ + void logGrantRolesToRole(ClientBasic* client, + const RoleName& role, + const std::vector& roles); + + /** + * Logs the result of a revokeRolesFromRole command. + */ + void logRevokeRolesFromRole(ClientBasic* client, + const RoleName& role, + const std::vector& roles); + + /** + * Logs the result of a grantPrivilegesToRole command. + */ + void logGrantPrivilegesToRole(ClientBasic* client, + const RoleName& role, + const PrivilegeVector& privileges); + + /** + * Logs the result of a revokePrivilegesFromRole command. + */ + void logRevokePrivilegesFromRole(ClientBasic* client, + const RoleName& role, + const PrivilegeVector& privileges); + + /** + * Logs the result of a replSet(Re)config command. + */ + void logReplSetReconfig(ClientBasic* client, + const BSONObj* oldConfig, + const BSONObj* newConfig); + + /** + * Logs the result of an ApplicationMessage command. + */ + void logApplicationMessage(ClientBasic* client, + const StringData& msg); + + /** + * Logs the result of a shutdown command. + */ + void logShutdown(ClientBasic* client); + + /** + * Logs the result of a createIndex command. + */ + void logCreateIndex(ClientBasic* client, + const BSONObj* indexSpec, + const StringData& indexname, + const StringData& nsname); + + /** + * Logs the result of a createCollection command. + */ + void logCreateCollection(ClientBasic* client, + const StringData& nsname); + + /** + * Logs the result of a createDatabase command. + */ + void logCreateDatabase(ClientBasic* client, + const StringData& dbname); + + + /** + * Logs the result of a dropIndex command. + */ + void logDropIndex(ClientBasic* client, + const StringData& indexname, + const StringData& nsname); + + /** + * Logs the result of a dropCollection command. + */ + void logDropCollection(ClientBasic* client, + const StringData& nsname); + + /** + * Logs the result of a dropDatabase command. + */ + void logDropDatabase(ClientBasic* client, + const StringData& dbname); + + /** + * Logs a collection rename event. + */ + void logRenameCollection(ClientBasic* client, + const StringData& source, + const StringData& target); + + /** + * Logs the result of a enableSharding command. + */ + void logEnableSharding(ClientBasic* client, + const StringData& dbname); + + /** + * Logs the result of a addShard command. + */ + void logAddShard(ClientBasic* client, + const StringData& name, + const std::string& servers, + long long maxSize); + + /** + * Logs the result of a removeShard command. + */ + void logRemoveShard(ClientBasic* client, + const StringData& shardname); + + /** + * Logs the result of a shardCollection command. + */ + void logShardCollection(ClientBasic* client, + const StringData& ns, + const BSONObj& keyPattern, + bool unique); + + + /* + * Appends an array of user/db pairs and an array of role/db pairs + * to the provided Document. The users and roles are extracted from the current client. + * They are to be the impersonated users and roles for a Command run by an internal user. + */ + void appendImpersonatedUsers(BSONObjBuilder* cmd); + const char cmdOptionImpersonatedUsers[] = "impersonatedUsers"; + const char cmdOptionImpersonatedRoles[] = "impersonatedRoles"; + + /* + * Looks for an 'impersonatedUsers' field. This field is used by mongos to + * transmit the usernames of the currently authenticated user when it runs commands + * on a shard using internal user authentication. Auditing uses this information + * to properly ascribe users to actions. This is necessary only for implicit actions that + * mongos cannot properly audit itself; examples are implicit collection and database creation. + * This function requires that the field is the last field in the bson object; it edits the + * command BSON to efficiently remove the field before returning. + * + * cmdObj [in, out]: If any impersonated users field exists, it will be parsed and removed. + * authSession [in]: current authorization session + * parsedUserNames [out]: populated with parsed usernames + * fieldIsPresent [out]: true if impersonatedUsers field was present in the object + */ + void parseAndRemoveImpersonatedUsersField( + BSONObj cmdObj, + AuthorizationSession* authSession, + std::vector* parsedUserNames, + bool* fieldIsPresent); + + /* + * Looks for an 'impersonatedRoles' field. This field is used by mongos to + * transmit the roles of the currently authenticated user when it runs commands + * on a shard using internal user authentication. Auditing uses this information + * to properly ascribe user roles to actions. This is necessary only for implicit actions that + * mongos cannot properly audit itself; examples are implicit collection and database creation. + * This function requires that the field is the last field in the bson object; it edits the + * command BSON to efficiently remove the field before returning. + * + * cmdObj [in, out]: If any impersonated roles field exists, it will be parsed and removed. + * authSession [in]: current authorization session + * parsedRoleNames [out]: populated with parsed user rolenames + * fieldIsPresent [out]: true if impersonatedRoles field was present in the object + */ + void parseAndRemoveImpersonatedRolesField( + BSONObj cmdObj, + AuthorizationSession* authSession, + std::vector* parsedRoleNames, + bool* fieldIsPresent); + +} // namespace audit +} // namespace mongo diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript index e736fb3b665..c10c05d3278 100644 --- a/src/mongo/db/auth/SConscript +++ b/src/mongo/db/auth/SConscript @@ -2,41 +2,70 @@ Import("env") -env.StaticLibrary('serverauth', ['mongo_authentication_session.cpp']) +env.Library('serverauth', ['mongo_authentication_session.cpp']) env.Command(['action_type.h', 'action_type.cpp'], ['generate_action_types.py', 'action_types.txt'], '$PYTHON $SOURCES $TARGETS') # Just the data structures used -env.StaticLibrary('authcore', ['action_set.cpp', - 'action_type.cpp', - 'auth_external_state.cpp', - 'authorization_manager.cpp', - 'principal.cpp', - 'principal_set.cpp', - 'privilege.cpp', - 'privilege_set.cpp'], - LIBDEPS=['$BUILD_DIR/mongo/base/base', - '$BUILD_DIR/mongo/bson', - '$BUILD_DIR/mongo/stringutils']) - -env.StaticLibrary('authservercommon', - ['auth_external_state_server_common.cpp', - 'auth_server_parameters.cpp', - 'security_key.cpp'], - LIBDEPS=['authcore']) - -env.StaticLibrary('authmongod', - ['auth_external_state_d.cpp', - 'auth_index_d.cpp'], - LIBDEPS=['authservercommon']) - -env.StaticLibrary('authmongos', - ['auth_external_state_s.cpp'], - LIBDEPS=['authservercommon']) +env.Library('authcore', ['action_set.cpp', + 'action_type.cpp', + 'authorization_manager.cpp', + 'authorization_session.cpp', + 'authz_documents_update_guard.cpp', + 'authz_manager_external_state.cpp', + 'authz_manager_external_state_local.cpp', + 'authz_session_external_state.cpp', + 'privilege.cpp', + 'privilege_parser.cpp', + 'resource_pattern.cpp', + 'role_graph.cpp', + 'role_graph_update.cpp', + 'role_name.cpp', + 'role_graph_builtin_roles.cpp', + 'user.cpp', + 'user_document_parser.cpp', + 'user_management_commands_parser.cpp', + 'user_name.cpp', + 'user_set.cpp'], + LIBDEPS=['$BUILD_DIR/mongo/auth_helpers', + '$BUILD_DIR/mongo/base/base', + '$BUILD_DIR/mongo/bson', + '$BUILD_DIR/mongo/db/common', + '$BUILD_DIR/mongo/db/ops/update_driver', + '$BUILD_DIR/mongo/md5', + '$BUILD_DIR/mongo/stringutils']) + +env.Library('authservercommon', + ['auth_server_parameters.cpp', + 'authorization_manager_global.cpp', + 'authz_session_external_state_server_common.cpp', + 'security_key.cpp'], + LIBDEPS=['authcore']) + +env.Library('authmongod', + ['authz_manager_external_state_d.cpp', + 'authz_session_external_state_d.cpp', + 'auth_index_d.cpp'], + LIBDEPS=['authservercommon', '$BUILD_DIR/mongo/server_parameters']) + +env.Library('authmongos', + ['authz_manager_external_state_s.cpp', + 'authz_session_external_state_s.cpp', + 'user_cache_invalidator_job.cpp'], + LIBDEPS=['authservercommon']) + +env.Library('authmocks', + ['authz_manager_external_state_mock.cpp'], + LIBDEPS=['$BUILD_DIR/mongo/db/ops/update_driver', + '$BUILD_DIR/mongo/expressions']) env.CppUnitTest('action_set_test', 'action_set_test.cpp', LIBDEPS=['authcore']) -env.CppUnitTest('principal_set_test', 'principal_set_test.cpp', LIBDEPS=['authcore']) -env.CppUnitTest('privilege_set_test', 'privilege_set_test.cpp', LIBDEPS=['authcore']) +env.CppUnitTest('privilege_parser_test', 'privilege_parser_test.cpp', LIBDEPS=['authcore']) +env.CppUnitTest('role_graph_test', 'role_graph_test.cpp', LIBDEPS=['authcore']) +env.CppUnitTest('user_document_parser_test', 'user_document_parser_test.cpp', LIBDEPS=['authcore']) +env.CppUnitTest('user_set_test', 'user_set_test.cpp', LIBDEPS=['authcore']) env.CppUnitTest('authorization_manager_test', 'authorization_manager_test.cpp', - LIBDEPS=['authcore']) + LIBDEPS=['authcore', 'authmocks']) +env.CppUnitTest('authorization_session_test', 'authorization_session_test.cpp', + LIBDEPS=['authcore', 'authmocks']) diff --git a/src/mongo/db/auth/action_set.cpp b/src/mongo/db/auth/action_set.cpp index b7c17ae1996..8eff5dd74eb 100644 --- a/src/mongo/db/auth/action_set.cpp +++ b/src/mongo/db/auth/action_set.cpp @@ -28,10 +28,18 @@ namespace mongo { void ActionSet::addAction(const ActionType& action) { + if (action == ActionType::anyAction) { + addAllActions(); + return; + } _actions.set(action.getIdentifier(), true); } void ActionSet::addAllActionsFromSet(const ActionSet& actions) { + if (actions.contains(ActionType::anyAction)) { + addAllActions(); + return; + } _actions |= actions._actions; } @@ -41,10 +49,14 @@ namespace mongo { void ActionSet::removeAction(const ActionType& action) { _actions.set(action.getIdentifier(), false); + _actions.set(ActionType::anyAction.getIdentifier(), false); } void ActionSet::removeAllActionsFromSet(const ActionSet& other) { _actions &= ~other._actions; + if (!other.empty()) { + _actions.set(ActionType::anyAction.getIdentifier(), false); + } } void ActionSet::removeAllActions() { @@ -63,15 +75,24 @@ namespace mongo { ActionSet* result) { std::vector actionsList; splitStringDelim(actionsString, &actionsList, ','); + return parseActionSetFromStringVector(actionsList, result); + } + + Status ActionSet::parseActionSetFromStringVector(const std::vector& actionsVector, + ActionSet* result) { ActionSet actions; - for (size_t i = 0; i < actionsList.size(); i++) { + for (size_t i = 0; i < actionsVector.size(); i++) { ActionType action; - Status status = ActionType::parseActionFromString(actionsList[i], &action); + Status status = ActionType::parseActionFromString(actionsVector[i], &action); if (status != Status::OK()) { ActionSet empty; *result = empty; return status; } + if (action == ActionType::anyAction) { + actions.addAllActions(); + break; + } actions.addAction(action); } *result = actions; @@ -79,6 +100,9 @@ namespace mongo { } std::string ActionSet::toString() const { + if (contains(ActionType::anyAction)) { + return ActionType::anyAction.toString(); + } StringBuilder str; bool addedOne = false; for (int i = 0; i < ActionType::actionTypeEndValue; i++) { @@ -94,4 +118,19 @@ namespace mongo { return str.str(); } + std::vector ActionSet::getActionsAsStrings() const { + std::vector result; + if (contains(ActionType::anyAction)) { + result.push_back(ActionType::anyAction.toString()); + return result; + } + for (int i = 0; i < ActionType::actionTypeEndValue; i++) { + ActionType action(i); + if (contains(action)) { + result.push_back(ActionType::actionToString(action)); + } + } + return result; + } + } // namespace mongo diff --git a/src/mongo/db/auth/action_set.h b/src/mongo/db/auth/action_set.h index 2cb8997a943..97db1bdebd0 100644 --- a/src/mongo/db/auth/action_set.h +++ b/src/mongo/db/auth/action_set.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include "mongo/base/status.h" #include "mongo/db/auth/action_type.h" @@ -24,7 +25,9 @@ namespace mongo { /* * An ActionSet is a bitmask of ActionTypes that represents a set of actions. - * These are the actions that a Privilege can grant a principal to perform on a resource. + * These are the actions that a Privilege can grant a user to perform on a resource. + * If the special ActionType::anyAction is granted to this set, it automatically sets all bits + * in the bitmask, indicating that it contains all possible actions. */ class ActionSet { public: @@ -35,12 +38,16 @@ namespace mongo { void addAllActionsFromSet(const ActionSet& actionSet); void addAllActions(); + // Removes action from the set. Also removes the "anyAction" action, if present. + // Note: removing the "anyAction" action does *not* remove all other actions. void removeAction(const ActionType& action); void removeAllActionsFromSet(const ActionSet& actionSet); void removeAllActions(); bool empty() const { return _actions.none(); } + bool equals(const ActionSet& other) const { return this->_actions == other._actions; } + bool contains(const ActionType& action) const; // Returns true only if this ActionSet contains all the actions present in the 'other' @@ -50,14 +57,26 @@ namespace mongo { // Returns the string representation of this ActionSet std::string toString() const; + // Returns a vector of strings representing the actions in the ActionSet. + std::vector getActionsAsStrings() const; + // Takes a comma-separated string of action type string representations and returns // an int bitmask of the actions. static Status parseActionSetFromString(const std::string& actionsString, ActionSet* result); + // Takes a vector of action type string representations and returns an ActionSet of the + // actions. + static Status parseActionSetFromStringVector(const std::vector& actionsVector, + ActionSet* result); + private: // bitmask of actions this privilege grants std::bitset _actions; }; + static inline bool operator==(const ActionSet& lhs, const ActionSet& rhs) { + return lhs.equals(rhs); + } + } // namespace mongo diff --git a/src/mongo/db/auth/action_set_test.cpp b/src/mongo/db/auth/action_set_test.cpp index ccf50fbaba5..f91e7c142c5 100644 --- a/src/mongo/db/auth/action_set_test.cpp +++ b/src/mongo/db/auth/action_set_test.cpp @@ -99,5 +99,50 @@ namespace { ASSERT_FALSE(set3.isSupersetOf(set2)); } + TEST(ActionSetTest, anyAction) { + ActionSet set; + + ASSERT_OK(ActionSet::parseActionSetFromString("anyAction", &set)); + ASSERT_TRUE(set.contains(ActionType::find)); + ASSERT_TRUE(set.contains(ActionType::insert)); + ASSERT_TRUE(set.contains(ActionType::anyAction)); + + set.removeAllActions(); + set.addAllActions(); + ASSERT_TRUE(set.contains(ActionType::find)); + ASSERT_TRUE(set.contains(ActionType::insert)); + ASSERT_TRUE(set.contains(ActionType::anyAction)); + + set.removeAllActions(); + set.addAction(ActionType::anyAction); + ASSERT_TRUE(set.contains(ActionType::find)); + ASSERT_TRUE(set.contains(ActionType::insert)); + ASSERT_TRUE(set.contains(ActionType::anyAction)); + + set.removeAction(ActionType::find); + ASSERT_FALSE(set.contains(ActionType::find)); + ASSERT_TRUE(set.contains(ActionType::insert)); + ASSERT_FALSE(set.contains(ActionType::anyAction)); + + set.addAction(ActionType::find); + ASSERT_TRUE(set.contains(ActionType::find)); + ASSERT_TRUE(set.contains(ActionType::insert)); + ASSERT_FALSE(set.contains(ActionType::anyAction)); + + set.addAction(ActionType::anyAction); + ASSERT_TRUE(set.contains(ActionType::find)); + ASSERT_TRUE(set.contains(ActionType::insert)); + ASSERT_TRUE(set.contains(ActionType::anyAction)); + + ASSERT_EQUALS("anyAction", set.toString()); + + set.removeAction(ActionType::anyAction); + ASSERT_TRUE(set.contains(ActionType::find)); + ASSERT_TRUE(set.contains(ActionType::insert)); + ASSERT_FALSE(set.contains(ActionType::anyAction)); + + ASSERT_NOT_EQUALS("anyAction", set.toString()); + } + } // namespace } // namespace mongo diff --git a/src/mongo/db/auth/action_types.txt b/src/mongo/db/auth/action_types.txt index 057084293a3..13bda600c66 100644 --- a/src/mongo/db/auth/action_types.txt +++ b/src/mongo/db/auth/action_types.txt @@ -1,15 +1,22 @@ -# -*- python-mode -*- +# -*- mode: python -*- # List describing the ActionTypes that should be created. # Please note that the order of the elements is not guaranteed to be the same across versions. # This means that the integer value assigned to each ActionType and used internally in ActionSet # also may change between versions. ["addShard", -"captrunc", -"clean", -"clone", -"cloneCollectionLocalSource", -"cloneCollectionTarget", +"anyAction", # Special ActionType that represents *all* actions +"appendOplogNote", +"applicationMessage", +"auditLogRotate", # Not used for permissions checks, but to id the event in logs. +"authCheck", # Not used for permissions checks, but to id the authorization-checking event in logs. +"authSchemaUpgrade", +"authenticate", # Not used for permission checks, but to id authentication events in logs. +"changeCustomData", +"changePassword", +"changeOwnPassword", +"changeOwnCustomData", +"cleanupOrphaned", "closeAllDatabases", "collMod", "collStats", @@ -17,19 +24,26 @@ "connPoolStats", "connPoolSync", "convertToCapped", -"copyDBTarget", "cpuProfiler", "createCollection", +"createDatabase", # Not used for permissions checks, but to id the event in logs. +"createIndex", # Not used for permissions checks, but to id the event in logs. +"createRole", +"createUser", "cursorInfo", "dbHash", "dbStats", "diagLogging", +"dropAllRolesFromDatabase", # Not used for permissions checks, but to id the event in logs. +"dropAllUsersFromDatabase", # Not used for permissions checks, but to id the event in logs. "dropCollection", "dropDatabase", -"dropIndexes", +"dropIndex", +"dropRole", +"dropUser", "emptycapped", +"enableProfiler", "enableSharding", -"ensureIndex", "find", "flushRouterConfig", "fsync", @@ -38,63 +52,58 @@ "getParameter", "getShardMap", "getShardVersion", -"handshake", +"grantRole", +"grantPrivilegesToRole", # Not used for permissions checks, but to id the event in logs. +"grantRolesToRole", # Not used for permissions checks, but to id the event in logs. +"grantRolesToUser", # Not used for permissions checks, but to id the event in logs. "hostInfo", -"indexRead", +"impersonate", "indexStats", "inprog", "insert", +"internal", # Special action type that represents internal actions +"invalidateUserCache", "killCursors", "killop", "listDatabases", "listShards", "logRotate", -"mapReduceShardedFinish", "moveChunk", -"movePrimary", "netstat", -"profileEnable", -"profileRead", +"planCacheIndexFilter", # view/update index filters +"planCacheRead", # view contents of plan cache +"planCacheWrite", # clear cache, drop cache entry, pin/unpin/shun plans "reIndex", "remove", "removeShard", +"renameCollection", # Not used for permissions checks, but to id the event in logs. "renameCollectionSameDB", "repairDatabase", -"replSetElect", -"replSetFreeze", -"replSetFresh", -"replSetGetRBID", +"replSetConfigure", "replSetGetStatus", "replSetHeartbeat", -"replSetInitiate", -"replSetMaintenance", -"replSetReconfig", -"replSetStepDown", -"replSetSyncFrom", +"replSetReconfig", # Not used for permissions checks, but to id the event in logs. +"replSetStateChange", "resync", +"revokeRole", +"revokePrivilegesFromRole", # Not used for permissions checks, but to id the event in logs. +"revokeRolesFromRole", # Not used for permissions checks, but to id the event in logs. +"revokeRolesFromUser", # Not used for permissions checks, but to id the event in logs. "serverStatus", "setParameter", -"setShardVersion", -"shardCollection", +"shardCollection", # Not used for permissions checks, but to id the event in logs. "shardingState", "shutdown", -"split", "splitChunk", "splitVector", "storageDetails", "top", "touch", "unlock", -"unsetSharding", "update", -"userAdmin", +"updateRole", # Not used for permissions checks, but to id the event in logs. +"updateUser", # Not used for permissions checks, but to id the event in logs. "validate", -"writebacklisten", -"writeBacksQueued", -"_migrateClone", -"_recvChunkAbort", -"_recvChunkCommit", -"_recvChunkStart", -"_recvChunkStatus", -"_transferMods"] +"viewRole", +"viewUser"] diff --git a/src/mongo/db/auth/auth_external_state.cpp b/src/mongo/db/auth/auth_external_state.cpp deleted file mode 100644 index 946b75b322a..00000000000 --- a/src/mongo/db/auth/auth_external_state.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/auth/auth_external_state.h" - -#include "mongo/base/status.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/namespacestring.h" - -namespace mongo { - - AuthExternalState::AuthExternalState() {} - AuthExternalState::~AuthExternalState() {} - - Status AuthExternalState::getPrivilegeDocument(const std::string& dbname, - const PrincipalName& principalName, - BSONObj* result) { - - if (dbname == StringData("$external", StringData::LiteralTag()) || - dbname == AuthorizationManager::SERVER_RESOURCE_NAME || - dbname == AuthorizationManager::CLUSTER_RESOURCE_NAME) { - return Status(ErrorCodes::UserNotFound, - mongoutils::str::stream() << "No privilege documents stored in the " << - dbname << " user source."); - } - - if (!NamespaceString::validDBName(dbname)) { - return Status(ErrorCodes::BadValue, "Bad database name \"" + dbname + "\""); - } - - if (dbname == StringData("local", StringData::LiteralTag()) && - principalName.getUser() == internalSecurity.user) { - - if (internalSecurity.pwd.empty()) { - return Status(ErrorCodes::UserNotFound, - "key file must be used to log in with internal user", - 15889); - } - *result = BSON(AuthorizationManager::USER_NAME_FIELD_NAME << - internalSecurity.user << - AuthorizationManager::PASSWORD_FIELD_NAME << - internalSecurity.pwd).getOwned(); - return Status::OK(); - } - - std::string usersNamespace = dbname + ".system.users"; - - BSONObj userBSONObj; - BSONObjBuilder queryBuilder; - queryBuilder.append(AuthorizationManager::USER_NAME_FIELD_NAME, principalName.getUser()); - if (principalName.getDB() == dbname) { - queryBuilder.appendNull(AuthorizationManager::USER_SOURCE_FIELD_NAME); - } - else { - queryBuilder.append(AuthorizationManager::USER_SOURCE_FIELD_NAME, - principalName.getDB()); - } - - bool found = _findUser(usersNamespace, queryBuilder.obj(), &userBSONObj); - if (!found) { - return Status(ErrorCodes::UserNotFound, - mongoutils::str::stream() << "auth: couldn't find user " << - principalName.toString() << ", " << usersNamespace, - 0); - } - - *result = userBSONObj.getOwned(); - return Status::OK(); - } - - bool AuthExternalState::_hasPrivilegeDocument(const std::string& dbname) const { - std::string usersNamespace = dbname + ".system.users"; - - BSONObj userBSONObj; - BSONObj query; - return _findUser(usersNamespace, query, &userBSONObj); - } - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state.h b/src/mongo/db/auth/auth_external_state.h deleted file mode 100644 index 44efe8b97e4..00000000000 --- a/src/mongo/db/auth/auth_external_state.h +++ /dev/null @@ -1,74 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/status.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/auth/principal_name.h" - -namespace mongo { - - /** - * Public interface for a class that encapsulates all the information related to system state - * not stored in AuthorizationManager. This is primarily to make AuthorizationManager easier - * to test. There are two classes that implement this interface, AuthExternalStateImpl, which - * is what's used for the actual system, and AuthExternalStateMock, which is used in the tests. - */ - class AuthExternalState { - MONGO_DISALLOW_COPYING(AuthExternalState); - - public: - - virtual ~AuthExternalState(); - - // Returns true if this connection should be treated as if it has full access to do - // anything, regardless of the current auth state. Currently the reasons why this could be - // are that auth isn't enabled, the connection is from localhost and there are no admin - // users, or the connection is a "god" connection. - // NOTE: _checkShouldAllowLocalhost MUST be called at least once before any call to - // shouldIgnoreAuthChecks or we could ignore auth checks incorrectly. - virtual bool shouldIgnoreAuthChecks() const = 0; - - // Should be called at the beginning of every new request. This performs the checks - // necessary to determine if localhost connections should be given full access. - virtual void startRequest() = 0; - - // Gets the privilege information document for "principalName" on "dbname". - // - // On success, returns Status::OK() and stores a shared-ownership copy of the document into - // "result". - Status getPrivilegeDocument(const std::string& dbname, - const PrincipalName& principalName, - BSONObj* result); - - protected: - AuthExternalState(); // This class should never be instantiated directly. - - // Queries the userNamespace with the given query and returns the privilegeDocument found - // in *result. Returns true if it finds a document matching the query, or false if not. - virtual bool _findUser(const std::string& usersNamespace, - const BSONObj& query, - BSONObj* result) const = 0; - - // Returns true if there exists at least one privilege document in the given database. - bool _hasPrivilegeDocument(const std::string& dbname) const; - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state_d.cpp b/src/mongo/db/auth/auth_external_state_d.cpp deleted file mode 100644 index a29aa8027d4..00000000000 --- a/src/mongo/db/auth/auth_external_state_d.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/auth/auth_external_state_d.h" - -#include "mongo/base/status.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/client.h" -#include "mongo/db/dbhelpers.h" -#include "mongo/db/d_concurrency.h" -#include "mongo/db/instance.h" -#include "mongo/db/jsobj.h" - -namespace mongo { - - AuthExternalStateMongod::AuthExternalStateMongod() {} - AuthExternalStateMongod::~AuthExternalStateMongod() {} - - void AuthExternalStateMongod::startRequest() { - if (!Lock::isLocked()) { - _checkShouldAllowLocalhost(); - } - } - - bool AuthExternalStateMongod::_findUser(const string& usersNamespace, - const BSONObj& query, - BSONObj* result) const { - Client::GodScope gs; - Client::ReadContext ctx(usersNamespace); - - return Helpers::findOne(usersNamespace, query, *result); - } - - bool AuthExternalStateMongod::shouldIgnoreAuthChecks() const { - return cc().isGod() || AuthExternalStateServerCommon::shouldIgnoreAuthChecks(); - } - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state_d.h b/src/mongo/db/auth/auth_external_state_d.h deleted file mode 100644 index d6af37abe57..00000000000 --- a/src/mongo/db/auth/auth_external_state_d.h +++ /dev/null @@ -1,45 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/status.h" -#include "mongo/db/auth/auth_external_state_server_common.h" - -namespace mongo { - - /** - * The implementation of AuthExternalState functionality for mongod. - */ - class AuthExternalStateMongod : public AuthExternalStateServerCommon { - MONGO_DISALLOW_COPYING(AuthExternalStateMongod); - - public: - AuthExternalStateMongod(); - virtual ~AuthExternalStateMongod(); - - virtual bool shouldIgnoreAuthChecks() const; - - virtual void startRequest(); - - protected: - virtual bool _findUser(const string& usersNamespace, - const BSONObj& query, - BSONObj* result) const; - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state_mock.h b/src/mongo/db/auth/auth_external_state_mock.h deleted file mode 100644 index 47e8cb59cf4..00000000000 --- a/src/mongo/db/auth/auth_external_state_mock.h +++ /dev/null @@ -1,54 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/status.h" -#include "mongo/db/auth/auth_external_state.h" - -namespace mongo { - - /** - * Mock of the AuthExternalState class used only for testing. - */ - class AuthExternalStateMock : public AuthExternalState { - MONGO_DISALLOW_COPYING(AuthExternalStateMock); - - public: - AuthExternalStateMock() : _returnValue(false) {} - - virtual bool shouldIgnoreAuthChecks() const { - return _returnValue; - } - - void setReturnValueForShouldIgnoreAuthChecks(bool returnValue) { - _returnValue = returnValue; - } - - virtual bool _findUser(const std::string& usersNamespace, - const BSONObj& query, - BSONObj* result) const { - return false; - } - - virtual void startRequest() {} - - private: - bool _returnValue; - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state_s.cpp b/src/mongo/db/auth/auth_external_state_s.cpp deleted file mode 100644 index b7167cef509..00000000000 --- a/src/mongo/db/auth/auth_external_state_s.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/auth/auth_external_state_s.h" - -#include - -#include "mongo/base/status.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/jsobj.h" -#include "mongo/s/grid.h" - -namespace mongo { - - AuthExternalStateMongos::AuthExternalStateMongos() {} - AuthExternalStateMongos::~AuthExternalStateMongos() {} - - void AuthExternalStateMongos::startRequest() { - _checkShouldAllowLocalhost(); - } - - namespace { - ScopedDbConnection* getConnectionForUsersCollection(const std::string& ns) { - // - // Note: The connection mechanism here is *not* ideal, and should not be used elsewhere. - // If the primary for the collection moves, this approach may throw rather than handle - // version exceptions. - // - - DBConfigPtr config = grid.getDBConfig(ns); - Shard s = config->getShard(ns); - - return ScopedDbConnection::getInternalScopedDbConnection(s.getConnString(), 30.0); - } - } - - bool AuthExternalStateMongos::_findUser(const string& usersNamespace, - const BSONObj& query, - BSONObj* result) const { - scoped_ptr conn(getConnectionForUsersCollection(usersNamespace)); - *result = conn->get()->findOne(usersNamespace, query).getOwned(); - conn->done(); - return !result->isEmpty(); - } - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state_s.h b/src/mongo/db/auth/auth_external_state_s.h deleted file mode 100644 index f5955615306..00000000000 --- a/src/mongo/db/auth/auth_external_state_s.h +++ /dev/null @@ -1,43 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/status.h" -#include "mongo/db/auth/auth_external_state_server_common.h" - -namespace mongo { - - /** - * The implementation of AuthExternalState functionality for mongos. - */ - class AuthExternalStateMongos : public AuthExternalStateServerCommon { - MONGO_DISALLOW_COPYING(AuthExternalStateMongos); - - public: - AuthExternalStateMongos(); - virtual ~AuthExternalStateMongos(); - - virtual void startRequest(); - - protected: - virtual bool _findUser(const string& usersNamespace, - const BSONObj& query, - BSONObj* result) const; - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state_server_common.cpp b/src/mongo/db/auth/auth_external_state_server_common.cpp deleted file mode 100644 index 2f34c3c1b1b..00000000000 --- a/src/mongo/db/auth/auth_external_state_server_common.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/auth/auth_external_state_server_common.h" - -#include "mongo/base/status.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/client.h" -#include "mongo/db/server_parameters.h" -#include "mongo/util/debug_util.h" - -namespace mongo { - -namespace { - MONGO_EXPORT_STARTUP_SERVER_PARAMETER(enableLocalhostAuthBypass, bool, true); -} // namespace - - // NOTE: we default _allowLocalhost to true under the assumption that _checkShouldAllowLocalhost - // will always be called before any calls to shouldIgnoreAuthChecks. If this is not the case, - // it could cause a security hole. - AuthExternalStateServerCommon::AuthExternalStateServerCommon() : _allowLocalhost(true) {} - AuthExternalStateServerCommon::~AuthExternalStateServerCommon() {} - - void AuthExternalStateServerCommon::_checkShouldAllowLocalhost() { - if (noauth) - return; - // If we know that an admin user exists, don't re-check. - if (!_allowLocalhost) - return; - // Don't bother checking if we're not on a localhost connection - if (!ClientBasic::getCurrent()->getIsLocalHostConnection()) { - _allowLocalhost = false; - return; - } - - _allowLocalhost = !_hasPrivilegeDocument("admin"); - if (_allowLocalhost) { - ONCE { - log() << "note: no users configured in admin.system.users, allowing localhost " - "access" << std::endl; - } - } - } - - bool AuthExternalStateServerCommon::shouldIgnoreAuthChecks() const { - ClientBasic* client = ClientBasic::getCurrent(); - return noauth || - (enableLocalhostAuthBypass &&client->getIsLocalHostConnection() && _allowLocalhost); - } - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_external_state_server_common.h b/src/mongo/db/auth/auth_external_state_server_common.h deleted file mode 100644 index 3e1e36ac7b5..00000000000 --- a/src/mongo/db/auth/auth_external_state_server_common.h +++ /dev/null @@ -1,50 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/status.h" -#include "mongo/db/auth/auth_external_state.h" - -namespace mongo { - - /** - * The implementation of AuthExternalState functionality common to mongod and mongos. - */ - class AuthExternalStateServerCommon : public AuthExternalState { - MONGO_DISALLOW_COPYING(AuthExternalStateServerCommon); - - public: - virtual ~AuthExternalStateServerCommon(); - - virtual bool shouldIgnoreAuthChecks() const; - - protected: - AuthExternalStateServerCommon(); - - // Checks whether or not localhost connections should be given full access and stores the - // result in _allowLocalhost. Currently localhost connections are only given full access - // if there are no users in the admin database. - virtual void _checkShouldAllowLocalhost(); - - private: - - bool _allowLocalhost; - - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/auth_index_d.cpp b/src/mongo/db/auth/auth_index_d.cpp index 3ce6c0de43f..8c59f8e9ca3 100644 --- a/src/mongo/db/auth/auth_index_d.cpp +++ b/src/mongo/db/auth/auth_index_d.cpp @@ -12,17 +12,33 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/auth/auth_index_d.h" #include "mongo/base/init.h" +#include "mongo/base/status.h" #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/client.h" #include "mongo/db/dbhelpers.h" -#include "mongo/db/index_update.h" #include "mongo/db/jsobj.h" -#include "mongo/db/namespace_details.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/catalog/collection.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -30,79 +46,83 @@ namespace mongo { namespace authindex { namespace { - BSONObj oldSystemUsersKeyPattern; - BSONObj extendedSystemUsersKeyPattern; - std::string extendedSystemUsersIndexName; + BSONObj v1SystemUsersKeyPattern; + BSONObj v3SystemUsersKeyPattern; + BSONObj v3SystemRolesKeyPattern; + std::string v3SystemUsersIndexName; + std::string v3SystemRolesIndexName; MONGO_INITIALIZER(AuthIndexKeyPatterns)(InitializerContext*) { - oldSystemUsersKeyPattern = BSON(AuthorizationManager::USER_NAME_FIELD_NAME << 1); - extendedSystemUsersKeyPattern = BSON(AuthorizationManager::USER_NAME_FIELD_NAME << 1 << - AuthorizationManager::USER_SOURCE_FIELD_NAME << 1); - extendedSystemUsersIndexName = std::string(str::stream() << - AuthorizationManager::USER_NAME_FIELD_NAME << - "_1_" << - AuthorizationManager::USER_SOURCE_FIELD_NAME << - "_1"); + v1SystemUsersKeyPattern = BSON("user" << 1 << "userSource" << 1); + v3SystemUsersKeyPattern = BSON(AuthorizationManager::USER_NAME_FIELD_NAME << 1 << + AuthorizationManager::USER_DB_FIELD_NAME << 1); + v3SystemRolesKeyPattern = BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << 1 << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << 1); + v3SystemUsersIndexName = std::string( + str::stream() << + AuthorizationManager::USER_NAME_FIELD_NAME << "_1_" << + AuthorizationManager::USER_DB_FIELD_NAME << "_1"); + v3SystemRolesIndexName = std::string( + str::stream() << + AuthorizationManager::ROLE_NAME_FIELD_NAME << "_1_" << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << "_1"); + return Status::OK(); } - void configureSystemUsersIndexes(const StringData& dbname) { - std::string systemUsers = dbname.toString() + ".system.users"; - Client::WriteContext wctx(systemUsers); - - createSystemIndexes(systemUsers); +} // namespace - NamespaceDetails* nsd = nsdetails(systemUsers.c_str()); - if (nsd == NULL) + void configureSystemIndexes(const StringData& dbname) { + int authzVersion; + Status status = getGlobalAuthorizationManager()->getAuthorizationVersion(&authzVersion); + if (!status.isOK()) { return; - - NamespaceDetails::IndexIterator indexIter = nsd->ii(); - std::vector namedIndexesToDrop; - - while (indexIter.more()) { - IndexDetails& idetails = indexIter.next(); - if (idetails.keyPattern() == oldSystemUsersKeyPattern) - namedIndexesToDrop.push_back(idetails.indexName()); } - for (size_t i = 0; i < namedIndexesToDrop.size(); ++i) { - std::string errmsg; - BSONObjBuilder infoBuilder; - if (dropIndexes(nsd, - systemUsers.c_str(), - namedIndexesToDrop[i].c_str(), - errmsg, - infoBuilder, - false)) { - log() << "Dropped index " << namedIndexesToDrop[i] << " with key pattern " << - oldSystemUsersKeyPattern << " from " << systemUsers << - " because it is incompatible with extended form privilege documents." << endl; + if (dbname == "admin" && authzVersion == AuthorizationManager::schemaVersion26Final) { + NamespaceString systemUsers(dbname, "system.users"); + + // Make sure the old unique index from v2.4 on system.users doesn't exist. + Client::WriteContext wctx(systemUsers); + Collection* collection = wctx.ctx().db()->getCollection(NamespaceString(systemUsers)); + if (!collection) { + return; } - else { - // Only reason should be orphaned index, which dropIndexes logged. + IndexCatalog* indexCatalog = collection->getIndexCatalog(); + IndexDescriptor* oldIndex = NULL; + while ((oldIndex = indexCatalog->findIndexByKeyPattern(v1SystemUsersKeyPattern))) { + indexCatalog->dropIndex(oldIndex); } } } -} // namespace - - void configureSystemIndexes(const StringData& dbname) { - configureSystemUsersIndexes(dbname); - } - void createSystemIndexes(const NamespaceString& ns) { - if (ns.coll == "system.users") { + void createSystemIndexes(Collection* collection) { + invariant( collection ); + const NamespaceString& ns = collection->ns(); + if (ns == AuthorizationManager::usersCollectionNamespace) { + try { + Helpers::ensureIndex(collection, + v3SystemUsersKeyPattern, + true, // unique + v3SystemUsersIndexName.c_str()); + } catch (const DBException& e) { + if (e.getCode() == ASSERT_ID_DUPKEY) { + log() << "Duplicate key exception while trying to build unique index on " << + ns << ". This is likely due to problems during the upgrade process " << + endl; + } + throw; + } + } else if (ns == AuthorizationManager::rolesCollectionNamespace) { try { - Helpers::ensureIndex(ns.ns().c_str(), - extendedSystemUsersKeyPattern, + Helpers::ensureIndex(collection, + v3SystemRolesKeyPattern, true, // unique - extendedSystemUsersIndexName.c_str()); + v3SystemRolesIndexName.c_str()); } catch (const DBException& e) { if (e.getCode() == ASSERT_ID_DUPKEY) { log() << "Duplicate key exception while trying to build unique index on " << - ns << ". You most likely have user documents with duplicate \"user\" " - "fields. To resolve this, start up with a version of MongoDB prior to " - "2.4, drop the duplicate user documents, then start up again with the " - "current version." << endl; + ns << "." << endl; } throw; } diff --git a/src/mongo/db/auth/auth_index_d.h b/src/mongo/db/auth/auth_index_d.h index 730adb25b7e..24843f6f0f0 100644 --- a/src/mongo/db/auth/auth_index_d.h +++ b/src/mongo/db/auth/auth_index_d.h @@ -12,31 +12,44 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "mongo/base/string_data.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" namespace mongo { + + class Collection; + namespace authindex { /** - * Ensures that exactly the appropriate indexes are present on system collections supporting - * authentication and authorization in database "dbname". - * - * It is appropriate to call this function on new or existing databases, though it is primarily - * intended for use on existing databases. Under no circumstances may it be called on databases - * with running operations. + * Creates the appropriate indexes on _new_ system collections supporting authentication and + * authorization. */ - void configureSystemIndexes(const StringData& dbname); + void createSystemIndexes(Collection* collection); /** - * Creates the appropriate indexes on _new_ system collections supporting authentication and - * authorization. + * Ensures that exactly the appropriate indexes to support authentication and authorization + * are present for the given database. + * + * It is appropriate to call this function on new or existing databases, though it is + * primarily intended for use on existing databases. */ - void createSystemIndexes(const NamespaceString& ns); + void configureSystemIndexes(const StringData& dbname); } // namespace authindex } // namespace mongo diff --git a/src/mongo/db/auth/auth_server_parameters.cpp b/src/mongo/db/auth/auth_server_parameters.cpp index 85ae1f11ca8..307c8ece6f8 100644 --- a/src/mongo/db/auth/auth_server_parameters.cpp +++ b/src/mongo/db/auth/auth_server_parameters.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/base/init.h" diff --git a/src/mongo/db/auth/authorization_manager.cpp b/src/mongo/db/auth/authorization_manager.cpp index 1cf8efede39..d85b4841804 100644 --- a/src/mongo/db/auth/authorization_manager.cpp +++ b/src/mongo/db/auth/authorization_manager.cpp @@ -12,814 +12,1425 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ +#include "mongo/platform/basic.h" + #include "mongo/db/auth/authorization_manager.h" +#include +#include #include #include #include "mongo/base/init.h" #include "mongo/base/status.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/bson/util/bson_extract.h" +#include "mongo/client/auth_helpers.h" #include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/auth_external_state.h" -#include "mongo/db/auth/principal.h" -#include "mongo/db/auth/principal_set.h" +#include "mongo/db/auth/authz_documents_update_guard.h" +#include "mongo/db/auth/authz_manager_external_state.h" #include "mongo/db/auth/privilege.h" -#include "mongo/db/auth/privilege_set.h" -#include "mongo/db/client.h" +#include "mongo/db/auth/role_graph.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_document_parser.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/user_name_hash.h" #include "mongo/db/jsobj.h" -#include "mongo/db/namespacestring.h" +#include "mongo/platform/compiler.h" +#include "mongo/platform/unordered_map.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" +#include "mongo/util/map_util.h" #include "mongo/util/mongoutils/str.h" namespace mongo { - // this is a config setting, set at startup and not changing after initialization. - bool noauth = true; - - AuthInfo::AuthInfo() { - user = "__system"; - } AuthInfo internalSecurity; - const std::string AuthorizationManager::SERVER_RESOURCE_NAME = "$SERVER"; - const std::string AuthorizationManager::CLUSTER_RESOURCE_NAME = "$CLUSTER"; - const std::string AuthorizationManager::USER_NAME_FIELD_NAME = "user"; - const std::string AuthorizationManager::USER_SOURCE_FIELD_NAME = "userSource"; - const std::string AuthorizationManager::PASSWORD_FIELD_NAME = "pwd"; + MONGO_INITIALIZER_WITH_PREREQUISITES(SetupInternalSecurityUser, MONGO_NO_PREREQUISITES)( + InitializerContext* context) { - bool AuthorizationManager::_doesSupportOldStylePrivileges = true; - -namespace { - const std::string ADMIN_DBNAME = "admin"; - const std::string LOCAL_DBNAME = "local"; - - const std::string ROLES_FIELD_NAME = "roles"; - const std::string OTHER_DB_ROLES_FIELD_NAME = "otherDBRoles"; - const std::string READONLY_FIELD_NAME = "readOnly"; - - const std::string SYSTEM_ROLE_READ = "read"; - const std::string SYSTEM_ROLE_READ_WRITE = "readWrite"; - const std::string SYSTEM_ROLE_USER_ADMIN = "userAdmin"; - const std::string SYSTEM_ROLE_DB_ADMIN = "dbAdmin"; - const std::string SYSTEM_ROLE_CLUSTER_ADMIN = "clusterAdmin"; - const std::string SYSTEM_ROLE_READ_ANY_DB = "readAnyDatabase"; - const std::string SYSTEM_ROLE_READ_WRITE_ANY_DB = "readWriteAnyDatabase"; - const std::string SYSTEM_ROLE_USER_ADMIN_ANY_DB = "userAdminAnyDatabase"; - const std::string SYSTEM_ROLE_DB_ADMIN_ANY_DB = "dbAdminAnyDatabase"; - -} // namespace + User* user = new User(UserName("__system", "local")); - // ActionSets for the various system roles. These ActionSets contain all the actions that - // a user of each system role is granted. - ActionSet readRoleActions; - ActionSet readWriteRoleActions; - ActionSet userAdminRoleActions; - ActionSet dbAdminRoleActions; - ActionSet clusterAdminRoleActions; - // Can only be performed by internal connections. Nothing ever explicitly grants these actions, - // but they're included when calling addAllActions on an ActionSet, which is how internal - // connections are granted their privileges. - ActionSet internalActions; - // Old-style user roles - ActionSet compatibilityReadOnlyActions; - ActionSet compatibilityReadWriteActions; - ActionSet compatibilityReadOnlyAdminActions; - ActionSet compatibilityReadWriteAdminActions; - - // This sets up the system role ActionSets. This is what determines what actions each role - // is authorized to perform - MONGO_INITIALIZER(AuthorizationSystemRoles)(InitializerContext* context) { - // Read role - readRoleActions.addAction(ActionType::cloneCollectionLocalSource); - readRoleActions.addAction(ActionType::collStats); - readRoleActions.addAction(ActionType::dbHash); - readRoleActions.addAction(ActionType::dbStats); - readRoleActions.addAction(ActionType::find); - readRoleActions.addAction(ActionType::indexRead); - readRoleActions.addAction(ActionType::killCursors); - - // Read-write role - readWriteRoleActions.addAllActionsFromSet(readRoleActions); - readWriteRoleActions.addAction(ActionType::cloneCollectionTarget); - readWriteRoleActions.addAction(ActionType::convertToCapped); - readWriteRoleActions.addAction(ActionType::createCollection); // db admin gets this also - readWriteRoleActions.addAction(ActionType::dropCollection); - readWriteRoleActions.addAction(ActionType::dropIndexes); - readWriteRoleActions.addAction(ActionType::emptycapped); - readWriteRoleActions.addAction(ActionType::ensureIndex); - readWriteRoleActions.addAction(ActionType::insert); - readWriteRoleActions.addAction(ActionType::remove); - readWriteRoleActions.addAction(ActionType::renameCollectionSameDB); // db admin gets this also - readWriteRoleActions.addAction(ActionType::update); - - // User admin role - userAdminRoleActions.addAction(ActionType::userAdmin); - - // DB admin role - dbAdminRoleActions.addAction(ActionType::clean); - dbAdminRoleActions.addAction(ActionType::cloneCollectionLocalSource); - dbAdminRoleActions.addAction(ActionType::collMod); - dbAdminRoleActions.addAction(ActionType::collStats); - dbAdminRoleActions.addAction(ActionType::compact); - dbAdminRoleActions.addAction(ActionType::convertToCapped); - dbAdminRoleActions.addAction(ActionType::createCollection); // read_write gets this also - dbAdminRoleActions.addAction(ActionType::dbStats); - dbAdminRoleActions.addAction(ActionType::dropCollection); - dbAdminRoleActions.addAction(ActionType::dropIndexes); - dbAdminRoleActions.addAction(ActionType::ensureIndex); - dbAdminRoleActions.addAction(ActionType::indexRead); - dbAdminRoleActions.addAction(ActionType::indexStats); - dbAdminRoleActions.addAction(ActionType::profileEnable); - dbAdminRoleActions.addAction(ActionType::profileRead); - dbAdminRoleActions.addAction(ActionType::reIndex); - dbAdminRoleActions.addAction(ActionType::renameCollectionSameDB); // read_write gets this also - dbAdminRoleActions.addAction(ActionType::storageDetails); - dbAdminRoleActions.addAction(ActionType::validate); - - // We separate clusterAdmin read-only and read-write actions for backwards - // compatibility with old-style read-only admin users. This separation is not exposed to - // the user, and could go away once we stop supporting old-style privilege documents. - ActionSet clusterAdminRoleReadActions; - ActionSet clusterAdminRoleWriteActions; - - // Cluster admin role - clusterAdminRoleReadActions.addAction(ActionType::connPoolStats); - clusterAdminRoleReadActions.addAction(ActionType::connPoolSync); - clusterAdminRoleReadActions.addAction(ActionType::getCmdLineOpts); - clusterAdminRoleReadActions.addAction(ActionType::getLog); - clusterAdminRoleReadActions.addAction(ActionType::getParameter); - clusterAdminRoleReadActions.addAction(ActionType::getShardMap); - clusterAdminRoleReadActions.addAction(ActionType::getShardVersion); - clusterAdminRoleReadActions.addAction(ActionType::hostInfo); - clusterAdminRoleReadActions.addAction(ActionType::listDatabases); - clusterAdminRoleReadActions.addAction(ActionType::listShards); - clusterAdminRoleReadActions.addAction(ActionType::logRotate); - clusterAdminRoleReadActions.addAction(ActionType::netstat); - clusterAdminRoleReadActions.addAction(ActionType::replSetFreeze); - clusterAdminRoleReadActions.addAction(ActionType::replSetGetStatus); - clusterAdminRoleReadActions.addAction(ActionType::replSetMaintenance); - clusterAdminRoleReadActions.addAction(ActionType::replSetStepDown); - clusterAdminRoleReadActions.addAction(ActionType::replSetSyncFrom); - clusterAdminRoleReadActions.addAction(ActionType::setParameter); - clusterAdminRoleReadActions.addAction(ActionType::setShardVersion); // TODO: should this be internal? - clusterAdminRoleReadActions.addAction(ActionType::serverStatus); - clusterAdminRoleReadActions.addAction(ActionType::splitVector); - clusterAdminRoleReadActions.addAction(ActionType::shutdown); - clusterAdminRoleReadActions.addAction(ActionType::top); - clusterAdminRoleReadActions.addAction(ActionType::touch); - clusterAdminRoleReadActions.addAction(ActionType::unlock); - clusterAdminRoleReadActions.addAction(ActionType::unsetSharding); - clusterAdminRoleReadActions.addAction(ActionType::writeBacksQueued); - - clusterAdminRoleWriteActions.addAction(ActionType::addShard); - clusterAdminRoleWriteActions.addAction(ActionType::closeAllDatabases); - clusterAdminRoleWriteActions.addAction(ActionType::cpuProfiler); - clusterAdminRoleWriteActions.addAction(ActionType::cursorInfo); - clusterAdminRoleWriteActions.addAction(ActionType::diagLogging); - clusterAdminRoleWriteActions.addAction(ActionType::dropDatabase); // TODO: Should there be a CREATE_DATABASE also? - clusterAdminRoleWriteActions.addAction(ActionType::enableSharding); - clusterAdminRoleWriteActions.addAction(ActionType::flushRouterConfig); - clusterAdminRoleWriteActions.addAction(ActionType::fsync); - clusterAdminRoleWriteActions.addAction(ActionType::inprog); - clusterAdminRoleWriteActions.addAction(ActionType::killop); - clusterAdminRoleWriteActions.addAction(ActionType::moveChunk); - clusterAdminRoleWriteActions.addAction(ActionType::movePrimary); - clusterAdminRoleWriteActions.addAction(ActionType::removeShard); - clusterAdminRoleWriteActions.addAction(ActionType::repairDatabase); - clusterAdminRoleWriteActions.addAction(ActionType::replSetInitiate); - clusterAdminRoleWriteActions.addAction(ActionType::replSetReconfig); - clusterAdminRoleWriteActions.addAction(ActionType::resync); - clusterAdminRoleWriteActions.addAction(ActionType::shardCollection); - clusterAdminRoleWriteActions.addAction(ActionType::shardingState); - clusterAdminRoleWriteActions.addAction(ActionType::split); - clusterAdminRoleWriteActions.addAction(ActionType::splitChunk); - - clusterAdminRoleActions.addAllActionsFromSet(clusterAdminRoleReadActions); - clusterAdminRoleActions.addAllActionsFromSet(clusterAdminRoleWriteActions); - clusterAdminRoleActions.addAction(ActionType::killCursors); - - // Old-style user actions, for backwards compatibility - compatibilityReadOnlyActions.addAllActionsFromSet(readRoleActions); - - compatibilityReadWriteActions.addAllActionsFromSet(readWriteRoleActions); - compatibilityReadWriteActions.addAllActionsFromSet(dbAdminRoleActions); - compatibilityReadWriteActions.addAllActionsFromSet(userAdminRoleActions); - compatibilityReadWriteActions.addAction(ActionType::clone); - compatibilityReadWriteActions.addAction(ActionType::copyDBTarget); - compatibilityReadWriteActions.addAction(ActionType::dropDatabase); - compatibilityReadWriteActions.addAction(ActionType::repairDatabase); - - compatibilityReadOnlyAdminActions.addAllActionsFromSet(compatibilityReadOnlyActions); - compatibilityReadOnlyAdminActions.addAllActionsFromSet(clusterAdminRoleReadActions); - - compatibilityReadWriteAdminActions.addAllActionsFromSet(compatibilityReadWriteActions); - compatibilityReadWriteAdminActions.addAllActionsFromSet(compatibilityReadOnlyAdminActions); - compatibilityReadWriteAdminActions.addAllActionsFromSet(clusterAdminRoleWriteActions); - - // Internal commands - internalActions.addAction(ActionType::clone); - internalActions.addAction(ActionType::handshake); - internalActions.addAction(ActionType::mapReduceShardedFinish); - internalActions.addAction(ActionType::replSetElect); - internalActions.addAction(ActionType::replSetFresh); - internalActions.addAction(ActionType::replSetGetRBID); - internalActions.addAction(ActionType::replSetHeartbeat); - internalActions.addAction(ActionType::writebacklisten); - internalActions.addAction(ActionType::_migrateClone); - internalActions.addAction(ActionType::_recvChunkAbort); - internalActions.addAction(ActionType::_recvChunkCommit); - internalActions.addAction(ActionType::_recvChunkStart); - internalActions.addAction(ActionType::_recvChunkStatus); - internalActions.addAction(ActionType::_transferMods); + user->incrementRefCount(); // Pin this user so the ref count never drops below 1. + ActionSet allActions; + allActions.addAllActions(); + PrivilegeVector privileges; + RoleGraph::generateUniversalPrivileges(&privileges); + user->addPrivileges(privileges); + internalSecurity.user = user; return Status::OK(); } - void AuthorizationManager::setSupportOldStylePrivilegeDocuments(bool enabled) { - _doesSupportOldStylePrivileges = enabled; - } - - static inline Status _oldPrivilegeFormatNotSupported() { - return Status(ErrorCodes::UnsupportedFormat, - "Support for compatibility-form privilege documents disabled; " - "All system.users entries must contain a 'roles' field"); - } - - static inline Status _badValue(const char* reason, int location) { - return Status(ErrorCodes::BadValue, reason, location); - } - - static inline Status _badValue(const std::string& reason, int location) { - return Status(ErrorCodes::BadValue, reason, location); - } + const std::string AuthorizationManager::USER_NAME_FIELD_NAME = "user"; + const std::string AuthorizationManager::USER_DB_FIELD_NAME = "db"; + const std::string AuthorizationManager::ROLE_NAME_FIELD_NAME = "role"; + const std::string AuthorizationManager::ROLE_SOURCE_FIELD_NAME = "db"; + const std::string AuthorizationManager::PASSWORD_FIELD_NAME = "pwd"; + const std::string AuthorizationManager::V1_USER_NAME_FIELD_NAME = "user"; + const std::string AuthorizationManager::V1_USER_SOURCE_FIELD_NAME = "userSource"; + + const NamespaceString AuthorizationManager::adminCommandNamespace("admin.$cmd"); + const NamespaceString AuthorizationManager::rolesCollectionNamespace("admin.system.roles"); + const NamespaceString AuthorizationManager::usersAltCollectionNamespace( + "admin.system.new_users"); + const NamespaceString AuthorizationManager::usersBackupCollectionNamespace( + "admin.system.backup_users"); + const NamespaceString AuthorizationManager::usersCollectionNamespace("admin.system.users"); + const NamespaceString AuthorizationManager::versionCollectionNamespace("admin.system.version"); + const NamespaceString AuthorizationManager::defaultTempUsersCollectionNamespace( + "admin.tempusers"); + const NamespaceString AuthorizationManager::defaultTempRolesCollectionNamespace( + "admin.temproles"); + + const BSONObj AuthorizationManager::versionDocumentQuery = BSON("_id" << "authSchema"); + + const std::string AuthorizationManager::schemaVersionFieldName = "currentVersion"; + +#ifndef _MSC_EXTENSIONS + const int AuthorizationManager::schemaVersion24; + const int AuthorizationManager::schemaVersion26Upgrade; + const int AuthorizationManager::schemaVersion26Final; +#endif - static inline StringData makeStringDataFromBSONElement(const BSONElement& element) { - return StringData(element.valuestr(), element.valuestrsize() - 1); - } + bool AuthorizationManager::_doesSupportOldStylePrivileges = true; - static Status _checkRolesArray(const BSONElement& rolesElement) { - if (rolesElement.type() != Array) { - return _badValue("Role fields must be an array when present in system.users entries", - 0); - } - for (BSONObjIterator iter(rolesElement.embeddedObject()); iter.more(); iter.next()) { - BSONElement element = *iter; - if (element.type() != String || makeStringDataFromBSONElement(element).empty()) { - return _badValue("Roles must be non-empty strings.", 0); + /** + * Guard object for synchronizing accesses to data cached in AuthorizationManager instances. + * This guard allows one thread to access the cache at a time, and provides an exception-safe + * mechanism for a thread to release the cache mutex while performing network or disk operations + * while allowing other readers to proceed. + * + * There are two ways to use this guard. One may simply instantiate the guard like a + * std::lock_guard, and perform reads or writes of the cache. + * + * Alternatively, one may instantiate the guard, examine the cache, and then enter into an + * update mode by first wait()ing until otherUpdateInFetchPhase() is false, and then + * calling beginFetchPhase(). At this point, other threads may acquire the guard in the simple + * manner and do reads, but other threads may not enter into a fetch phase. During the fetch + * phase, the thread should perform required network or disk activity to determine what update + * it will make to the cache. Then, it should call endFetchPhase(), to reacquire the user cache + * mutex. At that point, the thread can make its modifications to the cache and let the guard + * go out of scope. + * + * All updates by guards using a fetch-phase are totally ordered with respect to one another, + * and all guards using no fetch phase are totally ordered with respect to one another, but + * there is not a total ordering among all guard objects. + * + * The cached data has an associated counter, called the cache generation. If the cache + * generation changes while a guard is in fetch phase, the fetched data should not be stored + * into the cache, because some invalidation event occurred during the fetch phase. + * + * NOTE: It is not safe to enter fetch phase while holding a database lock. Fetch phase + * operations are allowed to acquire database locks themselves, so entering fetch while holding + * a database lock may lead to deadlock. + */ + class AuthorizationManager::CacheGuard { + MONGO_DISALLOW_COPYING(CacheGuard); + public: + enum FetchSynchronization { + fetchSynchronizationAutomatic, + fetchSynchronizationManual + }; + + /** + * Constructs a cache guard, locking the mutex that synchronizes user cache accesses. + */ + CacheGuard(AuthorizationManager* authzManager, + const FetchSynchronization sync = fetchSynchronizationAutomatic) : + _isThisGuardInFetchPhase(false), + _authzManager(authzManager), + _lock(authzManager->_cacheMutex) { + + if (fetchSynchronizationAutomatic == sync) { + synchronizeWithFetchPhase(); } } - return Status::OK(); - } - Status AuthorizationManager::checkValidPrivilegeDocument(const StringData& dbname, - const BSONObj& doc) { - BSONElement userElement = doc[USER_NAME_FIELD_NAME]; - BSONElement userSourceElement = doc[USER_SOURCE_FIELD_NAME]; - BSONElement passwordElement = doc[PASSWORD_FIELD_NAME]; - BSONElement rolesElement = doc[ROLES_FIELD_NAME]; - BSONElement otherDBRolesElement = doc[OTHER_DB_ROLES_FIELD_NAME]; - BSONElement readOnlyElement = doc[READONLY_FIELD_NAME]; + /** + * Releases the mutex that synchronizes user cache access, if held, and notifies + * any threads waiting for their own opportunity to update the user cache. + */ + ~CacheGuard() { + if (!_lock.owns_lock()) { + _lock.lock(); + } + if (_isThisGuardInFetchPhase) { + fassert(17190, _authzManager->_isFetchPhaseBusy); + _authzManager->_isFetchPhaseBusy = false; + _authzManager->_fetchPhaseIsReady.notify_all(); + } + } - // Validate the "user" element. - if (userElement.type() != String) - return _badValue("system.users entry needs 'user' field to be a string", 14051); - if (makeStringDataFromBSONElement(userElement).empty()) - return _badValue("system.users entry needs 'user' field to be non-empty", 14053); + /** + * Returns true of the authzManager reports that it is in fetch phase. + */ + bool otherUpdateInFetchPhase() { return _authzManager->_isFetchPhaseBusy; } + + /** + * Waits on the _authzManager->_fetchPhaseIsReady condition. + */ + void wait() { + fassert(17222, !_isThisGuardInFetchPhase); + _authzManager->_fetchPhaseIsReady.wait(_lock); + } - // Must set exactly one of "userSource" and "pwd" fields. - if (userSourceElement.eoo() == passwordElement.eoo()) { - return _badValue("system.users entry must have either a 'pwd' field or a 'userSource' " - "field, but not both", 0); + /** + * Enters fetch phase, releasing the _authzManager->_cacheMutex after recording the current + * cache generation. + */ + void beginFetchPhase() { + fassert(17191, !_authzManager->_isFetchPhaseBusy); + _isThisGuardInFetchPhase = true; + _authzManager->_isFetchPhaseBusy = true; + _startGeneration = _authzManager->_cacheGeneration; + _lock.unlock(); } - if (!_doesSupportOldStylePrivileges && rolesElement.eoo()) { - return _oldPrivilegeFormatNotSupported(); + /** + * Exits the fetch phase, reacquiring the _authzManager->_cacheMutex. + */ + void endFetchPhase() { + _lock.lock(); + // We do not clear _authzManager->_isFetchPhaseBusy or notify waiters until + // ~CacheGuard(), for two reasons. First, there's no value to notifying the waiters + // before you're ready to release the mutex, because they'll just go to sleep on the + // mutex. Second, in order to meaningfully check the preconditions of + // isSameCacheGeneration(), we need a state that means "fetch phase was entered and now + // has been exited." That state is _isThisGuardInFetchPhase == true and + // _lock.owns_lock() == true. } - // Cannot have both "roles" and "readOnly" elements. - if (!rolesElement.eoo() && !readOnlyElement.eoo()) { - return _badValue("system.users entry must not have both 'roles' and 'readOnly' fields", - 0); + /** + * Returns true if _authzManager->_cacheGeneration remained the same while this guard was + * in fetch phase. Behavior is undefined if this guard never entered fetch phase. + * + * If this returns true, do not update the cached data with this + */ + bool isSameCacheGeneration() const { + fassert(17223, _isThisGuardInFetchPhase); + fassert(17231, _lock.owns_lock()); + return _startGeneration == _authzManager->_cacheGeneration; } - // Validate the "pwd" element, if present. - if (!passwordElement.eoo()) { - if (passwordElement.type() != String) - return _badValue("system.users entry needs 'pwd' field to be a string", 14052); - if (makeStringDataFromBSONElement(passwordElement).empty()) - return _badValue("system.users entry needs 'pwd' field to be non-empty", 14054); + private: + void synchronizeWithFetchPhase() { + while (otherUpdateInFetchPhase()) + wait(); + fassert(17192, !_authzManager->_isFetchPhaseBusy); + _isThisGuardInFetchPhase = true; + _authzManager->_isFetchPhaseBusy = true; } - // Validate the "userSource" element, if present. - if (!userSourceElement.eoo()) { - if (userSourceElement.type() != String || - makeStringDataFromBSONElement(userSourceElement).empty()) { + OID _startGeneration; + bool _isThisGuardInFetchPhase; + AuthorizationManager* _authzManager; + boost::unique_lock _lock; + }; + + AuthorizationManager::AuthorizationManager(AuthzManagerExternalState* externalState) : + _authEnabled(false), + _externalState(externalState), + _version(schemaVersionInvalid), + _isFetchPhaseBusy(false) { + _updateCacheGeneration_inlock(); + } - return _badValue("system.users entry needs 'userSource' field to be a non-empty " - "string, if present", 0); - } - if (userSourceElement.str() == dbname) { - return _badValue(mongoutils::str::stream() << "'" << dbname << - "' is not a valid value for the userSource field in " << - dbname << ".system.users entries", - 0); - } - if (rolesElement.eoo()) { - return _badValue("system.users entry needs 'roles' field if 'userSource' field " - "is present.", 0); - } + AuthorizationManager::~AuthorizationManager() { + for (unordered_map::iterator it = _userCache.begin(); + it != _userCache.end(); ++it) { + fassert(17265, it->second != internalSecurity.user); + delete it->second ; } + } - // Validate the "roles" element. - if (!rolesElement.eoo()) { - Status status = _checkRolesArray(rolesElement); - if (!status.isOK()) + Status AuthorizationManager::getAuthorizationVersion(int* version) { + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + int newVersion = _version; + if (schemaVersionInvalid == newVersion) { + while (guard.otherUpdateInFetchPhase()) + guard.wait(); + guard.beginFetchPhase(); + Status status = _externalState->getStoredAuthorizationVersion(&newVersion); + guard.endFetchPhase(); + if (!status.isOK()) { + warning() << "Problem fetching the stored schema version of authorization data: " + << status; + *version = schemaVersionInvalid; return status; - } - - if (!otherDBRolesElement.eoo()) { - if (dbname != ADMIN_DBNAME) { - return _badValue("Only admin.system.users entries may contain 'otherDBRoles' " - "fields", 0); - } - if (rolesElement.eoo()) { - return _badValue("system.users entries with 'otherDBRoles' fields must contain " - "'roles' fields", 0); } - if (otherDBRolesElement.type() != Object) { - return _badValue("'otherDBRoles' field must be an object when present in " - "system.users entries", 0); - } - for (BSONObjIterator iter(otherDBRolesElement.embeddedObject()); - iter.more(); iter.next()) { - Status status = _checkRolesArray(*iter); - if (!status.isOK()) - return status; + if (guard.isSameCacheGeneration()) { + _version = newVersion; } } - + *version = newVersion; return Status::OK(); } - AuthorizationManager::AuthorizationManager(AuthExternalState* externalState) { - _externalState.reset(externalState); + void AuthorizationManager::setSupportOldStylePrivilegeDocuments(bool enabled) { + _doesSupportOldStylePrivileges = enabled; } - AuthorizationManager::~AuthorizationManager(){} + bool AuthorizationManager::getSupportOldStylePrivilegeDocuments() { + return _doesSupportOldStylePrivileges; + } - ActionSet AuthorizationManager::getAllUserActions() { - ActionSet allActions; - allActions.addAllActionsFromSet(readRoleActions); - allActions.addAllActionsFromSet(readWriteRoleActions); - allActions.addAllActionsFromSet(userAdminRoleActions); - allActions.addAllActionsFromSet(dbAdminRoleActions); - allActions.addAllActionsFromSet(clusterAdminRoleActions); - return allActions; + OID AuthorizationManager::getCacheGeneration() { + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + return _cacheGeneration; } - void AuthorizationManager::startRequest() { - _externalState->startRequest(); + void AuthorizationManager::setAuthEnabled(bool enabled) { + _authEnabled = enabled; } - void AuthorizationManager::addAuthorizedPrincipal(Principal* principal) { + bool AuthorizationManager::isAuthEnabled() const { + return _authEnabled; + } - // Log out any already-logged-in user on the same database as "principal". - logoutDatabase(principal->getName().getDB().toString()); // See SERVER-8144. + bool AuthorizationManager::hasAnyPrivilegeDocuments() const { + return _externalState->hasAnyPrivilegeDocuments(); + } - _authenticatedPrincipals.add(principal); - if (!principal->isImplicitPrivilegeAcquisitionEnabled()) - return; + Status AuthorizationManager::writeAuthSchemaVersionIfNeeded() { + Status status = _externalState->updateOne( + AuthorizationManager::versionCollectionNamespace, + AuthorizationManager::versionDocumentQuery, + BSON("$set" << BSON(AuthorizationManager::schemaVersionFieldName << + AuthorizationManager::schemaVersion26Final)), + true, // upsert + BSONObj()); // write concern + if (status == ErrorCodes::NoMatchingDocument) { // SERVER-11492 + status = Status::OK(); + } + return status; + } - const std::string dbname = principal->getName().getDB().toString(); - if (dbname == StringData("local", StringData::LiteralTag()) && - principal->getName().getUser() == internalSecurity.user) { + Status AuthorizationManager::insertPrivilegeDocument(const std::string& dbname, + const BSONObj& userObj, + const BSONObj& writeConcern) const { + return _externalState->insertPrivilegeDocument(dbname, userObj, writeConcern); + } - // Grant full access to internal user - ActionSet allActions; - allActions.addAllActions(); - acquirePrivilege(Privilege(PrivilegeSet::WILDCARD_RESOURCE, allActions), - principal->getName()); - return; - } + Status AuthorizationManager::updatePrivilegeDocument(const UserName& user, + const BSONObj& updateObj, + const BSONObj& writeConcern) const { + return _externalState->updatePrivilegeDocument(user, updateObj, writeConcern); + } - _acquirePrivilegesForPrincipalFromDatabase(ADMIN_DBNAME, principal->getName()); - principal->markDatabaseAsProbed(ADMIN_DBNAME); - _acquirePrivilegesForPrincipalFromDatabase(dbname, principal->getName()); - principal->markDatabaseAsProbed(dbname); + Status AuthorizationManager::removePrivilegeDocuments(const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) const { + return _externalState->removePrivilegeDocuments(query, writeConcern, numRemoved); } - void AuthorizationManager::_acquirePrivilegesForPrincipalFromDatabase( - const std::string& dbname, const PrincipalName& principal) { + Status AuthorizationManager::removeRoleDocuments(const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) const { + Status status = _externalState->remove(rolesCollectionNamespace, + query, + writeConcern, + numRemoved); + if (status.code() == ErrorCodes::UnknownError) { + return Status(ErrorCodes::RoleModificationFailed, status.reason()); + } + return status; + } - BSONObj privilegeDocument; - Status status = getPrivilegeDocument(dbname, principal, &privilegeDocument); + Status AuthorizationManager::insertRoleDocument(const BSONObj& roleObj, + const BSONObj& writeConcern) const { + Status status = _externalState->insert(rolesCollectionNamespace, + roleObj, + writeConcern); if (status.isOK()) { - status = acquirePrivilegesFromPrivilegeDocument(dbname, principal, privilegeDocument); + return status; } - if (!status.isOK() && status != ErrorCodes::UserNotFound) { - log() << "Privilege acquisition failed for " << principal << " in database " << - dbname << ": " << status.reason() << " (" << status.codeString() << ")" << endl; + if (status.code() == ErrorCodes::DuplicateKey) { + std::string name = roleObj[AuthorizationManager::ROLE_NAME_FIELD_NAME].String(); + std::string source = roleObj[AuthorizationManager::ROLE_SOURCE_FIELD_NAME].String(); + return Status(ErrorCodes::DuplicateKey, + mongoutils::str::stream() << "Role \"" << name << "@" << source << + "\" already exists"); } + if (status.code() == ErrorCodes::UnknownError) { + return Status(ErrorCodes::RoleModificationFailed, status.reason()); + } + return status; } - Principal* AuthorizationManager::lookupPrincipal(const PrincipalName& name) { - return _authenticatedPrincipals.lookup(name); + Status AuthorizationManager::updateRoleDocument(const RoleName& role, + const BSONObj& updateObj, + const BSONObj& writeConcern) const { + Status status = _externalState->updateOne( + rolesCollectionNamespace, + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << role.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << role.getDB()), + updateObj, + false, + writeConcern); + if (status.isOK()) { + return status; + } + if (status.code() == ErrorCodes::NoMatchingDocument) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role " << role.getFullName() << + " not found"); + } + if (status.code() == ErrorCodes::UnknownError) { + return Status(ErrorCodes::RoleModificationFailed, status.reason()); + } + return status; } - void AuthorizationManager::logoutDatabase(const std::string& dbname) { - Principal* principal = _authenticatedPrincipals.lookupByDBName(dbname); - if (!principal) - return; - _acquiredPrivileges.revokePrivilegesFromPrincipal(principal->getName()); - _authenticatedPrincipals.removeByDBName(dbname); + Status AuthorizationManager::queryAuthzDocument( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& projection, + const boost::function& resultProcessor) { + return _externalState->query(collectionName, query, projection, resultProcessor); } - PrincipalSet::NameIterator AuthorizationManager::getAuthenticatedPrincipalNames() { - return _authenticatedPrincipals.getNames(); + Status AuthorizationManager::updateAuthzDocuments(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched) const { + return _externalState->update(collectionName, + query, + updatePattern, + upsert, + multi, + writeConcern, + nMatched); } - Status AuthorizationManager::acquirePrivilege(const Privilege& privilege, - const PrincipalName& authorizingPrincipal) { - if (!_authenticatedPrincipals.lookup(authorizingPrincipal)) { - return Status(ErrorCodes::UserNotFound, - mongoutils::str::stream() - << "No authenticated principle found with name: " - << authorizingPrincipal.getUser() - << " from database " - << authorizingPrincipal.getDB(), - 0); + Status AuthorizationManager::getBSONForPrivileges(const PrivilegeVector& privileges, + mutablebson::Element resultArray) { + for (PrivilegeVector::const_iterator it = privileges.begin(); + it != privileges.end(); ++it) { + std::string errmsg; + ParsedPrivilege privilege; + if (!ParsedPrivilege::privilegeToParsedPrivilege(*it, &privilege, &errmsg)) { + return Status(ErrorCodes::BadValue, errmsg); + } + resultArray.appendObject("privileges", privilege.toBSON()); } - _acquiredPrivileges.grantPrivilege(privilege, authorizingPrincipal); return Status::OK(); } - void AuthorizationManager::grantInternalAuthorization(const std::string& principalName) { - Principal* principal = new Principal(PrincipalName(principalName, "local")); - ActionSet actions; - actions.addAllActions(); + Status AuthorizationManager::getBSONForRole(RoleGraph* graph, + const RoleName& roleName, + mutablebson::Element result) { + if (!graph->roleExists(roleName)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << roleName.getFullName() << + "does not name an existing role"); + } + std::string id = mongoutils::str::stream() << roleName.getDB() << "." << roleName.getRole(); + result.appendString("_id", id); + result.appendString(ROLE_NAME_FIELD_NAME, roleName.getRole()); + result.appendString(ROLE_SOURCE_FIELD_NAME, roleName.getDB()); + + // Build privileges array + mutablebson::Element privilegesArrayElement = + result.getDocument().makeElementArray("privileges"); + result.pushBack(privilegesArrayElement); + const PrivilegeVector& privileges = graph->getDirectPrivileges(roleName); + Status status = getBSONForPrivileges(privileges, privilegesArrayElement); + if (!status.isOK()) { + return status; + } - addAuthorizedPrincipal(principal); - fassert(16581, acquirePrivilege(Privilege(PrivilegeSet::WILDCARD_RESOURCE, actions), - principal->getName()).isOK()); - } + // Build roles array + mutablebson::Element rolesArrayElement = result.getDocument().makeElementArray("roles"); + result.pushBack(rolesArrayElement); + for (RoleNameIterator roles = graph->getDirectSubordinates(roleName); + roles.more(); + roles.next()) { + + const RoleName& subRole = roles.get(); + mutablebson::Element roleObj = result.getDocument().makeElementObject(""); + roleObj.appendString(ROLE_NAME_FIELD_NAME, subRole.getRole()); + roleObj.appendString(ROLE_SOURCE_FIELD_NAME, subRole.getDB()); + rolesArrayElement.pushBack(roleObj); + } - bool AuthorizationManager::hasInternalAuthorization() { - ActionSet allActions; - allActions.addAllActions(); - return _acquiredPrivileges.hasPrivilege(Privilege(PrivilegeSet::WILDCARD_RESOURCE, - allActions)); + return Status::OK(); } - ActionSet AuthorizationManager::getActionsForOldStyleUser(const std::string& dbname, - bool readOnly) { - if (dbname == ADMIN_DBNAME || dbname == LOCAL_DBNAME) { - if (readOnly) { - return compatibilityReadOnlyAdminActions; - } else { - return compatibilityReadWriteAdminActions; - } - } else { - if (readOnly) { - return compatibilityReadOnlyActions; - } else { - return compatibilityReadWriteActions; + static const RoleName userAdminAnyDatabase("userAdminAnyDatabase", "admin"); + static void _initializeUserPrivilegesFromRolesV1(User* user) { + PrivilegeVector privileges; + for (RoleNameIterator roles = user->getRoles(); roles.more(); roles.next()) { + RoleGraph::addPrivilegesForBuiltinRole(roles.get(), &privileges); + if (roles.get() == userAdminAnyDatabase) { + // Giving schemaVersion24 users with userAdminAnyDatabase these privileges allows + // them to conduct a manual upgrade from schemaVersion24 to schemaVersion26Final. + ActionSet actions; + actions.addAction(ActionType::find); + actions.addAction(ActionType::insert); + actions.addAction(ActionType::update); + actions.addAction(ActionType::remove); + actions.addAction(ActionType::createIndex); + actions.addAction(ActionType::dropIndex); + Privilege::addPrivilegeToPrivilegeVector( + &privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::versionCollectionNamespace), + actions)); + Privilege::addPrivilegeToPrivilegeVector( + &privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersAltCollectionNamespace), + actions)); + Privilege::addPrivilegeToPrivilegeVector( + &privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersBackupCollectionNamespace), + actions)); } } + user->addPrivileges(privileges); } - Status AuthorizationManager::acquirePrivilegesFromPrivilegeDocument( - const std::string& dbname, const PrincipalName& principal, const BSONObj& privilegeDocument) { - if (!_authenticatedPrincipals.lookup(principal)) { - return Status(ErrorCodes::UserNotFound, - mongoutils::str::stream() - << "No authenticated principle found with name: " - << principal.getUser() - << " from database " - << principal.getDB(), + Status AuthorizationManager::_initializeUserFromPrivilegeDocument( + User* user, const BSONObj& privDoc) { + V2UserDocumentParser parser; + std::string userName = parser.extractUserNameFromUserDocument(privDoc); + if (userName != user->getName().getUser()) { + return Status(ErrorCodes::BadValue, + mongoutils::str::stream() << "User name from privilege document \"" + << userName + << "\" doesn't match name of provided User \"" + << user->getName().getUser() + << "\"", 0); } - return buildPrivilegeSet(dbname, principal, privilegeDocument, &_acquiredPrivileges); - } - Status AuthorizationManager::buildPrivilegeSet(const std::string& dbname, - const PrincipalName& principal, - const BSONObj& privilegeDocument, - PrivilegeSet* result) { - if (!privilegeDocument.hasField(ROLES_FIELD_NAME)) { - // Old-style (v2.2 and prior) privilege document - if (_doesSupportOldStylePrivileges) { - return _buildPrivilegeSetFromOldStylePrivilegeDocument(dbname, - principal, - privilegeDocument, - result); - } - else { - return _oldPrivilegeFormatNotSupported(); - } + Status status = parser.initializeUserCredentialsFromUserDocument(user, privDoc); + if (!status.isOK()) { + return status; } - else { - return _buildPrivilegeSetFromExtendedPrivilegeDocument( - dbname, principal, privilegeDocument, result); + status = parser.initializeUserRolesFromUserDocument(privDoc, user); + if (!status.isOK()) { + return status; + } + status = parser.initializeUserIndirectRolesFromUserDocument(privDoc, user); + if (!status.isOK()) { + return status; } + status = parser.initializeUserPrivilegesFromUserDocument(privDoc, user); + return Status::OK(); } - Status AuthorizationManager::_buildPrivilegeSetFromOldStylePrivilegeDocument( - const std::string& dbname, - const PrincipalName& principal, - const BSONObj& privilegeDocument, - PrivilegeSet* result) { - if (!(privilegeDocument.hasField(USER_NAME_FIELD_NAME) && - privilegeDocument.hasField(PASSWORD_FIELD_NAME))) { + Status AuthorizationManager::getUserDescription(const UserName& userName, BSONObj* result) { + return _externalState->getUserDescription(userName, result); + } - return Status(ErrorCodes::UnsupportedFormat, - mongoutils::str::stream() << "Invalid old-style privilege document " - "received when trying to extract privileges: " - << privilegeDocument, - 0); + Status AuthorizationManager::getRoleDescription(const RoleName& roleName, + bool showPrivileges, + BSONObj* result) { + return _externalState->getRoleDescription(roleName, showPrivileges, result); + } + + Status AuthorizationManager::getRoleDescriptionsForDB(const std::string dbname, + bool showPrivileges, + bool showBuiltinRoles, + vector* result) { + return _externalState->getRoleDescriptionsForDB(dbname, + showPrivileges, + showBuiltinRoles, + result); + } + + Status AuthorizationManager::acquireUser(const UserName& userName, User** acquiredUser) { + if (userName == internalSecurity.user->getName()) { + *acquiredUser = internalSecurity.user; + return Status::OK(); } - if (privilegeDocument[USER_NAME_FIELD_NAME].str() != principal.getUser()) { - return Status(ErrorCodes::BadValue, - mongoutils::str::stream() << "Principal name from privilege document \"" - << privilegeDocument[USER_NAME_FIELD_NAME].str() - << "\" doesn't match name of provided Principal \"" - << principal.getUser() - << "\"", - 0); + + unordered_map::iterator it; + + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + while ((_userCache.end() == (it = _userCache.find(userName))) && + guard.otherUpdateInFetchPhase()) { + + guard.wait(); + } + + if (it != _userCache.end()) { + fassert(16914, it->second); + fassert(17003, it->second->isValid()); + fassert(17008, it->second->getRefCount() > 0); + it->second->incrementRefCount(); + *acquiredUser = it->second; + return Status::OK(); } - bool readOnly = privilegeDocument[READONLY_FIELD_NAME].trueValue(); - ActionSet actions = getActionsForOldStyleUser(dbname, readOnly); - std::string resourceName = (dbname == ADMIN_DBNAME || dbname == LOCAL_DBNAME) ? - PrivilegeSet::WILDCARD_RESOURCE : dbname; - result->grantPrivilege(Privilege(resourceName, actions), principal); + std::auto_ptr user; + + int authzVersion = _version; + guard.beginFetchPhase(); + + // Number of times to retry a user document that fetches due to transient + // AuthSchemaIncompatible errors. These errors should only ever occur during and shortly + // after schema upgrades. + static const int maxAcquireRetries = 2; + Status status = Status::OK(); + for (int i = 0; i < maxAcquireRetries; ++i) { + if (authzVersion == schemaVersionInvalid) { + Status status = _externalState->getStoredAuthorizationVersion(&authzVersion); + if (!status.isOK()) + return status; + } + + switch (authzVersion) { + default: + status = Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Illegal value for authorization data schema version, " << + authzVersion); + break; + case schemaVersion26Final: + case schemaVersion26Upgrade: + status = _fetchUserV2(userName, &user); + break; + case schemaVersion24: + status = _fetchUserV1(userName, &user); + break; + } + if (status.isOK()) + break; + if (status != ErrorCodes::AuthSchemaIncompatible) + return status; + + authzVersion = schemaVersionInvalid; + } + if (!status.isOK()) + return status; + + guard.endFetchPhase(); + + user->incrementRefCount(); + // NOTE: It is not safe to throw an exception from here to the end of the method. + if (guard.isSameCacheGeneration()) { + _userCache.insert(make_pair(userName, user.get())); + if (_version == schemaVersionInvalid) + _version = authzVersion; + } + else { + // If the cache generation changed while this thread was in fetch mode, the data + // associated with the user may now be invalid, so we must mark it as such. The caller + // may still opt to use the information for a short while, but not indefinitely. + user->invalidate(); + } + *acquiredUser = user.release(); return Status::OK(); } - /** - * Adds to "outPrivileges" the privileges associated with having the named "role" on "dbname". - * - * Returns non-OK status if "role" is not a defined role in "dbname". - */ - static void _addPrivilegesForSystemRole(const std::string& dbname, - const std::string& role, - std::vector* outPrivileges) { - const bool isAdminDB = (dbname == ADMIN_DBNAME); + Status AuthorizationManager::_fetchUserV2(const UserName& userName, + std::auto_ptr* acquiredUser) { + BSONObj userObj; + Status status = getUserDescription(userName, &userObj); + if (!status.isOK()) { + return status; + } + + // Put the new user into an auto_ptr temporarily in case there's an error while + // initializing the user. + std::auto_ptr user(new User(userName)); - if (role == SYSTEM_ROLE_READ) { - outPrivileges->push_back(Privilege(dbname, readRoleActions)); + status = _initializeUserFromPrivilegeDocument(user.get(), userObj); + if (!status.isOK()) { + return status; } - else if (role == SYSTEM_ROLE_READ_WRITE) { - outPrivileges->push_back(Privilege(dbname, readWriteRoleActions)); + acquiredUser->reset(user.release()); + return Status::OK(); + } + + Status AuthorizationManager::_fetchUserV1(const UserName& userName, + std::auto_ptr* acquiredUser) { + + BSONObj privDoc; + V1UserDocumentParser parser; + const bool isExternalUser = (userName.getDB() == "$external"); + const bool isAdminUser = (userName.getDB() == "admin"); + + std::auto_ptr user(new User(userName)); + user->setSchemaVersion1(); + user->markProbedV1("$external"); + if (isExternalUser) { + User::CredentialData creds; + creds.isExternal = true; + user->setCredentials(creds); } - else if (role == SYSTEM_ROLE_USER_ADMIN) { - outPrivileges->push_back(Privilege(dbname, userAdminRoleActions)); + else { + // Users from databases other than "$external" must have an associated privilege + // document in their database. + Status status = _externalState->getPrivilegeDocumentV1( + userName.getDB(), userName, &privDoc); + if (!status.isOK()) + return status; + + status = parser.initializeUserRolesFromUserDocument( + user.get(), privDoc, userName.getDB()); + if (!status.isOK()) + return status; + + status = parser.initializeUserCredentialsFromUserDocument(user.get(), privDoc); + if (!status.isOK()) + return status; + user->markProbedV1(userName.getDB()); } - else if (role == SYSTEM_ROLE_DB_ADMIN) { - outPrivileges->push_back(Privilege(dbname, dbAdminRoleActions)); + if (!isAdminUser) { + // Users from databases other than "admin" probe the "admin" database at login, to + // ensure that the acquire any privileges derived from "otherDBRoles" fields in + // admin.system.users. + Status status = _externalState->getPrivilegeDocumentV1("admin", userName, &privDoc); + if (status.isOK()) { + status = parser.initializeUserRolesFromUserDocument(user.get(), privDoc, "admin"); + if (!status.isOK()) + return status; + } + else if (status != ErrorCodes::UserNotFound) { + return status; + } + user->markProbedV1("admin"); } - else if (isAdminDB && role == SYSTEM_ROLE_READ_ANY_DB) { - outPrivileges->push_back(Privilege(PrivilegeSet::WILDCARD_RESOURCE, readRoleActions)); + + _initializeUserPrivilegesFromRolesV1(user.get()); + acquiredUser->reset(user.release()); + return Status::OK(); + } + + Status AuthorizationManager::acquireV1UserProbedForDb( + const UserName& userName, const StringData& dbname, User** acquiredUser) { + + if (userName == internalSecurity.user->getName()) { + *acquiredUser = internalSecurity.user; + return Status::OK(); } - else if (isAdminDB && role == SYSTEM_ROLE_READ_WRITE_ANY_DB) { - outPrivileges->push_back( - Privilege(PrivilegeSet::WILDCARD_RESOURCE, readWriteRoleActions)); + + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + std::auto_ptr user; + { + unordered_map::iterator it; + while ((_userCache.end() == (it = _userCache.find(userName))) && + guard.otherUpdateInFetchPhase()) { + + guard.wait(); + } + + if (_userCache.end() != it) { + User* cachedUser = it->second; + fassert(17225, cachedUser->isValid()); + if ((cachedUser->getSchemaVersion() != schemaVersion24) || + cachedUser->hasProbedV1(dbname)) { + + cachedUser->incrementRefCount(); + *acquiredUser = cachedUser; + return Status::OK(); + } + // We clone cachedUser for two reasons. First, because it is not OK to mutate a + // User object that may have been returned from acquireUser() or + // acquireV1UserProbedForDb(). Second, because outside of this scope (or more + // precisely, after calling guard.wait() or guard.beginFetchPhase(), after the + // scope), references to data in the _userCache for which we do not know the + // refcount is greater than zero are invalid. + user.reset(cachedUser->clone()); + } } - else if (isAdminDB && role == SYSTEM_ROLE_USER_ADMIN_ANY_DB) { - outPrivileges->push_back( - Privilege(PrivilegeSet::WILDCARD_RESOURCE, userAdminRoleActions)); + while (guard.otherUpdateInFetchPhase()) + guard.wait(); + guard.beginFetchPhase(); + + if (!user.get()) { + Status status = _fetchUserV1(userName, &user); + if (status == ErrorCodes::AuthSchemaIncompatible) { + // Must early-return from this if block, because we end the fetch phase. Since the + // auth schema is incompatible with schemaVersion24, make a best effort to do the + // schemaVersion26(Upgrade|Final) user acquisition, and return. + status = _fetchUserV2(userName, &user); + guard.endFetchPhase(); + if (status.isOK()) { + // Not safe to throw from here until the function returns. + if (guard.isSameCacheGeneration()) { + _invalidateUserCache_inlock(); + _userCache.insert(make_pair(userName, user.get())); + } + else { + user->invalidate(); + } + user->incrementRefCount(); + *acquiredUser = user.release(); + } + return status; + } + if (!status.isOK()) + return status; } - else if (isAdminDB && role == SYSTEM_ROLE_DB_ADMIN_ANY_DB) { - outPrivileges->push_back( - Privilege(PrivilegeSet::WILDCARD_RESOURCE, dbAdminRoleActions)); + + if (!user->hasProbedV1(dbname)) { + BSONObj privDoc; + Status status = _externalState->getPrivilegeDocumentV1(dbname, userName, &privDoc); + if (status.isOK()) { + V1UserDocumentParser parser; + status = parser.initializeUserRolesFromUserDocument(user.get(), privDoc, dbname); + if (!status.isOK()) + return status; + _initializeUserPrivilegesFromRolesV1(user.get()); + user->markProbedV1(dbname); + } + else if (status == ErrorCodes::UserNotFound) { + user->markProbedV1(dbname); + } else { + return status; + } } - else if (isAdminDB && role == SYSTEM_ROLE_CLUSTER_ADMIN) { - outPrivileges->push_back( - Privilege(PrivilegeSet::WILDCARD_RESOURCE, clusterAdminRoleActions)); + + guard.endFetchPhase(); + user->incrementRefCount(); + // NOTE: It is not safe to throw an exception from here to the end of the method. + *acquiredUser = user.release(); + if (guard.isSameCacheGeneration()) { + unordered_map::iterator it = _userCache.find(userName); + if (it != _userCache.end()) { + it->second->invalidate(); + it->second = *acquiredUser; + } + else { + _userCache.insert(make_pair(userName, *acquiredUser)); + } } else { - warning() << "No such role, \"" << role << "\", in database " << dbname << - ". No privileges will be acquired from this role" << endl; + // If the cache generation changed while this thread was in fetch mode, the data + // associated with the user may now be invalid, so we must mark it as such. The caller + // may still opt to use the information for a short while, but not indefinitely. + (*acquiredUser)->invalidate(); } + return Status::OK(); } - /** - * Given a database name and a BSONElement representing an array of roles, populates - * "outPrivileges" with the privileges associated with the given roles on the named database. - * - * Returns Status::OK() on success. - */ - static Status _getPrivilegesFromRoles(const std::string& dbname, - const BSONElement& rolesElement, - std::vector* outPrivileges) { + void AuthorizationManager::releaseUser(User* user) { + if (user == internalSecurity.user) { + return; + } - static const char privilegesTypeMismatchMessage[] = - "Roles must be enumerated in an array of strings."; + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + user->decrementRefCount(); + if (user->getRefCount() == 0) { + // If it's been invalidated then it's not in the _userCache anymore. + if (user->isValid()) { + MONGO_COMPILER_VARIABLE_UNUSED bool erased = _userCache.erase(user->getName()); + dassert(erased); + } + delete user; + } + } - if (dbname == PrivilegeSet::WILDCARD_RESOURCE) { - return Status(ErrorCodes::BadValue, - PrivilegeSet::WILDCARD_RESOURCE + " is an invalid database name."); + void AuthorizationManager::invalidateUserByName(const UserName& userName) { + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + _updateCacheGeneration_inlock(); + unordered_map::iterator it = _userCache.find(userName); + if (it == _userCache.end()) { + return; } - if (rolesElement.type() != Array) - return Status(ErrorCodes::TypeMismatch, privilegesTypeMismatchMessage); + User* user = it->second; + _userCache.erase(it); + user->invalidate(); + } - for (BSONObjIterator iter(rolesElement.embeddedObject()); iter.more(); iter.next()) { - BSONElement roleElement = *iter; - if (roleElement.type() != String) - return Status(ErrorCodes::TypeMismatch, privilegesTypeMismatchMessage); - _addPrivilegesForSystemRole(dbname, roleElement.str(), outPrivileges); + void AuthorizationManager::invalidateUsersFromDB(const std::string& dbname) { + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + _updateCacheGeneration_inlock(); + unordered_map::iterator it = _userCache.begin(); + while (it != _userCache.end()) { + User* user = it->second; + if (user->getName().getDB() == dbname) { + _userCache.erase(it++); + user->invalidate(); + } else { + ++it; + } } - return Status::OK(); } - Status AuthorizationManager::_buildPrivilegeSetFromExtendedPrivilegeDocument( - const std::string& dbname, - const PrincipalName& principal, - const BSONObj& privilegeDocument, - PrivilegeSet* result) { + void AuthorizationManager::invalidateUserCache() { + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + _invalidateUserCache_inlock(); + } - if (!privilegeDocument[READONLY_FIELD_NAME].eoo()) { - return Status(ErrorCodes::UnsupportedFormat, - "Privilege documents may not contain both \"readonly\" and " - "\"roles\" fields"); + void AuthorizationManager::_invalidateUserCache_inlock() { + _updateCacheGeneration_inlock(); + for (unordered_map::iterator it = _userCache.begin(); + it != _userCache.end(); ++it) { + fassert(17266, it->second != internalSecurity.user); + it->second->invalidate(); } + _userCache.clear(); - std::vector acquiredPrivileges; + // Reread the schema version before acquiring the next user. + _version = schemaVersionInvalid; + } - // Acquire privileges on "dbname". - Status status = _getPrivilegesFromRoles( - dbname, privilegeDocument[ROLES_FIELD_NAME], &acquiredPrivileges); + Status AuthorizationManager::initialize() { + invalidateUserCache(); + Status status = _externalState->initialize(); if (!status.isOK()) return status; - // If "dbname" is the admin database, handle the otherDBPrivileges field, which - // grants privileges on databases other than "dbname". - BSONElement otherDbPrivileges = privilegeDocument[OTHER_DB_ROLES_FIELD_NAME]; - if (dbname == ADMIN_DBNAME) { - switch (otherDbPrivileges.type()) { - case EOO: - break; - case Object: { - for (BSONObjIterator iter(otherDbPrivileges.embeddedObject()); - iter.more(); iter.next()) { - - BSONElement rolesElement = *iter; - status = _getPrivilegesFromRoles( - rolesElement.fieldName(), rolesElement, &acquiredPrivileges); - if (!status.isOK()) - return status; - } - break; + return Status::OK(); + } + + bool AuthorizationManager::tryAcquireAuthzUpdateLock(const StringData& why) { + return _externalState->tryAcquireAuthzUpdateLock(why); + } + + void AuthorizationManager::releaseAuthzUpdateLock() { + return _externalState->releaseAuthzUpdateLock(); + } + +namespace { + + /** + * Logs that the auth schema upgrade failed because of "status" and returns "status". + */ + Status logUpgradeFailed(const Status& status) { + log() << "Auth schema upgrade failed with " << status; + return status; + } + + /** + * Upserts a schemaVersion26Upgrade user document in the usersAltCollectionNamespace + * according to the schemaVersion24 user document "oldUserDoc" from database "sourceDB". + * + * Throws a DBException on errors. + */ + void upgradeProcessUser(AuthzManagerExternalState* externalState, + const StringData& sourceDB, + const BSONObj& oldUserDoc, + const BSONObj& writeConcern) { + + uassert(17387, + mongoutils::str::stream() << "While preparing to upgrade user doc from the 2.4 " + "user data schema to the 2.6 schema, found a user doc with a " + "\"credentials\" field, indicating that the doc already has the new " + "schema. Make sure that all documents in admin.system.users have the same " + "user data schema and that the version document in admin.system.version " + "indicates the correct schema version. User doc found: " << + oldUserDoc.toString(), + !oldUserDoc.hasField("credentials")); + + uassert(17386, + mongoutils::str::stream() << "While preparing to upgrade user doc from " + "the 2.4 user data schema to the 2.6 schema, found a user doc " + "that doesn't conform to the 2.4 *or* 2.6 schema. Doc found: " + << oldUserDoc.toString(), + oldUserDoc.hasField("user") && + (oldUserDoc.hasField("userSource") || oldUserDoc.hasField("pwd"))); + + std::string oldUserSource; + uassertStatusOK(bsonExtractStringFieldWithDefault( + oldUserDoc, + "userSource", + sourceDB, + &oldUserSource)); + + if (oldUserSource == "local") + return; // Skips users from "local" database, which cannot be upgraded. + + const std::string oldUserName = oldUserDoc["user"].String(); + BSONObj query = BSON("_id" << oldUserSource + "." + oldUserName); + + BSONObjBuilder updateBuilder; + { + BSONObjBuilder toSetBuilder(updateBuilder.subobjStart("$set")); + toSetBuilder << "user" << oldUserName << "db" << oldUserSource; + BSONElement pwdElement = oldUserDoc["pwd"]; + if (!pwdElement.eoo()) { + toSetBuilder << "credentials" << BSON("MONGODB-CR" << pwdElement.String()); } - default: - return Status(ErrorCodes::TypeMismatch, - "Field \"otherDBRoles\" must be an object, if present."); + else if (oldUserSource == "$external") { + toSetBuilder << "credentials" << BSON("external" << true); } } - else if (!otherDbPrivileges.eoo()) { - return Status(ErrorCodes::BadValue, "Only the admin database may contain a field " - "called \"otherDBRoles\""); + { + BSONObjBuilder pushAllBuilder(updateBuilder.subobjStart("$pushAll")); + BSONArrayBuilder rolesBuilder(pushAllBuilder.subarrayStart("roles")); + + const bool readOnly = oldUserDoc["readOnly"].trueValue(); + const BSONElement rolesElement = oldUserDoc["roles"]; + if (readOnly) { + // Handles the cases where there is a truthy readOnly field, which is a 2.2-style + // read-only user. + if (sourceDB == "admin") { + rolesBuilder << BSON("role" << "readAnyDatabase" << "db" << "admin"); + } + else { + rolesBuilder << BSON("role" << "read" << "db" << sourceDB); + } + } + else if (rolesElement.eoo()) { + // Handles the cases where the readOnly field is absent or falsey, but the + // user is known to be 2.2-style because it lacks a roles array. + if (sourceDB == "admin") { + rolesBuilder << BSON("role" << "root" << "db" << "admin"); + } + else { + rolesBuilder << BSON("role" << "dbOwner" << "db" << sourceDB); + } + } + else { + // Handles 2.4-style user documents, with roles arrays and (optionally, in admin db) + // otherDBRoles objects. + uassert(17252, + "roles field in v2.4 user documents must be an array", + rolesElement.type() == Array); + for (BSONObjIterator oldRoles(rolesElement.Obj()); + oldRoles.more(); + oldRoles.next()) { + + BSONElement roleElement = *oldRoles; + rolesBuilder << BSON("role" << roleElement.String() << "db" << sourceDB); + } + + BSONElement otherDBRolesElement = oldUserDoc["otherDBRoles"]; + if (sourceDB == "admin" && !otherDBRolesElement.eoo()) { + uassert(17253, + "otherDBRoles field in v2.4 user documents must be an object.", + otherDBRolesElement.type() == Object); + + for (BSONObjIterator otherDBs(otherDBRolesElement.Obj()); + otherDBs.more(); + otherDBs.next()) { + + BSONElement otherDBRoles = *otherDBs; + if (otherDBRoles.fieldNameStringData() == "local") + continue; + uassert(17254, + "Member fields of otherDBRoles objects must be arrays.", + otherDBRoles.type() == Array); + for (BSONObjIterator oldRoles(otherDBRoles.Obj()); + oldRoles.more(); + oldRoles.next()) { + + BSONElement roleElement = *oldRoles; + rolesBuilder << BSON("role" << roleElement.String() << + "db" << otherDBRoles.fieldNameStringData()); + } + } + } + } } + BSONObj update = updateBuilder.obj(); + + uassertStatusOK(externalState->updateOne( + AuthorizationManager::usersAltCollectionNamespace, + query, + update, + true, + writeConcern)); + } - result->grantPrivileges(acquiredPrivileges, principal); - return Status::OK(); + /** + * For every schemaVersion24 user document in the system.users collection of "db", + * upserts the appropriate schemaVersion26Upgrade user document in usersAltCollectionNamespace. + */ + Status upgradeUsersFromDB(AuthzManagerExternalState* externalState, + const StringData& db, + const BSONObj& writeConcern) { + log() << "Auth schema upgrade processing schema version " << + AuthorizationManager::schemaVersion24 << " users from database " << db; + return externalState->query( + NamespaceString(db, "system.users"), + BSONObj(), + BSONObj(), + boost::bind(upgradeProcessUser, externalState, db, _1, writeConcern)); } - bool AuthorizationManager::checkAuthorization(const std::string& resource, - ActionType action) { - return checkAuthForPrivilege(Privilege(resource, action)).isOK(); + /** + * Inserts "document" into "collection", throwing a DBException on failure. + */ + void uassertInsertIntoCollection( + AuthzManagerExternalState* externalState, + const NamespaceString& collection, + const BSONObj& document, + const BSONObj& writeConcern) { + uassertStatusOK(externalState->insert(collection, document, writeConcern)); } - bool AuthorizationManager::checkAuthorization(const std::string& resource, - ActionSet actions) { - return checkAuthForPrivilege(Privilege(resource, actions)).isOK(); + /** + * Copies the contents of "sourceCollection" into "targetCollection", which must be a distinct + * collection. + */ + Status copyCollectionContents( + AuthzManagerExternalState* externalState, + const NamespaceString& targetCollection, + const NamespaceString& sourceCollection, + const BSONObj& writeConcern) { + return externalState->query( + sourceCollection, + BSONObj(), + BSONObj(), + boost::bind(uassertInsertIntoCollection, + externalState, + targetCollection, + _1, + writeConcern)); } - Status AuthorizationManager::checkAuthForQuery(const std::string& ns) { - NamespaceString namespaceString(ns); - verify(!namespaceString.isCommand()); - if (!checkAuthorization(ns, ActionType::find)) { - return Status(ErrorCodes::Unauthorized, - mongoutils::str::stream() << "not authorized for query on " << ns, - 0); + /** + * Upgrades auth schema from schemaVersion24 to schemaVersion26Upgrade. + * + * Assumes that the current version is schemaVersion24. + * + * - Backs up usersCollectionNamespace into usersBackupCollectionNamespace. + * - Empties usersAltCollectionNamespace. + * - Builds usersAltCollectionNamespace from the contents of every database's system.users + * collection. + * - Manipulates the schema version document appropriately. + * + * Upon successful completion, system is in schemaVersion26Upgrade. On failure, + * system is in schemaVersion24 or schemaVersion26Upgrade, but it is safe to re-run this + * method. + */ + Status buildNewUsersCollection( + AuthzManagerExternalState* externalState, + const BSONObj& writeConcern) { + + // Write an explicit schemaVersion24 into the schema version document, to facilitate + // recovery. + Status status = externalState->updateOne( + AuthorizationManager::versionCollectionNamespace, + AuthorizationManager::versionDocumentQuery, + BSON("$set" << BSON(AuthorizationManager::schemaVersionFieldName << + AuthorizationManager::schemaVersion24)), + true, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade erasing contents of " << + AuthorizationManager::usersBackupCollectionNamespace; + int numRemoved; + status = externalState->remove( + AuthorizationManager::usersBackupCollectionNamespace, + BSONObj(), + writeConcern, + &numRemoved); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade backing up " << + AuthorizationManager::usersCollectionNamespace << " into " << + AuthorizationManager::usersBackupCollectionNamespace; + status = copyCollectionContents( + externalState, + AuthorizationManager::usersBackupCollectionNamespace, + AuthorizationManager::usersCollectionNamespace, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade dropping indexes from " << + AuthorizationManager::usersAltCollectionNamespace; + status = externalState->dropIndexes(AuthorizationManager::usersAltCollectionNamespace, + writeConcern); + if (!status.isOK()) { + warning() << "Auth schema upgrade failed to drop indexes on " << + AuthorizationManager::usersAltCollectionNamespace << " (" << status << ")"; + } + + log() << "Auth schema upgrade erasing contents of " << + AuthorizationManager::usersAltCollectionNamespace; + status = externalState->remove( + AuthorizationManager::usersAltCollectionNamespace, + BSONObj(), + writeConcern, + &numRemoved); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade creating needed indexes of " << + AuthorizationManager::usersAltCollectionNamespace; + status = externalState->createIndex( + AuthorizationManager::usersAltCollectionNamespace, + BSON("user" << 1 << "db" << 1), + true, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); + + // Update usersAltCollectionNamespace from the contents of each database's system.users + // collection. + std::vector dbNames; + status = externalState->getAllDatabaseNames(&dbNames); + if (!status.isOK()) + return logUpgradeFailed(status); + for (size_t i = 0; i < dbNames.size(); ++i) { + const std::string& db = dbNames[i]; + status = upgradeUsersFromDB(externalState, db, writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); } + + // Switch to schemaVersion26Upgrade. Starting after this point, user information will be + // read from usersAltCollectionNamespace. + status = externalState->updateOne( + AuthorizationManager::versionCollectionNamespace, + AuthorizationManager::versionDocumentQuery, + BSON("$set" << BSON(AuthorizationManager::schemaVersionFieldName << + AuthorizationManager::schemaVersion26Upgrade)), + true, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); return Status::OK(); } - Status AuthorizationManager::checkAuthForInsert(const std::string& ns) { - NamespaceString namespaceString(ns); - if (!checkAuthorization(ns, ActionType::insert)) { - return Status(ErrorCodes::Unauthorized, - mongoutils::str::stream() << "not authorized for insert on " << ns, - 0); + /** + * Performs the upgrade to schemaVersion26Final from schemaVersion26Upgrade. + * + * Assumes that the current version is schemaVersion26Upgrade. + * + * - Erases contents and indexes of usersCollectionNamespace. + * - Erases contents and indexes of rolesCollectionNamespace. + * - Creates appropriate indexes on usersCollectionNamespace and rolesCollectionNamespace. + * - Copies usersAltCollectionNamespace to usersCollectionNamespace. + * - Manipulates the schema version document appropriately. + * + * Upon successful completion, system is in schemaVersion26Final. On failure, + * system is in schemaVersion26Upgrade or schemaVersion26Final, but it is safe to re-run this + * method. + */ + Status overwriteSystemUsersCollection( + AuthzManagerExternalState* externalState, + const BSONObj& writeConcern) { + log() << "Auth schema upgrade erasing version " << AuthorizationManager::schemaVersion24 << + " users from " << AuthorizationManager::usersCollectionNamespace; + Status status = externalState->dropIndexes(AuthorizationManager::usersCollectionNamespace, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); + int numRemoved; + status = externalState->remove( + AuthorizationManager::usersCollectionNamespace, + BSONObj(), + writeConcern, + &numRemoved); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade erasing " << AuthorizationManager::rolesCollectionNamespace; + status = externalState->dropIndexes(AuthorizationManager::rolesCollectionNamespace, + writeConcern); + if (!status.isOK()) { + warning() << "Auth schema upgrade failed to drop indexes on " << + AuthorizationManager::rolesCollectionNamespace << " (" << status << ")"; } + + status = externalState->remove( + AuthorizationManager::rolesCollectionNamespace, + BSONObj(), + writeConcern, + &numRemoved); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade creating needed indexes of " << + AuthorizationManager::rolesCollectionNamespace; + status = externalState->createIndex( + AuthorizationManager::rolesCollectionNamespace, + BSON("role" << 1 << "db" << 1), + true, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade creating needed indexes of " << + AuthorizationManager::usersCollectionNamespace; + status = externalState->createIndex( + AuthorizationManager::usersCollectionNamespace, + BSON("user" << 1 << "db" << 1), + true, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); + + log() << "Auth schema upgrade copying version " << + AuthorizationManager::schemaVersion26Final << " users from " << + AuthorizationManager::usersAltCollectionNamespace << " to " << + AuthorizationManager::usersCollectionNamespace; + + status = copyCollectionContents( + externalState, + AuthorizationManager::usersCollectionNamespace, + AuthorizationManager::usersAltCollectionNamespace, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); + + // Set the schema version in the schema version document, completing the process. + status = externalState->updateOne( + AuthorizationManager::versionCollectionNamespace, + AuthorizationManager::versionDocumentQuery, + BSON("$set" << BSON(AuthorizationManager::schemaVersionFieldName << + AuthorizationManager::schemaVersion26Final)), + true, + writeConcern); + if (!status.isOK()) + return logUpgradeFailed(status); return Status::OK(); } +} // namespace - Status AuthorizationManager::checkAuthForUpdate(const std::string& ns, bool upsert) { - NamespaceString namespaceString(ns); - if (!upsert) { - if (!checkAuthorization(ns, ActionType::update)) { - return Status(ErrorCodes::Unauthorized, - mongoutils::str::stream() << "not authorized for update on " << ns, - 0); - } + Status AuthorizationManager::upgradeSchemaStep(const BSONObj& writeConcern, bool* isDone) { + int authzVersion; + Status status = getAuthorizationVersion(&authzVersion); + if (!status.isOK()) { + return status; } - else { - ActionSet required; - required.addAction(ActionType::update); - required.addAction(ActionType::insert); - if (!checkAuthorization(ns, required)) { - return Status(ErrorCodes::Unauthorized, - mongoutils::str::stream() << "not authorized for upsert on " << ns, - 0); + + switch (authzVersion) { + case schemaVersion24: + *isDone = false; + return buildNewUsersCollection(_externalState.get(), writeConcern); + case schemaVersion26Upgrade: { + Status status = overwriteSystemUsersCollection(_externalState.get(), writeConcern); + if (status.isOK()) + *isDone = true; + return status; + } + case schemaVersion26Final: + *isDone = true; + return Status::OK(); + default: + return Status(ErrorCodes::AuthSchemaIncompatible, mongoutils::str::stream() << + "Do not know how to upgrade auth schema from version " << authzVersion); + } + } + + Status AuthorizationManager::upgradeSchema(int maxSteps, const BSONObj& writeConcern) { + + if (maxSteps < 1) { + return Status(ErrorCodes::BadValue, + "Minimum value for maxSteps parameter to upgradeSchema is 1"); + } + invalidateUserCache(); + for (int i = 0; i < maxSteps; ++i) { + bool isDone; + Status status = upgradeSchemaStep(writeConcern, &isDone); + invalidateUserCache(); + if (!status.isOK() || isDone) { + return status; } } - return Status::OK(); + return Status(ErrorCodes::OperationIncomplete, mongoutils::str::stream() << + "Auth schema upgrade incomplete after " << maxSteps << " successful steps."); } - Status AuthorizationManager::checkAuthForDelete(const std::string& ns) { - NamespaceString namespaceString(ns); - if (!checkAuthorization(ns, ActionType::remove)) { - return Status(ErrorCodes::Unauthorized, - mongoutils::str::stream() << "not authorized to remove from " << ns, - 0); +namespace { + bool isAuthzNamespace(const StringData& ns) { + if (nsToCollectionSubstring(ns) == "system.users") { + // SERVER-15360 Make sure that changes made to non-admin system.users collections on a + // 2.4 primary replicating to us get detected and cause the user cache to be invalidated + return true; } - return Status::OK(); + return (ns == AuthorizationManager::rolesCollectionNamespace.ns() || + ns == AuthorizationManager::usersCollectionNamespace.ns() || + ns == AuthorizationManager::versionCollectionNamespace.ns()); } - Status AuthorizationManager::checkAuthForGetMore(const std::string& ns) { - return checkAuthForQuery(ns); + bool isAuthzCollection(const StringData& coll) { + return (coll == AuthorizationManager::rolesCollectionNamespace.coll() || + coll == AuthorizationManager::usersCollectionNamespace.coll() || + coll == AuthorizationManager::versionCollectionNamespace.coll()); } - Privilege AuthorizationManager::_modifyPrivilegeForSpecialCases(const Privilege& privilege) { - ActionSet newActions; - newActions.addAllActionsFromSet(privilege.getActions()); - std::string collectionName = NamespaceString(privilege.getResource()).coll; - if (collectionName == "system.users") { - newActions.removeAction(ActionType::find); - newActions.removeAction(ActionType::insert); - newActions.removeAction(ActionType::update); - newActions.removeAction(ActionType::remove); - newActions.addAction(ActionType::userAdmin); - } else if (collectionName == "system.profile") { - newActions.removeAction(ActionType::find); - newActions.addAction(ActionType::profileRead); - } else if (collectionName == "system.indexes" && newActions.contains(ActionType::find)) { - newActions.removeAction(ActionType::find); - newActions.addAction(ActionType::indexRead); + bool loggedCommandOperatesOnAuthzData(const char* ns, const BSONObj& cmdObj) { + if (ns != AuthorizationManager::adminCommandNamespace.ns()) + return false; + const StringData cmdName(cmdObj.firstElement().fieldNameStringData()); + if (cmdName == "drop") { + return isAuthzCollection(StringData(cmdObj.firstElement().valuestr(), + cmdObj.firstElement().valuestrsize() - 1)); + } + else if (cmdName == "dropDatabase") { + return true; + } + else if (cmdName == "renameCollection") { + return isAuthzCollection(cmdObj.firstElement().str()) || + isAuthzCollection(cmdObj["to"].str()); + } + else if (cmdName == "dropIndexes" || cmdName == "deleteIndexes") { + return false; + } + else if (cmdName == "create") { + return false; + } + else { + return true; } - - return Privilege(privilege.getResource(), newActions); } - Status AuthorizationManager::checkAuthForPrivilege(const Privilege& privilege) { - if (_externalState->shouldIgnoreAuthChecks()) - return Status::OK(); - - return _probeForPrivilege(privilege); + bool appliesToAuthzData( + const char* op, + const char* ns, + const BSONObj& o) { + + switch (*op) { + case 'i': + case 'u': + case 'd': + return isAuthzNamespace(ns); + case 'c': + return loggedCommandOperatesOnAuthzData(ns, o); + break; + case 'n': + return false; + default: + return true; + } } - Status AuthorizationManager::checkAuthForPrivileges(const vector& privileges) { - if (_externalState->shouldIgnoreAuthChecks()) - return Status::OK(); - - for (size_t i = 0; i < privileges.size(); ++i) { - Status status = _probeForPrivilege(privileges[i]); - if (!status.isOK()) - return status; - } +} // namespace - return Status::OK(); + void AuthorizationManager::_updateCacheGeneration_inlock() { + _cacheGeneration = OID::gen(); } - Status AuthorizationManager::_probeForPrivilege(const Privilege& privilege) { - Privilege modifiedPrivilege = _modifyPrivilegeForSpecialCases(privilege); - if (_acquiredPrivileges.hasPrivilege(modifiedPrivilege)) - return Status::OK(); - - std::string dbname = nsToDatabase(modifiedPrivilege.getResource()); - for (PrincipalSet::iterator iter = _authenticatedPrincipals.begin(), - end = _authenticatedPrincipals.end(); - iter != end; ++iter) { - - Principal* principal = *iter; - if (!principal->isImplicitPrivilegeAcquisitionEnabled()) - continue; - if (principal->isDatabaseProbed(dbname)) - continue; - _acquirePrivilegesForPrincipalFromDatabase(dbname, principal->getName()); - principal->markDatabaseAsProbed(dbname); - if (_acquiredPrivileges.hasPrivilege(modifiedPrivilege)) - return Status::OK(); - } - return Status(ErrorCodes::Unauthorized, "unauthorized", 0); + void AuthorizationManager::logOp( + const char* op, + const char* ns, + const BSONObj& o, + BSONObj* o2, + bool* b) { + + _externalState->logOp(op, ns, o, o2, b); + if (appliesToAuthzData(op, ns, o)) { + CacheGuard guard(this, CacheGuard::fetchSynchronizationManual); + _invalidateUserCache_inlock(); + } } } // namespace mongo diff --git a/src/mongo/db/auth/authorization_manager.h b/src/mongo/db/auth/authorization_manager.h index a32710557dd..8b6a078dd8f 100644 --- a/src/mongo/db/auth/authorization_manager.h +++ b/src/mongo/db/auth/authorization_manager.h @@ -1,5 +1,5 @@ /** -* Copyright (C) 2012 10gen Inc. +* Copyright (C) 2013 10gen Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -12,206 +12,517 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once +#include +#include +#include +#include +#include #include -#include #include "mongo/base/disallow_copying.h" #include "mongo/base/status.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/bson/oid.h" #include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/auth_external_state.h" -#include "mongo/db/auth/principal.h" -#include "mongo/db/auth/principal_name.h" -#include "mongo/db/auth/principal_set.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/auth/privilege_set.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/auth/role_graph.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/user_name_hash.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/platform/unordered_map.h" namespace mongo { - // --noauth cmd line option - extern bool noauth; + class AuthzManagerExternalState; + class UserDocumentParser; /** * Internal secret key info. */ struct AuthInfo { - AuthInfo(); - string user; - string pwd; + User* user; }; extern AuthInfo internalSecurity; // set at startup and not changed after initialization. /** - * Contains all the authorization logic for a single client connection. It contains a set of - * the principals which have been authenticated, as well as a set of privileges that have been - * granted by those principals to perform various actions. - * An AuthorizationManager object is present within every mongo::Client object, therefore there - * is one per thread that corresponds to an incoming client connection. + * Contains server/cluster-wide information about Authorization. */ class AuthorizationManager { MONGO_DISALLOW_COPYING(AuthorizationManager); public: - static const std::string SERVER_RESOURCE_NAME; - static const std::string CLUSTER_RESOURCE_NAME; + // The newly constructed AuthorizationManager takes ownership of "externalState" + explicit AuthorizationManager(AuthzManagerExternalState* externalState); + + ~AuthorizationManager(); static const std::string USER_NAME_FIELD_NAME; - static const std::string USER_SOURCE_FIELD_NAME; + static const std::string USER_DB_FIELD_NAME; + static const std::string ROLE_NAME_FIELD_NAME; + static const std::string ROLE_SOURCE_FIELD_NAME; // TODO: rename to ROLE_DB_FIELD_NAME static const std::string PASSWORD_FIELD_NAME; - + static const std::string V1_USER_NAME_FIELD_NAME; + static const std::string V1_USER_SOURCE_FIELD_NAME; + + static const NamespaceString adminCommandNamespace; + static const NamespaceString rolesCollectionNamespace; + static const NamespaceString usersAltCollectionNamespace; + static const NamespaceString usersBackupCollectionNamespace; + static const NamespaceString usersCollectionNamespace; + static const NamespaceString versionCollectionNamespace; + static const NamespaceString defaultTempUsersCollectionNamespace; // for mongorestore + static const NamespaceString defaultTempRolesCollectionNamespace; // for mongorestore + + /** + * Query to match the auth schema version document in the versionCollectionNamespace. + */ + static const BSONObj versionDocumentQuery; + + /** + * Name of the field in the auth schema version document containing the current schema + * version. + */ + static const std::string schemaVersionFieldName; + + /** + * Value used to represent that the schema version is not cached or invalid. + */ + static const int schemaVersionInvalid = 0; + + /** + * Auth schema version for MongoDB v2.4 and prior. + */ + static const int schemaVersion24 = 1; + + /** + * Auth schema version for MongoDB v2.6 during the upgrade process. Same as + * schemaVersion26Final, except that user documents are found in admin.new.users, and user + * management commands are disabled. + */ + static const int schemaVersion26Upgrade = 2; + + /** + * Auth schema version for MongoDB 2.6. Users are stored in admin.system.users, + * roles in admin.system.roles. + */ + static const int schemaVersion26Final = 3; + + // TODO: Make the following functions no longer static. + + /** + * Sets whether or not we allow old style (pre v2.4) privilege documents for this whole + * server. Only relevant prior to upgrade. + */ static void setSupportOldStylePrivilegeDocuments(bool enabled); + /** + * Returns true if we allow old style privilege privilege documents for this whole server. + */ + static bool getSupportOldStylePrivilegeDocuments(); + + /** + * Takes a vector of privileges and fills the output param "resultArray" with a BSON array + * representation of the privileges. + */ + static Status getBSONForPrivileges(const PrivilegeVector& privileges, + mutablebson::Element resultArray); + + /** + * Takes a role name and a role graph and fills the output param "result" with a BSON + * representation of the role object. + * This function does no locking - it is up to the caller to synchronize access to the + * role graph. + * Note: The passed in RoleGraph can't be marked const because some of its accessors can + * actually modify it internally (to set up built-in roles). + */ + static Status getBSONForRole(/*const*/ RoleGraph* graph, + const RoleName& roleName, + mutablebson::Element result); + + + /** + * Sets whether or not access control enforcement is enabled for this manager. + */ + void setAuthEnabled(bool enabled); + + /** + * Returns true if access control is enabled for this manager . + */ + bool isAuthEnabled() const; + + /** + * Returns via the output parameter "version" the version number of the authorization + * system. Returns Status::OK() if it was able to successfully fetch the current + * authorization version. If it has problems fetching the most up to date version it + * returns a non-OK status. When returning a non-OK status, *version will be set to + * schemaVersionInvalid (0). + */ + Status getAuthorizationVersion(int* version); + + /** + * Returns the user cache generation identifier. + */ + OID getCacheGeneration(); + + // Returns true if there exists at least one privilege document in the system. + bool hasAnyPrivilegeDocuments() const; + + /** + * Updates the auth schema version document to reflect that the system is upgraded to + * schemaVersion26Final. + * + * Do not call if getAuthorizationVersion() reports a value other than schemaVersion26Final. + */ + Status writeAuthSchemaVersionIfNeeded(); + + /** + * Creates the given user object in the given database. + * 'writeConcern' contains the arguments to be passed to getLastError to block for + * successful completion of the write. + */ + Status insertPrivilegeDocument(const std::string& dbname, + const BSONObj& userObj, + const BSONObj& writeConcern) const; + + /** + * Updates the given user object with the given update modifier. + * 'writeConcern' contains the arguments to be passed to getLastError to block for + * successful completion of the write. + */ + Status updatePrivilegeDocument(const UserName& user, + const BSONObj& updateObj, + const BSONObj& writeConcern) const; + + /* + * Removes users for the given database matching the given query. + * Writes into *numRemoved the number of user documents that were modified. + * 'writeConcern' contains the arguments to be passed to getLastError to block for + * successful completion of the write. + */ + Status removePrivilegeDocuments(const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) const; + + /** + * Creates the given role object in the given database. + * 'writeConcern' contains the arguments to be passed to getLastError to block for + * successful completion of the write. + */ + Status insertRoleDocument(const BSONObj& roleObj, const BSONObj& writeConcern) const; + + /** + * Updates the given role object with the given update modifier. + * 'writeConcern' contains the arguments to be passed to getLastError to block for + * successful completion of the write. + */ + Status updateRoleDocument(const RoleName& role, + const BSONObj& updateObj, + const BSONObj& writeConcern) const; + + /** + * Updates documents matching "query" according to "updatePattern" in "collectionName". + * Should only be called on collections with authorization documents in them + * (ie admin.system.users and admin.system.roles). + */ + Status updateAuthzDocuments(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched) const; + + /* + * Removes roles matching the given query. + * Writes into *numRemoved the number of role documents that were modified. + * 'writeConcern' contains the arguments to be passed to getLastError to block for + * successful completion of the write. + */ + Status removeRoleDocuments(const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) const; + + /** + * Finds all documents matching "query" in "collectionName". For each document returned, + * calls the function resultProcessor on it. + * Should only be called on collections with authorization documents in them + * (ie admin.system.users and admin.system.roles). + */ + Status queryAuthzDocument(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& projection, + const boost::function& resultProcessor); + // Checks to see if "doc" is a valid privilege document, assuming it is stored in the // "system.users" collection of database "dbname". // // Returns Status::OK() if the document is good, or Status(ErrorCodes::BadValue), otherwise. - static Status checkValidPrivilegeDocument(const StringData& dbname, const BSONObj& doc); - - // Takes ownership of the externalState. - explicit AuthorizationManager(AuthExternalState* externalState); - ~AuthorizationManager(); - - // Should be called at the beginning of every new request. This performs the checks - // necessary to determine if localhost connections should be given full access. - // TODO: try to eliminate the need for this call. - void startRequest(); - - // Adds "principal" to the authorization manager, and takes ownership of it. - void addAuthorizedPrincipal(Principal* principal); - - // Returns the authenticated principal with the given name. Returns NULL - // if no such user is found. - // Ownership of the returned Principal remains with _authenticatedPrincipals - Principal* lookupPrincipal(const PrincipalName& name); - - // Gets an iterator over the names of all authenticated principals stored in this manager. - PrincipalSet::NameIterator getAuthenticatedPrincipalNames(); - - // Removes any authenticated principals whose authorization credentials came from the given - // database, and revokes any privileges that were granted via that principal. - void logoutDatabase(const std::string& dbname); - - // Grant this connection the given privilege. - Status acquirePrivilege(const Privilege& privilege, - const PrincipalName& authorizingPrincipal); - - // Adds a new principal with the given principal name and authorizes it with full access. - // Used to grant internal threads full access. - void grantInternalAuthorization(const std::string& principalName); - - // Checks if this connection has been authenticated as an internal user. - bool hasInternalAuthorization(); - - // Checks if this connection has the privileges required to perform the given action - // on the given resource. Contains all the authorization logic including handling things - // like the localhost exception. Returns true if the action may proceed on the resource. - // Note: this may acquire a database read lock (for automatic privilege acquisition). - bool checkAuthorization(const std::string& resource, ActionType action); - - // Same as above but takes an ActionSet instead of a single ActionType. Returns true if - // all of the actions may proceed on the resource. - bool checkAuthorization(const std::string& resource, ActionSet actions); - - // Parses the privilege documents and acquires all privileges that the privilege document - // grants - Status acquirePrivilegesFromPrivilegeDocument(const std::string& dbname, - const PrincipalName& principal, - const BSONObj& privilegeDocument); - - // Returns the privilege document with the given user name in the given database. Currently - // this information comes from the system.users collection in that database. - Status getPrivilegeDocument(const std::string& dbname, - const PrincipalName& userName, - BSONObj* result) { - return _externalState->getPrivilegeDocument(dbname, userName, result); - } - - // Checks if this connection has the privileges necessary to perform a query on the given - // namespace. - Status checkAuthForQuery(const std::string& ns); - - // Checks if this connection has the privileges necessary to perform an update on the given - // namespace. - Status checkAuthForUpdate(const std::string& ns, bool upsert); - - // Checks if this connection has the privileges necessary to perform an insert to the given - // namespace. - Status checkAuthForInsert(const std::string& ns); - - // Checks if this connection has the privileges necessary to perform a delete on the given - // namespace. - Status checkAuthForDelete(const std::string& ns); - - // Checks if this connection has the privileges necessary to perform a getMore on the given - // namespace. - Status checkAuthForGetMore(const std::string& ns); - - // Checks if this connection is authorized for the given Privilege. - Status checkAuthForPrivilege(const Privilege& privilege); - - // Checks if this connection is authorized for all the given Privileges. - Status checkAuthForPrivileges(const vector& privileges); + Status checkValidPrivilegeDocument(const StringData& dbname, const BSONObj& doc); // Given a database name and a readOnly flag return an ActionSet describing all the actions // that an old-style user with those attributes should be given. - static ActionSet getActionsForOldStyleUser(const std::string& dbname, bool readOnly); - - // Parses the privilege document and returns a PrivilegeSet of all the Privileges that - // the privilege document grants. - static Status buildPrivilegeSet(const std::string& dbname, - const PrincipalName& principal, - const BSONObj& privilegeDocument, - PrivilegeSet* result); - - // Returns an ActionSet of all actions that can be be granted to users. This does not - // include internal-only actions. - static ActionSet getAllUserActions(); + ActionSet getActionsForOldStyleUser(const std::string& dbname, bool readOnly) const; + + /** + * Writes into "result" a document describing the named user and returns Status::OK(). The + * description includes the user credentials and customData, if present, the user's role + * membership and delegation information, a full list of the user's privileges, and a full + * list of the user's roles, including those roles held implicitly through other roles + * (indirect roles). In the event that some of this information is inconsistent, the + * document will contain a "warnings" array, with string messages describing + * inconsistencies. + * + * If the user does not exist, returns ErrorCodes::UserNotFound. + */ + Status getUserDescription(const UserName& userName, BSONObj* result); + + /** + * Writes into "result" a document describing the named role and returns Status::OK(). The + * description includes the roles in which the named role has membership and a full list of + * the roles of which the named role is a member, including those roles memberships held + * implicitly through other roles (indirect roles). If "showPrivileges" is true, then the + * description documents will also include a full list of the role's privileges. + * In the event that some of this information is inconsistent, the document will contain a + * "warnings" array, with string messages describing inconsistencies. + * + * If the role does not exist, returns ErrorCodes::RoleNotFound. + */ + Status getRoleDescription(const RoleName& roleName, bool showPrivileges, BSONObj* result); + + /** + * Writes into "result" documents describing the roles that are defined on the given + * database. Each role description document includes the other roles in which the role has + * membership and a full list of the roles of which the named role is a member, + * including those roles memberships held implicitly through other roles (indirect roles). + * If showPrivileges is true, then the description documents will also include a full list + * of the role's privileges. If showBuiltinRoles is true, then the result array will + * contain description documents for all the builtin roles for the given database, if it + * is false the result will just include user defined roles. + * In the event that some of the information in a given role description is inconsistent, + * the document will contain a "warnings" array, with string messages describing + * inconsistencies. + */ + Status getRoleDescriptionsForDB(const std::string dbname, + bool showPrivileges, + bool showBuiltinRoles, + vector* result); + + /** + * Returns the User object for the given userName in the out parameter "acquiredUser". + * If the user cache already has a user object for this user, it increments the refcount + * on that object and gives out a pointer to it. If no user object for this user name + * exists yet in the cache, reads the user's privilege document from disk, builds up + * a User object, sets the refcount to 1, and gives that out. The returned user may + * be invalid by the time the caller gets access to it. + * The AuthorizationManager retains ownership of the returned User object. + * On non-OK Status return values, acquiredUser will not be modified. + */ + Status acquireUser(const UserName& userName, User** acquiredUser); + + /** + * Decrements the refcount of the given User object. If the refcount has gone to zero, + * deletes the User. Caller must stop using its pointer to "user" after calling this. + */ + void releaseUser(User* user); + + /** + * Returns a User object for a V1-style user with the given "userName" in "*acquiredUser", + * On success, "acquiredUser" will have any privileges that the named user has on + * database "dbname". + * + * Bumps the returned **acquiredUser's reference count on success. + */ + Status acquireV1UserProbedForDb( + const UserName& userName, const StringData& dbname, User** acquiredUser); + + /** + * Marks the given user as invalid and removes it from the user cache. + */ + void invalidateUserByName(const UserName& user); + + /** + * Invalidates all users who's source is "dbname" and removes them from the user cache. + */ + void invalidateUsersFromDB(const std::string& dbname); + + /** + * Initializes the authorization manager. Depending on what version the authorization + * system is at, this may involve building up the user cache and/or the roles graph. + * Call this function at startup and after resynchronizing a slave/secondary. + */ + Status initialize(); + + /** + * Invalidates all of the contents of the user cache. + */ + void invalidateUserCache(); + + /** + * Parses privDoc and fully initializes the user object (credentials, roles, and privileges) + * with the information extracted from the privilege document. + * This should never be called from outside the AuthorizationManager - the only reason it's + * public instead of private is so it can be unit tested. + */ + Status _initializeUserFromPrivilegeDocument(User* user, const BSONObj& privDoc); + + /** + * Tries to acquire the global lock guarding modifications to all persistent data related + * to authorization, namely the admin.system.users, admin.system.roles, and + * admin.system.version collections. This serializes all writers to the authorization + * documents, but does not impact readers. + */ + bool tryAcquireAuthzUpdateLock(const StringData& why); + + /** + * Releases the lock guarding modifications to persistent authorization data, which must + * already be held. + */ + void releaseAuthzUpdateLock(); + + /** + * Performs one step in the process of upgrading the stored authorization data to the + * newest schema. + * + * On success, returns Status::OK(), and *isDone will indicate whether there are more + * steps to perform. + * + * If the authorization data is already fully upgraded, returns Status::OK and sets *isDone + * to true, so this is safe to call on a fully upgraded system. + * + * On failure, returns a status other than Status::OK(). In this case, is is typically safe + * to try again. + */ + Status upgradeSchemaStep(const BSONObj& writeConcern, bool* isDone); + + /** + * Performs up to maxSteps steps in the process of upgrading the stored authorization data + * to the newest schema. Behaves as if by repeatedly calling upgradeSchemaStep up to + * maxSteps times until either it completes the upgrade or returns a non-OK status. + * + * Invalidates the user cache before the first step and after each attempted step. + * + * Returns Status::OK() to indicate that the upgrade process has completed successfully. + * Returns ErrorCodes::OperationIncomplete to indicate that progress was made, but that more + * steps must be taken to complete the process. Other returns indicate a failure to make + * progress performing the upgrade, and the specific code and message in the returned status + * may provide additional information. + */ + Status upgradeSchema(int maxSteps, const BSONObj& writeConcern); + + /** + * Hook called by replication code to let the AuthorizationManager observe changes + * to relevant collections. + */ + void logOp(const char* opstr, + const char* ns, + const BSONObj& obj, + BSONObj* patt, + bool* b); private: - // Finds the set of privileges attributed to "principal" in database "dbname", - // and adds them to the set of acquired privileges. - void _acquirePrivilegesForPrincipalFromDatabase(const std::string& dbname, - const PrincipalName& principal); - - // Checks to see if the given privilege is allowed, performing implicit privilege - // acquisition if enabled and necessary to resolve the privilege. - Status _probeForPrivilege(const Privilege& privilege); - - // Parses the old-style (pre 2.4) privilege document and returns a PrivilegeSet of all the - // Privileges that the privilege document grants. - static Status _buildPrivilegeSetFromOldStylePrivilegeDocument( - const std::string& dbname, - const PrincipalName& principal, - const BSONObj& privilegeDocument, - PrivilegeSet* result); - - // Parses extended-form (2.4+) privilege documents and returns a PrivilegeSet of all the - // privileges that the document grants. - // - // The document, "privilegeDocument", is assumed to describe privileges for "principal", and - // to come from database "dbname". - static Status _buildPrivilegeSetFromExtendedPrivilegeDocument( - const std::string& dbname, - const PrincipalName& principal, - const BSONObj& privilegeDocument, - PrivilegeSet* result); - - // Returns a new privilege that has replaced the actions needed to handle special casing - // certain namespaces like system.users and system.profile. - Privilege _modifyPrivilegeForSpecialCases(const Privilege& privilege); + /** + * Type used to guard accesses and updates to the user cache. + */ + class CacheGuard; + friend class AuthorizationManager::CacheGuard; + + /** + * Invalidates all User objects in the cache and removes them from the cache. + * Should only be called when already holding _cacheMutex. + */ + void _invalidateUserCache_inlock(); + + /** + * Updates _cacheGeneration to a new OID + */ + void _updateCacheGeneration_inlock(); + + /** + * Fetches user information from a v2-schema user document for the named user, + * and stores a pointer to a new user object into *acquiredUser on success. + */ + Status _fetchUserV2(const UserName& userName, std::auto_ptr* acquiredUser); + + /** + * Fetches user information from a v1-schema user document for the named user, possibly + * examining system.users collections from userName.getDB() and admin.system.users in the + * process. Stores a pointer to a new user object into *acquiredUser on success. + */ + Status _fetchUserV1(const UserName& userName, std::auto_ptr* acquiredUser); static bool _doesSupportOldStylePrivileges; - scoped_ptr _externalState; - - // All the privileges that have been acquired by the authenticated principals. - PrivilegeSet _acquiredPrivileges; - // All principals who have been authenticated on this connection - PrincipalSet _authenticatedPrincipals; + /** + * True if access control enforcement is enabled in this AuthorizationManager. + * + * Defaults to false. Changes to its value are not synchronized, so it should only be set + * at initalization-time. + */ + bool _authEnabled; + + scoped_ptr _externalState; + + /** + * Cached value of the authorization schema version. + * + * May be set by acquireUser() and getAuthorizationVersion(). Invalidated by + * invalidateUserCache(). + * + * Reads and writes guarded by CacheGuard. + */ + int _version; + + /** + * Caches User objects with information about user privileges, to avoid the need to + * go to disk to read user privilege documents whenever possible. Every User object + * has a reference count - the AuthorizationManager must not delete a User object in the + * cache unless its reference count is zero. + */ + unordered_map _userCache; + + /** + * Current generation of cached data. Updated every time part of the cache gets + * invalidated. Protected by CacheGuard. + */ + OID _cacheGeneration; + + /** + * True if there is an update to the _userCache in progress, and that update is currently in + * the "fetch phase", during which it does not hold the _cacheMutex. + * + * Manipulated via CacheGuard. + */ + bool _isFetchPhaseBusy; + + /** + * Protects _userCache, _cacheGeneration, _version and _isFetchPhaseBusy. Manipulated + * via CacheGuard. + */ + boost::mutex _cacheMutex; + + /** + * Condition used to signal that it is OK for another CacheGuard to enter a fetch phase. + * Manipulated via CacheGuard. + */ + boost::condition_variable _fetchPhaseIsReady; }; } // namespace mongo diff --git a/src/mongo/db/auth/authorization_manager_global.cpp b/src/mongo/db/auth/authorization_manager_global.cpp new file mode 100644 index 00000000000..68e5cd1e7cb --- /dev/null +++ b/src/mongo/db/auth/authorization_manager_global.cpp @@ -0,0 +1,94 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/platform/basic.h" + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/init.h" +#include "mongo/client/auth_helpers.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/server_parameters.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +namespace { + AuthorizationManager* globalAuthManager = NULL; + + class AuthzVersionParameter : public ServerParameter { + MONGO_DISALLOW_COPYING(AuthzVersionParameter); + public: + AuthzVersionParameter(ServerParameterSet* sps, const std::string& name); + virtual void append(BSONObjBuilder& b, const std::string& name); + virtual Status set(const BSONElement& newValueElement); + virtual Status setFromString(const std::string& str); + }; + + MONGO_INITIALIZER_GENERAL(AuthzSchemaParameter, + MONGO_NO_PREREQUISITES, + ("BeginStartupOptionParsing"))(InitializerContext*) { + new AuthzVersionParameter(ServerParameterSet::getGlobal(), + auth::schemaVersionServerParameter); + return Status::OK(); + } + + AuthzVersionParameter::AuthzVersionParameter(ServerParameterSet* sps, const std::string& name) : + ServerParameter(sps, name, false, false) {} + + void AuthzVersionParameter::append(BSONObjBuilder& b, const std::string& name) { + int authzVersion; + uassertStatusOK(getGlobalAuthorizationManager()->getAuthorizationVersion(&authzVersion)); + b.append(name, authzVersion); + } + + Status AuthzVersionParameter::set(const BSONElement& newValueElement) { + return Status(ErrorCodes::InternalError, "set called on unsettable server parameter"); + } + + Status AuthzVersionParameter::setFromString(const std::string& newValueString) { + return Status(ErrorCodes::InternalError, "set called on unsettable server parameter"); + } +} // namespace + + void setGlobalAuthorizationManager(AuthorizationManager* authManager) { + fassert(16841, globalAuthManager == NULL); + globalAuthManager = authManager; + } + + void clearGlobalAuthorizationManager() { + fassert(16843, globalAuthManager != NULL); + delete globalAuthManager; + globalAuthManager = NULL; + } + + AuthorizationManager* getGlobalAuthorizationManager() { + fassert(16842, globalAuthManager != NULL); + return globalAuthManager; + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authorization_manager_global.h b/src/mongo/db/auth/authorization_manager_global.h new file mode 100644 index 00000000000..0b13533265e --- /dev/null +++ b/src/mongo/db/auth/authorization_manager_global.h @@ -0,0 +1,47 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/auth/authorization_manager.h" + +namespace mongo { + + // Gets the singleton AuthorizationManager object for this server process. + AuthorizationManager* getGlobalAuthorizationManager(); + + // Sets the singleton AuthorizationManager object for this server process. + // Must be called once at startup and then never again (unless clearGlobalAuthorizationManager + // is called, at which point this can be called again, but should only happen in tests). + void setGlobalAuthorizationManager(AuthorizationManager* authManager); + + // Sets the singleton AuthorizationManager object for this server process to NULL. + // Should only be used in tests. + void clearGlobalAuthorizationManager(); + +} // namespace mongo diff --git a/src/mongo/db/auth/authorization_manager_test.cpp b/src/mongo/db/auth/authorization_manager_test.cpp index 9df59339dab..e0833660d98 100644 --- a/src/mongo/db/auth/authorization_manager_test.cpp +++ b/src/mongo/db/auth/authorization_manager_test.cpp @@ -18,10 +18,15 @@ */ #include "mongo/base/status.h" -#include "mongo/db/auth/auth_external_state_mock.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authz_session_external_state_mock.h" +#include "mongo/db/auth/authz_manager_external_state_mock.h" #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/jsobj.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" #include "mongo/unittest/unittest.h" #include "mongo/util/map_util.h" @@ -31,565 +36,554 @@ namespace mongo { namespace { - TEST(AuthorizationManagerTest, AcquirePrivilegeAndCheckAuthorization) { - Principal* principal = new Principal(PrincipalName("Spencer", "test")); + TEST(RoleParsingTest, BuildRoleBSON) { + RoleGraph graph; + RoleName roleA("roleA", "dbA"); + RoleName roleB("roleB", "dbB"); + RoleName roleC("roleC", "dbC"); ActionSet actions; + actions.addAction(ActionType::find); actions.addAction(ActionType::insert); - Privilege writePrivilege("test", actions); - Privilege allDBsWritePrivilege("*", actions); - AuthExternalStateMock* externalState = new AuthExternalStateMock(); - AuthorizationManager authManager(externalState); - - ASSERT_FALSE(authManager.checkAuthorization("test", ActionType::insert)); - externalState->setReturnValueForShouldIgnoreAuthChecks(true); - ASSERT_TRUE(authManager.checkAuthorization("test", ActionType::insert)); - externalState->setReturnValueForShouldIgnoreAuthChecks(false); - ASSERT_FALSE(authManager.checkAuthorization("test", ActionType::insert)); - - ASSERT_EQUALS(ErrorCodes::UserNotFound, - authManager.acquirePrivilege(writePrivilege, principal->getName())); - authManager.addAuthorizedPrincipal(principal); - ASSERT_OK(authManager.acquirePrivilege(writePrivilege, principal->getName())); - ASSERT_TRUE(authManager.checkAuthorization("test", ActionType::insert)); - - ASSERT_FALSE(authManager.checkAuthorization("otherDb", ActionType::insert)); - ASSERT_OK(authManager.acquirePrivilege(allDBsWritePrivilege, principal->getName())); - ASSERT_TRUE(authManager.checkAuthorization("otherDb", ActionType::insert)); - // Auth checks on a collection should be applied to the database name. - ASSERT_TRUE(authManager.checkAuthorization("otherDb.collectionName", ActionType::insert)); - - authManager.logoutDatabase("test"); - ASSERT_FALSE(authManager.checkAuthorization("test", ActionType::insert)); - } - TEST(AuthorizationManagerTest, GetPrivilegesFromPrivilegeDocumentCompatible) { - PrincipalName principal ("Spencer", "test"); - BSONObj invalid; - BSONObj readWrite = BSON("user" << "Spencer" << "pwd" << "passwordHash"); - BSONObj readOnly = BSON("user" << "Spencer" << "pwd" << "passwordHash" << - "readOnly" << true); - - PrivilegeSet privilegeSet; - ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, - AuthorizationManager::buildPrivilegeSet("test", - principal, - invalid, - &privilegeSet).code()); - - ASSERT_OK(AuthorizationManager::buildPrivilegeSet("test", - principal, - readOnly, - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::insert))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - - ASSERT_OK(AuthorizationManager::buildPrivilegeSet("test", - principal, - readWrite, - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::insert))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::userAdmin))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::compact))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::shutdown))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::addShard))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("*", ActionType::find))); - - ASSERT_OK(AuthorizationManager::buildPrivilegeSet("admin", - principal, - readOnly, - &privilegeSet)); - // Should grant privileges on *. - ASSERT(privilegeSet.hasPrivilege(Privilege("*", ActionType::find))); - - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::insert))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("*", ActionType::insert))); - - ASSERT_OK(AuthorizationManager::buildPrivilegeSet("admin", - principal, - readWrite, - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("*", ActionType::insert))); + ASSERT_OK(graph.createRole(roleA)); + ASSERT_OK(graph.createRole(roleB)); + ASSERT_OK(graph.createRole(roleC)); + + ASSERT_OK(graph.addRoleToRole(roleA, roleC)); + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_OK(graph.addRoleToRole(roleB, roleC)); + + ASSERT_OK(graph.addPrivilegeToRole( + roleA, Privilege(ResourcePattern::forAnyNormalResource(), actions))); + ASSERT_OK(graph.addPrivilegeToRole( + roleB, Privilege(ResourcePattern::forExactNamespace(NamespaceString("dbB.foo")), + actions))); + ASSERT_OK(graph.addPrivilegeToRole( + roleC, Privilege(ResourcePattern::forClusterResource(), actions))); + ASSERT_OK(graph.recomputePrivilegeData()); + + + // Role A + mutablebson::Document doc; + ASSERT_OK(AuthorizationManager::getBSONForRole(&graph, roleA, doc.root())); + BSONObj roleDoc = doc.getObject(); + + ASSERT_EQUALS("dbA.roleA", roleDoc["_id"].String()); + ASSERT_EQUALS("roleA", roleDoc["role"].String()); + ASSERT_EQUALS("dbA", roleDoc["db"].String()); + + vector privs = roleDoc["privileges"].Array(); + ASSERT_EQUALS(1U, privs.size()); + ASSERT_EQUALS("", privs[0].Obj()["resource"].Obj()["db"].String()); + ASSERT_EQUALS("", privs[0].Obj()["resource"].Obj()["collection"].String()); + ASSERT(privs[0].Obj()["resource"].Obj()["cluster"].eoo()); + vector actionElements = privs[0].Obj()["actions"].Array(); + ASSERT_EQUALS(2U, actionElements.size()); + ASSERT_EQUALS("find", actionElements[0].String()); + ASSERT_EQUALS("insert", actionElements[1].String()); + + vector roles = roleDoc["roles"].Array(); + ASSERT_EQUALS(2U, roles.size()); + ASSERT_EQUALS("roleC", roles[0].Obj()["role"].String()); + ASSERT_EQUALS("dbC", roles[0].Obj()["db"].String()); + ASSERT_EQUALS("roleB", roles[1].Obj()["role"].String()); + ASSERT_EQUALS("dbB", roles[1].Obj()["db"].String()); + + // Role B + doc.reset(); + ASSERT_OK(AuthorizationManager::getBSONForRole(&graph, roleB, doc.root())); + roleDoc = doc.getObject(); + + ASSERT_EQUALS("dbB.roleB", roleDoc["_id"].String()); + ASSERT_EQUALS("roleB", roleDoc["role"].String()); + ASSERT_EQUALS("dbB", roleDoc["db"].String()); + + privs = roleDoc["privileges"].Array(); + ASSERT_EQUALS(1U, privs.size()); + ASSERT_EQUALS("dbB", privs[0].Obj()["resource"].Obj()["db"].String()); + ASSERT_EQUALS("foo", privs[0].Obj()["resource"].Obj()["collection"].String()); + ASSERT(privs[0].Obj()["resource"].Obj()["cluster"].eoo()); + actionElements = privs[0].Obj()["actions"].Array(); + ASSERT_EQUALS(2U, actionElements.size()); + ASSERT_EQUALS("find", actionElements[0].String()); + ASSERT_EQUALS("insert", actionElements[1].String()); + + roles = roleDoc["roles"].Array(); + ASSERT_EQUALS(1U, roles.size()); + ASSERT_EQUALS("roleC", roles[0].Obj()["role"].String()); + ASSERT_EQUALS("dbC", roles[0].Obj()["db"].String()); + + // Role C + doc.reset(); + ASSERT_OK(AuthorizationManager::getBSONForRole(&graph, roleC, doc.root())); + roleDoc = doc.getObject(); + + ASSERT_EQUALS("dbC.roleC", roleDoc["_id"].String()); + ASSERT_EQUALS("roleC", roleDoc["role"].String()); + ASSERT_EQUALS("dbC", roleDoc["db"].String()); + + privs = roleDoc["privileges"].Array(); + ASSERT_EQUALS(1U, privs.size()); + ASSERT(privs[0].Obj()["resource"].Obj()["cluster"].Bool()); + ASSERT(privs[0].Obj()["resource"].Obj()["db"].eoo()); + ASSERT(privs[0].Obj()["resource"].Obj()["collection"].eoo()); + actionElements = privs[0].Obj()["actions"].Array(); + ASSERT_EQUALS(2U, actionElements.size()); + ASSERT_EQUALS("find", actionElements[0].String()); + ASSERT_EQUALS("insert", actionElements[1].String()); + + roles = roleDoc["roles"].Array(); + ASSERT_EQUALS(0U, roles.size()); } - class PrivilegeDocumentParsing : public ::mongo::unittest::Test { + class AuthorizationManagerTest : public ::mongo::unittest::Test { public: - PrivilegeDocumentParsing() : user("spencer", "test") {} - - PrincipalName user; - PrivilegeSet privilegeSet; - }; - - TEST_F(PrivilegeDocumentParsing, VerifyRolesFieldMustBeAnArray) { - ASSERT_NOT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << "read"), - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyInvalidRoleGrantsNoPrivileges) { - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("frim")), - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyInvalidRoleStillAllowsOtherRoles) { - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read" << "frim")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyCannotGrantClusterAdminRoleFromNonAdminDatabase) { - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read" << "clusterAdmin")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::shutdown))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::dropDatabase))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyCannotGrantClusterReadFromNonAdminDatabase) { - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read" << "readAnyDatabase")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::find))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyCannotGrantClusterReadWriteFromNonAdminDatabase) { - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read" << "readWriteAnyDatabase")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::insert))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::insert))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyCannotGrantClusterUserAdminFromNonAdminDatabase) { - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read" << "userAdminAnyDatabase")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::userAdmin))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::userAdmin))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyCannotGrantClusterDBAdminFromNonAdminDatabase) { - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read" << "dbAdminAnyDatabase")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::clean))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::clean))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyOtherDBRolesMustBeAnObjectOfArraysOfStrings) { - ASSERT_NOT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read") << - "otherDBRoles" << BSON_ARRAY("read")), - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::find))); - - ASSERT_NOT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read") << - "otherDBRoles" << BSON("test2" << "read")), - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::find))); - } - - TEST_F(PrivilegeDocumentParsing, VerifyCannotGrantPrivilegesOnOtherDatabasesNormally) { - // Cannot grant privileges on other databases, except from admin database. - ASSERT_NOT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("read") << - "otherDBRoles" << BSON("test2" << BSON_ARRAY("read"))), - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::find))); - } - - TEST_F(PrivilegeDocumentParsing, SuccessfulSimpleReadGrant) { - // Grant read on test. - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << "roles" << BSON_ARRAY("read")), - &privilegeSet)); - - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::find))); - } - - TEST_F(PrivilegeDocumentParsing, SuccessfulSimpleUserAdminTest) { - // Grant userAdmin on "test" database. - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "test", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("userAdmin")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::userAdmin))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::userAdmin))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::userAdmin))); - } - - TEST_F(PrivilegeDocumentParsing, GrantUserAdminOnAdmin) { - // Grant userAdmin on admin. - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("userAdmin")), - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::userAdmin))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::userAdmin))); - ASSERT(privilegeSet.hasPrivilege(Privilege("admin", ActionType::userAdmin))); - } - - TEST_F(PrivilegeDocumentParsing, GrantUserAdminOnTestViaAdmin) { - // Grant userAdmin on test via admin. - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSONArrayBuilder().arr() << - "otherDBRoles" << BSON("test" << BSON_ARRAY("userAdmin"))), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::userAdmin))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::userAdmin))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::userAdmin))); - } - - TEST_F(PrivilegeDocumentParsing, SuccessfulClusterAdminTest) { - // Grant userAdminAnyDatabase. - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("userAdminAnyDatabase")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::userAdmin))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test2", ActionType::userAdmin))); - ASSERT(privilegeSet.hasPrivilege(Privilege("admin", ActionType::userAdmin))); - } + virtual ~AuthorizationManagerTest() { + authzManager->invalidateUserCache(); + } + void setUp() { + externalState = new AuthzManagerExternalStateMock(); + externalState->setAuthzVersion(AuthorizationManager::schemaVersion26Final); + authzManager.reset(new AuthorizationManager(externalState)); + externalState->setAuthorizationManager(authzManager.get()); + authzManager->setAuthEnabled(true); + } - TEST_F(PrivilegeDocumentParsing, GrantClusterReadWrite) { - // Grant readWrite on everything via the admin database. - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("readWriteAnyDatabase")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test2", ActionType::find))); - ASSERT(privilegeSet.hasPrivilege(Privilege("admin", ActionType::find))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::insert))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test2", ActionType::insert))); - ASSERT(privilegeSet.hasPrivilege(Privilege("admin", ActionType::insert))); - } + scoped_ptr authzManager; + AuthzManagerExternalStateMock* externalState; + }; - TEST_F(PrivilegeDocumentParsing, ProhibitGrantOnWildcard) { - // Cannot grant readWrite to everythign using "otherDBRoles". - ASSERT_NOT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSONArrayBuilder().arr() << - "otherDBRoles" << BSON("*" << BSON_ARRAY("readWrite"))), - &privilegeSet)); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::find))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test", ActionType::insert))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("test2", ActionType::insert))); - ASSERT(!privilegeSet.hasPrivilege(Privilege("admin", ActionType::insert))); + TEST_F(AuthorizationManagerTest, testAcquireV0User) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion24); + + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "v0RW" << "pwd" << "password"), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("admin.system.users"), + BSON("user" << "v0AdminRO" << + "pwd" << "password" << + "readOnly" << true), + BSONObj())); + + ASSERT_OK(authzManager->initialize()); + User* v0RW; + ASSERT_OK(authzManager->acquireUser(UserName("v0RW", "test"), &v0RW)); + ASSERT_EQUALS(UserName("v0RW", "test"), v0RW->getName()); + ASSERT(v0RW->isValid()); + ASSERT_EQUALS(1U, v0RW->getRefCount()); + RoleNameIterator roles = v0RW->getRoles(); + ASSERT_EQUALS(RoleName("dbOwner", "test"), roles.next()); + ASSERT_FALSE(roles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(v0RW); + + User* v0AdminRO; + ASSERT_OK(authzManager->acquireUser(UserName("v0AdminRO", "admin"), &v0AdminRO)); + ASSERT(UserName("v0AdminRO", "admin") == v0AdminRO->getName()); + ASSERT(v0AdminRO->isValid()); + ASSERT_EQUALS((uint32_t)1, v0AdminRO->getRefCount()); + RoleNameIterator adminRoles = v0AdminRO->getRoles(); + ASSERT_EQUALS(RoleName("readAnyDatabase", "admin"), adminRoles.next()); + ASSERT_FALSE(adminRoles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(v0AdminRO); } - TEST_F(PrivilegeDocumentParsing, GrantClusterAdmin) { - // Grant cluster admin - ASSERT_OK(AuthorizationManager::buildPrivilegeSet( - "admin", - user, - BSON("user" << "spencer" << "pwd" << "" << - "roles" << BSON_ARRAY("clusterAdmin")), - &privilegeSet)); - ASSERT(privilegeSet.hasPrivilege(Privilege("test", ActionType::dropDatabase))); - ASSERT(privilegeSet.hasPrivilege(Privilege("test2", ActionType::dropDatabase))); - ASSERT(privilegeSet.hasPrivilege(Privilege("admin", ActionType::dropDatabase))); - ASSERT(privilegeSet.hasPrivilege(Privilege("$SERVER", ActionType::shutdown))); - ASSERT(privilegeSet.hasPrivilege(Privilege("$CLUSTER", ActionType::moveChunk))); + TEST_F(AuthorizationManagerTest, testAcquireV1User) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion24); + + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "v1read" << + "pwd" << "password" << + "roles" << BSON_ARRAY("read")), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("admin.system.users"), + BSON("user" << "v1cluster" << + "pwd" << "password" << + "roles" << BSON_ARRAY("clusterAdmin")), + BSONObj())); + + User* v1read; + ASSERT_OK(authzManager->acquireUser(UserName("v1read", "test"), &v1read)); + ASSERT_EQUALS(UserName("v1read", "test"), v1read->getName()); + ASSERT(v1read->isValid()); + ASSERT_EQUALS((uint32_t)1, v1read->getRefCount()); + + RoleNameIterator roles = v1read->getRoles(); + ASSERT_EQUALS(RoleName("read", "test"), roles.next()); + ASSERT_FALSE(roles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(v1read); + + User* v1cluster; + ASSERT_OK(authzManager->acquireUser(UserName("v1cluster", "admin"), &v1cluster)); + ASSERT_EQUALS(UserName("v1cluster", "admin"), v1cluster->getName()); + ASSERT(v1cluster->isValid()); + ASSERT_EQUALS((uint32_t)1, v1cluster->getRefCount()); + RoleNameIterator clusterRoles = v1cluster->getRoles(); + ASSERT_EQUALS(RoleName("clusterAdmin", "admin"), clusterRoles.next()); + ASSERT_FALSE(clusterRoles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(v1cluster); } - TEST(AuthorizationManagerTest, GetPrivilegesFromPrivilegeDocumentInvalid) { - BSONObj oldAndNewMixed = BSON("user" << "spencer" << - "pwd" << "passwordHash" << - "readOnly" << false << - "roles" << BSON_ARRAY("write" << "userAdmin")); + TEST_F(AuthorizationManagerTest, initializeAllV1UserData) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion24); + + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "readOnly" << + "pwd" << "password" << + "roles" << BSON_ARRAY("read")), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("admin.system.users"), + BSON("user" << "clusterAdmin" << + "userSource" << "$external" << + "roles" << BSON_ARRAY("clusterAdmin")), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "readWriteMultiDB" << + "pwd" << "password" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("test2.system.users"), + BSON("user" << "readWriteMultiDB" << + "userSource" << "test" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj())); + + Status status = authzManager->initialize(); + ASSERT_OK(status); + + User* readOnly; + ASSERT_OK(authzManager->acquireUser(UserName("readOnly", "test"), &readOnly)); + ASSERT_EQUALS(UserName("readOnly", "test"), readOnly->getName()); + ASSERT(readOnly->isValid()); + ASSERT_EQUALS(1U, readOnly->getRefCount()); + RoleNameIterator roles = readOnly->getRoles(); + ASSERT_EQUALS(RoleName("read", "test"), roles.next()); + ASSERT_FALSE(roles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(readOnly); + + User* clusterAdmin; + ASSERT_OK(authzManager->acquireUser(UserName("clusterAdmin", "$external"), &clusterAdmin)); + ASSERT_EQUALS(UserName("clusterAdmin", "$external"), clusterAdmin->getName()); + ASSERT(clusterAdmin->isValid()); + ASSERT_EQUALS(1U, clusterAdmin->getRefCount()); + RoleNameIterator clusterRoles = clusterAdmin->getRoles(); + ASSERT_EQUALS(RoleName("clusterAdmin", "admin"), clusterRoles.next()); + ASSERT_FALSE(clusterRoles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(clusterAdmin); + + User* multiDB; + status = authzManager->acquireUser(UserName("readWriteMultiDB", "test2"), &multiDB); + ASSERT_NOT_OK(status); + ASSERT(status.code() == ErrorCodes::UserNotFound); + + ASSERT_OK(authzManager->acquireUser(UserName("readWriteMultiDB", "test"), &multiDB)); + ASSERT_EQUALS(UserName("readWriteMultiDB", "test"), multiDB->getName()); + ASSERT(multiDB->isValid()); + ASSERT_EQUALS(1U, multiDB->getRefCount()); + User* multiDBProbed; + ASSERT_OK(authzManager->acquireV1UserProbedForDb( + UserName("readWriteMultiDB", "test"), + "test2", + &multiDBProbed)); + authzManager->releaseUser(multiDB); + multiDB = multiDBProbed; + ASSERT_EQUALS(UserName("readWriteMultiDB", "test"), multiDB->getName()); + ASSERT(multiDB->isValid()); + ASSERT_EQUALS(1U, multiDB->getRefCount()); + + RoleNameIterator multiDBRoles = multiDB->getRoles(); + ASSERT(multiDBRoles.more()); + RoleName role = multiDBRoles.next(); + if (role == RoleName("readWrite", "test")) { + ASSERT(multiDBRoles.more()); + ASSERT_EQUALS(RoleName("readWrite", "test2"), multiDBRoles.next()); + } else { + ASSERT_EQUALS(RoleName("readWrite", "test2"), role); + ASSERT(multiDBRoles.more()); + ASSERT_EQUALS(RoleName("readWrite", "test"), multiDBRoles.next()); + } + ASSERT_FALSE(multiDBRoles.more()); - PrincipalName principal("spencer", "anydb"); - PrivilegeSet result; - ASSERT_NOT_OK(AuthorizationManager::buildPrivilegeSet( - "anydb", principal, oldAndNewMixed, &result)); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(multiDB); } - TEST(AuthorizationManagerTest, DocumentValidationCompatibility) { - Status (*check)(const StringData&, const BSONObj&) = - &AuthorizationManager::checkValidPrivilegeDocument; - - // Good documents, with and without "readOnly" fields. - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a"))); - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << "readOnly" << 1))); - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << "readOnly" << false))); - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << "readOnly" << "yes"))); - - // Must have a "pwd" field. - ASSERT_NOT_OK(check("test", BSON("user" << "andy"))); - - // "pwd" field must be a string. - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << 100))); - - // "pwd" field string must not be empty. - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << ""))); - // Must have a "user" field. - ASSERT_NOT_OK(check("test", BSON("pwd" << "a"))); - - // "user" field must be a string. - ASSERT_NOT_OK(check("test", BSON("user" << 100 << "pwd" << "a"))); - - // "user" field string must not be empty. - ASSERT_NOT_OK(check("test", BSON("user" << "" << "pwd" << "a"))); + TEST_F(AuthorizationManagerTest, testAcquireV2User) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion26Final); + + ASSERT_OK(externalState->insertPrivilegeDocument( + "admin", + BSON("user" << "v2read" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "password") << + "roles" << BSON_ARRAY(BSON("role" << "read" << "db" << "test"))), + BSONObj())); + ASSERT_OK(externalState->insertPrivilegeDocument( + "admin", + BSON("user" << "v2cluster" << + "db" << "admin" << + "credentials" << BSON("MONGODB-CR" << "password") << + "roles" << BSON_ARRAY(BSON("role" << "clusterAdmin" << "db" << "admin"))), + BSONObj())); + + User* v2read; + ASSERT_OK(authzManager->acquireUser(UserName("v2read", "test"), &v2read)); + ASSERT_EQUALS(UserName("v2read", "test"), v2read->getName()); + ASSERT(v2read->isValid()); + ASSERT_EQUALS(1U, v2read->getRefCount()); + RoleNameIterator roles = v2read->getRoles(); + ASSERT_EQUALS(RoleName("read", "test"), roles.next()); + ASSERT_FALSE(roles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(v2read); + + User* v2cluster; + ASSERT_OK(authzManager->acquireUser(UserName("v2cluster", "admin"), &v2cluster)); + ASSERT_EQUALS(UserName("v2cluster", "admin"), v2cluster->getName()); + ASSERT(v2cluster->isValid()); + ASSERT_EQUALS(1U, v2cluster->getRefCount()); + RoleNameIterator clusterRoles = v2cluster->getRoles(); + ASSERT_EQUALS(RoleName("clusterAdmin", "admin"), clusterRoles.next()); + ASSERT_FALSE(clusterRoles.more()); + // Make sure user's refCount is 0 at the end of the test to avoid an assertion failure + authzManager->releaseUser(v2cluster); } - - class CompatibilityModeDisabler { + class AuthzUpgradeTest : public AuthorizationManagerTest { public: - CompatibilityModeDisabler() { - AuthorizationManager::setSupportOldStylePrivilegeDocuments(false); - } - ~CompatibilityModeDisabler() { - AuthorizationManager::setSupportOldStylePrivilegeDocuments(true); + void setUpV1UserData() { + + // Docs for "readOnly@test" + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "readOnly" << + "pwd" << "password" << + "roles" << BSON_ARRAY("read")), + BSONObj())); + + // Docs for "clusterAdmin@$external" + ASSERT_OK(externalState->insert(NamespaceString("admin.system.users"), + BSON("user" << "clusterAdmin" << + "userSource" << "$external" << + "roles" << BSON_ARRAY("clusterAdmin")), + BSONObj())); + + // Docs for "readWriteMultiDB@test" + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "readWriteMultiDB" << + "pwd" << "password" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("test2.system.users"), + BSON("user" << "readWriteMultiDB" << + "userSource" << "test" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj())); + + // Docs for otherdbroles@test + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "otherdbroles" << + "pwd" << "password" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("admin.system.users"), + BSON("user" << "otherdbroles" << + "userSource" << "test" << + "roles" << BSONArray() << + "otherDBRoles" << BSON("test3" << BSON_ARRAY("readWrite"))), + BSONObj())); + + // Docs for mixedroles@test + ASSERT_OK(externalState->insert(NamespaceString("test.system.users"), + BSON("user" << "mixedroles" << + "pwd" << "password" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("admin.system.users"), + BSON("user" << "mixedroles" << + "userSource" << "test" << + "roles" << BSONArray() << + "otherDBRoles" << BSON("test3" << BSON_ARRAY("readWrite" << + "dbAdmin") << + "test2" << BSON_ARRAY("readWrite"))), + BSONObj())); + ASSERT_OK(externalState->insert(NamespaceString("test2.system.users"), + BSON("user" << "mixedroles" << + "userSource" << "test" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj())); + + ASSERT_OK(authzManager->initialize()); } - }; - - TEST(AuthorizationManagerTest, DisableCompatibilityMode) { - Status (*check)(const StringData&, const BSONObj&) = - &AuthorizationManager::checkValidPrivilegeDocument; - - CompatibilityModeDisabler disabler; - - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << "a"))); - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << "readOnly" << 1))); - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << "readOnly" << false))); - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << "readOnly" << "yes"))); - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSON_ARRAY("dbAdmin" << "read")))); - } - - TEST(AuthorizationManagerTest, DocumentValidationExtended) { - Status (*check)(const StringData&, const BSONObj&) = - &AuthorizationManager::checkValidPrivilegeDocument; - - // Document describing new-style user on "test". - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSON_ARRAY("read")))); - - // Document giving roles on "test" to a user from "test2". - ASSERT_OK(check("test", BSON("user" << "andy" << "userSource" << "test2" << - "roles" << BSON_ARRAY("read")))); - - // Cannot have "userSource" field value == dbname. - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "userSource" << "test" << - "roles" << BSON_ARRAY("read")))); - - // Cannot have both "userSource" and "pwd" - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "userSource" << "test2" << - "pwd" << "a" << "roles" << BSON_ARRAY("read")))); - - // Cannot have an otherDBRoles field except in the admin database. - ASSERT_NOT_OK(check("test", - BSON("user" << "andy" << "userSource" << "test2" << - "roles" << BSON_ARRAY("read") << - "otherDBRoles" << BSON("test2" << BSON_ARRAY("readWrite"))))); - - ASSERT_OK(check("admin", - BSON("user" << "andy" << "userSource" << "test2" << - "roles" << BSON_ARRAY("read") << - "otherDBRoles" << BSON("test2" << BSON_ARRAY("readWrite"))))); - - // Must have "roles" to have "otherDBRoles". - ASSERT_NOT_OK(check("admin", - BSON("user" << "andy" << "pwd" << "a" << - "otherDBRoles" << BSON("test2" << BSON_ARRAY("readWrite"))))); - - ASSERT_OK(check("admin", - BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSONArrayBuilder().arr() << - "otherDBRoles" << BSON("test2" << BSON_ARRAY("readWrite"))))); - - // "otherDBRoles" may be empty. - ASSERT_OK(check("admin", - BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSONArrayBuilder().arr() << - "otherDBRoles" << BSONObjBuilder().obj()))); - - // Cannot omit "roles" if "userSource" is present. - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "userSource" << "test2"))); - - // Cannot have both "roles" and "readOnly". - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << "readOnly" << 1 << - "roles" << BSON_ARRAY("read")))); - - // Roles must be strings, not empty. - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSON_ARRAY("read" << "")))); - - ASSERT_NOT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSON_ARRAY(1 << "read")))); - - // Multiple roles OK. - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSON_ARRAY("dbAdmin" << "read")))); - - // Empty roles list OK. - ASSERT_OK(check("test", BSON("user" << "andy" << "pwd" << "a" << - "roles" << BSONArrayBuilder().arr()))); - } + void validateV1AdminUserData(const NamespaceString& collectionName) { + BSONObj doc; + + // Verify that the expected users are present. + ASSERT_EQUALS(3U, externalState->getCollectionContents(collectionName).size()); + ASSERT_OK(externalState->findOne(collectionName, + BSON("user" << "clusterAdmin" << + "userSource" << "$external"), + &doc)); + ASSERT_EQUALS("clusterAdmin", doc["user"].str()); + ASSERT_EQUALS("$external", doc["userSource"].str()); + ASSERT_TRUE(doc["pwd"].eoo()); + ASSERT_EQUALS(1U, doc["roles"].Array().size()); + ASSERT_EQUALS("clusterAdmin", doc["roles"].Array()[0].str()); + + ASSERT_OK(externalState->findOne(collectionName, + BSON("user" << "otherdbroles" << + "userSource" << "test"), + &doc)); + ASSERT_TRUE(doc["pwd"].eoo()); + ASSERT_EQUALS(0U, doc["roles"].Array().size()); + + ASSERT_OK(externalState->findOne(collectionName, + BSON("user" << "mixedroles" << + "userSource" << "test"), + &doc)); + ASSERT_TRUE(doc["pwd"].eoo()); + ASSERT_EQUALS(0U, doc["roles"].Array().size()); + } - class AuthExternalStateImplictPriv : public AuthExternalStateMock { - public: - virtual bool _findUser(const string& usersNamespace, - const BSONObj& query, - BSONObj* result) const { - - NamespaceString nsstring(usersNamespace); - std::string user = query[AuthorizationManager::USER_NAME_FIELD_NAME].String(); - std::string userSource; - if (!query[AuthorizationManager::USER_SOURCE_FIELD_NAME].trueValue()) { - userSource = nsstring.db; + void validateV2UserData() { + BSONObj doc; + + // Verify that the admin.system.version document reflects correct upgrade. + ASSERT_OK(externalState->findOne( + AuthorizationManager::versionCollectionNamespace, + BSON("_id" << "authSchema" << + AuthorizationManager::schemaVersionFieldName << + AuthorizationManager::schemaVersion26Final), + &doc)); + ASSERT_EQUALS(2, doc.nFields()); + ASSERT_EQUALS(1U, externalState->getCollectionContents( + AuthorizationManager::versionCollectionNamespace).size()); + + // Verify that the expected users are present. + ASSERT_EQUALS(5U, externalState->getCollectionContents( + AuthorizationManager::usersAltCollectionNamespace).size()); + ASSERT_EQUALS(5U, externalState->getCollectionContents( + AuthorizationManager::usersCollectionNamespace).size()); + + // "readOnly@test" user + ASSERT_OK(externalState->findOne(AuthorizationManager::usersCollectionNamespace, + BSON("user" << "readOnly" << "db" << "test"), + &doc)); + ASSERT_EQUALS("readOnly", doc["user"].str()); + ASSERT_EQUALS("test", doc["db"].str()); + ASSERT_EQUALS("password", doc["credentials"]["MONGODB-CR"].str()); + ASSERT_EQUALS(1U, doc["roles"].Array().size()); + + // "clusterAdmin@$external" user + ASSERT_OK(externalState->findOne( + AuthorizationManager::usersCollectionNamespace, + BSON("user" << "clusterAdmin" << "db" << "$external"), + &doc)); + ASSERT_EQUALS("clusterAdmin", doc["user"].str()); + ASSERT_EQUALS("$external", doc["db"].str()); + ASSERT_EQUALS(1U, doc["roles"].Array().size()); + + // "readWriteMultiDB@test" user + ASSERT_OK(externalState->findOne( + AuthorizationManager::usersCollectionNamespace, + BSON("user" << "readWriteMultiDB" << "db" << "test"), + &doc)); + ASSERT_EQUALS("readWriteMultiDB", doc["user"].str()); + ASSERT_EQUALS("test", doc["db"].str()); + ASSERT_EQUALS("password", doc["credentials"]["MONGODB-CR"].str()); + ASSERT_EQUALS(2U, doc["roles"].Array().size()); + + // "otherdbroles@test" user + ASSERT_OK(externalState->findOne( + AuthorizationManager::usersCollectionNamespace, + BSON("user" << "otherdbroles" << "db" << "test"), + &doc)); + ASSERT_EQUALS("test.otherdbroles", doc["_id"].str()); + ASSERT_EQUALS("password", doc["credentials"]["MONGODB-CR"].str()); + std::vector roles = doc["roles"].Array(); + std::set > rolePairs; + for (size_t i = 0; i < roles.size(); ++i) { + BSONElement roleElement = roles[i]; + rolePairs.insert(make_pair(roleElement["role"].str(), roleElement["db"].str())); } - else { - userSource = query[AuthorizationManager::USER_SOURCE_FIELD_NAME].String(); + ASSERT_EQUALS(2U, rolePairs.size()); + ASSERT_EQUALS(1U, rolePairs.count(make_pair("readWrite", "test"))); + ASSERT_EQUALS(1U, rolePairs.count(make_pair("readWrite", "test3"))); + + // "mixedroles@test" user + ASSERT_OK(externalState->findOne( + AuthorizationManager::usersCollectionNamespace, + BSON("user" << "mixedroles" << "db" << "test"), + &doc)); + ASSERT_EQUALS("test.mixedroles", doc["_id"].str()); + ASSERT_EQUALS("password", doc["credentials"]["MONGODB-CR"].str()); + rolePairs.clear(); + roles = doc["roles"].Array(); + for (size_t i = 0; i < roles.size(); ++i) { + BSONElement roleElement = roles[i]; + rolePairs.insert(make_pair(roleElement["role"].str(), roleElement["db"].str())); } - *result = mapFindWithDefault(_privilegeDocs, - std::make_pair(nsstring.db, - PrincipalName(user, userSource)), - BSON("invalid" << 1)); - return !(*result)["invalid"].trueValue(); + ASSERT_EQUALS(4U, rolePairs.size()); + ASSERT_EQUALS(1U, rolePairs.count(make_pair("readWrite", "test"))); + ASSERT_EQUALS(1U, rolePairs.count(make_pair("readWrite", "test2"))); + ASSERT_EQUALS(1U, rolePairs.count(make_pair("readWrite", "test3"))); + ASSERT_EQUALS(1U, rolePairs.count(make_pair("dbAdmin", "test3"))); } - void addPrivilegeDocument(const string& dbname, - const PrincipalName& user, - const BSONObj& doc) { - - ASSERT(_privilegeDocs.insert(std::make_pair(std::make_pair(dbname, user), - doc.getOwned())).second); + void upgradeAuthCollections() { + ASSERT_OK(authzManager->upgradeSchema(10, BSONObj())); } - - private: - std::map, BSONObj > _privilegeDocs; }; - class ImplicitPriviligesTest : public ::mongo::unittest::Test { - public: - AuthExternalStateImplictPriv* state; - scoped_ptr authman; + TEST_F(AuthzUpgradeTest, upgradeUserDataFromV1ToV2Clean) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion24); + setUpV1UserData(); + upgradeAuthCollections(); - void setUp() { - state = new AuthExternalStateImplictPriv; - authman.reset(new AuthorizationManager(state)); - } - }; + validateV2UserData(); + validateV1AdminUserData(AuthorizationManager::usersBackupCollectionNamespace); + } + + TEST_F(AuthzUpgradeTest, upgradeUserDataFromV1ToV2TakesTwoSteps) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion24); + setUpV1UserData(); + ASSERT_EQUALS(ErrorCodes::OperationIncomplete, + authzManager->upgradeSchema(1, BSONObj())); + ASSERT_OK(authzManager->upgradeSchema(1, BSONObj())); + } + + TEST_F(AuthzUpgradeTest, upgradeUserDataFromV1ToV2WithSysVerDoc) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion24); + setUpV1UserData(); + upgradeAuthCollections(); + + validateV1AdminUserData(AuthorizationManager::usersBackupCollectionNamespace); + validateV2UserData(); + } - TEST_F(ImplicitPriviligesTest, ImplicitAcquireFromSomeDatabases) { - state->addPrivilegeDocument("test", PrincipalName("andy", "test"), - BSON("user" << "andy" << - "pwd" << "a" << - "roles" << BSON_ARRAY("readWrite"))); - state->addPrivilegeDocument("test2", PrincipalName("andy", "test"), - BSON("user" << "andy" << - "userSource" << "test" << - "roles" << BSON_ARRAY("read"))); - state->addPrivilegeDocument("admin", PrincipalName("andy", "test"), - BSON("user" << "andy" << - "userSource" << "test" << - "roles" << BSON_ARRAY("clusterAdmin") << - "otherDBRoles" << BSON("test3" << BSON_ARRAY("dbAdmin")))); - - ASSERT(!authman->checkAuthorization("test.foo", ActionType::find)); - ASSERT(!authman->checkAuthorization("test.foo", ActionType::insert)); - ASSERT(!authman->checkAuthorization("test.foo", ActionType::collMod)); - ASSERT(!authman->checkAuthorization("test2.foo", ActionType::find)); - ASSERT(!authman->checkAuthorization("test2.foo", ActionType::insert)); - ASSERT(!authman->checkAuthorization("test2.foo", ActionType::collMod)); - ASSERT(!authman->checkAuthorization("test3.foo", ActionType::find)); - ASSERT(!authman->checkAuthorization("test3.foo", ActionType::insert)); - ASSERT(!authman->checkAuthorization("test3.foo", ActionType::collMod)); - ASSERT(!authman->checkAuthorization("admin.foo", ActionType::find)); - ASSERT(!authman->checkAuthorization("admin.foo", ActionType::insert)); - ASSERT(!authman->checkAuthorization("admin.foo", ActionType::collMod)); - ASSERT(!authman->checkAuthorization("$SERVER", ActionType::shutdown)); - - Principal* principal = new Principal(PrincipalName("andy", "test")); - principal->setImplicitPrivilegeAcquisition(true); - authman->addAuthorizedPrincipal(principal); - - ASSERT(authman->checkAuthorization("test.foo", ActionType::find)); - ASSERT(authman->checkAuthorization("test.foo", ActionType::insert)); - ASSERT(!authman->checkAuthorization("test.foo", ActionType::collMod)); - ASSERT(authman->checkAuthorization("test2.foo", ActionType::find)); - ASSERT(!authman->checkAuthorization("test2.foo", ActionType::insert)); - ASSERT(!authman->checkAuthorization("test2.foo", ActionType::collMod)); - ASSERT(!authman->checkAuthorization("test3.foo", ActionType::find)); - ASSERT(!authman->checkAuthorization("test3.foo", ActionType::insert)); - ASSERT(authman->checkAuthorization("test3.foo", ActionType::collMod)); - ASSERT(!authman->checkAuthorization("admin.foo", ActionType::find)); - ASSERT(!authman->checkAuthorization("admin.foo", ActionType::insert)); - ASSERT(!authman->checkAuthorization("admin.foo", ActionType::collMod)); - ASSERT(authman->checkAuthorization("$SERVER", ActionType::shutdown)); + TEST_F(AuthzUpgradeTest, upgradeUserDataFromV1ToV2FailsWithBadInitialVersionDoc) { + externalState->setAuthzVersion(AuthorizationManager::schemaVersion24); + setUpV1UserData(); + externalState->setAuthzVersion(AuthorizationManager::schemaVersion26Final); + bool done; + ASSERT_OK(authzManager->upgradeSchemaStep(BSONObj(), &done)); + ASSERT_TRUE(done); + validateV1AdminUserData(AuthorizationManager::usersCollectionNamespace); + int numRemoved; + ASSERT_OK(externalState->remove(AuthorizationManager::versionCollectionNamespace, + BSONObj(), + BSONObj(), + &numRemoved)); + upgradeAuthCollections(); + validateV1AdminUserData(AuthorizationManager::usersBackupCollectionNamespace); + validateV2UserData(); } } // namespace diff --git a/src/mongo/db/auth/authorization_session.cpp b/src/mongo/db/auth/authorization_session.cpp new file mode 100644 index 00000000000..432e6c3b22f --- /dev/null +++ b/src/mongo/db/auth/authorization_session.cpp @@ -0,0 +1,532 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authorization_session.h" + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authz_session_external_state.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/security_key.h" +#include "mongo/db/client.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + +namespace { + const std::string ADMIN_DBNAME = "admin"; +} // namespace + + AuthorizationSession::AuthorizationSession(AuthzSessionExternalState* externalState) + : _impersonationFlag(false) { + _externalState.reset(externalState); + } + + AuthorizationSession::~AuthorizationSession() { + for (UserSet::iterator it = _authenticatedUsers.begin(); + it != _authenticatedUsers.end(); ++it) { + getAuthorizationManager().releaseUser(*it); + } + } + + AuthorizationManager& AuthorizationSession::getAuthorizationManager() { + return _externalState->getAuthorizationManager(); + } + + void AuthorizationSession::startRequest() { + _externalState->startRequest(); + _refreshUserInfoAsNeeded(); + } + + Status AuthorizationSession::addAndAuthorizeUser(const UserName& userName) { + User* user; + Status status = getAuthorizationManager().acquireUser(userName, &user); + if (!status.isOK()) { + return status; + } + + // Calling add() on the UserSet may return a user that was replaced because it was from the + // same database. + User* replacedUser = _authenticatedUsers.add(user); + if (replacedUser) { + getAuthorizationManager().releaseUser(replacedUser); + } + + // If there are any users and roles in the impersonation data, clear it out. + clearImpersonatedUserData(); + + _buildAuthenticatedRolesVector(); + return Status::OK(); + } + + User* AuthorizationSession::lookupUser(const UserName& name) { + return _authenticatedUsers.lookup(name); + } + + void AuthorizationSession::logoutDatabase(const std::string& dbname) { + User* removedUser = _authenticatedUsers.removeByDBName(dbname); + if (removedUser) { + getAuthorizationManager().releaseUser(removedUser); + } + clearImpersonatedUserData(); + _buildAuthenticatedRolesVector(); + } + + UserNameIterator AuthorizationSession::getAuthenticatedUserNames() { + return _authenticatedUsers.getNames(); + } + + RoleNameIterator AuthorizationSession::getAuthenticatedRoleNames() { + return makeRoleNameIterator(_authenticatedRoleNames.begin(), + _authenticatedRoleNames.end()); + } + + std::string AuthorizationSession::getAuthenticatedUserNamesToken() { + std::string ret; + for (UserNameIterator nameIter = getAuthenticatedUserNames(); + nameIter.more(); + nameIter.next()) { + ret += '\0'; // Using a NUL byte which isn't valid in usernames to separate them. + ret += nameIter->getFullName(); + } + + return ret; + } + + void AuthorizationSession::grantInternalAuthorization() { + _authenticatedUsers.add(internalSecurity.user); + _buildAuthenticatedRolesVector(); + } + + Status AuthorizationSession::checkAuthForQuery(const NamespaceString& ns, + const BSONObj& query) { + if (MONGO_unlikely(ns.isCommand())) { + return Status(ErrorCodes::InternalError, mongoutils::str::stream() << + "Checking query auth on command namespace " << ns.ns()); + } + if (!isAuthorizedForActionsOnNamespace(ns, ActionType::find)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "not authorized for query on " << ns.ns()); + } + return Status::OK(); + } + + Status AuthorizationSession::checkAuthForGetMore(const NamespaceString& ns, + long long cursorID) { + if (!isAuthorizedForActionsOnNamespace(ns, ActionType::find)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "not authorized for getmore on " << ns.ns()); + } + return Status::OK(); + } + + Status AuthorizationSession::checkAuthForInsert(const NamespaceString& ns, + const BSONObj& document) { + if (ns.coll() == StringData("system.indexes", StringData::LiteralTag())) { + BSONElement nsElement = document["ns"]; + if (nsElement.type() != String) { + return Status(ErrorCodes::Unauthorized, "Cannot authorize inserting into " + "system.indexes documents without a string-typed \"ns\" field."); + } + NamespaceString indexNS(nsElement.str()); + if (!isAuthorizedForActionsOnNamespace(indexNS, ActionType::createIndex)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "not authorized to create index on " << + indexNS.ns()); + } + } else { + if (!isAuthorizedForActionsOnNamespace(ns, ActionType::insert)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "not authorized for insert on " << + ns.ns()); + } + } + + return Status::OK(); + } + + Status AuthorizationSession::checkAuthForUpdate(const NamespaceString& ns, + const BSONObj& query, + const BSONObj& update, + bool upsert) { + if (!upsert) { + if (!isAuthorizedForActionsOnNamespace(ns, ActionType::update)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "not authorized for update on " << + ns.ns()); + } + } + else { + ActionSet required; + required.addAction(ActionType::update); + required.addAction(ActionType::insert); + if (!isAuthorizedForActionsOnNamespace(ns, required)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "not authorized for upsert on " << + ns.ns()); + } + } + return Status::OK(); + } + + Status AuthorizationSession::checkAuthForDelete(const NamespaceString& ns, + const BSONObj& query) { + if (!isAuthorizedForActionsOnNamespace(ns, ActionType::remove)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "not authorized to remove from " << ns.ns()); + } + return Status::OK(); + } + + Status AuthorizationSession::checkAuthorizedToGrantPrivilege(const Privilege& privilege) { + const ResourcePattern& resource = privilege.getResourcePattern(); + if (resource.isDatabasePattern() || resource.isExactNamespacePattern()) { + if (!isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(resource.databaseToMatch()), + ActionType::grantRole)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to grant privileges on the " + << resource.databaseToMatch() << "database"); + } + } else if (!isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName("admin"), ActionType::grantRole)) { + return Status(ErrorCodes::Unauthorized, + "To grant privileges affecting multiple databases or the cluster," + " must be authorized to grant roles from the admin database"); + } + return Status::OK(); + } + + + Status AuthorizationSession::checkAuthorizedToRevokePrivilege(const Privilege& privilege) { + const ResourcePattern& resource = privilege.getResourcePattern(); + if (resource.isDatabasePattern() || resource.isExactNamespacePattern()) { + if (!isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(resource.databaseToMatch()), + ActionType::revokeRole)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to revoke privileges on the " + << resource.databaseToMatch() << "database"); + } + } else if (!isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName("admin"), ActionType::revokeRole)) { + return Status(ErrorCodes::Unauthorized, + "To revoke privileges affecting multiple databases or the cluster," + " must be authorized to revoke roles from the admin database"); + } + return Status::OK(); + } + + bool AuthorizationSession::isAuthorizedToGrantRole(const RoleName& role) { + return isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(role.getDB()), + ActionType::grantRole); + } + + bool AuthorizationSession::isAuthorizedToRevokeRole(const RoleName& role) { + return isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(role.getDB()), + ActionType::revokeRole); + } + + bool AuthorizationSession::isAuthorizedForPrivilege(const Privilege& privilege) { + if (_externalState->shouldIgnoreAuthChecks()) + return true; + + return _isAuthorizedForPrivilege(privilege); + } + + bool AuthorizationSession::isAuthorizedForPrivileges(const vector& privileges) { + if (_externalState->shouldIgnoreAuthChecks()) + return true; + + for (size_t i = 0; i < privileges.size(); ++i) { + if (!_isAuthorizedForPrivilege(privileges[i])) + return false; + } + + return true; + } + + bool AuthorizationSession::isAuthorizedForActionsOnResource(const ResourcePattern& resource, + ActionType action) { + return isAuthorizedForPrivilege(Privilege(resource, action)); + } + + bool AuthorizationSession::isAuthorizedForActionsOnResource(const ResourcePattern& resource, + const ActionSet& actions) { + return isAuthorizedForPrivilege(Privilege(resource, actions)); + } + + bool AuthorizationSession::isAuthorizedForActionsOnNamespace(const NamespaceString& ns, + ActionType action) { + return isAuthorizedForPrivilege(Privilege(ResourcePattern::forExactNamespace(ns), action)); + } + + bool AuthorizationSession::isAuthorizedForActionsOnNamespace(const NamespaceString& ns, + const ActionSet& actions) { + return isAuthorizedForPrivilege(Privilege(ResourcePattern::forExactNamespace(ns), actions)); + } + + static const int resourceSearchListCapacity = 5; + /** + * Builds from "target" an exhaustive list of all ResourcePatterns that match "target". + * + * Stores the resulting list into resourceSearchList, and returns the length. + * + * The seach lists are as follows, depending on the type of "target": + * + * target is ResourcePattern::forAnyResource(): + * searchList = { ResourcePattern::forAnyResource(), ResourcePattern::forAnyResource() } + * target is the ResourcePattern::forClusterResource(): + * searchList = { ResourcePattern::forAnyResource(), ResourcePattern::forClusterResource() } + * target is a database, db: + * searchList = { ResourcePattern::forAnyResource(), + * ResourcePattern::forAnyNormalResource(), + * db } + * target is a non-system collection, db.coll: + * searchList = { ResourcePattern::forAnyResource(), + * ResourcePattern::forAnyNormalResource(), + * db, + * coll, + * db.coll } + * target is a system collection, db.system.coll: + * searchList = { ResourcePattern::forAnyResource(), + * system.coll, + * db.system.coll } + */ + static int buildResourceSearchList( + const ResourcePattern& target, + ResourcePattern resourceSearchList[resourceSearchListCapacity]) { + + int size = 0; + resourceSearchList[size++] = ResourcePattern::forAnyResource(); + if (target.isExactNamespacePattern()) { + if (!target.ns().isSystem()) { + resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); + resourceSearchList[size++] = ResourcePattern::forDatabaseName(target.ns().db()); + } + resourceSearchList[size++] = ResourcePattern::forCollectionName(target.ns().coll()); + } + else if (target.isDatabasePattern()) { + resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); + } + resourceSearchList[size++] = target; + dassert(size <= resourceSearchListCapacity); + return size; + } + + bool AuthorizationSession::isAuthorizedToChangeOwnPasswordAsUser(const UserName& userName) { + User* user = lookupUser(userName); + if (!user) { + return false; + } + ResourcePattern resourceSearchList[resourceSearchListCapacity]; + const int resourceSearchListLength = + buildResourceSearchList(ResourcePattern::forDatabaseName(userName.getDB()), + resourceSearchList); + + ActionSet actions; + for (int i = 0; i < resourceSearchListLength; ++i) { + actions.addAllActionsFromSet(user->getActionsForResource(resourceSearchList[i])); + } + return actions.contains(ActionType::changeOwnPassword); + } + + bool AuthorizationSession::isAuthorizedToChangeOwnCustomDataAsUser(const UserName& userName) { + User* user = lookupUser(userName); + if (!user) { + return false; + } + ResourcePattern resourceSearchList[resourceSearchListCapacity]; + const int resourceSearchListLength = + buildResourceSearchList(ResourcePattern::forDatabaseName(userName.getDB()), + resourceSearchList); + + ActionSet actions; + for (int i = 0; i < resourceSearchListLength; ++i) { + actions.addAllActionsFromSet(user->getActionsForResource(resourceSearchList[i])); + } + return actions.contains(ActionType::changeOwnCustomData); + } + + bool AuthorizationSession::isAuthenticatedAsUserWithRole(const RoleName& roleName) { + for (UserSet::iterator it = _authenticatedUsers.begin(); + it != _authenticatedUsers.end(); ++it) { + if ((*it)->hasRole(roleName)) { + return true; + } + } + return false; + } + + void AuthorizationSession::_refreshUserInfoAsNeeded() { + AuthorizationManager& authMan = getAuthorizationManager(); + UserSet::iterator it = _authenticatedUsers.begin(); + while (it != _authenticatedUsers.end()) { + User* user = *it; + + if (!user->isValid()) { + // Make a good faith effort to acquire an up-to-date user object, since the one + // we've cached is marked "out-of-date." + UserName name = user->getName(); + User* updatedUser; + + Status status = authMan.acquireUser(name, &updatedUser); + switch (status.code()) { + case ErrorCodes::OK: { + // Success! Replace the old User object with the updated one. + fassert(17067, _authenticatedUsers.replaceAt(it, updatedUser) == user); + authMan.releaseUser(user); + LOG(1) << "Updated session cache of user information for " << name; + break; + } + case ErrorCodes::UserNotFound: { + // User does not exist anymore; remove it from _authenticatedUsers. + fassert(17068, _authenticatedUsers.removeAt(it) == user); + authMan.releaseUser(user); + log() << "Removed deleted user " << name << + " from session cache of user information."; + continue; // No need to advance "it" in this case. + } + default: + // Unrecognized error; assume that it's transient, and continue working with the + // out-of-date privilege data. + warning() << "Could not fetch updated user privilege information for " << + name << "; continuing to use old information. Reason is " << status; + break; + } + } + ++it; + } + _buildAuthenticatedRolesVector(); + } + + void AuthorizationSession::_buildAuthenticatedRolesVector() { + _authenticatedRoleNames.clear(); + for (UserSet::iterator it = _authenticatedUsers.begin(); + it != _authenticatedUsers.end(); + ++it) { + RoleNameIterator roles = (*it)->getIndirectRoles(); + while (roles.more()) { + RoleName roleName = roles.next(); + _authenticatedRoleNames.push_back(RoleName(roleName.getRole(), + roleName.getDB())); + } + } + } + + bool AuthorizationSession::_isAuthorizedForPrivilege(const Privilege& privilege) { + const ResourcePattern& target(privilege.getResourcePattern()); + + ResourcePattern resourceSearchList[resourceSearchListCapacity]; + const int resourceSearchListLength = buildResourceSearchList(target, resourceSearchList); + + ActionSet unmetRequirements = privilege.getActions(); + + for (UserSet::iterator it = _authenticatedUsers.begin(); + it != _authenticatedUsers.end(); ++it) { + User* user = *it; + + if (user->getSchemaVersion() == AuthorizationManager::schemaVersion24 && + (target.isDatabasePattern() || target.isExactNamespacePattern()) && + !user->hasProbedV1(target.databaseToMatch())) { + + UserName name = user->getName(); + User* updatedUser; + Status status = getAuthorizationManager().acquireV1UserProbedForDb( + name, + target.databaseToMatch(), + &updatedUser); + if (status.isOK()) { + if (user != updatedUser) { + LOG(1) << "Updated session cache with privileges on the " << + target.databaseToMatch() << " database for V1 user " << name; + fassert(17226, _authenticatedUsers.replaceAt(it, updatedUser) == user); + } + getAuthorizationManager().releaseUser(user); + user = updatedUser; + } + else if (status != ErrorCodes::UserNotFound) { + warning() << "Could not fetch updated user privilege information for V1-style " + "user " << name << "; continuing to use old information. Reason is " + << status; + } + } + + for (int i = 0; i < resourceSearchListLength; ++i) { + ActionSet userActions = user->getActionsForResource(resourceSearchList[i]); + unmetRequirements.removeAllActionsFromSet(userActions); + + if (unmetRequirements.empty()) + return true; + } + } + + return false; + } + + void AuthorizationSession::setImpersonatedUserData(std::vector usernames, + std::vector roles) { + _impersonatedUserNames = usernames; + _impersonatedRoleNames = roles; + _impersonationFlag = true; + } + + UserNameIterator AuthorizationSession::getImpersonatedUserNames() { + return makeUserNameIterator(_impersonatedUserNames.begin(), + _impersonatedUserNames.end()); + } + + RoleNameIterator AuthorizationSession::getImpersonatedRoleNames() { + return makeRoleNameIterator(_impersonatedRoleNames.begin(), + _impersonatedRoleNames.end()); + } + + // Clear the vectors of impersonated usernames and roles. + void AuthorizationSession::clearImpersonatedUserData() { + _impersonatedUserNames.clear(); + _impersonatedRoleNames.clear(); + _impersonationFlag = false; + } + + + bool AuthorizationSession::isImpersonating() const { + return _impersonationFlag; + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authorization_session.h b/src/mongo/db/auth/authorization_session.h new file mode 100644 index 00000000000..cfea0dae093 --- /dev/null +++ b/src/mongo/db/auth/authorization_session.h @@ -0,0 +1,232 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authz_session_external_state.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/user_set.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { + + /** + * Contains all the authorization logic for a single client connection. It contains a set of + * the users which have been authenticated, as well as a set of privileges that have been + * granted to those users to perform various actions. + * + * An AuthorizationSession object is present within every mongo::ClientBasic object. + * + * Users in the _authenticatedUsers cache may get marked as invalid by the AuthorizationManager, + * for instance if their privileges are changed by a user or role modification command. At the + * beginning of every user-initiated operation startRequest() gets called which updates + * the cached information about any users who have been marked as invalid. This guarantees that + * every operation looks at one consistent view of each user for every auth check required over + * the lifetime of the operation. + */ + class AuthorizationSession { + MONGO_DISALLOW_COPYING(AuthorizationSession); + public: + + // Takes ownership of the externalState. + explicit AuthorizationSession(AuthzSessionExternalState* externalState); + ~AuthorizationSession(); + + AuthorizationManager& getAuthorizationManager(); + + // Should be called at the beginning of every new request. This performs the checks + // necessary to determine if localhost connections should be given full access. + // TODO: try to eliminate the need for this call. + void startRequest(); + + /** + * Adds the User identified by "UserName" to the authorization session, acquiring privileges + * for it in the process. + */ + Status addAndAuthorizeUser(const UserName& userName); + + // Returns the authenticated user with the given name. Returns NULL + // if no such user is found. + // The user remains in the _authenticatedUsers set for this AuthorizationSession, + // and ownership of the user stays with the AuthorizationManager + User* lookupUser(const UserName& name); + + // Gets an iterator over the names of all authenticated users stored in this manager. + UserNameIterator getAuthenticatedUserNames(); + + // Gets an iterator over the roles of all authenticated users stored in this manager. + RoleNameIterator getAuthenticatedRoleNames(); + + // Returns a std::string representing all logged-in users on the current session. + // WARNING: this std::string will contain NUL bytes so don't call c_str()! + std::string getAuthenticatedUserNamesToken(); + + // Removes any authenticated principals whose authorization credentials came from the given + // database, and revokes any privileges that were granted via that principal. + void logoutDatabase(const std::string& dbname); + + // Adds the internalSecurity user to the set of authenticated users. + // Used to grant internal threads full access. + void grantInternalAuthorization(); + + // Checks if this connection has the privileges necessary to perform the given query on the + // given namespace. + Status checkAuthForQuery(const NamespaceString& ns, const BSONObj& query); + + // Checks if this connection has the privileges necessary to perform a getMore on the given + // cursor in the given namespace. + Status checkAuthForGetMore(const NamespaceString& ns, long long cursorID); + + // Checks if this connection has the privileges necessary to perform the given update on the + // given namespace. + Status checkAuthForUpdate(const NamespaceString& ns, + const BSONObj& query, + const BSONObj& update, + bool upsert); + + // Checks if this connection has the privileges necessary to insert the given document + // to the given namespace. Correctly interprets inserts to system.indexes and performs + // the proper auth checks for index building. + Status checkAuthForInsert(const NamespaceString& ns, const BSONObj& document); + + // Checks if this connection has the privileges necessary to perform a delete on the given + // namespace. + Status checkAuthForDelete(const NamespaceString& ns, const BSONObj& query); + + // Checks if this connection has the privileges necessary to grant the given privilege + // to a role. + Status checkAuthorizedToGrantPrivilege(const Privilege& privilege); + + // Checks if this connection has the privileges necessary to revoke the given privilege + // from a role. + Status checkAuthorizedToRevokePrivilege(const Privilege& privilege); + + // Utility function for isAuthorizedForActionsOnResource( + // ResourcePattern::forDatabaseName(role.getDB()), ActionType::grantAnyRole) + bool isAuthorizedToGrantRole(const RoleName& role); + + // Utility function for isAuthorizedForActionsOnResource( + // ResourcePattern::forDatabaseName(role.getDB()), ActionType::grantAnyRole) + bool isAuthorizedToRevokeRole(const RoleName& role); + + // Returns true if the current session is authenticated as the given user and that user + // is allowed to change his/her own password + bool isAuthorizedToChangeOwnPasswordAsUser(const UserName& userName); + + // Returns true if the current session is authenticated as the given user and that user + // is allowed to change his/her own customData. + bool isAuthorizedToChangeOwnCustomDataAsUser(const UserName& userName); + + // Returns true if any of the authenticated users on this session have the given role. + // NOTE: this does not refresh any of the users even if they are marked as invalid. + bool isAuthenticatedAsUserWithRole(const RoleName& roleName); + + // Returns true if this session is authorized for the given Privilege. + // + // Contains all the authorization logic including handling things like the localhost + // exception. + bool isAuthorizedForPrivilege(const Privilege& privilege); + + // Like isAuthorizedForPrivilege, above, except returns true if the session is authorized + // for all of the listed privileges. + bool isAuthorizedForPrivileges(const vector& privileges); + + // Utility function for isAuthorizedForPrivilege(Privilege(resource, action)). + bool isAuthorizedForActionsOnResource(const ResourcePattern& resource, ActionType action); + + // Utility function for isAuthorizedForPrivilege(Privilege(resource, actions)). + bool isAuthorizedForActionsOnResource(const ResourcePattern& resource, + const ActionSet& actions); + + // Utility function for + // isAuthorizedForActionsOnResource(ResourcePattern::forExactNamespace(ns), action). + bool isAuthorizedForActionsOnNamespace(const NamespaceString& ns, ActionType action); + + // Utility function for + // isAuthorizedForActionsOnResource(ResourcePattern::forExactNamespace(ns), actions). + bool isAuthorizedForActionsOnNamespace(const NamespaceString& ns, const ActionSet& actions); + + // Replaces the data for users that a system user is impersonating with new data. + // The auditing system adds these users and their roles to each audit record in the log. + void setImpersonatedUserData(std::vector usernames, std::vector roles); + + // Gets an iterator over the names of all users that the system user is impersonating. + UserNameIterator getImpersonatedUserNames(); + + // Gets an iterator over the roles of all users that the system user is impersonating. + RoleNameIterator getImpersonatedRoleNames(); + + // Clears the data for impersonated users. + void clearImpersonatedUserData(); + + // Tells whether impersonation is active or not. This state is set when + // setImpersonatedUserData is called and cleared when clearImpersonatedUserData is + // called. + bool isImpersonating() const; + + private: + + // If any users authenticated on this session are marked as invalid this updates them with + // up-to-date information. May require a read lock on the "admin" db to read the user data. + void _refreshUserInfoAsNeeded(); + + // Builds a vector of all roles held by users who are authenticated on this connection. The + // vector is stored in _authenticatedRoleNames. This function is called when users are + // logged in or logged out, as well as when the user cache is determined to be out of date. + void _buildAuthenticatedRolesVector(); + + // Checks if this connection is authorized for the given Privilege, ignoring whether or not + // we should even be doing authorization checks in general. Note: this may acquire a read + // lock on the admin database (to update out-of-date user privilege information). + bool _isAuthorizedForPrivilege(const Privilege& privilege); + + scoped_ptr _externalState; + + // All Users who have been authenticated on this connection. + UserSet _authenticatedUsers; + // The roles of the authenticated users. This vector is generated when the authenticated + // users set is changed. + std::vector _authenticatedRoleNames; + + // A vector of impersonated UserNames and a vector of those users' RoleNames. + // These are used in the auditing system. They are not used for authz checks. + std::vector _impersonatedUserNames; + std::vector _impersonatedRoleNames; + bool _impersonationFlag; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authorization_session_test.cpp b/src/mongo/db/auth/authorization_session_test.cpp new file mode 100644 index 00000000000..4e749e8bdec --- /dev/null +++ b/src/mongo/db/auth/authorization_session_test.cpp @@ -0,0 +1,545 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests of the AuthorizationSession type. + */ + +#include "mongo/base/status.h" +#include "mongo/db/auth/authz_session_external_state_mock.h" +#include "mongo/db/auth/authz_manager_external_state_mock.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/map_util.h" + +#define ASSERT_NULL(EXPR) ASSERT_FALSE(EXPR) +#define ASSERT_NON_NULL(EXPR) ASSERT_TRUE(EXPR) + +namespace mongo { +namespace { + + class FailureCapableAuthzManagerExternalStateMock : + public AuthzManagerExternalStateMock { + public: + FailureCapableAuthzManagerExternalStateMock() : _findsShouldFail(false) {} + virtual ~FailureCapableAuthzManagerExternalStateMock() {} + + void setFindsShouldFail(bool enable) { _findsShouldFail = enable; } + + virtual Status findOne(const NamespaceString& collectionName, + const BSONObj& query, + BSONObj* result) { + if (_findsShouldFail && + collectionName == AuthorizationManager::usersCollectionNamespace) { + + return Status(ErrorCodes::UnknownError, + "findOne on admin.system.users set to fail in mock."); + } + return AuthzManagerExternalStateMock::findOne(collectionName, query, result); + } + + private: + bool _findsShouldFail; + }; + + class AuthorizationSessionTest : public ::mongo::unittest::Test { + public: + FailureCapableAuthzManagerExternalStateMock* managerState; + AuthzSessionExternalStateMock* sessionState; + scoped_ptr authzManager; + scoped_ptr authzSession; + + void setUp() { + managerState = new FailureCapableAuthzManagerExternalStateMock(); + managerState->setAuthzVersion(AuthorizationManager::schemaVersion26Final); + authzManager.reset(new AuthorizationManager(managerState)); + sessionState = new AuthzSessionExternalStateMock(authzManager.get()); + authzSession.reset(new AuthorizationSession(sessionState)); + authzManager->setAuthEnabled(true); + } + }; + + const ResourcePattern testDBResource(ResourcePattern::forDatabaseName("test")); + const ResourcePattern otherDBResource(ResourcePattern::forDatabaseName("other")); + const ResourcePattern adminDBResource(ResourcePattern::forDatabaseName("admin")); + const ResourcePattern testFooCollResource( + ResourcePattern::forExactNamespace(NamespaceString("test.foo"))); + const ResourcePattern otherFooCollResource( + ResourcePattern::forExactNamespace(NamespaceString("other.foo"))); + const ResourcePattern thirdFooCollResource( + ResourcePattern::forExactNamespace(NamespaceString("third.foo"))); + const ResourcePattern adminFooCollResource( + ResourcePattern::forExactNamespace(NamespaceString("admin.foo"))); + const ResourcePattern testUsersCollResource( + ResourcePattern::forExactNamespace(NamespaceString("test.system.users"))); + const ResourcePattern otherUsersCollResource( + ResourcePattern::forExactNamespace(NamespaceString("other.system.users"))); + const ResourcePattern thirdUsersCollResource( + ResourcePattern::forExactNamespace(NamespaceString("third.system.users"))); + const ResourcePattern testIndexesCollResource( + ResourcePattern::forExactNamespace(NamespaceString("test.system.indexes"))); + const ResourcePattern otherIndexesCollResource( + ResourcePattern::forExactNamespace(NamespaceString("other.system.indexes"))); + const ResourcePattern thirdIndexesCollResource( + ResourcePattern::forExactNamespace(NamespaceString("third.system.indexes"))); + const ResourcePattern testProfileCollResource( + ResourcePattern::forExactNamespace(NamespaceString("test.system.profile"))); + const ResourcePattern otherProfileCollResource( + ResourcePattern::forExactNamespace(NamespaceString("other.system.profile"))); + const ResourcePattern thirdProfileCollResource( + ResourcePattern::forExactNamespace(NamespaceString("third.system.profile"))); + + TEST_F(AuthorizationSessionTest, AddUserAndCheckAuthorization) { + // Check that disabling auth checks works + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + sessionState->setReturnValueForShouldIgnoreAuthChecks(true); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + sessionState->setReturnValueForShouldIgnoreAuthChecks(false); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + + // Check that you can't authorize a user that doesn't exist. + ASSERT_EQUALS(ErrorCodes::UserNotFound, + authzSession->addAndAuthorizeUser(UserName("spencer", "test"))); + + // Add a user with readWrite and dbAdmin on the test DB + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "readWrite" << + "db" << "test") << + BSON("role" << "dbAdmin" << + "db" << "test"))), + BSONObj())); + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("spencer", "test"))); + + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testDBResource, ActionType::dbStats)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + + // Add an admin user with readWriteAnyDatabase + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "admin" << + "db" << "admin" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "readWriteAnyDatabase" << + "db" << "admin"))), + BSONObj())); + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("admin", "admin"))); + + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace( + NamespaceString("anydb.somecollection")), + ActionType::insert)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + otherDBResource, ActionType::insert)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::collMod)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + + authzSession->logoutDatabase("test"); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::collMod)); + + authzSession->logoutDatabase("admin"); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::collMod)); + } + + TEST_F(AuthorizationSessionTest, DuplicateRolesOK) { + // Add a user with doubled-up readWrite and single dbAdmin on the test DB + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "readWrite" << + "db" << "test") << + BSON("role" << "dbAdmin" << + "db" << "test") << + BSON("role" << "readWrite" << + "db" << "test"))), + BSONObj())); + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("spencer", "test"))); + + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testDBResource, ActionType::dbStats)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + } + + TEST_F(AuthorizationSessionTest, SystemCollectionsAccessControl) { + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "rw" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "readWrite" << + "db" << "test") << + BSON("role" << "dbAdmin" << + "db" << "test"))), + BSONObj())); + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "useradmin" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "userAdmin" << + "db" << "test"))), + BSONObj())); + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "rwany" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "readWriteAnyDatabase" << + "db" << "admin") << + BSON("role" << "dbAdminAnyDatabase" << + "db" << "admin"))), + BSONObj())); + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "useradminany" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "userAdminAnyDatabase" << + "db" << "admin"))), + BSONObj())); + + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("rwany", "test"))); + + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testIndexesCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testProfileCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + otherIndexesCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + otherProfileCollResource, ActionType::find)); + + // Logging in as useradminany@test implicitly logs out rwany@test. + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("useradminany", "test"))); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::insert)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::insert)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testIndexesCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testProfileCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherIndexesCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherProfileCollResource, ActionType::find)); + + // Logging in as rw@test implicitly logs out useradminany@test. + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("rw", "test"))); + + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testIndexesCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testProfileCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherIndexesCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherProfileCollResource, ActionType::find)); + + + // Logging in as useradmin@test implicitly logs out rw@test. + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("useradmin", "test"))); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testUsersCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherUsersCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testIndexesCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testProfileCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherIndexesCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + otherProfileCollResource, ActionType::find)); + } + + TEST_F(AuthorizationSessionTest, InvalidateUser) { + // Add a readWrite user + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "readWrite" << + "db" << "test"))), + BSONObj())); + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("spencer", "test"))); + + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + + User* user = authzSession->lookupUser(UserName("spencer", "test")); + ASSERT(user->isValid()); + + // Change the user to be read-only + int ignored; + managerState->remove( + AuthorizationManager::usersCollectionNamespace, + BSONObj(), + BSONObj(), + &ignored); + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "read" << + "db" << "test"))), + BSONObj())); + + // Make sure that invalidating the user causes the session to reload its privileges. + authzManager->invalidateUserByName(user->getName()); + authzSession->startRequest(); // Refreshes cached data for invalid users + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + + user = authzSession->lookupUser(UserName("spencer", "test")); + ASSERT(user->isValid()); + + // Delete the user. + managerState->remove( + AuthorizationManager::usersCollectionNamespace, + BSONObj(), + BSONObj(), + &ignored); + // Make sure that invalidating the user causes the session to reload its privileges. + authzManager->invalidateUserByName(user->getName()); + authzSession->startRequest(); // Refreshes cached data for invalid users + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT_FALSE(authzSession->lookupUser(UserName("spencer", "test"))); + } + + TEST_F(AuthorizationSessionTest, UseOldUserInfoInFaceOfConnectivityProblems) { + // Add a readWrite user + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "readWrite" << + "db" << "test"))), + BSONObj())); + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("spencer", "test"))); + + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + + User* user = authzSession->lookupUser(UserName("spencer", "test")); + ASSERT(user->isValid()); + + // Change the user to be read-only + int ignored; + managerState->setFindsShouldFail(true); + managerState->remove( + AuthorizationManager::usersCollectionNamespace, + BSONObj(), + BSONObj(), + &ignored); + ASSERT_OK(managerState->insertPrivilegeDocument("admin", + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "read" << + "db" << "test"))), + BSONObj())); + + // Even though the user's privileges have been reduced, since we've configured user + // document lookup to fail, the authz session should continue to use its known out-of-date + // privilege data. + authzManager->invalidateUserByName(user->getName()); + authzSession->startRequest(); // Refreshes cached data for invalid users + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + + // Once we configure document lookup to succeed again, authorization checks should + // observe the new values. + managerState->setFindsShouldFail(false); + authzSession->startRequest(); // Refreshes cached data for invalid users + ASSERT_TRUE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT_FALSE(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + } + + + TEST_F(AuthorizationSessionTest, ImplicitAcquireFromSomeDatabasesWithV1Users) { + managerState->setAuthzVersion(AuthorizationManager::schemaVersion24); + + managerState->insert(NamespaceString("test.system.users"), + BSON("user" << "andy" << + "pwd" << "a" << + "roles" << BSON_ARRAY("readWrite")), + BSONObj()); + managerState->insert(NamespaceString("other.system.users"), + BSON("user" << "andy" << + "userSource" << "test" << + "roles" << BSON_ARRAY("read")), + BSONObj()); + managerState->insert(NamespaceString("admin.system.users"), + BSON("user" << "andy" << + "userSource" << "test" << + "roles" << BSON_ARRAY("clusterAdmin") << + "otherDBRoles" << BSON("third" << BSON_ARRAY("dbAdmin"))), + BSONObj()); + ASSERT_OK(authzManager->initialize()); + + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::shutdown)); + + ASSERT_OK(authzSession->addAndAuthorizeUser(UserName("andy", "test"))); + + User* user = authzSession->lookupUser(UserName("andy", "test")); + ASSERT(UserName("andy", "test") == user->getName()); + + ASSERT(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT(authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::collMod)); + ASSERT(authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::insert)); + ASSERT(authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::collMod)); + ASSERT(authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::shutdown)); + + authzSession->logoutDatabase("test"); + + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + testFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + otherFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + thirdFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::find)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::insert)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + adminFooCollResource, ActionType::collMod)); + ASSERT(!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::shutdown)); + } + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/auth/authz_documents_update_guard.cpp b/src/mongo/db/auth/authz_documents_update_guard.cpp new file mode 100644 index 00000000000..e1dcaad9d5e --- /dev/null +++ b/src/mongo/db/auth/authz_documents_update_guard.cpp @@ -0,0 +1,57 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_documents_update_guard.h" + +#include "mongo/db/auth/authorization_manager.h" + + +namespace mongo { + + AuthzDocumentsUpdateGuard::AuthzDocumentsUpdateGuard(AuthorizationManager* authzManager) + : _authzManager(authzManager), _lockedForUpdate(false) {} + + AuthzDocumentsUpdateGuard::~AuthzDocumentsUpdateGuard() { + if (_lockedForUpdate) { + unlock(); + } + } + + bool AuthzDocumentsUpdateGuard::tryLock(const StringData& why) { + fassert(17126, !_lockedForUpdate); + _lockedForUpdate = _authzManager->tryAcquireAuthzUpdateLock(why); + return _lockedForUpdate; + } + + void AuthzDocumentsUpdateGuard::unlock() { + fassert(17127, _lockedForUpdate); + _authzManager->releaseAuthzUpdateLock(); + _lockedForUpdate = false; + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_documents_update_guard.h b/src/mongo/db/auth/authz_documents_update_guard.h new file mode 100644 index 00000000000..d07f8cfda2c --- /dev/null +++ b/src/mongo/db/auth/authz_documents_update_guard.h @@ -0,0 +1,69 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" + +namespace mongo { + + class AuthorizationManager; + + /* + * Guard object for locking the lock that serializes all writes to the persistent authorization + * documents. + */ + class AuthzDocumentsUpdateGuard { + MONGO_DISALLOW_COPYING(AuthzDocumentsUpdateGuard); + public: + explicit AuthzDocumentsUpdateGuard(AuthorizationManager* authzManager); + ~AuthzDocumentsUpdateGuard(); + + /** + * Tries to acquire the global lock guarding modifications to all persistent data related + * to authorization, namely the admin.system.users, admin.system.roles, and + * admin.system.version collections. This serializes all writers to the authorization + * documents, but does not impact readers. + * Returns whether or not it was successful at acquiring the lock. + */ + bool tryLock(const StringData& why); + + /** + * Releases the lock guarding modifications to persistent authorization data, which must + * already be held. + */ + void unlock(); + + private: + AuthorizationManager* _authzManager; + // True if the Guard has locked the lock that guards modifications to authz documents. + bool _lockedForUpdate; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state.cpp b/src/mongo/db/auth/authz_manager_external_state.cpp new file mode 100644 index 00000000000..e5f63496799 --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state.cpp @@ -0,0 +1,195 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_manager_external_state.h" + +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/security_key.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + AuthzManagerExternalState::AuthzManagerExternalState() {} + AuthzManagerExternalState::~AuthzManagerExternalState() {} + + Status AuthzManagerExternalState::getPrivilegeDocumentV1(const StringData& dbname, + const UserName& userName, + BSONObj* result) { + if (userName == internalSecurity.user->getName()) { + return Status(ErrorCodes::InternalError, + "Requested privilege document for the internal user"); + } + + if (!NamespaceString::validDBName(dbname)) { + return Status(ErrorCodes::BadValue, + mongoutils::str::stream() << "Bad database name \"" << dbname << "\""); + } + + const bool isUserFromTargetDB = (dbname == userName.getDB()); + + // Build the query needed to get the privilege document + + BSONObjBuilder queryBuilder; + const NamespaceString usersNamespace(dbname, "system.users"); + queryBuilder.append(AuthorizationManager::V1_USER_NAME_FIELD_NAME, userName.getUser()); + if (isUserFromTargetDB) { + queryBuilder.appendNull(AuthorizationManager::V1_USER_SOURCE_FIELD_NAME); + } + else { + queryBuilder.append(AuthorizationManager::V1_USER_SOURCE_FIELD_NAME, userName.getDB()); + } + + // Query for the privilege document + BSONObj userBSONObj; + Status found = findOne(usersNamespace, queryBuilder.done(), &userBSONObj); + if (!found.isOK()) { + if (found.code() == ErrorCodes::NoMatchingDocument) { + // Return more detailed status that includes user name. + return Status(ErrorCodes::UserNotFound, + mongoutils::str::stream() << "auth: couldn't find user " << + userName.toString() << ", " << usersNamespace.ns(), + 0); + } else { + return found; + } + } + + if (isUserFromTargetDB) { + if (userBSONObj[AuthorizationManager::PASSWORD_FIELD_NAME].eoo()) { + return Status(ErrorCodes::AuthSchemaIncompatible, mongoutils::str::stream() << + "User documents with schema version " << + AuthorizationManager::schemaVersion24 << + " must have a \"" << + AuthorizationManager::PASSWORD_FIELD_NAME << + "\" field."); + } + } + + *result = userBSONObj.getOwned(); + return Status::OK(); + } + + bool AuthzManagerExternalState::hasAnyPrivilegeDocuments() { + BSONObj userBSONObj; + Status status = findOne( + AuthorizationManager::usersCollectionNamespace, + BSONObj(), + &userBSONObj); + // If the status is NoMatchingDocument, there are no privilege documents. + // If it's OK, there are. Otherwise, we were unable to complete the query, + // so best to assume that there _are_ privilege documents. This might happen + // if the node contaning the users collection becomes transiently unavailable. + // See SERVER-12616, for example. + return status != ErrorCodes::NoMatchingDocument; + } + + + Status AuthzManagerExternalState::insertPrivilegeDocument(const string& dbname, + const BSONObj& userObj, + const BSONObj& writeConcern) { + Status status = insert(NamespaceString("admin.system.users"), userObj, writeConcern); + if (status.isOK()) { + return status; + } + if (status.code() == ErrorCodes::DuplicateKey) { + std::string name = userObj[AuthorizationManager::USER_NAME_FIELD_NAME].String(); + std::string source = userObj[AuthorizationManager::USER_DB_FIELD_NAME].String(); + return Status(ErrorCodes::DuplicateKey, + mongoutils::str::stream() << "User \"" << name << "@" << source << + "\" already exists"); + } + if (status.code() == ErrorCodes::UnknownError) { + return Status(ErrorCodes::UserModificationFailed, status.reason()); + } + return status; + } + + Status AuthzManagerExternalState::updatePrivilegeDocument( + const UserName& user, const BSONObj& updateObj, const BSONObj& writeConcern) { + Status status = updateOne( + NamespaceString("admin.system.users"), + BSON(AuthorizationManager::USER_NAME_FIELD_NAME << user.getUser() << + AuthorizationManager::USER_DB_FIELD_NAME << user.getDB()), + updateObj, + false, + writeConcern); + if (status.isOK()) { + return status; + } + if (status.code() == ErrorCodes::NoMatchingDocument) { + return Status(ErrorCodes::UserNotFound, + mongoutils::str::stream() << "User " << user.getFullName() << + " not found"); + } + if (status.code() == ErrorCodes::UnknownError) { + return Status(ErrorCodes::UserModificationFailed, status.reason()); + } + return status; + } + + Status AuthzManagerExternalState::removePrivilegeDocuments(const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) { + Status status = remove(NamespaceString("admin.system.users"), + query, + writeConcern, + numRemoved); + if (status.code() == ErrorCodes::UnknownError) { + return Status(ErrorCodes::UserModificationFailed, status.reason()); + } + return status; + } + + Status AuthzManagerExternalState::updateOne( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + const BSONObj& writeConcern) { + int nMatched; + Status status = update(collectionName, + query, + updatePattern, + upsert, + false, + writeConcern, + &nMatched); + if (!status.isOK()) { + return status; + } + dassert(nMatched == 1 || nMatched == 0); + if (nMatched == 0) { + return Status(ErrorCodes::NoMatchingDocument, "No document found"); + } + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state.h b/src/mongo/db/auth/authz_manager_external_state.h new file mode 100644 index 00000000000..8656847036d --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state.h @@ -0,0 +1,264 @@ +/* +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { + + /** + * Public interface for a class that encapsulates all the information related to system + * state not stored in AuthorizationManager. This is primarily to make AuthorizationManager + * easier to test as well as to allow different implementations for mongos and mongod. + */ + class AuthzManagerExternalState { + MONGO_DISALLOW_COPYING(AuthzManagerExternalState); + + public: + + virtual ~AuthzManagerExternalState(); + + /** + * Initializes the external state object. Must be called after construction and before + * calling other methods. Object may not be used after this method returns something other + * than Status::OK(). + */ + virtual Status initialize() = 0; + + /** + * Retrieves the schema version of the persistent data describing users and roles. + * Will leave *outVersion unmodified on non-OK status return values. + */ + virtual Status getStoredAuthorizationVersion(int* outVersion) = 0; + + /** + * Writes into "result" a document describing the named user and returns Status::OK(). The + * description includes the user credentials, if present, the user's role membership and + * delegation information, a full list of the user's privileges, and a full list of the + * user's roles, including those roles held implicitly through other roles (indirect roles). + * In the event that some of this information is inconsistent, the document will contain a + * "warnings" array, with string messages describing inconsistencies. + * + * If the user does not exist, returns ErrorCodes::UserNotFound. + */ + virtual Status getUserDescription(const UserName& userName, BSONObj* result) = 0; + + /** + * Writes into "result" a document describing the named role and returns Status::OK(). The + * description includes the roles in which the named role has membership and a full list of + * the roles of which the named role is a member, including those roles memberships held + * implicitly through other roles (indirect roles). If "showPrivileges" is true, then the + * description documents will also include a full list of the role's privileges. + * In the event that some of this information is inconsistent, the document will contain a + * "warnings" array, with string messages describing inconsistencies. + * + * If the role does not exist, returns ErrorCodes::RoleNotFound. + */ + virtual Status getRoleDescription(const RoleName& roleName, + bool showPrivileges, + BSONObj* result) = 0; + + /** + * Writes into "result" documents describing the roles that are defined on the given + * database. Each role description document includes the other roles in which the role has + * membership and a full list of the roles of which the named role is a member, + * including those roles memberships held implicitly through other roles (indirect roles). + * If showPrivileges is true, then the description documents will also include a full list + * of the role's privileges. If showBuiltinRoles is true, then the result array will + * contain description documents for all the builtin roles for the given database, if it + * is false the result will just include user defined roles. + * In the event that some of the information in a given role description is inconsistent, + * the document will contain a "warnings" array, with string messages describing + * inconsistencies. + */ + virtual Status getRoleDescriptionsForDB(const std::string dbname, + bool showPrivileges, + bool showBuiltinRoles, + vector* result) = 0; + + /** + * Gets the privilege document for "userName" stored in the system.users collection of + * database "dbname". Useful only for schemaVersion24 user documents. For newer schema + * versions, use getUserDescription(). + * + * On success, returns Status::OK() and stores a shared-ownership copy of the document into + * "result". + */ + Status getPrivilegeDocumentV1( + const StringData& dbname, const UserName& userName, BSONObj* result); + + /** + * Returns true if there exists at least one privilege document in the system. + */ + bool hasAnyPrivilegeDocuments(); + + /** + * Creates the given user object in the given database. + * + * TODO(spencer): remove dbname argument once users are only written into the admin db + */ + Status insertPrivilegeDocument(const std::string& dbname, + const BSONObj& userObj, + const BSONObj& writeConcern); + + /** + * Updates the given user object with the given update modifier. + */ + Status updatePrivilegeDocument(const UserName& user, + const BSONObj& updateObj, + const BSONObj& writeConcern); + + /** + * Removes users for the given database matching the given query. + * Writes into *numRemoved the number of user documents that were modified. + */ + Status removePrivilegeDocuments(const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved); + + /** + * Puts into the *dbnames vector the name of every database in the cluster. + * May take a global lock, so should only be called during startup. + */ + virtual Status getAllDatabaseNames(std::vector* dbnames) = 0; + + /** + * Finds a document matching "query" in "collectionName", and store a shared-ownership + * copy into "result". + * + * Returns Status::OK() on success. If no match is found, returns + * ErrorCodes::NoMatchingDocument. Other errors returned as appropriate. + */ + virtual Status findOne(const NamespaceString& collectionName, + const BSONObj& query, + BSONObj* result) = 0; + + /** + * Finds all documents matching "query" in "collectionName". For each document returned, + * calls the function resultProcessor on it. + */ + virtual Status query(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& projection, + const boost::function& resultProcessor) = 0; + + /** + * Inserts "document" into "collectionName". + * If there is a duplicate key error, returns a Status with code DuplicateKey. + */ + virtual Status insert(const NamespaceString& collectionName, + const BSONObj& document, + const BSONObj& writeConcern) = 0; + + /** + * Update one document matching "query" according to "updatePattern" in "collectionName". + * + * If "upsert" is true and no document matches "query", inserts one using "query" as a + * template. + * If "upsert" is false and no document matches "query", return a Status with the code + * NoMatchingDocument. The Status message in that case is not very descriptive and should + * not be displayed to the end user. + */ + virtual Status updateOne(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + const BSONObj& writeConcern); + + /** + * Updates documents matching "query" according to "updatePattern" in "collectionName". + */ + virtual Status update(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched) = 0; + + /** + * Removes all documents matching "query" from "collectionName". + */ + virtual Status remove(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) = 0; + + /** + * Creates an index with the given pattern on "collectionName". + */ + virtual Status createIndex(const NamespaceString& collectionName, + const BSONObj& pattern, + bool unique, + const BSONObj& writeConcern) = 0; + + /** + * Drops indexes other than the _id index on "collectionName". + */ + virtual Status dropIndexes(const NamespaceString& collectionName, + const BSONObj& writeConcern) = 0; + + /** + * Tries to acquire the global lock guarding modifications to all persistent data related + * to authorization, namely the admin.system.users, admin.system.roles, and + * admin.system.version collections. This serializes all writers to the authorization + * documents, but does not impact readers. + */ + virtual bool tryAcquireAuthzUpdateLock(const StringData& why) = 0; + + /** + * Releases the lock guarding modifications to persistent authorization data, which must + * already be held. + */ + virtual void releaseAuthzUpdateLock() = 0; + + virtual void logOp( + const char* op, + const char* ns, + const BSONObj& o, + BSONObj* o2, + bool* b) {} + + + protected: + AuthzManagerExternalState(); // This class should never be instantiated directly. + + static const long long _authzUpdateLockAcquisitionTimeoutMillis = 5000; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_d.cpp b/src/mongo/db/auth/authz_manager_external_state_d.cpp new file mode 100644 index 00000000000..6263a0d67f0 --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_d.cpp @@ -0,0 +1,263 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_manager_external_state_d.h" + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/client.h" +#include "mongo/db/dbhelpers.h" +#include "mongo/db/instance.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + AuthzManagerExternalStateMongod::AuthzManagerExternalStateMongod() {} + AuthzManagerExternalStateMongod::~AuthzManagerExternalStateMongod() {} + + Status AuthzManagerExternalStateMongod::_getUserDocument(const UserName& userName, + BSONObj* userDoc) { + + Client::ReadContext ctx("admin"); + int authzVersion; + Status status = getStoredAuthorizationVersion(&authzVersion); + if (!status.isOK()) + return status; + + switch (authzVersion) { + case AuthorizationManager::schemaVersion26Upgrade: + case AuthorizationManager::schemaVersion26Final: + break; + default: + return Status(ErrorCodes::AuthSchemaIncompatible, mongoutils::str::stream() << + "Unsupported schema version for getUserDescription(): " << + authzVersion); + } + + status = findOne( + (authzVersion == AuthorizationManager::schemaVersion26Final ? + AuthorizationManager::usersCollectionNamespace : + AuthorizationManager::usersAltCollectionNamespace), + BSON(AuthorizationManager::USER_NAME_FIELD_NAME << userName.getUser() << + AuthorizationManager::USER_DB_FIELD_NAME << userName.getDB()), + userDoc); + if (status == ErrorCodes::NoMatchingDocument) { + status = Status(ErrorCodes::UserNotFound, mongoutils::str::stream() << + "Could not find user " << userName.getFullName()); + } + return status; + } + + Status AuthzManagerExternalStateMongod::query( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& projection, + const boost::function& resultProcessor) { + try { + DBDirectClient client; + client.query(resultProcessor, collectionName.ns(), query, &projection); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongod::getAllDatabaseNames( + std::vector* dbnames) { + Lock::GlobalRead lk; + getDatabaseNames(*dbnames); + return Status::OK(); + } + + Status AuthzManagerExternalStateMongod::findOne( + const NamespaceString& collectionName, + const BSONObj& query, + BSONObj* result) { + + Client::ReadContext ctx(collectionName.ns()); + BSONObj found; + if (Helpers::findOne(collectionName.ns(), + query, + found)) { + *result = found.getOwned(); + return Status::OK(); + } + return Status(ErrorCodes::NoMatchingDocument, mongoutils::str::stream() << + "No document in " << collectionName.ns() << " matches " << query); + } + + Status AuthzManagerExternalStateMongod::insert( + const NamespaceString& collectionName, + const BSONObj& document, + const BSONObj& writeConcern) { + try { + DBDirectClient client; + client.insert(collectionName, document); + + // Handle write concern + BSONObjBuilder gleBuilder; + gleBuilder.append("getLastError", 1); + gleBuilder.appendElements(writeConcern); + BSONObj res; + client.runCommand("admin", gleBuilder.done(), res); + string errstr = client.getLastErrorString(res); + if (errstr.empty()) { + return Status::OK(); + } + if (res.hasField("code") && res["code"].Int() == ASSERT_ID_DUPKEY) { + return Status(ErrorCodes::DuplicateKey, errstr); + } + return Status(ErrorCodes::UnknownError, errstr); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongod::update(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched) { + try { + DBDirectClient client; + client.update(collectionName, query, updatePattern, upsert, multi); + + // Handle write concern + BSONObjBuilder gleBuilder; + gleBuilder.append("getLastError", 1); + gleBuilder.appendElements(writeConcern); + BSONObj res; + client.runCommand("admin", gleBuilder.done(), res); + string err = client.getLastErrorString(res); + if (!err.empty()) { + return Status(ErrorCodes::UnknownError, err); + } + + *nMatched = res["n"].numberInt(); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongod::remove( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) { + try { + DBDirectClient client; + client.remove(collectionName, query); + + // Handle write concern + BSONObjBuilder gleBuilder; + gleBuilder.append("getLastError", 1); + gleBuilder.appendElements(writeConcern); + BSONObj res; + client.runCommand("admin", gleBuilder.done(), res); + string errstr = client.getLastErrorString(res); + if (!errstr.empty()) { + return Status(ErrorCodes::UnknownError, errstr); + } + + *numRemoved = res["n"].numberInt(); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongod::createIndex( + const NamespaceString& collectionName, + const BSONObj& pattern, + bool unique, + const BSONObj& writeConcern) { + DBDirectClient client; + try { + if (client.ensureIndex(collectionName.ns(), + pattern, + unique)) { + BSONObjBuilder gleBuilder; + gleBuilder.append("getLastError", 1); + gleBuilder.appendElements(writeConcern); + BSONObj res; + client.runCommand("admin", gleBuilder.done(), res); + string errstr = client.getLastErrorString(res); + if (!errstr.empty()) { + return Status(ErrorCodes::UnknownError, errstr); + } + } + return Status::OK(); + } + catch (const DBException& ex) { + return ex.toStatus(); + } + } + + Status AuthzManagerExternalStateMongod::dropIndexes( + const NamespaceString& collectionName, + const BSONObj& writeConcern) { + DBDirectClient client; + try { + client.dropIndexes(collectionName.ns()); + BSONObjBuilder gleBuilder; + gleBuilder.append("getLastError", 1); + gleBuilder.appendElements(writeConcern); + BSONObj res; + client.runCommand("admin", gleBuilder.done(), res); + string errstr = client.getLastErrorString(res); + if (!errstr.empty()) { + return Status(ErrorCodes::UnknownError, errstr); + } + return Status::OK(); + } + catch (const DBException& ex) { + return ex.toStatus(); + } + } + + bool AuthzManagerExternalStateMongod::tryAcquireAuthzUpdateLock(const StringData& why) { + LOG(2) << "Attempting to lock user data for: " << why << endl; + return _authzDataUpdateLock.timed_lock( + boost::posix_time::milliseconds(_authzUpdateLockAcquisitionTimeoutMillis)); + } + + void AuthzManagerExternalStateMongod::releaseAuthzUpdateLock() { + return _authzDataUpdateLock.unlock(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_d.h b/src/mongo/db/auth/authz_manager_external_state_d.h new file mode 100644 index 00000000000..df7d26c1ec8 --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_d.h @@ -0,0 +1,91 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authz_manager_external_state_local.h" +#include "mongo/db/auth/role_graph.h" +#include "mongo/db/auth/user_name.h" + +namespace mongo { + + /** + * The implementation of AuthzManagerExternalState functionality for mongod. + */ + class AuthzManagerExternalStateMongod : public AuthzManagerExternalStateLocal { + MONGO_DISALLOW_COPYING(AuthzManagerExternalStateMongod); + + public: + AuthzManagerExternalStateMongod(); + virtual ~AuthzManagerExternalStateMongod(); + + virtual Status getAllDatabaseNames(std::vector* dbnames); + + virtual Status findOne(const NamespaceString& collectionName, + const BSONObj& query, + BSONObj* result); + virtual Status query(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& projection, + const boost::function& resultProcessor); + virtual Status insert(const NamespaceString& collectionName, + const BSONObj& document, + const BSONObj& writeConcern); + virtual Status update(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched); + virtual Status remove(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved); + virtual Status createIndex(const NamespaceString& collectionName, + const BSONObj& pattern, + bool unique, + const BSONObj& writeConcern); + virtual Status dropIndexes(const NamespaceString& collectionName, + const BSONObj& writeConcern); + virtual bool tryAcquireAuthzUpdateLock(const StringData& why); + virtual void releaseAuthzUpdateLock(); + + private: + virtual Status _getUserDocument(const UserName& userName, BSONObj* userDoc); + + boost::timed_mutex _authzDataUpdateLock; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_local.cpp b/src/mongo/db/auth/authz_manager_external_state_local.cpp new file mode 100644 index 00000000000..ce9f095c1c1 --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_local.cpp @@ -0,0 +1,392 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_manager_external_state_local.h" + +#include "mongo/base/status.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/bson/util/bson_extract.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user_document_parser.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + AuthzManagerExternalStateLocal::AuthzManagerExternalStateLocal() : + _roleGraphState(roleGraphStateInitial) {} + AuthzManagerExternalStateLocal::~AuthzManagerExternalStateLocal() {} + + Status AuthzManagerExternalStateLocal::initialize() { + Status status = _initializeRoleGraph(); + if (!status.isOK()) { + if (status == ErrorCodes::GraphContainsCycle) { + error() << "Cycle detected in admin.system.roles; role inheritance disabled. " + "Remove the listed cycle and any others to re-enable role inheritance. " << + status.reason(); + } + else { + error() << "Could not generate role graph from admin.system.roles; " + "only system roles available: " << status; + } + } + + return Status::OK(); + } + + Status AuthzManagerExternalStateLocal::getStoredAuthorizationVersion(int* outVersion) { + BSONObj versionDoc; + Status status = findOne(AuthorizationManager::versionCollectionNamespace, + AuthorizationManager::versionDocumentQuery, + &versionDoc); + if (status.isOK()) { + BSONElement versionElement = versionDoc[AuthorizationManager::schemaVersionFieldName]; + if (versionElement.isNumber()) { + *outVersion = versionElement.numberInt(); + return Status::OK(); + } + else if (versionElement.eoo()) { + return Status(ErrorCodes::NoSuchKey, mongoutils::str::stream() << + "No " << AuthorizationManager::schemaVersionFieldName << + " field in version document."); + } + else { + return Status(ErrorCodes::TypeMismatch, mongoutils::str::stream() << + "Could not determine schema version of authorization data. " + "Bad (non-numeric) type " << typeName(versionElement.type()) << + " (" << versionElement.type() << ") for " << + AuthorizationManager::schemaVersionFieldName << + " field in version document"); + } + } + else if (status == ErrorCodes::NoMatchingDocument) { + if (hasAnyPrivilegeDocuments()) { + *outVersion = AuthorizationManager::schemaVersion24; + } + else { + *outVersion = AuthorizationManager::schemaVersion26Final; + } + return Status::OK(); + } + else { + return status; + } + } + +namespace { + void addRoleNameToObjectElement(mutablebson::Element object, const RoleName& role) { + fassert(17153, object.appendString(AuthorizationManager::ROLE_NAME_FIELD_NAME, + role.getRole())); + fassert(17154, object.appendString(AuthorizationManager::ROLE_SOURCE_FIELD_NAME, + role.getDB())); + } + + void addRoleNameObjectsToArrayElement(mutablebson::Element array, RoleNameIterator roles) { + for (; roles.more(); roles.next()) { + mutablebson::Element roleElement = array.getDocument().makeElementObject(""); + addRoleNameToObjectElement(roleElement, roles.get()); + fassert(17155, array.pushBack(roleElement)); + } + } + + void addPrivilegeObjectsOrWarningsToArrayElement(mutablebson::Element privilegesElement, + mutablebson::Element warningsElement, + const PrivilegeVector& privileges) { + std::string errmsg; + for (size_t i = 0; i < privileges.size(); ++i) { + ParsedPrivilege pp; + if (ParsedPrivilege::privilegeToParsedPrivilege(privileges[i], &pp, &errmsg)) { + fassert(17156, privilegesElement.appendObject("", pp.toBSON())); + } else { + fassert(17157, + warningsElement.appendString( + "", + std::string(mongoutils::str::stream() << + "Skipped privileges on resource " << + privileges[i].getResourcePattern().toString() << + ". Reason: " << errmsg))); + } + } + } +} // namespace + + Status AuthzManagerExternalStateLocal::getUserDescription( + const UserName& userName, + BSONObj* result) { + + BSONObj userDoc; + Status status = _getUserDocument(userName, &userDoc); + if (!status.isOK()) + return status; + + BSONElement directRolesElement; + status = bsonExtractTypedField(userDoc, "roles", Array, &directRolesElement); + if (!status.isOK()) + return status; + std::vector directRoles; + status = V2UserDocumentParser::parseRoleVector(BSONArray(directRolesElement.Obj()), + &directRoles); + if (!status.isOK()) + return status; + + unordered_set indirectRoles; + PrivilegeVector allPrivileges; + bool isRoleGraphInconsistent; + { + boost::lock_guard lk(_roleGraphMutex); + isRoleGraphInconsistent = _roleGraphState == roleGraphStateConsistent; + for (size_t i = 0; i < directRoles.size(); ++i) { + const RoleName& role(directRoles[i]); + indirectRoles.insert(role); + if (isRoleGraphInconsistent) { + for (RoleNameIterator subordinates = _roleGraph.getIndirectSubordinates(role); + subordinates.more(); + subordinates.next()) { + + indirectRoles.insert(subordinates.get()); + } + } + const PrivilegeVector& rolePrivileges( + isRoleGraphInconsistent ? + _roleGraph.getAllPrivileges(role) : + _roleGraph.getDirectPrivileges(role)); + for (PrivilegeVector::const_iterator priv = rolePrivileges.begin(), + end = rolePrivileges.end(); + priv != end; + ++priv) { + + Privilege::addPrivilegeToPrivilegeVector(&allPrivileges, *priv); + } + } + } + + mutablebson::Document resultDoc(userDoc, mutablebson::Document::kInPlaceDisabled); + mutablebson::Element inheritedRolesElement = resultDoc.makeElementArray("inheritedRoles"); + mutablebson::Element privilegesElement = resultDoc.makeElementArray("inheritedPrivileges"); + mutablebson::Element warningsElement = resultDoc.makeElementArray("warnings"); + fassert(17159, resultDoc.root().pushBack(inheritedRolesElement)); + fassert(17158, resultDoc.root().pushBack(privilegesElement)); + if (!isRoleGraphInconsistent) { + fassert(17160, warningsElement.appendString( + "", "Role graph inconsistent, only direct privileges available.")); + } + addRoleNameObjectsToArrayElement(inheritedRolesElement, + makeRoleNameIteratorForContainer(indirectRoles)); + addPrivilegeObjectsOrWarningsToArrayElement( + privilegesElement, warningsElement, allPrivileges); + if (warningsElement.hasChildren()) { + fassert(17161, resultDoc.root().pushBack(warningsElement)); + } + *result = resultDoc.getObject(); + return Status::OK(); + } + + Status AuthzManagerExternalStateLocal::getRoleDescription(const RoleName& roleName, + bool showPrivileges, + BSONObj* result) { + boost::lock_guard lk(_roleGraphMutex); + return _getRoleDescription_inlock(roleName, showPrivileges, result); + } + + Status AuthzManagerExternalStateLocal::_getRoleDescription_inlock(const RoleName& roleName, + bool showPrivileges, + BSONObj* result) { + if (!_roleGraph.roleExists(roleName)) + return Status(ErrorCodes::RoleNotFound, "No role named " + roleName.toString()); + + mutablebson::Document resultDoc; + fassert(17162, resultDoc.root().appendString( + AuthorizationManager::ROLE_NAME_FIELD_NAME, roleName.getRole())); + fassert(17163, resultDoc.root().appendString( + AuthorizationManager::ROLE_SOURCE_FIELD_NAME, roleName.getDB())); + fassert(17267, + resultDoc.root().appendBool("isBuiltin", _roleGraph.isBuiltinRole(roleName))); + mutablebson::Element rolesElement = resultDoc.makeElementArray("roles"); + fassert(17164, resultDoc.root().pushBack(rolesElement)); + mutablebson::Element inheritedRolesElement = resultDoc.makeElementArray("inheritedRoles"); + fassert(17165, resultDoc.root().pushBack(inheritedRolesElement)); + mutablebson::Element privilegesElement = resultDoc.makeElementArray("privileges"); + mutablebson::Element inheritedPrivilegesElement = + resultDoc.makeElementArray("inheritedPrivileges"); + if (showPrivileges) { + fassert(17166, resultDoc.root().pushBack(privilegesElement)); + } + mutablebson::Element warningsElement = resultDoc.makeElementArray("warnings"); + + addRoleNameObjectsToArrayElement(rolesElement, _roleGraph.getDirectSubordinates(roleName)); + if (_roleGraphState == roleGraphStateConsistent) { + addRoleNameObjectsToArrayElement( + inheritedRolesElement, _roleGraph.getIndirectSubordinates(roleName)); + if (showPrivileges) { + addPrivilegeObjectsOrWarningsToArrayElement( + privilegesElement, + warningsElement, + _roleGraph.getDirectPrivileges(roleName)); + + addPrivilegeObjectsOrWarningsToArrayElement( + inheritedPrivilegesElement, + warningsElement, + _roleGraph.getAllPrivileges(roleName)); + + fassert(17323, resultDoc.root().pushBack(inheritedPrivilegesElement)); + } + } + else if (showPrivileges) { + warningsElement.appendString( + "", "Role graph state inconsistent; only direct privileges available."); + addPrivilegeObjectsOrWarningsToArrayElement( + privilegesElement, warningsElement, _roleGraph.getDirectPrivileges(roleName)); + } + if (warningsElement.hasChildren()) { + fassert(17167, resultDoc.root().pushBack(warningsElement)); + } + *result = resultDoc.getObject(); + return Status::OK(); + } + + Status AuthzManagerExternalStateLocal::getRoleDescriptionsForDB(const std::string dbname, + bool showPrivileges, + bool showBuiltinRoles, + vector* result) { + boost::lock_guard lk(_roleGraphMutex); + + for (RoleNameIterator it = _roleGraph.getRolesForDatabase(dbname); + it.more(); it.next()) { + if (!showBuiltinRoles && _roleGraph.isBuiltinRole(it.get())) { + continue; + } + BSONObj roleDoc; + Status status = _getRoleDescription_inlock(it.get(), showPrivileges, &roleDoc); + if (!status.isOK()) { + return status; + } + result->push_back(roleDoc); + } + return Status::OK(); + } + +namespace { + + /** + * Adds the role described in "doc" to "roleGraph". If the role cannot be added, due to + * some error in "doc", logs a warning. + */ + void addRoleFromDocumentOrWarn(RoleGraph* roleGraph, const BSONObj& doc) { + Status status = roleGraph->addRoleFromDocument(doc); + if (!status.isOK()) { + warning() << "Skipping invalid admin.system.roles document while calculating privileges" + " for user-defined roles: " << status << "; document " << doc; + } + } + + +} // namespace + + Status AuthzManagerExternalStateLocal::_initializeRoleGraph() { + boost::lock_guard lkInitialzeRoleGraph(_roleGraphMutex); + + _roleGraphState = roleGraphStateInitial; + _roleGraph = RoleGraph(); + + RoleGraph newRoleGraph; + Status status = query( + AuthorizationManager::rolesCollectionNamespace, + BSONObj(), + BSONObj(), + boost::bind(addRoleFromDocumentOrWarn, &newRoleGraph, _1)); + if (!status.isOK()) + return status; + + status = newRoleGraph.recomputePrivilegeData(); + + RoleGraphState newState; + if (status == ErrorCodes::GraphContainsCycle) { + error() << "Inconsistent role graph during authorization manager initialization. Only " + "direct privileges available. " << status.reason(); + newState = roleGraphStateHasCycle; + status = Status::OK(); + } + else if (status.isOK()) { + newState = roleGraphStateConsistent; + } + else { + newState = roleGraphStateInitial; + } + + if (status.isOK()) { + _roleGraph.swap(newRoleGraph); + _roleGraphState = newState; + } + return status; + } + + void AuthzManagerExternalStateLocal::logOp( + const char* op, + const char* ns, + const BSONObj& o, + BSONObj* o2, + bool* b) { + + if (ns == AuthorizationManager::rolesCollectionNamespace.ns() || + ns == AuthorizationManager::adminCommandNamespace.ns()) { + + boost::lock_guard lk(_roleGraphMutex); + Status status = _roleGraph.handleLogOp(op, NamespaceString(ns), o, o2); + + if (status == ErrorCodes::OplogOperationUnsupported) { + _roleGraph = RoleGraph(); + _roleGraphState = roleGraphStateInitial; + BSONObjBuilder oplogEntryBuilder; + oplogEntryBuilder << "op" << op << "ns" << ns << "o" << o; + if (o2) + oplogEntryBuilder << "o2" << *o2; + if (b) + oplogEntryBuilder << "b" << *b; + error() << "Unsupported modification to roles collection in oplog; " + "restart this process to reenable user-defined roles; " << status.reason() << + "; Oplog entry: " << oplogEntryBuilder.done(); + } + else if (!status.isOK()) { + warning() << "Skipping bad update to roles collection in oplog. " << status << + " Oplog entry: " << op; + } + status = _roleGraph.recomputePrivilegeData(); + if (status == ErrorCodes::GraphContainsCycle) { + _roleGraphState = roleGraphStateHasCycle; + error() << "Inconsistent role graph during authorization manager initialization. " + "Only direct privileges available. " << status.reason() << + " after applying oplog entry " << op; + } + else { + fassert(17183, status); + _roleGraphState = roleGraphStateConsistent; + } + } + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_local.h b/src/mongo/db/auth/authz_manager_external_state_local.h new file mode 100644 index 00000000000..f9c02dbd2ce --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_local.h @@ -0,0 +1,116 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authz_manager_external_state.h" +#include "mongo/db/auth/role_graph.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/db/auth/user_name.h" + +namespace mongo { + + /** + * Common implementation of AuthzManagerExternalState for systems where role + * and user information are stored locally. + */ + class AuthzManagerExternalStateLocal : public AuthzManagerExternalState { + MONGO_DISALLOW_COPYING(AuthzManagerExternalStateLocal); + + public: + virtual ~AuthzManagerExternalStateLocal(); + + virtual Status initialize(); + + virtual Status getStoredAuthorizationVersion(int* outVersion); + virtual Status getUserDescription(const UserName& userName, BSONObj* result); + virtual Status getRoleDescription(const RoleName& roleName, + bool showPrivileges, + BSONObj* result); + virtual Status getRoleDescriptionsForDB(const std::string dbname, + bool showPrivileges, + bool showBuiltinRoles, + vector* result); + + virtual void logOp( + const char* op, + const char* ns, + const BSONObj& o, + BSONObj* o2, + bool* b); + + protected: + AuthzManagerExternalStateLocal(); + + private: + enum RoleGraphState { + roleGraphStateInitial = 0, + roleGraphStateConsistent, + roleGraphStateHasCycle + }; + + /** + * Initializes the role graph from the contents of the admin.system.roles collection. + */ + Status _initializeRoleGraph(); + + /** + * Fetches the user document for "userName" from local storage, and stores it into "result". + */ + virtual Status _getUserDocument(const UserName& userName, BSONObj* result) = 0; + + Status _getRoleDescription_inlock(const RoleName& roleName, + bool showPrivileges, + BSONObj* result); + /** + * Eventually consistent, in-memory representation of all roles in the system (both + * user-defined and built-in). Synchronized via _roleGraphMutex. + */ + RoleGraph _roleGraph; + + /** + * State of _roleGraph, one of "initial", "consistent" and "has cycle". Synchronized via + * _roleGraphMutex. + */ + RoleGraphState _roleGraphState; + + /** + * Guards _roleGraphState and _roleGraph. + */ + boost::mutex _roleGraphMutex; + + boost::timed_mutex _authzDataUpdateLock; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_mock.cpp b/src/mongo/db/auth/authz_manager_external_state_mock.cpp new file mode 100644 index 00000000000..356fc1f6d2c --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_mock.cpp @@ -0,0 +1,370 @@ +/* +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_manager_external_state_mock.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/ops/update_driver.h" +#include "mongo/platform/unordered_set.h" +#include "mongo/util/map_util.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { +namespace { + void addRoleNameToObjectElement(mutablebson::Element object, const RoleName& role) { + fassert(17175, object.appendString(AuthorizationManager::ROLE_NAME_FIELD_NAME, role.getRole())); + fassert(17176, object.appendString(AuthorizationManager::ROLE_SOURCE_FIELD_NAME, role.getDB())); + } + + void addRoleNameObjectsToArrayElement(mutablebson::Element array, RoleNameIterator roles) { + for (; roles.more(); roles.next()) { + mutablebson::Element roleElement = array.getDocument().makeElementObject(""); + addRoleNameToObjectElement(roleElement, roles.get()); + fassert(17177, array.pushBack(roleElement)); + } + } + + void addPrivilegeObjectsOrWarningsToArrayElement(mutablebson::Element privilegesElement, + mutablebson::Element warningsElement, + const PrivilegeVector& privileges) { + std::string errmsg; + for (size_t i = 0; i < privileges.size(); ++i) { + ParsedPrivilege pp; + if (ParsedPrivilege::privilegeToParsedPrivilege(privileges[i], &pp, &errmsg)) { + fassert(17178, privilegesElement.appendObject("", pp.toBSON())); + } else { + fassert(17179, + warningsElement.appendString( + "", + std::string(mongoutils::str::stream() << + "Skipped privileges on resource " << + privileges[i].getResourcePattern().toString() << + ". Reason: " << errmsg))); + } + } + } +} // namespace + + AuthzManagerExternalStateMock::AuthzManagerExternalStateMock() : _authzManager(NULL) {} + AuthzManagerExternalStateMock::~AuthzManagerExternalStateMock() {} + + void AuthzManagerExternalStateMock::setAuthorizationManager( + AuthorizationManager* authzManager) { + _authzManager = authzManager; + } + + void AuthzManagerExternalStateMock::setAuthzVersion(int version) { + uassertStatusOK( + updateOne(AuthorizationManager::versionCollectionNamespace, + AuthorizationManager::versionDocumentQuery, + BSON("$set" << BSON(AuthorizationManager::schemaVersionFieldName << + version)), + true, + BSONObj())); + } + + Status AuthzManagerExternalStateMock::_getUserDocument(const UserName& userName, + BSONObj* userDoc) { + int authzVersion; + Status status = getStoredAuthorizationVersion(&authzVersion); + if (!status.isOK()) + return status; + + switch (authzVersion) { + case AuthorizationManager::schemaVersion26Upgrade: + case AuthorizationManager::schemaVersion26Final: + break; + default: + return Status(ErrorCodes::AuthSchemaIncompatible, mongoutils::str::stream() << + "Unsupported schema version for getUserDescription(): " << + authzVersion); + } + + status = findOne( + (authzVersion == AuthorizationManager::schemaVersion26Final ? + AuthorizationManager::usersCollectionNamespace : + AuthorizationManager::usersAltCollectionNamespace), + BSON(AuthorizationManager::USER_NAME_FIELD_NAME << userName.getUser() << + AuthorizationManager::USER_DB_FIELD_NAME << userName.getDB()), + userDoc); + if (status == ErrorCodes::NoMatchingDocument) { + status = Status(ErrorCodes::UserNotFound, mongoutils::str::stream() << + "Could not find user " << userName.getFullName()); + } + return status; + } + + Status AuthzManagerExternalStateMock::getAllDatabaseNames( + std::vector* dbnames) { + unordered_set dbnameSet; + NamespaceDocumentMap::const_iterator it; + for (it = _documents.begin(); it != _documents.end(); ++it) { + dbnameSet.insert(it->first.db().toString()); + } + *dbnames = std::vector(dbnameSet.begin(), dbnameSet.end()); + return Status::OK(); + } + + Status AuthzManagerExternalStateMock::_findUser( + const std::string& usersNamespace, + const BSONObj& query, + BSONObj* result) { + if (!findOne(NamespaceString(usersNamespace), query, result).isOK()) { + return Status(ErrorCodes::UserNotFound, + "No matching user for query " + query.toString()); + } + return Status::OK(); + } + + Status AuthzManagerExternalStateMock::findOne( + const NamespaceString& collectionName, + const BSONObj& query, + BSONObj* result) { + BSONObjCollection::iterator iter; + Status status = _findOneIter(collectionName, query, &iter); + if (!status.isOK()) + return status; + *result = iter->copy(); + return Status::OK(); + } + + Status AuthzManagerExternalStateMock::query( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj&, + const boost::function& resultProcessor) { + std::vector iterVector; + Status status = _queryVector(collectionName, query, &iterVector); + if (!status.isOK()) { + return status; + } + try { + for (std::vector::iterator it = iterVector.begin(); + it != iterVector.end(); ++it) { + resultProcessor(**it); + } + } + catch (const DBException& ex) { + status = ex.toStatus(); + } + return status; + } + + Status AuthzManagerExternalStateMock::insert( + const NamespaceString& collectionName, + const BSONObj& document, + const BSONObj&) { + BSONObj toInsert; + if (document["_id"].eoo()) { + BSONObjBuilder docWithIdBuilder; + docWithIdBuilder.append("_id", OID::gen()); + docWithIdBuilder.appendElements(document); + toInsert = docWithIdBuilder.obj(); + } + else { + toInsert = document.copy(); + } + _documents[collectionName].push_back(toInsert); + if (_authzManager) { + _authzManager->logOp( + "i", + collectionName.ns().c_str(), + toInsert, + NULL, + NULL); + } + return Status::OK(); + } + + Status AuthzManagerExternalStateMock::updateOne( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + const BSONObj& writeConcern) { + + namespace mmb = mutablebson; + UpdateDriver::Options updateOptions; + UpdateDriver driver(updateOptions); + Status status = driver.parse(updatePattern); + if (!status.isOK()) + return status; + + BSONObjCollection::iterator iter; + status = _findOneIter(collectionName, query, &iter); + mmb::Document document; + if (status.isOK()) { + document.reset(*iter, mmb::Document::kInPlaceDisabled); + BSONObj logObj; + status = driver.update(StringData(), &document, &logObj); + if (!status.isOK()) + return status; + BSONObj newObj = document.getObject().copy(); + *iter = newObj; + BSONObj idQuery = driver.makeOplogEntryQuery(newObj, false); + if (_authzManager) { + _authzManager->logOp( + "u", + collectionName.ns().c_str(), + logObj, + &idQuery, + NULL); + } + return Status::OK(); + } + else if (status == ErrorCodes::NoMatchingDocument && upsert) { + if (query.hasField("_id")) { + document.root().appendElement(query["_id"]); + } + status = driver.populateDocumentWithQueryFields(query, document); + if (!status.isOK()) { + return status; + } + status = driver.update(StringData(), &document); + if (!status.isOK()) { + return status; + } + return insert(collectionName, document.getObject(), writeConcern); + } + else { + return status; + } + } + + Status AuthzManagerExternalStateMock::update(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched) { + return Status(ErrorCodes::InternalError, + "AuthzManagerExternalStateMock::update not implemented in mock."); + } + + Status AuthzManagerExternalStateMock::remove( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj&, + int* numRemoved) { + int n = 0; + BSONObjCollection::iterator iter; + while (_findOneIter(collectionName, query, &iter).isOK()) { + BSONObj idQuery = (*iter)["_id"].wrap(); + _documents[collectionName].erase(iter); + ++n; + if (_authzManager) { + _authzManager->logOp( + "d", + collectionName.ns().c_str(), + idQuery, + NULL, + NULL); + } + } + *numRemoved = n; + return Status::OK(); + } + + Status AuthzManagerExternalStateMock::createIndex( + const NamespaceString& collectionName, + const BSONObj& pattern, + bool unique, + const BSONObj&) { + return Status::OK(); + } + + Status AuthzManagerExternalStateMock::dropIndexes( + const NamespaceString& collectionName, + const BSONObj& writeConcern) { + return Status::OK(); + } + + bool AuthzManagerExternalStateMock::tryAcquireAuthzUpdateLock(const StringData&) { + return true; + } + + void AuthzManagerExternalStateMock::releaseAuthzUpdateLock() {} + + std::vector AuthzManagerExternalStateMock::getCollectionContents( + const NamespaceString& collectionName) { + return mapFindWithDefault(_documents, collectionName, std::vector()); + } + + Status AuthzManagerExternalStateMock::_findOneIter( + const NamespaceString& collectionName, + const BSONObj& query, + BSONObjCollection::iterator* result) { + std::vector iterVector; + Status status = _queryVector(collectionName, query, &iterVector); + if (!status.isOK()) { + return status; + } + if (!iterVector.size()) { + return Status(ErrorCodes::NoMatchingDocument, "No matching document"); + } + *result = iterVector.front(); + return Status::OK(); + } + + Status AuthzManagerExternalStateMock::_queryVector( + const NamespaceString& collectionName, + const BSONObj& query, + std::vector* result) { + + StatusWithMatchExpression parseResult = MatchExpressionParser::parse(query); + if (!parseResult.isOK()) { + return parseResult.getStatus(); + } + MatchExpression* matcher = parseResult.getValue(); + + NamespaceDocumentMap::iterator mapIt = _documents.find(collectionName); + if (mapIt == _documents.end()) + return Status::OK(); + + for (BSONObjCollection::iterator vecIt = mapIt->second.begin(); + vecIt != mapIt->second.end(); + ++vecIt) { + + if (matcher->matchesBSON(*vecIt)) { + result->push_back(vecIt); + } + } + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_mock.h b/src/mongo/db/auth/authz_manager_external_state_mock.h new file mode 100644 index 00000000000..886157e0efe --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_mock.h @@ -0,0 +1,128 @@ +/* +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authz_manager_external_state_local.h" +#include "mongo/db/auth/role_graph.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { + + class AuthorizationManager; + + /** + * Mock of the AuthzManagerExternalState class used only for testing. + */ + class AuthzManagerExternalStateMock : public AuthzManagerExternalStateLocal { + MONGO_DISALLOW_COPYING(AuthzManagerExternalStateMock); + + public: + + AuthzManagerExternalStateMock(); + virtual ~AuthzManagerExternalStateMock(); + + void setAuthorizationManager(AuthorizationManager* authzManager); + void setAuthzVersion(int version); + + virtual Status getAllDatabaseNames(std::vector* dbnames); + + virtual Status findOne(const NamespaceString& collectionName, + const BSONObj& query, + BSONObj* result); + + virtual Status query(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& projection, // Currently unused in mock + const boost::function& resultProcessor); + + // This implementation does not understand uniqueness constraints. + virtual Status insert(const NamespaceString& collectionName, + const BSONObj& document, + const BSONObj& writeConcern); + + // This implementation does not understand uniqueness constraints, ignores writeConcern, + // and only correctly handles some upsert behaviors. + virtual Status updateOne(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + const BSONObj& writeConcern); + virtual Status update(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched); + virtual Status remove(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved); + virtual Status createIndex(const NamespaceString& collectionName, + const BSONObj& pattern, + bool unique, + const BSONObj& writeConcern); + virtual Status dropIndexes(const NamespaceString& collectionName, + const BSONObj& writeConcern); + virtual bool tryAcquireAuthzUpdateLock(const StringData& why); + virtual void releaseAuthzUpdateLock(); + + Status _findUser(const std::string& usersNamespace, + const BSONObj& query, + BSONObj* result); + std::vector getCollectionContents(const NamespaceString& collectionName); + + private: + typedef std::vector BSONObjCollection; + typedef std::map NamespaceDocumentMap; + + virtual Status _getUserDocument(const UserName& userName, BSONObj* userDoc); + + Status _findOneIter(const NamespaceString& collectionName, + const BSONObj& query, + BSONObjCollection::iterator* result); + + Status _queryVector(const NamespaceString& collectionName, + const BSONObj& query, + std::vector* result); + + + AuthorizationManager* _authzManager; // For reporting logOps. + NamespaceDocumentMap _documents; // Mock database. + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_s.cpp b/src/mongo/db/auth/authz_manager_external_state_s.cpp new file mode 100644 index 00000000000..a49d3f6decb --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_s.cpp @@ -0,0 +1,357 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_manager_external_state_s.h" + +#include +#include +#include + +#include "mongo/client/auth_helpers.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/jsobj.h" +#include "mongo/s/cluster_write.h" +#include "mongo/s/config.h" +#include "mongo/s/distlock.h" +#include "mongo/s/type_database.h" +#include "mongo/s/grid.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + AuthzManagerExternalStateMongos::AuthzManagerExternalStateMongos() {} + + AuthzManagerExternalStateMongos::~AuthzManagerExternalStateMongos() {} + + Status AuthzManagerExternalStateMongos::initialize() { + return Status::OK(); + } + + namespace { + ScopedDbConnection* getConnectionForAuthzCollection(const NamespaceString& ns) { + // + // Note: The connection mechanism here is *not* ideal, and should not be used elsewhere. + // If the primary for the collection moves, this approach may throw rather than handle + // version exceptions. + // + + DBConfigPtr config = grid.getDBConfig(ns.ns()); + Shard s = config->getShard(ns.ns()); + + return new ScopedDbConnection(s.getConnString(), 30.0); + } + } + + Status AuthzManagerExternalStateMongos::getStoredAuthorizationVersion(int* outVersion) { + scoped_ptr conn(getConnectionForAuthzCollection( + AuthorizationManager::usersCollectionNamespace)); + Status status = auth::getRemoteStoredAuthorizationVersion(conn->get(), outVersion); + conn->done(); + return status; + } + + Status AuthzManagerExternalStateMongos::getUserDescription(const UserName& userName, + BSONObj* result) { + try { + scoped_ptr conn(getConnectionForAuthzCollection( + AuthorizationManager::usersCollectionNamespace)); + BSONObj cmdResult; + conn->get()->runCommand( + "admin", + BSON("usersInfo" << + BSON_ARRAY(BSON(AuthorizationManager::USER_NAME_FIELD_NAME << + userName.getUser() << + AuthorizationManager::USER_DB_FIELD_NAME << + userName.getDB())) << + "showPrivileges" << true << + "showCredentials" << true), + cmdResult); + if (!cmdResult["ok"].trueValue()) { + int code = cmdResult["code"].numberInt(); + if (code == 0) code = ErrorCodes::UnknownError; + return Status(ErrorCodes::Error(code), cmdResult["errmsg"].str()); + } + + std::vector foundUsers = cmdResult["users"].Array(); + if (foundUsers.size() == 0) { + return Status(ErrorCodes::UserNotFound, + "User \"" + userName.toString() + "\" not found"); + } + if (foundUsers.size() > 1) { + return Status(ErrorCodes::UserDataInconsistent, + mongoutils::str::stream() << "Found multiple users on the \"" << + userName.getDB() << "\" database with name \"" << + userName.getUser() << "\""); + } + *result = foundUsers[0].Obj().getOwned(); + conn->done(); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongos::getRoleDescription(const RoleName& roleName, + bool showPrivileges, + BSONObj* result) { + try { + scoped_ptr conn(getConnectionForAuthzCollection( + AuthorizationManager::rolesCollectionNamespace)); + BSONObj cmdResult; + conn->get()->runCommand( + "admin", + BSON("rolesInfo" << + BSON_ARRAY(BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << + roleName.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + roleName.getDB())) << + "showPrivileges" << showPrivileges), + cmdResult); + if (!cmdResult["ok"].trueValue()) { + int code = cmdResult["code"].numberInt(); + if (code == 0) code = ErrorCodes::UnknownError; + return Status(ErrorCodes::Error(code), cmdResult["errmsg"].str()); + } + + std::vector foundRoles = cmdResult["roles"].Array(); + if (foundRoles.size() == 0) { + return Status(ErrorCodes::RoleNotFound, + "Role \"" + roleName.toString() + "\" not found"); + } + if (foundRoles.size() > 1) { + return Status(ErrorCodes::RoleDataInconsistent, + mongoutils::str::stream() << "Found multiple roles on the \"" << + roleName.getDB() << "\" database with name \"" << + roleName.getRole() << "\""); + } + *result = foundRoles[0].Obj().getOwned(); + conn->done(); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongos::getRoleDescriptionsForDB(const std::string dbname, + bool showPrivileges, + bool showBuiltinRoles, + vector* result) { + try { + scoped_ptr conn(getConnectionForAuthzCollection( + AuthorizationManager::rolesCollectionNamespace)); + BSONObj cmdResult; + conn->get()->runCommand( + dbname, + BSON("rolesInfo" << 1 << + "showPrivileges" << showPrivileges << + "showBuiltinRoles" << showBuiltinRoles), + cmdResult); + if (!cmdResult["ok"].trueValue()) { + int code = cmdResult["code"].numberInt(); + if (code == 0) code = ErrorCodes::UnknownError; + return Status(ErrorCodes::Error(code), cmdResult["errmsg"].str()); + } + for (BSONObjIterator it(cmdResult["roles"].Obj()); it.more(); it.next()) { + result->push_back((*it).Obj().getOwned()); + } + conn->done(); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongos::findOne( + const NamespaceString& collectionName, + const BSONObj& queryDoc, + BSONObj* result) { + try { + scoped_ptr conn(getConnectionForAuthzCollection(collectionName)); + Query query(queryDoc); + query.readPref(ReadPreference_PrimaryPreferred, BSONArray()); + *result = conn->get()->findOne(collectionName, query).getOwned(); + conn->done(); + if (result->isEmpty()) { + return Status(ErrorCodes::NoMatchingDocument, mongoutils::str::stream() << + "No document in " << collectionName.ns() << " matches " << queryDoc); + } + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongos::query( + const NamespaceString& collectionName, + const BSONObj& queryDoc, + const BSONObj& projection, + const boost::function& resultProcessor) { + try { + scoped_ptr conn(getConnectionForAuthzCollection(collectionName)); + Query query(queryDoc); + query.readPref(ReadPreference_PrimaryPreferred, BSONArray()); + conn->get()->query(resultProcessor, collectionName.ns(), query, &projection); + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongos::getAllDatabaseNames( + std::vector* dbnames) { + try { + scoped_ptr conn( + getConnectionForAuthzCollection(NamespaceString(DatabaseType::ConfigNS))); + auto_ptr c = conn->get()->query(DatabaseType::ConfigNS, Query()); + + while (c->more()) { + DatabaseType dbInfo; + std::string errmsg; + if (!dbInfo.parseBSON( c->nextSafe(), &errmsg) || !dbInfo.isValid( &errmsg )) { + return Status(ErrorCodes::FailedToParse, errmsg); + } + dbnames->push_back(dbInfo.getName()); + } + conn->done(); + dbnames->push_back("config"); // config db isn't listed in config.databases + return Status::OK(); + } catch (const DBException& e) { + return e.toStatus(); + } + } + + Status AuthzManagerExternalStateMongos::insert( + const NamespaceString& collectionName, + const BSONObj& document, + const BSONObj& writeConcern) { + return clusterInsert(collectionName, document, writeConcern, NULL); + } + + Status AuthzManagerExternalStateMongos::update(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched) { + BatchedCommandResponse response; + Status res = clusterUpdate(collectionName, + query, + updatePattern, + upsert, + multi, + writeConcern, + &response); + + if (res.isOK()) { + *nMatched = response.getN(); + } + + return res; + } + + Status AuthzManagerExternalStateMongos::remove( + const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved) { + BatchedCommandResponse response; + Status res = clusterDelete(collectionName, query, 0 /* limit */, writeConcern, &response); + + if (res.isOK()) { + *numRemoved = response.getN(); + } + + return res; + } + + Status AuthzManagerExternalStateMongos::createIndex( + const NamespaceString& collectionName, + const BSONObj& pattern, + bool unique, + const BSONObj& writeConcern) { + return clusterCreateIndex(collectionName, pattern, unique, writeConcern, NULL); + } + + Status AuthzManagerExternalStateMongos::dropIndexes( + const NamespaceString& collectionName, + const BSONObj& writeConcern) { + + scoped_ptr conn(getConnectionForAuthzCollection(collectionName)); + try { + conn->get()->dropIndexes(collectionName.ns()); + BSONObjBuilder gleBuilder; + gleBuilder.append("getLastError", 1); + gleBuilder.appendElements(writeConcern); + BSONObj res; + conn->get()->runCommand("admin", gleBuilder.done(), res); + string errstr = conn->get()->getLastErrorString(res); + if (!errstr.empty()) { + conn->done(); + return Status(ErrorCodes::UnknownError, errstr); + } + conn->done(); + return Status::OK(); + } + catch (const DBException& ex) { + return ex.toStatus(); + } + } + + bool AuthzManagerExternalStateMongos::tryAcquireAuthzUpdateLock(const StringData& why) { + boost::lock_guard lkLocal(_distLockGuard); + if (_authzDataUpdateLock.get()) { + return false; + } + + // Temporarily put into an auto_ptr just in case there is an exception thrown during + // lock acquisition. + std::auto_ptr lockHolder(new ScopedDistributedLock( + configServer.getConnectionString(), "authorizationData")); + lockHolder->setLockMessage(why.toString()); + + std::string errmsg; + if (!lockHolder->acquire(_authzUpdateLockAcquisitionTimeoutMillis, &errmsg)) { + warning() << + "Error while attempting to acquire distributed lock for user modification: " << + errmsg << endl; + return false; + } + _authzDataUpdateLock.reset(lockHolder.release()); + return true; + } + + void AuthzManagerExternalStateMongos::releaseAuthzUpdateLock() { + boost::lock_guard lkLocal(_distLockGuard); + _authzDataUpdateLock.reset(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_manager_external_state_s.h b/src/mongo/db/auth/authz_manager_external_state_s.h new file mode 100644 index 00000000000..76c49bdac78 --- /dev/null +++ b/src/mongo/db/auth/authz_manager_external_state_s.h @@ -0,0 +1,116 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authz_manager_external_state.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/s/distlock.h" + +namespace mongo { + + /** + * The implementation of AuthzManagerExternalState functionality for mongos. + */ + class AuthzManagerExternalStateMongos : public AuthzManagerExternalState{ + MONGO_DISALLOW_COPYING(AuthzManagerExternalStateMongos); + + public: + AuthzManagerExternalStateMongos(); + virtual ~AuthzManagerExternalStateMongos(); + + virtual Status initialize(); + virtual Status getStoredAuthorizationVersion(int* outVersion); + virtual Status getUserDescription(const UserName& userName, BSONObj* result); + virtual Status getRoleDescription(const RoleName& roleName, + bool showPrivileges, + BSONObj* result); + virtual Status getRoleDescriptionsForDB(const std::string dbname, + bool showPrivileges, + bool showBuiltinRoles, + vector* result); + + virtual Status getAllDatabaseNames(std::vector* dbnames); + + /** + * Implements findOne of the AuthzManagerExternalState interface + * + * NOTE: The data returned from this helper may be from any config server or replica set + * node. The first config server or primary node is preferred, when available. + */ + virtual Status findOne(const NamespaceString& collectionName, + const BSONObj& query, + BSONObj* result); + + /** + * Implements query of the AuthzManagerExternalState interface + * + * NOTE: The data returned from this helper may be from any config server or replica set + * node. The first config server or primary node is preferred, when available. + */ + virtual Status query(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& projection, + const boost::function& resultProcessor); + + virtual Status insert(const NamespaceString& collectionName, + const BSONObj& document, + const BSONObj& writeConcern); + virtual Status update(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& updatePattern, + bool upsert, + bool multi, + const BSONObj& writeConcern, + int* nMatched); + virtual Status remove(const NamespaceString& collectionName, + const BSONObj& query, + const BSONObj& writeConcern, + int* numRemoved); + virtual Status createIndex(const NamespaceString& collectionName, + const BSONObj& pattern, + bool unique, + const BSONObj& writeConcern); + virtual Status dropIndexes(const NamespaceString& collectionName, + const BSONObj& writeConcern); + virtual bool tryAcquireAuthzUpdateLock(const StringData& why); + virtual void releaseAuthzUpdateLock(); + + private: + boost::mutex _distLockGuard; // Guards access to _authzDataUpdateLock + scoped_ptr _authzDataUpdateLock; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state.cpp b/src/mongo/db/auth/authz_session_external_state.cpp new file mode 100644 index 00000000000..2a7db8c1ff1 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state.cpp @@ -0,0 +1,46 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_session_external_state.h" + +#include "mongo/base/status.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { + + AuthzSessionExternalState::AuthzSessionExternalState(AuthorizationManager* authzManager) : + _authzManager(authzManager) {} + AuthzSessionExternalState::~AuthzSessionExternalState() {} + + AuthorizationManager& AuthzSessionExternalState::getAuthorizationManager() { + return *_authzManager; + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state.h b/src/mongo/db/auth/authz_session_external_state.h new file mode 100644 index 00000000000..2f8b4d097c5 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state.h @@ -0,0 +1,76 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user_name.h" + +namespace mongo { + + class Principal; + + /** + * Public interface for a class that encapsulates all the session information related to system + * state not stored in AuthorizationSession. This is primarily to make AuthorizationSession + * easier to test as well as to allow different implementations in mongos and mongod. + */ + class AuthzSessionExternalState { + MONGO_DISALLOW_COPYING(AuthzSessionExternalState); + + public: + + virtual ~AuthzSessionExternalState(); + + AuthorizationManager& getAuthorizationManager(); + + // Returns true if this connection should be treated as if it has full access to do + // anything, regardless of the current auth state. Currently the reasons why this could be + // are that auth isn't enabled, the connection is from localhost and there are no admin + // users, or the connection is a "god" connection. + // NOTE: _checkShouldAllowLocalhost MUST be called at least once before any call to + // shouldIgnoreAuthChecks or we could ignore auth checks incorrectly. + virtual bool shouldIgnoreAuthChecks() const = 0; + + // Should be called at the beginning of every new request. This performs the checks + // necessary to determine if localhost connections should be given full access. + virtual void startRequest() = 0; + + protected: + // This class should never be instantiated directly. + AuthzSessionExternalState(AuthorizationManager* authzManager); + + AuthorizationManager* _authzManager; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state_d.cpp b/src/mongo/db/auth/authz_session_external_state_d.cpp new file mode 100644 index 00000000000..0156ccf6c9b --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state_d.cpp @@ -0,0 +1,58 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_session_external_state_d.h" + +#include "mongo/base/status.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/client.h" +#include "mongo/db/dbhelpers.h" +#include "mongo/db/d_concurrency.h" +#include "mongo/db/instance.h" +#include "mongo/db/jsobj.h" +#include "mongo/scripting/engine.h" + +namespace mongo { + + AuthzSessionExternalStateMongod::AuthzSessionExternalStateMongod( + AuthorizationManager* authzManager) : + AuthzSessionExternalStateServerCommon(authzManager) {} + AuthzSessionExternalStateMongod::~AuthzSessionExternalStateMongod() {} + + void AuthzSessionExternalStateMongod::startRequest() { + if (!Lock::isLocked()) { + _checkShouldAllowLocalhost(); + } + } + + bool AuthzSessionExternalStateMongod::shouldIgnoreAuthChecks() const { + return cc().isGod() || AuthzSessionExternalStateServerCommon::shouldIgnoreAuthChecks(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state_d.h b/src/mongo/db/auth/authz_session_external_state_d.h new file mode 100644 index 00000000000..f5b2c82cd03 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state_d.h @@ -0,0 +1,53 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authz_session_external_state_server_common.h" + +namespace mongo { + + /** + * The implementation of AuthzSessionExternalState functionality for mongod. + */ + class AuthzSessionExternalStateMongod : public AuthzSessionExternalStateServerCommon { + MONGO_DISALLOW_COPYING(AuthzSessionExternalStateMongod); + + public: + AuthzSessionExternalStateMongod(AuthorizationManager* authzManager); + virtual ~AuthzSessionExternalStateMongod(); + + virtual bool shouldIgnoreAuthChecks() const; + + virtual void startRequest(); + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state_mock.h b/src/mongo/db/auth/authz_session_external_state_mock.h new file mode 100644 index 00000000000..3db4df17de6 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state_mock.h @@ -0,0 +1,62 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authz_session_external_state.h" + +namespace mongo { + + /** + * Mock of the AuthzSessionExternalState class used only for testing. + */ + class AuthzSessionExternalStateMock : public AuthzSessionExternalState { + MONGO_DISALLOW_COPYING(AuthzSessionExternalStateMock); + + public: + AuthzSessionExternalStateMock(AuthorizationManager* authzManager) : + AuthzSessionExternalState(authzManager), _returnValue(false) {} + + virtual bool shouldIgnoreAuthChecks() const { + return _returnValue; + } + + void setReturnValueForShouldIgnoreAuthChecks(bool returnValue) { + _returnValue = returnValue; + } + + virtual void startRequest() {} + + private: + bool _returnValue; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state_s.cpp b/src/mongo/db/auth/authz_session_external_state_s.cpp new file mode 100644 index 00000000000..14801eae945 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state_s.cpp @@ -0,0 +1,50 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_session_external_state_s.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/jsobj.h" +#include "mongo/s/grid.h" + +namespace mongo { + + AuthzSessionExternalStateMongos::AuthzSessionExternalStateMongos( + AuthorizationManager* authzManager) : + AuthzSessionExternalStateServerCommon(authzManager) {} + AuthzSessionExternalStateMongos::~AuthzSessionExternalStateMongos() {} + + void AuthzSessionExternalStateMongos::startRequest() { + _checkShouldAllowLocalhost(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state_s.h b/src/mongo/db/auth/authz_session_external_state_s.h new file mode 100644 index 00000000000..6672957ced4 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state_s.h @@ -0,0 +1,51 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authz_session_external_state_server_common.h" + +namespace mongo { + + /** + * The implementation of AuthzSessionExternalState functionality for mongos. + */ + class AuthzSessionExternalStateMongos : public AuthzSessionExternalStateServerCommon { + MONGO_DISALLOW_COPYING(AuthzSessionExternalStateMongos); + + public: + AuthzSessionExternalStateMongos(AuthorizationManager* authzManager); + virtual ~AuthzSessionExternalStateMongos(); + + virtual void startRequest(); + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state_server_common.cpp b/src/mongo/db/auth/authz_session_external_state_server_common.cpp new file mode 100644 index 00000000000..80f24004533 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state_server_common.cpp @@ -0,0 +1,79 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authz_session_external_state_server_common.h" + +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/client.h" +#include "mongo/db/server_parameters.h" +#include "mongo/util/debug_util.h" + +namespace mongo { + +namespace { + MONGO_EXPORT_STARTUP_SERVER_PARAMETER(enableLocalhostAuthBypass, bool, true); +} // namespace + + // NOTE: we default _allowLocalhost to true under the assumption that _checkShouldAllowLocalhost + // will always be called before any calls to shouldIgnoreAuthChecks. If this is not the case, + // it could cause a security hole. + AuthzSessionExternalStateServerCommon::AuthzSessionExternalStateServerCommon( + AuthorizationManager* authzManager) : + AuthzSessionExternalState(authzManager), + _allowLocalhost(enableLocalhostAuthBypass) {} + AuthzSessionExternalStateServerCommon::~AuthzSessionExternalStateServerCommon() {} + + void AuthzSessionExternalStateServerCommon::_checkShouldAllowLocalhost() { + if (!_authzManager->isAuthEnabled()) + return; + // If we know that an admin user exists, don't re-check. + if (!_allowLocalhost) + return; + // Don't bother checking if we're not on a localhost connection + if (!ClientBasic::getCurrent()->getIsLocalHostConnection()) { + _allowLocalhost = false; + return; + } + + _allowLocalhost = !_authzManager->hasAnyPrivilegeDocuments(); + if (_allowLocalhost) { + ONCE { + log() << "note: no users configured in admin.system.users, allowing localhost " + "access" << std::endl; + } + } + } + + bool AuthzSessionExternalStateServerCommon::shouldIgnoreAuthChecks() const { + ClientBasic* client = ClientBasic::getCurrent(); + return !_authzManager->isAuthEnabled() || + (_allowLocalhost && client->getIsLocalHostConnection()); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/authz_session_external_state_server_common.h b/src/mongo/db/auth/authz_session_external_state_server_common.h new file mode 100644 index 00000000000..a621fc1f651 --- /dev/null +++ b/src/mongo/db/auth/authz_session_external_state_server_common.h @@ -0,0 +1,63 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authz_session_external_state.h" + +namespace mongo { + + /** + * The implementation of AuthzSessionExternalState functionality common to mongod and mongos. + */ + class AuthzSessionExternalStateServerCommon : public AuthzSessionExternalState { + MONGO_DISALLOW_COPYING(AuthzSessionExternalStateServerCommon); + + public: + virtual ~AuthzSessionExternalStateServerCommon(); + + virtual bool shouldIgnoreAuthChecks() const; + + protected: + AuthzSessionExternalStateServerCommon(AuthorizationManager* authzManager); + + // Checks whether or not localhost connections should be given full access and stores the + // result in _allowLocalhost. Currently localhost connections are only given full access + // if there are no users in the admin database. + virtual void _checkShouldAllowLocalhost(); + + private: + + bool _allowLocalhost; + + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/principal.cpp b/src/mongo/db/auth/principal.cpp deleted file mode 100644 index ad1b763cd28..00000000000 --- a/src/mongo/db/auth/principal.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/db/auth/principal.h" - -#include -#include - -#include "mongo/util/map_util.h" - -namespace mongo { - - Principal::Principal(const PrincipalName& name) : - _name(name), - _expirationTime(boost::posix_time::pos_infin), - _enableImplicitPrivileges(false) { - } - - Principal::~Principal() {} - - void Principal::setExpirationTime(boost::posix_time::ptime& expiration) { - _expirationTime = expiration; - } - - void Principal::setImplicitPrivilegeAcquisition(bool enabled) { - _enableImplicitPrivileges = enabled; - } - - bool Principal::isDatabaseProbed(const StringData& dbname) const { - return mapFindWithDefault(_probedDatabases, dbname, false); - } - - void Principal::markDatabaseAsProbed(const StringData& dbname) { - _probedDatabases[dbname] = true; - } - -} // namespace mongo diff --git a/src/mongo/db/auth/principal.h b/src/mongo/db/auth/principal.h deleted file mode 100644 index e58eadfd95c..00000000000 --- a/src/mongo/db/auth/principal.h +++ /dev/null @@ -1,70 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/string_data.h" -#include "mongo/db/auth/principal_name.h" -#include "mongo/util/string_map.h" - -namespace mongo { - - /** - * Represents an authenticated user. Every principal has a name, a time that the user's - * authentication expires, and a flag that describes whether or not privileges should be - * acquired implicitly. - * - * The implicit privilege acquisition flag defaults to disabled, and the expiration time - * defaults to never. - * - * This class does not do any locking/synchronization, the consumer will be responsible for - * synchronizing access. - */ - class Principal { - MONGO_DISALLOW_COPYING(Principal); - - public: - Principal(const PrincipalName& name, - const boost::posix_time::ptime& expirationTime); - explicit Principal(const PrincipalName& name); - ~Principal(); - - const PrincipalName& getName() const { return _name; } - - // Returns the expiration time of this principal information. - // No expiration is represented as boost::posix_time::pos_infin - const boost::posix_time::ptime& getExpirationTime() const { return _expirationTime; } - - // Returns true if this principal is configured for implicit acquisition of privileges. - bool isImplicitPrivilegeAcquisitionEnabled() const { return _enableImplicitPrivileges; } - - void setExpirationTime(boost::posix_time::ptime& expiration); - void setImplicitPrivilegeAcquisition(bool enabled); - - bool isDatabaseProbed(const StringData& dbname) const; - void markDatabaseAsProbed(const StringData& dbname); - - private: - PrincipalName _name; - boost::posix_time::ptime _expirationTime; - bool _enableImplicitPrivileges; - StringMap _probedDatabases; - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/principal_name.h b/src/mongo/db/auth/principal_name.h deleted file mode 100644 index 1e04d3d246f..00000000000 --- a/src/mongo/db/auth/principal_name.h +++ /dev/null @@ -1,76 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include "mongo/base/string_data.h" - -namespace mongo { - - /** - * Representation of a name of a principal (authenticatable user) in a MongoDB system. - * - * Consists of a "user name" part, and a "database name" part. - */ - class PrincipalName { - public: - PrincipalName() : _splitPoint(0) {} - PrincipalName(const StringData& user, const StringData& dbname) : - _fullName(user.toString() + "@" + dbname.toString()), - _splitPoint(user.size()) { - } - - /** - * Gets the user-name part of a principal name. - */ - StringData getUser() const { return StringData(_fullName).substr(0, _splitPoint); } - - /** - * Gets the database name part of a principal name. - */ - StringData getDB() const { return StringData(_fullName).substr(_splitPoint + 1); } - - /** - * Gets the full name of a principal as a string, formatted as "user@db". - * - * Allowed for keys in non-persistent data structures, such as std::map. - */ - const std::string& getFullName() const { return _fullName; } - - /** - * Stringifies the object, for logging/debugging. - */ - std::string toString() const { return getFullName(); } - - private: - std::string _fullName; // The full name, stored as a string. "user@db". - size_t _splitPoint; // The index of the "@" separating the user and db name parts. - }; - - static inline bool operator==(const PrincipalName& lhs, const PrincipalName& rhs) { - return lhs.getFullName() == rhs.getFullName(); - } - - static inline bool operator!=(const PrincipalName& lhs, const PrincipalName& rhs) { - return lhs.getFullName() != rhs.getFullName(); - } - - static inline bool operator<(const PrincipalName& lhs, const PrincipalName& rhs) { - return lhs.getFullName() < rhs.getFullName(); - } - -} // namespace mongo diff --git a/src/mongo/db/auth/principal_set.cpp b/src/mongo/db/auth/principal_set.cpp deleted file mode 100644 index 44dd5ee6b35..00000000000 --- a/src/mongo/db/auth/principal_set.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/pch.h" - -#include "mongo/db/auth/principal_set.h" - -#include -#include -#include - -#include "mongo/db/auth/principal.h" -#include "mongo/util/mongoutils/str.h" - -namespace mongo { - - PrincipalSet::PrincipalSet() {} - PrincipalSet::~PrincipalSet() { - for (std::vector::iterator it = _principals.begin(); - it != _principals.end(); ++it) { - delete *it; - } - } - - void PrincipalSet::add(Principal* principal) { - for (std::vector::iterator it = _principals.begin(); - it != _principals.end(); ++it) { - Principal* current = *it; - if (current->getName().getDB() == principal->getName().getDB()) { - // There can be only one principal per database. - delete current; - *it = principal; - return; - } - } - _principals.push_back(principal); - } - - void PrincipalSet::removeByDBName(const StringData& dbname) { - for (std::vector::iterator it = _principals.begin(); - it != _principals.end(); ++it) { - Principal* current = *it; - if (current->getName().getDB() == dbname) { - delete current; - _principals.erase(it); - break; - } - } - } - - Principal* PrincipalSet::lookup(const PrincipalName& name) const { - Principal* principal = lookupByDBName(name.getDB()); - if (principal && principal->getName() == name) { - return principal; - } - return NULL; - } - - Principal* PrincipalSet::lookupByDBName(const StringData& dbname) const { - for (std::vector::const_iterator it = _principals.begin(); - it != _principals.end(); ++it) { - Principal* current = *it; - if (current->getName().getDB() == dbname) { - return current; - } - } - return NULL; - } - -} // namespace mongo diff --git a/src/mongo/db/auth/principal_set.h b/src/mongo/db/auth/principal_set.h deleted file mode 100644 index b7b99e05a47..00000000000 --- a/src/mongo/db/auth/principal_set.h +++ /dev/null @@ -1,106 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/base/string_data.h" -#include "mongo/db/auth/principal.h" -#include "mongo/db/auth/principal_name.h" - -namespace mongo { - - /** - * A collection of authenticated principals. - * This class does not do any locking/synchronization, the consumer will be responsible for - * synchronizing access. - */ - class PrincipalSet { - MONGO_DISALLOW_COPYING(PrincipalSet); - public: - typedef std::vector::const_iterator iterator; - - /** - * Forward iterator over the names of the principals stored in a PrincipalSet. - * - * Instances are valid until the underlying vector is modified. - * - * more() must be the first method called after construction, and must be checked - * after each call to next() before calling any other methods. - */ - class NameIterator { - public: - explicit NameIterator(const std::vector& principals) : - _curr(principals.begin()), - _end(principals.end()) { - } - - NameIterator() {} - - bool more() { return _curr != _end; } - const PrincipalName& next() { - const PrincipalName& ret = get(); - ++_curr; - return ret; - } - - const PrincipalName& get() const { return (*_curr)->getName(); } - - const PrincipalName& operator*() const { return get(); } - const PrincipalName* operator->() const { return &get(); } - - private: - std::vector::const_iterator _curr; - std::vector::const_iterator _end; - }; - - PrincipalSet(); - ~PrincipalSet(); - - // If the principal is already present, this will replace the existing entry. - // The PrincipalSet takes ownership of the passed-in principal and is responsible for - // deleting it eventually - void add(Principal* principal); - - // Removes all principals whose authentication credentials came from dbname. - void removeByDBName(const StringData& dbname); - - // Returns the Principal with the given name, or NULL if not found. - // Ownership of the returned Principal remains with the PrincipalSet. The pointer - // returned is only guaranteed to remain valid until the next non-const method is called - // on the PrincipalSet. - Principal* lookup(const PrincipalName& name) const; - - // Gets the principal whose authentication credentials came from dbname, or NULL if none - // exist. There should be at most one such principal. - Principal* lookupByDBName(const StringData& dbname) const; - - // Gets an iterator over the names of the principals stored in the set. The iterator is - // valid until the next non-const method is called on the PrincipalSet. - NameIterator getNames() const { return NameIterator(_principals); } - - iterator begin() const { return _principals.begin(); } - iterator end() const { return _principals.end(); } - - private: - // The PrincipalSet maintains ownership of the Principals in it, and is responsible for - // deleting them when done with them. - std::vector _principals; - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/principal_set_test.cpp b/src/mongo/db/auth/principal_set_test.cpp deleted file mode 100644 index 9ca24cc06a1..00000000000 --- a/src/mongo/db/auth/principal_set_test.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Unit tests of the PrincipalSet type. - */ - -#include "mongo/db/auth/principal_set.h" -#include "mongo/db/auth/principal.h" -#include "mongo/db/auth/principal_name.h" -#include "mongo/unittest/unittest.h" - -#define ASSERT_NULL(EXPR) ASSERT_FALSE((EXPR)) - -namespace mongo { - - static inline std::ostream& operator<<(std::ostream& os, const PrincipalName& pname) { - return os << pname.toString(); - } - -namespace { - - TEST(PrincipalSetTest, BasicTest) { - PrincipalSet set; - - Principal* p1 = new Principal(PrincipalName("Bob", "test")); - Principal* p2 = new Principal(PrincipalName("George", "test")); - Principal* p3 = new Principal(PrincipalName("Bob", "test2")); - - ASSERT_NULL(set.lookup(PrincipalName("Bob", "test"))); - ASSERT_NULL(set.lookup(PrincipalName("George", "test"))); - ASSERT_NULL(set.lookup(PrincipalName("Bob", "test2"))); - ASSERT_NULL(set.lookupByDBName("test")); - ASSERT_NULL(set.lookupByDBName("test2")); - - set.add(p1); - - ASSERT_EQUALS(p1, set.lookup(PrincipalName("Bob", "test"))); - ASSERT_EQUALS(p1, set.lookupByDBName("test")); - ASSERT_NULL(set.lookup(PrincipalName("George", "test"))); - ASSERT_NULL(set.lookup(PrincipalName("Bob", "test2"))); - ASSERT_NULL(set.lookupByDBName("test2")); - - // This should not replace the existing user "Bob" because they are different databases - set.add(p3); - - ASSERT_EQUALS(p1, set.lookup(PrincipalName("Bob", "test"))); - ASSERT_EQUALS(p1, set.lookupByDBName("test")); - ASSERT_NULL(set.lookup(PrincipalName("George", "test"))); - ASSERT_EQUALS(p3, set.lookup(PrincipalName("Bob", "test2"))); - ASSERT_EQUALS(p3, set.lookupByDBName("test2")); - - set.add(p2); // This should replace Bob since they're on the same database - - ASSERT_NULL(set.lookup(PrincipalName("Bob", "test"))); - ASSERT_EQUALS(p2, set.lookup(PrincipalName("George", "test"))); - ASSERT_EQUALS(p2, set.lookupByDBName("test")); - ASSERT_EQUALS(p3, set.lookup(PrincipalName("Bob", "test2"))); - ASSERT_EQUALS(p3, set.lookupByDBName("test2")); - - set.removeByDBName("test"); - - ASSERT_NULL(set.lookup(PrincipalName("Bob", "test"))); - ASSERT_NULL(set.lookup(PrincipalName("George", "test"))); - ASSERT_NULL(set.lookupByDBName("test")); - ASSERT_EQUALS(p3, set.lookup(PrincipalName("Bob", "test2"))); - ASSERT_EQUALS(p3, set.lookupByDBName("test2")); - } - - TEST(PrincipalSetTest, IterateNames) { - PrincipalSet pset; - PrincipalSet::NameIterator iter = pset.getNames(); - ASSERT(!iter.more()); - - pset.add(new Principal(PrincipalName("bob", "test"))); - - iter = pset.getNames(); - ASSERT(iter.more()); - ASSERT_EQUALS(*iter, PrincipalName("bob", "test")); - ASSERT_EQUALS(iter.next(), PrincipalName("bob", "test")); - ASSERT(!iter.more()); - } - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/auth/privilege.cpp b/src/mongo/db/auth/privilege.cpp index 88401c91ad5..a1d0dab7bfe 100644 --- a/src/mongo/db/auth/privilege.cpp +++ b/src/mongo/db/auth/privilege.cpp @@ -15,22 +15,39 @@ #include "mongo/db/auth/privilege.h" -#include - #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/principal.h" namespace mongo { - Privilege::Privilege(const std::string& resource, const ActionType& action) : + void Privilege::addPrivilegeToPrivilegeVector(PrivilegeVector* privileges, + const Privilege& privilegeToAdd) { + for (PrivilegeVector::iterator it = privileges->begin(); it != privileges->end(); ++it) { + if (it->getResourcePattern() == privilegeToAdd.getResourcePattern()) { + it->addActions(privilegeToAdd.getActions()); + return; + } + } + // No privilege exists yet for this resource + privileges->push_back(privilegeToAdd); + } + + Privilege::Privilege(const ResourcePattern& resource, const ActionType& action) : _resource(resource) { _actions.addAction(action); } - Privilege::Privilege(const std::string& resource, const ActionSet& actions) : + Privilege::Privilege(const ResourcePattern& resource, const ActionSet& actions) : _resource(resource), _actions(actions) {} + void Privilege::addActions(const ActionSet& actionsToAdd) { + _actions.addAllActionsFromSet(actionsToAdd); + } + + void Privilege::removeActions(const ActionSet& actionsToRemove) { + _actions.removeAllActionsFromSet(actionsToRemove); + } + bool Privilege::includesAction(const ActionType& action) const { return _actions.contains(action); } diff --git a/src/mongo/db/auth/privilege.h b/src/mongo/db/auth/privilege.h index eef476ea224..710e18e6be8 100644 --- a/src/mongo/db/auth/privilege.h +++ b/src/mongo/db/auth/privilege.h @@ -15,28 +15,45 @@ #pragma once -#include +#include #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/principal.h" +#include "mongo/db/auth/privilege_parser.h" +#include "mongo/db/auth/resource_pattern.h" namespace mongo { + class Privilege; + typedef std::vector PrivilegeVector; + /** - * A representation of the permission to perform a set of actions on a specific resource. + * A representation of the permission to perform a set of actions on a resource. */ class Privilege { public: + /** + * Adds "privilegeToAdd" to "privileges", de-duping "privilegeToAdd" if the vector already + * contains a privilege on the same resource. + * + * This method is the preferred way to add privileges to privilege vectors. + */ + static void addPrivilegeToPrivilegeVector(PrivilegeVector* privileges, + const Privilege& privilegeToAdd); + - Privilege(const std::string& resource, const ActionType& action); - Privilege(const std::string& resource, const ActionSet& actions); + Privilege() {}; + Privilege(const ResourcePattern& resource, const ActionType& action); + Privilege(const ResourcePattern& resource, const ActionSet& actions); ~Privilege() {} - const std::string& getResource() const { return _resource; } + const ResourcePattern& getResourcePattern() const { return _resource; } const ActionSet& getActions() const { return _actions; } + void addActions(const ActionSet& actionsToAdd); + void removeActions(const ActionSet& actionsToRemove); + // Checks if the given action is present in the Privilege. bool includesAction(const ActionType& action) const; // Checks if the given actions are present in the Privilege. @@ -44,7 +61,7 @@ namespace mongo { private: - std::string _resource; + ResourcePattern _resource; ActionSet _actions; // bitmask of actions this privilege grants }; diff --git a/src/mongo/db/auth/privilege_parser.cpp b/src/mongo/db/auth/privilege_parser.cpp new file mode 100644 index 00000000000..88dfbf961b0 --- /dev/null +++ b/src/mongo/db/auth/privilege_parser.cpp @@ -0,0 +1,453 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/auth/privilege_parser.h" + +#include + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/field_parser.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + using mongoutils::str::stream; + + const BSONField ParsedResource::anyResource("anyResource"); + const BSONField ParsedResource::cluster("cluster"); + const BSONField ParsedResource::db("db"); + const BSONField ParsedResource::collection("collection"); + + ParsedResource::ParsedResource() { + clear(); + } + + ParsedResource::~ParsedResource() { + } + + bool ParsedResource::isValid(std::string* errMsg) const { + std::string dummy; + if (errMsg == NULL) { + errMsg = &dummy; + } + + int numCandidateTypes = 0; + if (isAnyResourceSet()) ++numCandidateTypes; + if (isClusterSet()) ++numCandidateTypes; + if (isDbSet() || isCollectionSet()) ++numCandidateTypes; + + if (isDbSet() != isCollectionSet()) { + *errMsg = stream() << "resource must set both " << db.name() << " and " << + collection.name() << " or neither, but not exactly one."; + return false; + } + if (numCandidateTypes != 1) { + *errMsg = stream() << "resource must have exactly " << db.name() << " and " << + collection.name() << " set, or have only " << cluster.name() << " set " << + " or have only " << anyResource.name() << " set"; + return false; + } + if (isAnyResourceSet() && !getAnyResource()) { + *errMsg = stream() << anyResource.name() << " must be true when specified"; + return false; + } + if (isClusterSet() && !getCluster()) { + *errMsg = stream() << cluster.name() << " must be true when specified"; + return false; + } + if (isDbSet() && (!NamespaceString::validDBName(getDb()) && !getDb().empty())) { + *errMsg = stream() << getDb() << " is not a valid database name"; + return false; + } + if (isCollectionSet() && (!NamespaceString::validCollectionName(getCollection()) && + !getCollection().empty())) { + *errMsg = stream() << getCollection() << " is not a valid collection name"; + return false; + } + return true; + } + + BSONObj ParsedResource::toBSON() const { + BSONObjBuilder builder; + + if (_isAnyResourceSet) builder.append(anyResource(), _anyResource); + + if (_isClusterSet) builder.append(cluster(), _cluster); + + if (_isDbSet) builder.append(db(), _db); + + if (_isCollectionSet) builder.append(collection(), _collection); + + return builder.obj(); + } + + bool ParsedResource::parseBSON(const BSONObj& source, string* errMsg) { + clear(); + + std::string dummy; + if (!errMsg) errMsg = &dummy; + + FieldParser::FieldState fieldState; + fieldState = FieldParser::extract(source, anyResource, &_anyResource, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isAnyResourceSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, cluster, &_cluster, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isClusterSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, db, &_db, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isDbSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, collection, &_collection, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isCollectionSet = fieldState == FieldParser::FIELD_SET; + + return true; + } + + void ParsedResource::clear() { + _anyResource = false; + _isAnyResourceSet = false; + + _cluster = false; + _isClusterSet = false; + + _db.clear(); + _isDbSet = false; + + _collection.clear(); + _isCollectionSet = false; + + } + + void ParsedResource::cloneTo(ParsedResource* other) const { + other->clear(); + + other->_anyResource = _anyResource; + other->_isAnyResourceSet = _isAnyResourceSet; + + other->_cluster = _cluster; + other->_isClusterSet = _isClusterSet; + + other->_db = _db; + other->_isDbSet = _isDbSet; + + other->_collection = _collection; + other->_isCollectionSet = _isCollectionSet; + } + + std::string ParsedResource::toString() const { + return toBSON().toString(); + } + + void ParsedResource::setAnyResource(bool anyResource) { + _anyResource = anyResource; + _isAnyResourceSet = true; + } + + void ParsedResource::unsetAnyResource() { + _isAnyResourceSet = false; + } + + bool ParsedResource::isAnyResourceSet() const { + return _isAnyResourceSet; + } + + bool ParsedResource::getAnyResource() const { + dassert(_isAnyResourceSet); + return _anyResource; + } + + void ParsedResource::setCluster(bool cluster) { + _cluster = cluster; + _isClusterSet = true; + } + + void ParsedResource::unsetCluster() { + _isClusterSet = false; + } + + bool ParsedResource::isClusterSet() const { + return _isClusterSet; + } + + bool ParsedResource::getCluster() const { + dassert(_isClusterSet); + return _cluster; + } + + void ParsedResource::setDb(const StringData& db) { + _db = db.toString(); + _isDbSet = true; + } + + void ParsedResource::unsetDb() { + _isDbSet = false; + } + + bool ParsedResource::isDbSet() const { + return _isDbSet; + } + + const std::string& ParsedResource::getDb() const { + dassert(_isDbSet); + return _db; + } + + void ParsedResource::setCollection(const StringData& collection) { + _collection = collection.toString(); + _isCollectionSet = true; + } + + void ParsedResource::unsetCollection() { + _isCollectionSet = false; + } + + bool ParsedResource::isCollectionSet() const { + return _isCollectionSet; + } + + const std::string& ParsedResource::getCollection() const { + dassert(_isCollectionSet); + return _collection; + } + + const BSONField > ParsedPrivilege::actions("actions"); + const BSONField ParsedPrivilege::resource("resource"); + + ParsedPrivilege::ParsedPrivilege() { + clear(); + } + + ParsedPrivilege::~ParsedPrivilege() { + } + + bool ParsedPrivilege::isValid(std::string* errMsg) const { + std::string dummy; + if (errMsg == NULL) { + errMsg = &dummy; + } + + // All the mandatory fields must be present. + if (!_isActionsSet || !_actions.size()) { + *errMsg = stream() << "missing " << actions.name() << " field"; + return false; + } + + if (!_isResourceSet) { + *errMsg = stream() << "missing " << resource.name() << " field"; + return false; + } + + return getResource().isValid(errMsg); + } + + BSONObj ParsedPrivilege::toBSON() const { + BSONObjBuilder builder; + + if (_isResourceSet) builder.append(resource(), _resource.toBSON()); + + if (_isActionsSet) { + BSONArrayBuilder actionsBuilder(builder.subarrayStart(actions())); + for (std::vector::const_iterator it = _actions.begin(); + it != _actions.end(); + ++it) { + actionsBuilder.append(*it); + } + actionsBuilder.doneFast(); + } + + return builder.obj().getOwned(); + } + + bool ParsedPrivilege::parseBSON(const BSONObj& source, string* errMsg) { + clear(); + + std::string dummy; + if (!errMsg) errMsg = &dummy; + + FieldParser::FieldState fieldState; + fieldState = FieldParser::extract(source, actions, &_actions, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isActionsSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, resource, &_resource, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isResourceSet = fieldState == FieldParser::FIELD_SET; + + return true; + } + + void ParsedPrivilege::clear() { + _actions.clear(); + _isActionsSet = false; + _resource.clear(); + _isResourceSet = false; + + } + + std::string ParsedPrivilege::toString() const { + return toBSON().toString(); + } + + void ParsedPrivilege::setActions(const std::vector& actions) { + for (std::vector::const_iterator it = actions.begin(); + it != actions.end(); + ++it) { + addToActions((*it)); + } + _isActionsSet = actions.size() > 0; + } + + void ParsedPrivilege::addToActions(const string& actions) { + _actions.push_back(actions); + _isActionsSet = true; + } + + void ParsedPrivilege::unsetActions() { + _actions.clear(); + _isActionsSet = false; + } + + bool ParsedPrivilege::isActionsSet() const { + return _isActionsSet; + } + + size_t ParsedPrivilege::sizeActions() const { + return _actions.size(); + } + + const std::vector& ParsedPrivilege::getActions() const { + dassert(_isActionsSet); + return _actions; + } + + const string& ParsedPrivilege::getActionsAt(size_t pos) const { + dassert(_isActionsSet); + dassert(_actions.size() > pos); + return _actions.at(pos); + } + + void ParsedPrivilege::setResource(const ParsedResource& resource) { + resource.cloneTo(&_resource); + _isResourceSet = true; + } + + void ParsedPrivilege::unsetResource() { + _isResourceSet = false; + } + + bool ParsedPrivilege::isResourceSet() const { + return _isResourceSet; + } + + const ParsedResource& ParsedPrivilege::getResource() const { + dassert(_isResourceSet); + return _resource; + } + + bool ParsedPrivilege::parsedPrivilegeToPrivilege(const ParsedPrivilege& parsedPrivilege, + Privilege* result, + std::string* errmsg) { + if (!parsedPrivilege.isValid(errmsg)) { + return false; + } + + // Build actions + ActionSet actions; + const vector& parsedActions = parsedPrivilege.getActions(); + Status status = ActionSet::parseActionSetFromStringVector(parsedActions, &actions); + if (!status.isOK()) { + *errmsg = status.reason(); + return false; + } + + // Build resource + ResourcePattern resource; + const ParsedResource& parsedResource = parsedPrivilege.getResource(); + if (parsedResource.isAnyResourceSet() && parsedResource.getAnyResource()) { + resource = ResourcePattern::forAnyResource(); + } else if (parsedResource.isClusterSet() && parsedResource.getCluster()) { + resource = ResourcePattern::forClusterResource(); + } else { + if (parsedResource.isDbSet() && !parsedResource.getDb().empty()) { + if (parsedResource.isCollectionSet() && !parsedResource.getCollection().empty()) { + resource = ResourcePattern::forExactNamespace( + NamespaceString(parsedResource.getDb(), + parsedResource.getCollection())); + } else { + resource = ResourcePattern::forDatabaseName(parsedResource.getDb()); + } + } else { + if (parsedResource.isCollectionSet() && !parsedResource.getCollection().empty()) { + resource = ResourcePattern::forCollectionName(parsedResource.getCollection()); + } else { + resource = ResourcePattern::forAnyNormalResource(); + } + } + } + + *result = Privilege(resource, actions); + return true; + } + + bool ParsedPrivilege::privilegeToParsedPrivilege(const Privilege& privilege, + ParsedPrivilege* result, + std::string* errmsg) { + ParsedResource parsedResource; + if (privilege.getResourcePattern().isExactNamespacePattern()) { + parsedResource.setDb(privilege.getResourcePattern().databaseToMatch()); + parsedResource.setCollection(privilege.getResourcePattern().collectionToMatch()); + } else if (privilege.getResourcePattern().isDatabasePattern()) { + parsedResource.setDb(privilege.getResourcePattern().databaseToMatch()); + parsedResource.setCollection(""); + } else if (privilege.getResourcePattern().isCollectionPattern()) { + parsedResource.setDb(""); + parsedResource.setCollection(privilege.getResourcePattern().collectionToMatch()); + } else if (privilege.getResourcePattern().isAnyNormalResourcePattern()) { + parsedResource.setDb(""); + parsedResource.setCollection(""); + } else if (privilege.getResourcePattern().isClusterResourcePattern()) { + parsedResource.setCluster(true); + } else if (privilege.getResourcePattern().isAnyResourcePattern()) { + parsedResource.setAnyResource(true); + } else { + *errmsg = stream() << privilege.getResourcePattern().toString() << + " is not a valid user-grantable resource pattern"; + return false; + } + + result->clear(); + result->setResource(parsedResource); + result->setActions(privilege.getActions().getActionsAsStrings()); + return result->isValid(errmsg); + } +} // namespace mongo diff --git a/src/mongo/db/auth/privilege_parser.h b/src/mongo/db/auth/privilege_parser.h new file mode 100644 index 00000000000..fc39566e311 --- /dev/null +++ b/src/mongo/db/auth/privilege_parser.h @@ -0,0 +1,196 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/string_data.h" +#include "mongo/db/jsobj.h" +#include "mongo/s/bson_serializable.h" + +namespace mongo { + + class Privilege; + + /** + * This class is used to parse documents describing resources as they are represented as part + * of privileges granted to roles in the role management commands. + */ + class ParsedResource : BSONSerializable { + MONGO_DISALLOW_COPYING(ParsedResource); + public: + + // + // schema declarations + // + + static const BSONField anyResource; + static const BSONField cluster; + static const BSONField db; + static const BSONField collection; + + // + // construction / destruction + // + + ParsedResource(); + ~ParsedResource(); + + /** Copies all the fields present in 'this' to 'other'. */ + void cloneTo(ParsedResource* other) const; + + // + // bson serializable interface implementation + // + + bool isValid(std::string* errMsg) const; + BSONObj toBSON() const; + bool parseBSON(const BSONObj& source, std::string* errMsg); + void clear(); + virtual std::string toString() const; + + // + // individual field accessors + // + + void setAnyResource(bool anyResource); + void unsetAnyResource(); + bool isAnyResourceSet() const; + bool getAnyResource() const; + + void setCluster(bool cluster); + void unsetCluster(); + bool isClusterSet() const; + bool getCluster() const; + + void setDb(const StringData& db); + void unsetDb(); + bool isDbSet() const; + const std::string& getDb() const; + + void setCollection(const StringData& collection); + void unsetCollection(); + bool isCollectionSet() const; + const std::string& getCollection() const; + + private: + // Convention: (M)andatory, (O)ptional + + // (O) Only present if the resource matches anything. + bool _anyResource; + bool _isAnyResourceSet; + + // (O) Only present if the resource is the cluster + bool _cluster; + bool _isClusterSet; + + // (O) database portion of the resource + string _db; + bool _isDbSet; + + // (O) collection portion of the resource + string _collection; + bool _isCollectionSet; + }; + + /** + * This class is used to parse documents describing privileges in the role managment commands. + */ + class ParsedPrivilege : BSONSerializable { + MONGO_DISALLOW_COPYING(ParsedPrivilege); + public: + + // + // schema declarations + // + + static const BSONField > actions; + static const BSONField resource; + + // + // construction / destruction + // + + ParsedPrivilege(); + ~ParsedPrivilege(); + + /** + * Takes a parsedPrivilege and turns it into a true Privilege object. + */ + static bool parsedPrivilegeToPrivilege(const ParsedPrivilege& parsedPrivilege, + Privilege* result, + std::string* errmsg); + /** + * Takes a Privilege object and turns it into a ParsedPrivilege. + */ + static bool privilegeToParsedPrivilege(const Privilege& privilege, + ParsedPrivilege* result, + std::string* errmsg); + + // + // bson serializable interface implementation + // + + bool isValid(std::string* errMsg) const; + BSONObj toBSON() const; + bool parseBSON(const BSONObj& source, std::string* errMsg); + void clear(); + std::string toString() const; + + // + // individual field accessors + // + + void setActions(const std::vector& actions); + void addToActions(const string& actions); + void unsetActions(); + bool isActionsSet() const; + size_t sizeActions() const; + const std::vector& getActions() const; + const string& getActionsAt(size_t pos) const; + + void setResource(const ParsedResource& resource); + void unsetResource(); + bool isResourceSet() const; + const ParsedResource& getResource() const; + + private: + // Convention: (M)andatory, (O)ptional + + // (M) Array of action types + std::vector _actions; + bool _isActionsSet; + + // (M) Object describing the resource pattern of this privilege + ParsedResource _resource; + bool _isResourceSet; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/privilege_parser_test.cpp b/src/mongo/db/auth/privilege_parser_test.cpp new file mode 100644 index 00000000000..798a197ae91 --- /dev/null +++ b/src/mongo/db/auth/privilege_parser_test.cpp @@ -0,0 +1,210 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * Unit tests of the ParsedPrivilege class. + */ + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/privilege_parser.h" +#include "mongo/db/server_options.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + + TEST(PrivilegeParserTest, IsValidTest) { + ParsedPrivilege parsedPrivilege; + std::string errmsg; + + // must have resource + parsedPrivilege.parseBSON(BSON("actions" << BSON_ARRAY("find")), &errmsg); + ASSERT_FALSE(parsedPrivilege.isValid(&errmsg)); + + // must have actions + parsedPrivilege.parseBSON(BSON("resource" << BSON("cluster" << true)), &errmsg); + ASSERT_FALSE(parsedPrivilege.isValid(&errmsg)); + + // resource can't have cluster as well as db or collection + parsedPrivilege.parseBSON(BSON("resource" << BSON("cluster" << true << + "db" << "" << + "collection" << "") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT_FALSE(parsedPrivilege.isValid(&errmsg)); + + // resource can't have db without collection + parsedPrivilege.parseBSON(BSON("resource" << BSON("db" << "") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT_FALSE(parsedPrivilege.isValid(&errmsg)); + + // resource can't have collection without db + parsedPrivilege.parseBSON(BSON("resource" << BSON("collection" << "") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT_FALSE(parsedPrivilege.isValid(&errmsg)); + + // Works with wildcard db and resource + parsedPrivilege.parseBSON(BSON("resource" << BSON("db" << "" << "collection" << "") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + + // Works with real db and collection + parsedPrivilege.parseBSON(BSON("resource" << BSON("db" << "test" << + "collection" << "foo") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + + // Works with cluster resource + parsedPrivilege.parseBSON(BSON("resource" << BSON("cluster" << true) << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + } + + TEST(PrivilegeParserTest, ConvertBetweenPrivilegeTest) { + ParsedPrivilege parsedPrivilege; + Privilege privilege; + std::string errmsg; + std::vector actionsVector; + actionsVector.push_back("find"); + + // Works with wildcard db and resource + parsedPrivilege.parseBSON(BSON("resource" << BSON("db" << "" << "collection" << "") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(ParsedPrivilege::parsedPrivilegeToPrivilege(parsedPrivilege, &privilege, &errmsg)); + ASSERT(privilege.getActions().contains(ActionType::find)); + ASSERT(!privilege.getActions().contains(ActionType::insert)); + ASSERT_EQUALS(privilege.getResourcePattern(), ResourcePattern::forAnyNormalResource()); + + ASSERT(ParsedPrivilege::privilegeToParsedPrivilege(privilege, &parsedPrivilege, &errmsg)); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(parsedPrivilege.isResourceSet()); + ASSERT_FALSE(parsedPrivilege.getResource().isClusterSet()); + ASSERT(parsedPrivilege.getResource().isDbSet()); + ASSERT(parsedPrivilege.getResource().isCollectionSet()); + ASSERT_EQUALS("", parsedPrivilege.getResource().getDb()); + ASSERT_EQUALS("", parsedPrivilege.getResource().getCollection()); + ASSERT(parsedPrivilege.isActionsSet()); + ASSERT(actionsVector == parsedPrivilege.getActions()); + + // Works with exact namespaces + parsedPrivilege.parseBSON(BSON("resource" << BSON("db" << "test" << + "collection" << "foo") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(ParsedPrivilege::parsedPrivilegeToPrivilege(parsedPrivilege, &privilege, &errmsg)); + ASSERT(privilege.getActions().contains(ActionType::find)); + ASSERT(!privilege.getActions().contains(ActionType::insert)); + ASSERT_EQUALS(privilege.getResourcePattern(), + ResourcePattern::forExactNamespace(NamespaceString("test.foo"))); + + ASSERT(ParsedPrivilege::privilegeToParsedPrivilege(privilege, &parsedPrivilege, &errmsg)); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(parsedPrivilege.isResourceSet()); + ASSERT_FALSE(parsedPrivilege.getResource().isClusterSet()); + ASSERT(parsedPrivilege.getResource().isDbSet()); + ASSERT(parsedPrivilege.getResource().isCollectionSet()); + ASSERT_EQUALS("test", parsedPrivilege.getResource().getDb()); + ASSERT_EQUALS("foo", parsedPrivilege.getResource().getCollection()); + ASSERT(parsedPrivilege.isActionsSet()); + ASSERT(actionsVector == parsedPrivilege.getActions()); + + // Works with database resource + parsedPrivilege.parseBSON(BSON("resource" << BSON("db" << "test" << + "collection" << "") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(ParsedPrivilege::parsedPrivilegeToPrivilege(parsedPrivilege, &privilege, &errmsg)); + ASSERT(privilege.getActions().contains(ActionType::find)); + ASSERT(!privilege.getActions().contains(ActionType::insert)); + ASSERT_EQUALS(privilege.getResourcePattern(), ResourcePattern::forDatabaseName("test")); + + ASSERT(ParsedPrivilege::privilegeToParsedPrivilege(privilege, &parsedPrivilege, &errmsg)); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(parsedPrivilege.isResourceSet()); + ASSERT_FALSE(parsedPrivilege.getResource().isClusterSet()); + ASSERT(parsedPrivilege.getResource().isDbSet()); + ASSERT(parsedPrivilege.getResource().isCollectionSet()); + ASSERT_EQUALS("test", parsedPrivilege.getResource().getDb()); + ASSERT_EQUALS("", parsedPrivilege.getResource().getCollection()); + ASSERT(parsedPrivilege.isActionsSet()); + ASSERT(actionsVector == parsedPrivilege.getActions()); + + // Works with collection resource + parsedPrivilege.parseBSON(BSON("resource" << BSON("db" << "" << + "collection" << "foo") << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(ParsedPrivilege::parsedPrivilegeToPrivilege(parsedPrivilege, &privilege, &errmsg)); + ASSERT(privilege.getActions().contains(ActionType::find)); + ASSERT(!privilege.getActions().contains(ActionType::insert)); + ASSERT_EQUALS(privilege.getResourcePattern(), ResourcePattern::forCollectionName("foo")); + + ASSERT(ParsedPrivilege::privilegeToParsedPrivilege(privilege, &parsedPrivilege, &errmsg)); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(parsedPrivilege.isResourceSet()); + ASSERT_FALSE(parsedPrivilege.getResource().isClusterSet()); + ASSERT(parsedPrivilege.getResource().isDbSet()); + ASSERT(parsedPrivilege.getResource().isCollectionSet()); + ASSERT_EQUALS("", parsedPrivilege.getResource().getDb()); + ASSERT_EQUALS("foo", parsedPrivilege.getResource().getCollection()); + ASSERT(parsedPrivilege.isActionsSet()); + ASSERT(actionsVector == parsedPrivilege.getActions()); + + // Works with cluster resource + parsedPrivilege.parseBSON(BSON("resource" << BSON("cluster" << true) << + "actions" << BSON_ARRAY("find")), + &errmsg); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(ParsedPrivilege::parsedPrivilegeToPrivilege(parsedPrivilege, &privilege, &errmsg)); + ASSERT(privilege.getActions().contains(ActionType::find)); + ASSERT(!privilege.getActions().contains(ActionType::insert)); + ASSERT_EQUALS(privilege.getResourcePattern(), ResourcePattern::forClusterResource()); + + ASSERT(ParsedPrivilege::privilegeToParsedPrivilege(privilege, &parsedPrivilege, &errmsg)); + ASSERT(parsedPrivilege.isValid(&errmsg)); + ASSERT(parsedPrivilege.isResourceSet()); + ASSERT(parsedPrivilege.getResource().isClusterSet()); + ASSERT(parsedPrivilege.getResource().getCluster()); + ASSERT_FALSE(parsedPrivilege.getResource().isDbSet()); + ASSERT_FALSE(parsedPrivilege.getResource().isCollectionSet()); + ASSERT(parsedPrivilege.isActionsSet()); + ASSERT(actionsVector == parsedPrivilege.getActions()); + } + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/auth/privilege_set.cpp b/src/mongo/db/auth/privilege_set.cpp deleted file mode 100644 index c641b65caea..00000000000 --- a/src/mongo/db/auth/privilege_set.cpp +++ /dev/null @@ -1,145 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/pch.h" - -#include "mongo/db/auth/privilege_set.h" - -#include -#include - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/principal.h" -#include "mongo/db/namespacestring.h" -#include "mongo/util/map_util.h" - -namespace mongo { - - const std::string PrivilegeSet::WILDCARD_RESOURCE = "*"; - - PrivilegeSet::PrivilegeSet() {} - PrivilegeSet::~PrivilegeSet() {} - - void PrivilegeSet::grantPrivilege(const Privilege& privilege, - const PrincipalName& authorizingPrincipal) { - grantPrivileges(std::vector(1, privilege), authorizingPrincipal); - } - - void PrivilegeSet::grantPrivileges(const std::vector& privileges, - const PrincipalName& authorizingPrincipal) { - StringMap& byResourceForPrincipal = _byPrincipal[authorizingPrincipal]; - for (std::vector::const_iterator iter = privileges.begin(), - end = privileges.end(); - iter != end; ++iter) { - - byResourceForPrincipal[iter->getResource()].addAllActionsFromSet(iter->getActions()); - - ResourcePrivilegeCacheEntry* entry = _lookupOrInsertEntry(iter->getResource()); - entry->actions.addAllActionsFromSet(iter->getActions()); - } - } - - void PrivilegeSet::revokePrivilegesFromPrincipal(const PrincipalName& principal) { - PrincipalPrivilegeMap::iterator principalEntry = _byPrincipal.find(principal); - if (principalEntry == _byPrincipal.end()) - return; - - // For every resource that "principal" authorizes, mark its entry in the _byResource table - // as dirty, so that it will be rebuilt on next consultation. - for (StringMap::const_iterator resourceEntry = principalEntry->second.begin(), - end = principalEntry->second.end(); - resourceEntry != end; ++resourceEntry) { - - _lookupOrInsertEntry(resourceEntry->first)->dirty = true; - } - - // Remove the princiapl from the _byPrincipal table. - _byPrincipal.erase(principalEntry); - } - - bool PrivilegeSet::hasPrivilege(const Privilege& desiredPrivilege) { - if (desiredPrivilege.getActions().empty()) - return true; - - StringData resourceSearchList[2]; - resourceSearchList[0] = WILDCARD_RESOURCE; - resourceSearchList[1] = nsToDatabaseSubstring(desiredPrivilege.getResource()); - - ActionSet unmetRequirements = desiredPrivilege.getActions(); - for (int i = 0; i < boost::size(resourceSearchList); ++i) { - ResourcePrivilegeCacheEntry* entry = _lookupEntry(resourceSearchList[i]); - if (NULL == entry) - continue; - if (entry->dirty) - _rebuildEntry(resourceSearchList[i], entry); - unmetRequirements.removeAllActionsFromSet(entry->actions); - if (unmetRequirements.empty()) - return true; - } - return false; - } - - bool PrivilegeSet::hasPrivileges(const std::vector& desiredPrivileges) { - for (std::vector::const_iterator iter = desiredPrivileges.begin(), - end = desiredPrivileges.end(); - iter != end; ++iter) { - - if (!hasPrivilege(*iter)) - return false; - } - return true; - } - - void PrivilegeSet::_rebuildEntry(const StringData& resource, - ResourcePrivilegeCacheEntry* entry) { - const ActionSet emptyActionSet; - entry->actions.removeAllActions(); - - for (PrincipalPrivilegeMap::const_iterator iter = _byPrincipal.begin(), - end = _byPrincipal.end(); - iter != end; ++iter) { - - entry->actions.addAllActionsFromSet( - mapFindWithDefault(iter->second, resource, emptyActionSet)); - } - - entry->dirty = false; - } - - PrivilegeSet::ResourcePrivilegeCacheEntry* PrivilegeSet::_lookupEntry( - const StringData& resource) { - - if (resource == WILDCARD_RESOURCE) - return &_globalPrivilegeEntry; - - ResourcePrivilegeCache::const_iterator iter = _byResource.find(resource); - if (iter != _byResource.end()) { - // StringMap doesn't have non-const iterators, so there is no way to lookup without - // inserting and get a mutable value, without const-cast. - return const_cast(&iter->second); - } - return NULL; - } - - PrivilegeSet::ResourcePrivilegeCacheEntry* PrivilegeSet::_lookupOrInsertEntry( - const StringData& resource) { - - if (resource == WILDCARD_RESOURCE) - return &_globalPrivilegeEntry; - return &_byResource[resource]; - } - -} // namespace mongo diff --git a/src/mongo/db/auth/privilege_set.h b/src/mongo/db/auth/privilege_set.h deleted file mode 100644 index b9b14c6b2d3..00000000000 --- a/src/mongo/db/auth/privilege_set.h +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/auth/principal_name.h" -#include "mongo/util/string_map.h" - -namespace mongo { - - /** - * A collection of privileges describing which authenticated principals bestow the client the - * ability to perform various actions on specific resources. Since every privilege comes from - * an authenticated principal, removing that principal removes all privileges granted by that - * principal. - * - * Resources are arranged hierarchically, with a wildcard resource, - * PrivilegeSet::WILDCARD_RESOURCE, matching any resource. In the current implementation, the - * only two levels of the hierarchy are the wildcard and one level below, which is analagous to - * the name of a database. It is future work to support collection or other sub-database - * resources. - * - * This class does not do any locking/synchronization, the consumer will be responsible for - * synchronizing access. - */ - class PrivilegeSet { - MONGO_DISALLOW_COPYING(PrivilegeSet); - public: - static const std::string WILDCARD_RESOURCE; - - PrivilegeSet(); - ~PrivilegeSet(); - - /** - * Adds the specified privilege to the set, associating it with the named principal. - * - * The privilege should be on a specific resource, or on the WILDCARD_RESOURCE. - */ - void grantPrivilege(const Privilege& privilege, const PrincipalName& authorizingPrincipal); - - /** - * Adds the specified privileges to the set, associating them with the named principal. - */ - void grantPrivileges(const std::vector& privileges, - const PrincipalName& authorizingPrincipal); - - /** - * Removes from the set all privileges associated with the given principal. - * - * If multiple princpals enable the same privilege, the set will continue to - * contain those privileges until all authorizing principals have had their - * privileges revoked from the set. - */ - void revokePrivilegesFromPrincipal(const PrincipalName& principal); - - /** - * Returns true if the set authorizes "desiredPrivilege". - * - * The set is considered to authorize "desiredPrivilege" if each action in - * "desiredPrivilege" is satisfied either on the database component of - * "desiredPrivilege.getResource()" or on WILDCARD_RESOURCE. - * - * TODO: Support checking for the privilege on the full resource name as well as the - * database component, to support sub-database granularity privilege assignment. - */ - bool hasPrivilege(const Privilege& desiredPrivilege); - - /** - * Same as hasPrivilege, except checks all the privileges in a vector. - */ - bool hasPrivileges(const std::vector& desiredPrivileges); - - private: - - /** - * Information about privileges held on a resource. - * - * Instances are stored in the _byResource map, and accelerate the fast path of - * hasPrivilege(). Privilege revocations via revokePrivilegesFromPrincipal() can make these - * entries invalid, at which point they are marked "dirty". Dirty entries are rebuilt via - * _rebuildEntry(), below, during execution of hasPrivilege(). - */ - class ResourcePrivilegeCacheEntry { - public: - ResourcePrivilegeCacheEntry() : actions(), dirty(false) {} - - // All actions enabled on the associated resource, provided that "dirty" is false. - ActionSet actions; - - // False if this data is consistent with the full privilege information, stored in the - // _byPrincipal map. - bool dirty; - }; - - /** - * Type of map from resource names to authorized actions. - */ - typedef StringMap ResourcePrivilegeCache; - - /** - * Type of map from principal identity to information about the principal's privileges. The - * values in the map are themselves maps from resource names to associated actions. - */ - typedef std::map > PrincipalPrivilegeMap; - - void _rebuildEntry(const StringData& resource, ResourcePrivilegeCacheEntry* summary); - - ResourcePrivilegeCacheEntry* _lookupEntry(const StringData& resource); - ResourcePrivilegeCacheEntry* _lookupOrInsertEntry(const StringData& resource); - - // Information about privileges available on all resources. - ResourcePrivilegeCacheEntry _globalPrivilegeEntry; - - // Cache of privilege information, by resource. - ResourcePrivilegeCache _byResource; - - // Directory of privilege information, by principal. - PrincipalPrivilegeMap _byPrincipal; - }; - -} // namespace mongo diff --git a/src/mongo/db/auth/privilege_set_test.cpp b/src/mongo/db/auth/privilege_set_test.cpp deleted file mode 100644 index 36af7b82205..00000000000 --- a/src/mongo/db/auth/privilege_set_test.cpp +++ /dev/null @@ -1,211 +0,0 @@ -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Unit tests of the PrivilegeSet type. - */ - -#include - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/privilege_set.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -namespace { - - // Convenience methods for outputing PrincipalName and construction ActionSets that make tests - // concise, but that we're reluctant to put into the types themselves. - - std::ostream& operator<<(std::ostream& os, const PrincipalName& pname) { - return os << pname.toString(); - } - - std::ostream& operator<<(std::ostream&os, const std::vector& ps) { - os << "[ "; - for (size_t i = 0; i < ps.size(); ++i) - os << ps[i] << ' '; - os << ']'; - return os; - } - - ActionSet operator|(const ActionSet& lhs, const ActionSet& rhs) { - ActionSet result = lhs; - result.addAllActionsFromSet(rhs); - return result; - } - - ActionSet operator|(const ActionSet& lhs, const ActionType& rhs) { - ActionSet result = lhs; - result.addAction(rhs); - return result; - } - - ActionSet operator|(const ActionType& lhs, const ActionType& rhs) { - ActionSet result; - result.addAction(lhs); - result.addAction(rhs); - return result; - } - - // Tests - - TEST(PrivilegeSetTest, PrivilegeSet) { - PrivilegeSet capSet; - PrincipalName user1("user1", "test"); - PrincipalName user2("user2", "test2"); - - // Initially, the capability set contains no privileges at all. - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::find))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("bar", ActionType::find))); - - // Grant find and update to "foo", only. - capSet.grantPrivilege(Privilege("foo", ActionType::find|ActionType::update), user1); - - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::find))); - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::find|ActionType::update))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::find|ActionType::remove))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("bar", ActionType::find))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::remove))); - - // Grant "userAdmin", "update" and "remove" on "foo" to user2, which changes the set of - // actions this privilege set will approve. - capSet.grantPrivilege( - Privilege("foo", ActionType::userAdmin|ActionType::update|ActionType::remove), - user2); - - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::userAdmin))); - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::update))); - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::userAdmin))); - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::find|ActionType::remove))); - - // Revoke user2's privileges. - capSet.revokePrivilegesFromPrincipal(user2); - - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::userAdmin))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::find|ActionType::remove))); - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::update))); - - // Revoke user2's privileges again; should be a no-op. - capSet.revokePrivilegesFromPrincipal(user2); - - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::userAdmin))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::find|ActionType::remove))); - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::update))); - - // Re-grant "userAdmin", "update" and "remove" on "foo" to user2. - capSet.grantPrivilege( - Privilege("foo", ActionType::userAdmin|ActionType::update|ActionType::remove), - user2); - - // The set still contains no capabilities on "bar". - ASSERT_FALSE(capSet.hasPrivilege(Privilege("bar", ActionType::find))); - - // Let user2 "find" on "bar". - capSet.grantPrivilege(Privilege("bar", ActionType::find), user2); - - ASSERT_TRUE(capSet.hasPrivilege(Privilege("bar", ActionType::find))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("bar", ActionType::update))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("bar", ActionType::remove))); - - // Let user1 "find" and "update" on "bar". - capSet.grantPrivilege(Privilege("bar", ActionType::update|ActionType::find), user1); - - ASSERT_TRUE(capSet.hasPrivilege(Privilege("bar", ActionType::find|ActionType::update))); - ASSERT_FALSE(capSet.hasPrivilege( - Privilege("bar", - ActionType::find|ActionType::update|ActionType::remove))); - - // Revoke user1's privileges. - capSet.revokePrivilegesFromPrincipal(user1); - - ASSERT_TRUE(capSet.hasPrivilege(Privilege("foo", ActionType::update))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::find))); - ASSERT_TRUE(capSet.hasPrivilege(Privilege("bar", ActionType::find))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("bar", ActionType::update))); - - // Revoke user2's privileges. - capSet.revokePrivilegesFromPrincipal(user2); - - ASSERT_FALSE(capSet.hasPrivilege(Privilege("foo", ActionType::update))); - ASSERT_FALSE(capSet.hasPrivilege(Privilege("bar", ActionType::find))); - } - - TEST(PrivilegeSetTest, WildcardPrivileges) { - // Tests acquisition and revocation of privileges on WILDCARD_RESOURCE. - - PrivilegeSet privSet; - - PrincipalName user("user", "db"); - Privilege wildcardFind("*", ActionType::find); - Privilege wildcardUpdate("*", ActionType::update); - Privilege wildcardFindAndUpdate("*", ActionType::find|ActionType::update); - Privilege fooFind("foo", ActionType::find); - Privilege fooUpdate("foo", ActionType::update); - Privilege fooFindAndUpdate("foo", ActionType::find|ActionType::update); - Privilege barFind("bar", ActionType::find); - Privilege barUpdate("bar", ActionType::update); - Privilege barFindAndUpdate("bar", ActionType::find|ActionType::update); - - // With no granted privileges, assert that hasPrivilege returns false. - ASSERT_FALSE(privSet.hasPrivilege(wildcardFind)); - ASSERT_FALSE(privSet.hasPrivilege(wildcardUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(wildcardFindAndUpdate)); - - ASSERT_FALSE(privSet.hasPrivilege(fooFind)); - ASSERT_FALSE(privSet.hasPrivilege(fooUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(fooFindAndUpdate)); - - ASSERT_FALSE(privSet.hasPrivilege(barFind)); - ASSERT_FALSE(privSet.hasPrivilege(barUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(barFindAndUpdate)); - - // Grant some privileges, and ensure that exactly those privileges are granted. - std::vector grantedPrivileges; - grantedPrivileges.push_back(wildcardFind); - grantedPrivileges.push_back(fooUpdate); - - privSet.grantPrivileges(grantedPrivileges, user); - - ASSERT_TRUE(privSet.hasPrivilege(wildcardFind)); - ASSERT_FALSE(privSet.hasPrivilege(wildcardUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(wildcardFindAndUpdate)); - - ASSERT_TRUE(privSet.hasPrivilege(fooFind)); - ASSERT_TRUE(privSet.hasPrivilege(fooUpdate)); - ASSERT_TRUE(privSet.hasPrivilege(fooFindAndUpdate)); - - ASSERT_TRUE(privSet.hasPrivilege(barFind)); - ASSERT_FALSE(privSet.hasPrivilege(barUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(barFindAndUpdate)); - - // Revoke the granted privileges, and assert that hasPrivilege returns false. - privSet.revokePrivilegesFromPrincipal(user); - - ASSERT_FALSE(privSet.hasPrivilege(wildcardFind)); - ASSERT_FALSE(privSet.hasPrivilege(wildcardUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(wildcardFindAndUpdate)); - - ASSERT_FALSE(privSet.hasPrivilege(fooFind)); - ASSERT_FALSE(privSet.hasPrivilege(fooUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(fooFindAndUpdate)); - - ASSERT_FALSE(privSet.hasPrivilege(barFind)); - ASSERT_FALSE(privSet.hasPrivilege(barUpdate)); - ASSERT_FALSE(privSet.hasPrivilege(barFindAndUpdate)); - } - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/auth/resource_pattern.cpp b/src/mongo/db/auth/resource_pattern.cpp new file mode 100644 index 00000000000..27d2a94b279 --- /dev/null +++ b/src/mongo/db/auth/resource_pattern.cpp @@ -0,0 +1,64 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include + +#include "mongo/db/auth/resource_pattern.h" + +#include "mongo/util/log.h" + +namespace mongo { + + std::string ResourcePattern::toString() const { + switch (_matchType) { + case matchNever: + return ""; + case matchClusterResource: + return ""; + case matchDatabaseName: + return ""; + case matchCollectionName: + return ""; + case matchExactNamespace: + return "<" + _ns.ns() + ">"; + case matchAnyNormalResource: + return ""; + case matchAnyResource: + return ""; + default: + return ""; + } + } + + std::ostream& operator<<(std::ostream& os, const ResourcePattern& pattern) { + return os << pattern.toString(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/resource_pattern.h b/src/mongo/db/auth/resource_pattern.h new file mode 100644 index 00000000000..3d2b5482da3 --- /dev/null +++ b/src/mongo/db/auth/resource_pattern.h @@ -0,0 +1,210 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/string_data.h" +#include "mongo/db/namespace_string.h" +#include "mongo/platform/hash_namespace.h" + +namespace mongo { + + /** + * Representation of names of various kinds of resources targetable by the access control + * system. + * + * Three of the types of name, "forDatabaseName", "forExactNamespace" and "forClusterResource", + * can represent concrete resources targeted for manipulation by database operations. All of + * the types also act as patterns, useful for matching against groups of concrete resources as + * part of the access control system. See buildResourceSearchList() in + * authorization_session.cpp for details. + */ + class ResourcePattern { + public: + /** + * Returns a pattern that matches absolutely any resource. + */ + static ResourcePattern forAnyResource() { + return ResourcePattern(matchAnyResource); + } + + /** + * Returns a pattern that matches any database or collection resource except collections for + * which ns.isSystem(). + */ + static ResourcePattern forAnyNormalResource() { + return ResourcePattern(matchAnyNormalResource); + } + + /** + * Returns a pattern that matches the "cluster" resource. + */ + static ResourcePattern forClusterResource() { + return ResourcePattern(matchClusterResource); + } + + /** + * Returns a pattern that matches the named database, and NamespaceStrings + * "ns" for which ns.isSystem() is false and ns.db() == dbname. + */ + static ResourcePattern forDatabaseName(const StringData& dbName) { + return ResourcePattern(matchDatabaseName, NamespaceString(dbName, "")); + } + + /** + * Returns a pattern that matches NamespaceStrings "ns" for which ns.coll() == + * collectionName. + */ + static ResourcePattern forCollectionName(const StringData& collectionName) { + return ResourcePattern(matchCollectionName, NamespaceString("", collectionName)); + } + + /** + * Returns a pattern that matches the given exact namespace string. + */ + static ResourcePattern forExactNamespace(const NamespaceString& ns) { + return ResourcePattern(matchExactNamespace, ns); + } + + /** + * Constructs a pattern that never matches. + */ + ResourcePattern() : _matchType(matchNever) {} + + /** + * Returns true if this pattern matches only exact namespaces. + */ + bool isExactNamespacePattern() const { + return _matchType == matchExactNamespace; + } + + /** + * Returns true if this pattern matches on the database name only. + */ + bool isDatabasePattern() const { + return _matchType == matchDatabaseName; + } + + /** + * Returns true if this pattern matches on the collection name only. + */ + bool isCollectionPattern() const { + return _matchType == matchCollectionName; + } + + /** + * Returns true if this pattern matches the cluster resource only. + */ + bool isClusterResourcePattern() const { + return _matchType == matchClusterResource; + } + + /** + * Returns true if this pattern matches only any normal resource. + */ + bool isAnyNormalResourcePattern() const { + return _matchType == matchAnyNormalResource; + } + + /** + * Returns true if this pattern matches any resource. + */ + bool isAnyResourcePattern() const { + return _matchType == matchAnyResource; + } + + /** + * Returns the namespace that this pattern matches. + * + * Behavior is undefined unless isExactNamespacePattern() is true. + */ + const NamespaceString& ns() const { return _ns; } + + /** + * Returns the database that this pattern matches. + * + * Behavior is undefined unless the pattern is of type matchDatabaseName or + * matchExactNamespace + */ + StringData databaseToMatch() const { return _ns.db(); } + + /** + * Returns the collection that this pattern matches. + * + * Behavior is undefined unless the pattern is of type matchCollectionName or + * matchExactNamespace + */ + StringData collectionToMatch() const { return _ns.coll(); } + + std::string toString() const; + + inline size_t hash() const { + // TODO: Choose a better hash function. + return MONGO_HASH_NAMESPACE::hash()(_ns.ns()) ^ _matchType; + } + + bool operator==(const ResourcePattern& other) const { + if (_matchType != other._matchType) + return false; + if (_ns != other._ns) + return false; + return true; + } + + private: + enum MatchType { + matchNever = 0, /// Matches no resource. + matchClusterResource = 1, /// Matches if the resource is the cluster resource. + matchDatabaseName = 2, /// Matches if the resource's database name is _ns.db(). + matchCollectionName = 3, /// Matches if the resource's collection name is _ns.coll(). + matchExactNamespace = 4, /// Matches if the resource's namespace name is _ns. + matchAnyNormalResource = 5, /// Matches all databases and non-system collections. + matchAnyResource = 6 /// Matches absolutely anything. + }; + + explicit ResourcePattern(MatchType type) : _matchType(type) {} + ResourcePattern(MatchType type, const NamespaceString& ns) : _matchType(type), _ns(ns) {} + + MatchType _matchType; + NamespaceString _ns; + }; + + std::ostream& operator<<(std::ostream& os, const ResourcePattern& pattern); + +} // namespace mongo + +MONGO_HASH_NAMESPACE_START + template <> struct hash { + size_t operator()(const mongo::ResourcePattern& resource) const { + return resource.hash(); + } + }; +MONGO_HASH_NAMESPACE_END diff --git a/src/mongo/db/auth/role_graph.cpp b/src/mongo/db/auth/role_graph.cpp new file mode 100644 index 00000000000..98ea177cc43 --- /dev/null +++ b/src/mongo/db/auth/role_graph.cpp @@ -0,0 +1,561 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/auth/role_graph.h" + +#include +#include +#include + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/platform/unordered_set.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + +namespace { + PrivilegeVector emptyPrivilegeVector; +} // namespace + + RoleGraph::RoleGraph() {}; + RoleGraph::RoleGraph(const RoleGraph& other) : _roleToSubordinates(other._roleToSubordinates), + _roleToIndirectSubordinates(other._roleToIndirectSubordinates), + _roleToMembers(other._roleToMembers), + _directPrivilegesForRole(other._directPrivilegesForRole), + _allPrivilegesForRole(other._allPrivilegesForRole), + _allRoles(other._allRoles) {} + RoleGraph::~RoleGraph() {}; + + void RoleGraph::swap(RoleGraph& other) { + using std::swap; + swap(this->_roleToSubordinates, other._roleToSubordinates); + swap(this->_roleToIndirectSubordinates, other._roleToIndirectSubordinates); + swap(this->_roleToMembers, other._roleToMembers); + swap(this->_directPrivilegesForRole, other._directPrivilegesForRole); + swap(this->_allPrivilegesForRole, other._allPrivilegesForRole); + swap(this->_allRoles, other._allRoles); + } + + void swap(RoleGraph& lhs, RoleGraph& rhs) { + lhs.swap(rhs); + } + + bool RoleGraph::roleExists(const RoleName& role) { + _createBuiltinRoleIfNeeded(role); + return _roleExistsDontCreateBuiltin(role); + } + + bool RoleGraph::_roleExistsDontCreateBuiltin(const RoleName& role) { + EdgeSet::const_iterator edgeIt = _roleToSubordinates.find(role); + if (edgeIt == _roleToSubordinates.end()) + return false; + edgeIt = _roleToMembers.find(role); + fassert(16825, edgeIt != _roleToMembers.end()); + + RolePrivilegeMap::const_iterator strIt = _directPrivilegesForRole.find(role); + if (strIt == _directPrivilegesForRole.end()) + return false; + strIt = _allPrivilegesForRole.find(role); + fassert(16826, strIt != _allPrivilegesForRole.end()); + return true; + } + + Status RoleGraph::createRole(const RoleName& role) { + if (roleExists(role)) { + return Status(ErrorCodes::DuplicateKey, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " already exists", + 0); + } + + _createRoleDontCheckIfRoleExists(role); + return Status::OK(); + } + + void RoleGraph::_createRoleDontCheckIfRoleExists(const RoleName& role) { + // Just reference the role in all the maps so that an entry gets created with empty + // containers for the value. + _roleToSubordinates[role]; + _roleToIndirectSubordinates[role]; + _roleToMembers[role]; + _directPrivilegesForRole[role]; + _allPrivilegesForRole[role]; + _allRoles.insert(role); + } + + Status RoleGraph::deleteRole(const RoleName& role) { + if (!roleExists(role)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not exist", + 0); + } + if (isBuiltinRole(role)) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot delete built-in role: " << + role.getFullName(), + 0); + } + + for (std::vector::iterator it = _roleToSubordinates[role].begin(); + it != _roleToSubordinates[role].end(); ++it) { + _roleToMembers[*it].erase(std::find(_roleToMembers[*it].begin(), + _roleToMembers[*it].end(), + role)); + } + for (std::vector::iterator it = _roleToMembers[role].begin(); + it != _roleToMembers[role].end(); ++it) { + _roleToSubordinates[*it].erase(std::find(_roleToSubordinates[*it].begin(), + _roleToSubordinates[*it].end(), + role)); + } + _roleToSubordinates.erase(role); + _roleToIndirectSubordinates.erase(role); + _roleToMembers.erase(role); + _directPrivilegesForRole.erase(role); + _allPrivilegesForRole.erase(role); + _allRoles.erase(role); + return Status::OK(); + } + + RoleNameIterator RoleGraph::getDirectSubordinates(const RoleName& role) { + if (!roleExists(role)) + return RoleNameIterator(NULL); + return makeRoleNameIteratorForContainer(_roleToSubordinates[role]); + } + + RoleNameIterator RoleGraph::getIndirectSubordinates(const RoleName& role) { + if (!roleExists(role)) + return RoleNameIterator(NULL); + return makeRoleNameIteratorForContainer(_roleToIndirectSubordinates[role]); + } + + RoleNameIterator RoleGraph::getDirectMembers(const RoleName& role) { + if (!roleExists(role)) + return RoleNameIterator(NULL); + return makeRoleNameIteratorForContainer(_roleToMembers[role]); + } + + const PrivilegeVector& RoleGraph::getDirectPrivileges(const RoleName& role) { + if (!roleExists(role)) + return emptyPrivilegeVector; + return _directPrivilegesForRole.find(role)->second; + } + + const PrivilegeVector& RoleGraph::getAllPrivileges(const RoleName& role) { + if (!roleExists(role)) + return emptyPrivilegeVector; + return _allPrivilegesForRole.find(role)->second; + } + + Status RoleGraph::addRoleToRole(const RoleName& recipient, const RoleName& role) { + if (!roleExists(recipient)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << recipient.getFullName() << + " does not exist"); + } + if (isBuiltinRole(recipient)) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot grant roles to built-in role: " << + role.getFullName()); + } + if (!roleExists(role)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not exist"); + } + + if (std::find(_roleToSubordinates[recipient].begin(), + _roleToSubordinates[recipient].end(), + role) == + _roleToSubordinates[recipient].end()) { + // Only add role if it's not already present + _roleToSubordinates[recipient].push_back(role); + _roleToMembers[role].push_back(recipient); + } + + return Status::OK(); + } + + Status RoleGraph::removeRoleFromRole(const RoleName& recipient, const RoleName& role) { + if (!roleExists(recipient)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << recipient.getFullName() << + " does not exist", + 0); + } + if (isBuiltinRole(recipient)) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot remove roles from built-in role: " << + role.getFullName(), + 0); + } + if (!roleExists(role)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not exist", + 0); + } + + std::vector::iterator itToRm = std::find(_roleToMembers[role].begin(), + _roleToMembers[role].end(), + recipient); + if (itToRm != _roleToMembers[role].end()) { + _roleToMembers[role].erase(itToRm); + } else { + return Status(ErrorCodes::RolesNotRelated, + mongoutils::str::stream() << recipient.getFullName() << " is not a member" + " of " << role.getFullName(), + 0); + } + + itToRm = std::find(_roleToSubordinates[recipient].begin(), + _roleToSubordinates[recipient].end(), + role); + fassert(16827, itToRm != _roleToSubordinates[recipient].end()); + _roleToSubordinates[recipient].erase(itToRm); + return Status::OK(); + } + + Status RoleGraph::removeAllRolesFromRole(const RoleName& victim) { + typedef std::vector RoleNameVector; + if (!roleExists(victim)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << victim.getFullName() << + " does not exist", + 0); + } + if (isBuiltinRole(victim)) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot remove roles from built-in role: " << + victim.getFullName(), + 0); + } + + RoleNameVector& subordinatesOfVictim = _roleToSubordinates[victim]; + for (RoleNameVector::const_iterator subordinateRole = subordinatesOfVictim.begin(), + end = subordinatesOfVictim.end(); + subordinateRole != end; + ++subordinateRole) { + + RoleNameVector& membersOfSubordinate = _roleToMembers[*subordinateRole]; + RoleNameVector::iterator toErase = std::find( + membersOfSubordinate.begin(), membersOfSubordinate.end(), victim); + fassert(17173, toErase != membersOfSubordinate.end()); + membersOfSubordinate.erase(toErase); + } + subordinatesOfVictim.clear(); + return Status::OK(); + } + + Status RoleGraph::addPrivilegeToRole(const RoleName& role, const Privilege& privilegeToAdd) { + if (!roleExists(role)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not exist", + 0); + } + if (isBuiltinRole(role)) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot grant privileges to built-in role: " + << role.getFullName(), + 0); + } + + _addPrivilegeToRoleNoChecks(role, privilegeToAdd); + return Status::OK(); + } + + void RoleGraph::_addPrivilegeToRoleNoChecks(const RoleName& role, + const Privilege& privilegeToAdd) { + Privilege::addPrivilegeToPrivilegeVector(&_directPrivilegesForRole[role], privilegeToAdd); + } + + // NOTE: Current runtime of this is O(n*m) where n is the size of the current PrivilegeVector + // for the given role, and m is the size of the privilegesToAdd vector. + // If this was a PrivilegeSet (sorted on resource) rather than a PrivilegeVector, we + // could do this in O(n+m) instead. + Status RoleGraph::addPrivilegesToRole(const RoleName& role, + const PrivilegeVector& privilegesToAdd) { + if (!roleExists(role)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not exist", + 0); + } + if (isBuiltinRole(role)) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot grant privileges to built-in role: " + << role.getFullName(), + 0); + } + + for (PrivilegeVector::const_iterator it = privilegesToAdd.begin(); + it != privilegesToAdd.end(); ++it) { + _addPrivilegeToRoleNoChecks(role, *it); + } + return Status::OK(); + } + + Status RoleGraph::removePrivilegeFromRole(const RoleName& role, + const Privilege& privilegeToRemove) { + if (!roleExists(role)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not exist", + 0); + } + if (isBuiltinRole(role)) { + return Status( + ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot remove privileges from built-in role: " << + role.getFullName()); + } + + PrivilegeVector& currentPrivileges = _directPrivilegesForRole[role]; + for (PrivilegeVector::iterator it = currentPrivileges.begin(); + it != currentPrivileges.end(); ++it) { + + Privilege& curPrivilege = *it; + if (curPrivilege.getResourcePattern() == privilegeToRemove.getResourcePattern()) { + ActionSet curActions = curPrivilege.getActions(); + + if (!curActions.isSupersetOf(privilegeToRemove.getActions())) { + // Didn't possess all the actions being removed. + return Status(ErrorCodes::PrivilegeNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not contain a privilege on " << + privilegeToRemove.getResourcePattern().toString() << + " with actions: " << + privilegeToRemove.getActions().toString(), + 0); + } + + curPrivilege.removeActions(privilegeToRemove.getActions()); + if (curPrivilege.getActions().empty()) { + currentPrivileges.erase(it); + } + return Status::OK(); + } + } + return Status(ErrorCodes::PrivilegeNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << " does not " + "contain any privileges on " << + privilegeToRemove.getResourcePattern().toString(), + 0); + } + + Status RoleGraph::removePrivilegesFromRole(const RoleName& role, + const PrivilegeVector& privilegesToRemove) { + for (PrivilegeVector::const_iterator it = privilegesToRemove.begin(); + it != privilegesToRemove.end(); ++it) { + Status status = removePrivilegeFromRole(role, *it); + if (!status.isOK()) { + return status; + } + } + return Status::OK(); + } + + Status RoleGraph::removeAllPrivilegesFromRole(const RoleName& role) { + if (!roleExists(role)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << role.getFullName() << + " does not exist", + 0); + } + if (isBuiltinRole(role)) { + return Status( + ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot remove privileges from built-in role: " << + role.getFullName()); + } + _directPrivilegesForRole[role].clear(); + return Status::OK(); + } + + Status RoleGraph::replaceRole(const RoleName& roleName, + const std::vector& roles, + const PrivilegeVector& privileges) { + Status status = removeAllPrivilegesFromRole(roleName); + if (status == ErrorCodes::RoleNotFound) { + fassert(17168, createRole(roleName)); + } + else if (!status.isOK()) { + return status; + } + fassert(17169, removeAllRolesFromRole(roleName)); + for (size_t i = 0; i < roles.size(); ++i) { + const RoleName& grantedRole = roles[i]; + status = createRole(grantedRole); + fassert(17170, status.isOK() || status == ErrorCodes::DuplicateKey); + fassert(17171, addRoleToRole(roleName, grantedRole)); + } + fassert(17172, addPrivilegesToRole(roleName, privileges)); + return Status::OK(); + } + + Status RoleGraph::recomputePrivilegeData() { + /* + * This method is used to recompute the "allPrivileges" vector for each node in the graph, + * as well as look for cycles. It is implemented by performing a depth-first traversal of + * the dependency graph, once for each node. "visitedRoles" tracks the set of role names + * ever visited, and it is used to prune each DFS. A node that has been visited once on any + * DFS is never visited again. Complexity of this implementation is O(n+m) where "n" is the + * number of nodes and "m" is the number of prerequisite edges. Space complexity is O(n), + * in both stack space and size of the "visitedRoles" set. + * + * "inProgressRoles" is used to detect and report cycles, as well as to keep track of roles + * we started visiting before realizing they had children that needed visiting first, so + * we can get back to them after visiting their children. + */ + + unordered_set visitedRoles; + for (EdgeSet::const_iterator it = _roleToSubordinates.begin(); + it != _roleToSubordinates.end(); ++it) { + Status status = _recomputePrivilegeDataHelper(it->first, visitedRoles); + if (!status.isOK()) { + return status; + } + } + return Status::OK(); + } + + Status RoleGraph::_recomputePrivilegeDataHelper(const RoleName& startingRole, + unordered_set& visitedRoles) { + if (visitedRoles.count(startingRole)) { + return Status::OK(); + } + + std::vector inProgressRoles; + inProgressRoles.push_back(startingRole); + while (inProgressRoles.size()) { + const RoleName currentRole = inProgressRoles.back(); + fassert(17277, !visitedRoles.count(currentRole)); + + if (!roleExists(currentRole)) { + return Status(ErrorCodes::RoleNotFound, + mongoutils::str::stream() << "Role: " << currentRole.getFullName() << + " does not exist", + 0); + } + + // Check for cycles + { + const std::vector::const_iterator begin = inProgressRoles.begin(); + // The currentRole will always be last so don't look there. + const std::vector::const_iterator end = --inProgressRoles.end(); + const std::vector::const_iterator firstOccurence = + std::find(begin, end, currentRole); + if (firstOccurence != end) { + std::ostringstream os; + os << "Cycle in dependency graph: "; + for (std::vector::const_iterator it = firstOccurence; + it != end; ++it) { + os << it->getFullName() << " -> "; + } + os << currentRole.getFullName(); + return Status(ErrorCodes::GraphContainsCycle, os.str()); + } + } + + // Make sure we've already visited all subordinate roles before worrying about this one. + const std::vector& currentRoleDirectRoles = _roleToSubordinates[currentRole]; + std::vector::const_iterator roleIt; + for (roleIt = currentRoleDirectRoles.begin(); + roleIt != currentRoleDirectRoles.end(); ++roleIt) { + const RoleName& childRole = *roleIt; + if (!visitedRoles.count(childRole)) { + inProgressRoles.push_back(childRole); + break; + } + } + // If roleIt didn't reach the end of currentRoleDirectRoles that means we found a child + // of currentRole that we haven't visited yet. + if (roleIt != currentRoleDirectRoles.end()) { + continue; + } + // At this point, we know that we've already visited all child roles of currentRole + // and thus their "all privileges" sets are correct and can be added to currentRole's + // "all privileges" set + + // Need to clear out the "all privileges" vector for the current role, and re-fill it + // with just the direct privileges for this role. + PrivilegeVector& currentRoleAllPrivileges = _allPrivilegesForRole[currentRole]; + currentRoleAllPrivileges = _directPrivilegesForRole[currentRole]; + + // Need to do the same thing for the indirect roles + unordered_set& currentRoleIndirectRoles = + _roleToIndirectSubordinates[currentRole]; + currentRoleIndirectRoles.clear(); + for (std::vector::const_iterator it = currentRoleDirectRoles.begin(); + it != currentRoleDirectRoles.end(); ++it) { + currentRoleIndirectRoles.insert(*it); + } + + // Recursively add children's privileges to current role's "all privileges" vector, and + // children's roles to current roles's "indirect roles" vector. + for (std::vector::const_iterator roleIt = currentRoleDirectRoles.begin(); + roleIt != currentRoleDirectRoles.end(); ++roleIt) { + // At this point, we already know that the "all privilege" set for the child is + // correct, so add those privileges to our "all privilege" set. + const RoleName& childRole = *roleIt; + + const PrivilegeVector& childsPrivileges = _allPrivilegesForRole[childRole]; + for (PrivilegeVector::const_iterator privIt = childsPrivileges.begin(); + privIt != childsPrivileges.end(); ++privIt) { + Privilege::addPrivilegeToPrivilegeVector(¤tRoleAllPrivileges, *privIt); + } + + // We also know that the "indirect roles" for the child is also correct, so we can + // add those roles to our "indirect roles" set. + const unordered_set& childsRoles = _roleToIndirectSubordinates[childRole]; + for (unordered_set::const_iterator childsRoleIt = childsRoles.begin(); + childsRoleIt != childsRoles.end(); ++childsRoleIt) { + currentRoleIndirectRoles.insert(*childsRoleIt); + } + } + + visitedRoles.insert(currentRole); + inProgressRoles.pop_back(); + } + return Status::OK(); + } + + RoleNameIterator RoleGraph::getRolesForDatabase(const std::string& dbname) { + _createBuiltinRolesForDBIfNeeded(dbname); + + std::set::const_iterator lower = _allRoles.lower_bound(RoleName("", dbname)); + std::string afterDB = dbname; + afterDB.push_back('\0'); + std::set::const_iterator upper = _allRoles.lower_bound(RoleName("", afterDB)); + return makeRoleNameIterator(lower, upper); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/role_graph.h b/src/mongo/db/auth/role_graph.h new file mode 100644 index 00000000000..af31bd3d931 --- /dev/null +++ b/src/mongo/db/auth/role_graph.h @@ -0,0 +1,313 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/db/namespace_string.h" +#include "mongo/platform/unordered_map.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + /** + * A graph of role and privilege relationships. + * + * This structure is used to store an in-memory representation of the admin.system.roledata + * collection, specifically the graph of which roles are members of other roles and what + * privileges each role has, both directly and transitively through membership in other roles. + * There are some restrictions on calls to getAllPrivileges(), specifically, one must call + * recomputePrivilegeData() before calling getAllPrivileges() if any of the mutation methods + * have been called on the instance since the later of its construction or the last call to + * recomputePrivilegeData() on the object. + */ + class RoleGraph { + public: + /** + * Adds to "privileges" the privileges associated with the named built-in role, and returns + * true. Returns false if "role" does not name a built-in role, and does not modify + * "privileges". Addition of new privileges is done as with + * Privilege::addPrivilegeToPrivilegeVector. + */ + static bool addPrivilegesForBuiltinRole(const RoleName& role, PrivilegeVector* privileges); + + RoleGraph(); + RoleGraph(const RoleGraph& other); + ~RoleGraph(); + + // Built-in roles for backwards compatibility with 2.2 and prior + static const std::string BUILTIN_ROLE_V0_READ; + static const std::string BUILTIN_ROLE_V0_READ_WRITE; + static const std::string BUILTIN_ROLE_V0_ADMIN_READ; + static const std::string BUILTIN_ROLE_V0_ADMIN_READ_WRITE; + + // Swaps the contents of this RoleGraph with those of "other" + void swap(RoleGraph& other); + + /** + * Adds to "privileges" the necessary privileges to do absolutely anything on the system. + */ + static void generateUniversalPrivileges(PrivilegeVector* privileges); + + /** + * Returns an iterator over the RoleNames of the "members" of the given role. + * Members of a role are roles that have been granted this role directly (roles that are + * members transitively through another role are not included). These are the "parents" of + * this node in the graph. + */ + RoleNameIterator getDirectMembers(const RoleName& role); + + /** + * Returns an iterator over the RoleNames of the "subordinates" of the given role. + * Subordinate roles are the roles that this role has been granted directly (roles + * that have been granted transitively through another role are not included). These are + * the "children" of this node in the graph. + */ + RoleNameIterator getDirectSubordinates(const RoleName& role); + + /** + * Returns an iterator that can be used to get a full list of roles that this role inherits + * privileges from. This includes its direct subordinate roles as well as the subordinates + * of its subordinates, and so on. + */ + RoleNameIterator getIndirectSubordinates(const RoleName& role); + + /** + * Returns an iterator that can be used to get a full list of roles (in lexicographical + * order) that are defined on the given database. + */ + RoleNameIterator getRolesForDatabase(const std::string& dbname); + + /** + * Returns a vector of the privileges that the given role has been directly granted. + * Privileges that have been granted transitively through this role's subordinate roles are + * not included. + */ + const PrivilegeVector& getDirectPrivileges(const RoleName& role); + + /** + * Returns a vector of all privileges that the given role contains. This includes both the + * privileges that have been granted to this role directly, as well as any privileges + * inherited from the role's subordinate roles. + */ + const PrivilegeVector& getAllPrivileges(const RoleName& role); + + /** + * Returns whether or not the given role exists in the role graph. Will implicitly + * add the role to the graph if it is a built-in role and isn't already in the graph. + */ + bool roleExists(const RoleName& role); + + /** + * Returns whether the given role corresponds to a built-in role. + */ + static bool isBuiltinRole(const RoleName& role); + + // Mutation functions + + /** + * Puts an entry into the RoleGraph for the given RoleName. + * Returns DuplicateKey if the role already exists. + */ + Status createRole(const RoleName& role); + + /** + * Deletes the given role by first removing it from the members/subordinates arrays for + * all other roles, and then by removing its own entries in the 4 member maps. + * Returns RoleNotFound if the role doesn't exist. + * Returns InvalidRoleModification if "role" is a built-in role. + */ + Status deleteRole(const RoleName& role); + + /** + * Grants "role" to "recipient". This leaves "recipient" as a member of "role" and "role" + * as a subordinate of "recipient". + * Returns RoleNotFound if either of "role" or "recipient" doesn't exist in + * the RoleGraph. + * Returns InvalidRoleModification if "recipient" is a built-in role. + */ + Status addRoleToRole(const RoleName& recipient, const RoleName& role); + + /** + * Revokes "role" from "recipient". + * Returns RoleNotFound if either of "role" or "recipient" doesn't exist in + * the RoleGraph. Returns RolesNotRelated if "recipient" is not currently a + * member of "role". + * Returns InvalidRoleModification if "role" is a built-in role. + */ + Status removeRoleFromRole(const RoleName& recipient, const RoleName& role); + + /** + * Removes all roles held by "victim". + * Returns RoleNotFound if "victim" doesn't exist in the role graph. + * Returns InvalidRoleModification if "victim" is a built-in role. + */ + Status removeAllRolesFromRole(const RoleName& victim); + + /** + * Grants "privilegeToAdd" to "role". + * Returns RoleNotFound if "role" doesn't exist in the role graph. + * Returns InvalidRoleModification if "role" is a built-in role. + */ + Status addPrivilegeToRole(const RoleName& role, const Privilege& privilegeToAdd); + + /** + * Grants Privileges from "privilegesToAdd" to "role". + * Returns RoleNotFound if "role" doesn't exist in the role graph. + * Returns InvalidRoleModification if "role" is a built-in role. + */ + Status addPrivilegesToRole(const RoleName& role, const PrivilegeVector& privilegesToAdd); + + /** + * Removes "privilegeToRemove" from "role". + * Returns RoleNotFound if "role" doesn't exist in the role graph. + * Returns PrivilegeNotFound if "role" doesn't contain the full privilege being removed. + * Returns InvalidRoleModification if "role" is a built-in role. + */ + Status removePrivilegeFromRole(const RoleName& role, + const Privilege& privilegeToRemove); + + /** + * Removes all privileges in the "privilegesToRemove" vector from "role". + * Returns RoleNotFound if "role" doesn't exist in the role graph. + * Returns InvalidRoleModification if "role" is a built-in role. + * Returns PrivilegeNotFound if "role" is missing any of the privileges being removed. If + * PrivilegeNotFound is returned then the graph may be in an inconsistent state and needs to + * be abandoned. + */ + Status removePrivilegesFromRole(const RoleName& role, + const PrivilegeVector& privilegesToRemove); + + /** + * Removes all privileges from "role". + * Returns RoleNotFound if "role" doesn't exist in the role graph. + * Returns InvalidRoleModification if "role" is a built-in role. + */ + Status removeAllPrivilegesFromRole(const RoleName& role); + + /** + * Updates the RoleGraph by adding the role named "roleName", with the given role + * memberships and privileges. If the name "roleName" already exists, it is replaced. Any + * subordinate roles mentioned in role.roles are created, if needed, with empty privilege + * and subordinate role lists. + * + * Should _only_ fail if the role to replace is a builtin role, in which + * case it will return ErrorCodes::InvalidRoleModification. + */ + Status replaceRole(const RoleName& roleName, + const std::vector& roles, + const PrivilegeVector& privileges); + + /** + * Adds the role described in "doc" the role graph. + */ + Status addRoleFromDocument(const BSONObj& doc); + + /** + * Applies to the RoleGraph the oplog operation described by the parameters. + * + * Returns Status::OK() on success, ErrorCodes::OplogOperationUnsupported if the oplog + * operation is not supported, and other codes (typically BadValue) if the oplog operation + * is ill-described. + */ + Status handleLogOp( + const char* op, + const NamespaceString& ns, + const BSONObj& o, + const BSONObj* o2); + + /** + * Recomputes the indirect (getAllPrivileges) data for this graph. + * + * Must be called between calls to any of the mutation functions and calls + * to getAllPrivileges(). + * + * Returns Status::OK() on success. If a cycle is detected, returns + * ErrorCodes::GraphContainsCycle, and the status message reveals the cycle. + */ + Status recomputePrivilegeData(); + + private: + // Helper method doing a topological DFS to compute the indirect privilege + // data and look for cycles + Status _recomputePrivilegeDataHelper(const RoleName& currentRole, + unordered_set& visitedRoles); + + /** + * If the role name given is not a built-in role, or it is but it's already in the role + * graph, then this does nothing. If it *is* a built-in role and this is the first time + * this function has been called for this role, it will add the role into the role graph. + */ + void _createBuiltinRoleIfNeeded(const RoleName& role); + + /** + * Adds the built-in roles for the given database name to the role graph if they aren't + * already present. + */ + void _createBuiltinRolesForDBIfNeeded(const std::string& dbname); + + /** + * Returns whether or not the given role exists strictly within the role graph. + */ + bool _roleExistsDontCreateBuiltin(const RoleName& role); + + /** + * Just creates the role in the role graph, without checking whether or not the role already + * exists. + */ + void _createRoleDontCheckIfRoleExists(const RoleName& role); + + /** + * Grants "privilegeToAdd" to "role". + * Doesn't do any checking as to whether the role exists or is a built-in role. + */ + void _addPrivilegeToRoleNoChecks(const RoleName& role, const Privilege& privilegeToAdd); + + + // Represents all the outgoing edges to other roles from any given role. + typedef unordered_map > EdgeSet; + // Maps a role name to a list of privileges associated with that role. + typedef unordered_map RolePrivilegeMap; + + EdgeSet _roleToSubordinates; + unordered_map > _roleToIndirectSubordinates; + EdgeSet _roleToMembers; + RolePrivilegeMap _directPrivilegesForRole; + RolePrivilegeMap _allPrivilegesForRole; + set _allRoles; + }; + + void swap(RoleGraph& lhs, RoleGraph& rhs); + +} // namespace mongo diff --git a/src/mongo/db/auth/role_graph_builtin_roles.cpp b/src/mongo/db/auth/role_graph_builtin_roles.cpp new file mode 100644 index 00000000000..d557b31b86a --- /dev/null +++ b/src/mongo/db/auth/role_graph_builtin_roles.cpp @@ -0,0 +1,789 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/auth/role_graph.h" + +#include "mongo/base/init.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/role_name.h" + +namespace mongo { + + const std::string RoleGraph::BUILTIN_ROLE_V0_READ = "read"; + const std::string RoleGraph::BUILTIN_ROLE_V0_READ_WRITE= "dbOwner"; + const std::string RoleGraph::BUILTIN_ROLE_V0_ADMIN_READ = "readAnyDatabase"; + const std::string RoleGraph::BUILTIN_ROLE_V0_ADMIN_READ_WRITE= "root"; + +namespace { + const std::string ADMIN_DBNAME = "admin"; + + const std::string BUILTIN_ROLE_READ = "read"; + const std::string BUILTIN_ROLE_READ_WRITE = "readWrite"; + const std::string BUILTIN_ROLE_USER_ADMIN = "userAdmin"; + const std::string BUILTIN_ROLE_DB_ADMIN = "dbAdmin"; + const std::string BUILTIN_ROLE_CLUSTER_ADMIN = "clusterAdmin"; + const std::string BUILTIN_ROLE_READ_ANY_DB = "readAnyDatabase"; + const std::string BUILTIN_ROLE_READ_WRITE_ANY_DB = "readWriteAnyDatabase"; + const std::string BUILTIN_ROLE_USER_ADMIN_ANY_DB = "userAdminAnyDatabase"; + const std::string BUILTIN_ROLE_DB_ADMIN_ANY_DB = "dbAdminAnyDatabase"; + const std::string BUILTIN_ROLE_ROOT = "root"; + const std::string BUILTIN_ROLE_INTERNAL = "__system"; + const std::string BUILTIN_ROLE_DB_OWNER = "dbOwner"; + const std::string BUILTIN_ROLE_CLUSTER_MONITOR = "clusterMonitor"; + const std::string BUILTIN_ROLE_HOST_MANAGEMENT = "hostManager"; + const std::string BUILTIN_ROLE_CLUSTER_MANAGEMENT = "clusterManager"; + const std::string BUILTIN_ROLE_BACKUP = "backup"; + const std::string BUILTIN_ROLE_RESTORE = "restore"; + + /// Actions that the "read" role may perform on a normal resources of a specific database, and + /// that the "readAnyDatabase" role may perform on normal resources of any database. + ActionSet readRoleActions; + + /// Actions that the "readWrite" role may perform on a normal resources of a specific database, + /// and that the "readWriteAnyDatabase" role may perform on normal resources of any database. + ActionSet readWriteRoleActions; + + /// Actions that the "userAdmin" role may perform on normal resources of a specific database, + /// and that the "userAdminAnyDatabase" role may perform on normal resources of any database. + ActionSet userAdminRoleActions; + + /// Actions that the "dbAdmin" role may perform on normal resources of a specific database, + // and that the "dbAdminAnyDatabase" role may perform on normal resources of any database. + ActionSet dbAdminRoleActions; + + /// Actions that the "clusterMonitor" role may perform on the cluster resource. + ActionSet clusterMonitorRoleClusterActions; + + /// Actions that the "clusterMonitor" role may perform on any database. + ActionSet clusterMonitorRoleDatabaseActions; + + /// Actions that the "hostManager" role may perform on the cluster resource. + ActionSet hostManagerRoleClusterActions; + + /// Actions that the "hostManager" role may perform on any database. + ActionSet hostManagerRoleDatabaseActions; + + /// Actions that the "clusterManager" role may perform on the cluster resource. + ActionSet clusterManagerRoleClusterActions; + + /// Actions that the "clusterManager" role may perform on any database + ActionSet clusterManagerRoleDatabaseActions; + + ActionSet& operator<<(ActionSet& target, ActionType source) { + target.addAction(source); + return target; + } + + void operator+=(ActionSet& target, const ActionSet& source) { + target.addAllActionsFromSet(source); + } + + // This sets up the built-in role ActionSets. This is what determines what actions each role + // is authorized to perform + MONGO_INITIALIZER(AuthorizationBuiltinRoles)(InitializerContext* context) { + // Read role + readRoleActions + << ActionType::collStats + << ActionType::dbHash + << ActionType::dbStats + << ActionType::find + << ActionType::killCursors + << ActionType::planCacheRead; + + // Read-write role + readWriteRoleActions += readRoleActions; + readWriteRoleActions + << ActionType::convertToCapped // db admin gets this also + << ActionType::createCollection // db admin gets this also + << ActionType::dropCollection + << ActionType::dropIndex + << ActionType::emptycapped + << ActionType::createIndex + << ActionType::insert + << ActionType::remove + << ActionType::renameCollectionSameDB // db admin gets this also + << ActionType::update; + + // User admin role + userAdminRoleActions + << ActionType::changeCustomData + << ActionType::changePassword + << ActionType::createUser + << ActionType::createRole + << ActionType::dropUser + << ActionType::dropRole + << ActionType::grantRole + << ActionType::revokeRole + << ActionType::viewUser + << ActionType::viewRole; + + + // DB admin role + dbAdminRoleActions + << ActionType::collMod + << ActionType::collStats // clusterMonitor gets this also + << ActionType::compact + << ActionType::convertToCapped // read_write gets this also + << ActionType::createCollection // read_write gets this also + << ActionType::dbStats // clusterMonitor gets this also + << ActionType::dropCollection + << ActionType::dropDatabase // clusterAdmin gets this also TODO(spencer): should readWriteAnyDatabase? + << ActionType::dropIndex + << ActionType::createIndex + << ActionType::indexStats + << ActionType::enableProfiler + << ActionType::planCacheIndexFilter + << ActionType::planCacheRead + << ActionType::planCacheWrite + << ActionType::reIndex + << ActionType::renameCollectionSameDB // read_write gets this also + << ActionType::repairDatabase + << ActionType::storageDetails + << ActionType::validate; + + // clusterMonitor role actions that target the cluster resource + clusterMonitorRoleClusterActions + << ActionType::connPoolStats + << ActionType::getCmdLineOpts + << ActionType::getLog + << ActionType::getParameter + << ActionType::getShardMap + << ActionType::hostInfo + << ActionType::listDatabases + << ActionType::listShards // clusterManager gets this also + << ActionType::netstat + << ActionType::replSetGetStatus // clusterManager gets this also + << ActionType::serverStatus + << ActionType::top + << ActionType::cursorInfo + << ActionType::inprog + << ActionType::shardingState; + + // clusterMonitor role actions that target a database (or collection) resource + clusterMonitorRoleDatabaseActions + << ActionType::collStats // dbAdmin gets this also + << ActionType::dbStats // dbAdmin gets this also + << ActionType::getShardVersion; + + // hostManager role actions that target the cluster resource + hostManagerRoleClusterActions + << ActionType::applicationMessage // clusterManager gets this also + << ActionType::connPoolSync + << ActionType::closeAllDatabases + << ActionType::cpuProfiler + << ActionType::logRotate + << ActionType::setParameter + << ActionType::shutdown + << ActionType::touch + << ActionType::unlock + << ActionType::diagLogging + << ActionType::flushRouterConfig // clusterManager gets this also + << ActionType::fsync + << ActionType::invalidateUserCache // userAdminAnyDatabase gets this also + << ActionType::killop + << ActionType::resync; // clusterManager gets this also + + // hostManager role actions that target the database resource + hostManagerRoleDatabaseActions + << ActionType::killCursors + << ActionType::repairDatabase; + + + // clusterManager role actions that target the cluster resource + clusterManagerRoleClusterActions + << ActionType::appendOplogNote // backup gets this also + << ActionType::applicationMessage // hostManager gets this also + << ActionType::replSetConfigure + << ActionType::replSetGetStatus // clusterMonitor gets this also + << ActionType::replSetStateChange + << ActionType::resync // hostManager gets this also + << ActionType::addShard + << ActionType::removeShard + << ActionType::listShards // clusterMonitor gets this also + << ActionType::flushRouterConfig // hostManager gets this also + << ActionType::cleanupOrphaned; + + clusterManagerRoleDatabaseActions + << ActionType::splitChunk + << ActionType::moveChunk + << ActionType::enableSharding + << ActionType::splitVector; + + return Status::OK(); + } + + void addReadOnlyDbPrivileges(PrivilegeVector* privileges, const StringData& dbName) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName(dbName), readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + NamespaceString(dbName, "system.indexes")), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace(NamespaceString(dbName, "system.js")), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + NamespaceString(dbName, "system.namespaces")), + readRoleActions)); + } + + void addReadWriteDbPrivileges(PrivilegeVector* privileges, const StringData& dbName) { + addReadOnlyDbPrivileges(privileges, dbName); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forDatabaseName(dbName), readWriteRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace(NamespaceString(dbName, "system.js")), + readWriteRoleActions)); + } + + void addUserAdminDbPrivileges(PrivilegeVector* privileges, const StringData& dbName) { + privileges->push_back( + Privilege(ResourcePattern::forDatabaseName(dbName), userAdminRoleActions)); + } + + void addDbAdminDbPrivileges(PrivilegeVector* privileges, const StringData& dbName) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forDatabaseName(dbName), dbAdminRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + NamespaceString(dbName, "system.indexes")), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + NamespaceString(dbName, "system.namespaces")), + readRoleActions)); + + ActionSet profileActions = readRoleActions; + profileActions.addAction(ActionType::convertToCapped); + profileActions.addAction(ActionType::createCollection); + profileActions.addAction(ActionType::dropCollection); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + NamespaceString(dbName, "system.profile")), + profileActions)); + } + + void addDbOwnerPrivileges(PrivilegeVector* privileges, const StringData& dbName) { + addReadWriteDbPrivileges(privileges, dbName); + addDbAdminDbPrivileges(privileges, dbName); + addUserAdminDbPrivileges(privileges, dbName); + } + + + void addReadOnlyAnyDbPrivileges(PrivilegeVector* privileges) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), ActionType::listDatabases)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.indexes"), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.js"), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.namespaces"), + readRoleActions)); + } + + void addReadWriteAnyDbPrivileges(PrivilegeVector* privileges) { + addReadOnlyAnyDbPrivileges(privileges); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), readWriteRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.js"), readWriteRoleActions)); + } + + void addUserAdminAnyDbPrivileges(PrivilegeVector* privileges) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), userAdminRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), ActionType::listDatabases)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), ActionType::authSchemaUpgrade)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), ActionType::invalidateUserCache)); + + + ActionSet readRoleAndIndexActions; + readRoleAndIndexActions += readRoleActions; + readRoleAndIndexActions << ActionType::createIndex << ActionType::dropIndex; + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.users"), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersCollectionNamespace), + readRoleAndIndexActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::rolesCollectionNamespace), + readRoleAndIndexActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::versionCollectionNamespace), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersAltCollectionNamespace), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersBackupCollectionNamespace), + readRoleActions)); + } + + void addDbAdminAnyDbPrivileges(PrivilegeVector* privileges) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), ActionType::listDatabases)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), dbAdminRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.indexes"), + readRoleActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.namespaces"), + readRoleActions)); + ActionSet profileActions = readRoleActions; + profileActions.addAction(ActionType::dropCollection); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.profile"), + profileActions)); + } + + void addClusterMonitorPrivileges(PrivilegeVector* privileges) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), clusterMonitorRoleClusterActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), + clusterMonitorRoleDatabaseActions)); + addReadOnlyDbPrivileges(privileges, "config"); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + NamespaceString("local.system.replset")), + ActionType::find)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.profile"), ActionType::find)); + } + + void addHostManagerPrivileges(PrivilegeVector* privileges) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), hostManagerRoleClusterActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), + hostManagerRoleDatabaseActions)); + } + + void addClusterManagerPrivileges(PrivilegeVector* privileges) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forClusterResource(), clusterManagerRoleClusterActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), + clusterManagerRoleDatabaseActions)); + addReadOnlyDbPrivileges(privileges, "config"); + + ActionSet configSettingsActions; + configSettingsActions << ActionType::insert << ActionType::update << ActionType::remove; + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace(NamespaceString("config", + "settings")), + configSettingsActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace(NamespaceString("local", + "system.replset")), + readRoleActions)); + } + + void addClusterAdminPrivileges(PrivilegeVector* privileges) { + addClusterMonitorPrivileges(privileges); + addHostManagerPrivileges(privileges); + addClusterManagerPrivileges(privileges); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), + ActionType::dropDatabase)); + } + + void addBackupPrivileges(PrivilegeVector* privileges) { + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyResource(), ActionType::collStats)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), ActionType::find)); + + ActionSet clusterActions; + clusterActions << ActionType::getParameter // To check authSchemaVersion + << ActionType::listDatabases + << ActionType::appendOplogNote; // For BRS + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forClusterResource(), clusterActions)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.indexes"), ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.namespaces"), + ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.js"), ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.users"), ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersAltCollectionNamespace), + ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersBackupCollectionNamespace), + ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::rolesCollectionNamespace), + ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::versionCollectionNamespace), + ActionType::find)); + } + + void addRestorePrivileges(PrivilegeVector* privileges) { + ActionSet actions; + actions << ActionType::insert + << ActionType::dropCollection + << ActionType::createIndex + << ActionType::createCollection + << ActionType::collMod; + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), actions)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.js"), actions)); + + // Need to be able to query system.namespaces to check existing collection options. + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.namespaces"), + ActionType::find)); + + // Privileges for user/role management + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forAnyNormalResource(), userAdminRoleActions)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::defaultTempUsersCollectionNamespace), + ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::defaultTempRolesCollectionNamespace), + ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersAltCollectionNamespace), + actions)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::usersBackupCollectionNamespace), + actions)); + + actions << ActionType::find; + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege( + ResourcePattern::forExactNamespace( + AuthorizationManager::versionCollectionNamespace), + actions)); + + // Need additional actions on system.users. + actions << ActionType::update << ActionType::remove; + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forCollectionName("system.users"), actions)); + + // Need to be able to run getParameter to check authSchemaVersion + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forClusterResource(), + ActionType::getParameter)); + + // Need to be able to create an index on the system.roles collection. + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace( + AuthorizationManager::rolesCollectionNamespace), + ActionType::createIndex)); + } + + void addRootRolePrivileges(PrivilegeVector* privileges) { + addClusterAdminPrivileges(privileges); + addUserAdminAnyDbPrivileges(privileges); + addDbAdminAnyDbPrivileges(privileges); + addReadWriteAnyDbPrivileges(privileges); + } + + void addInternalRolePrivileges(PrivilegeVector* privileges) { + RoleGraph::generateUniversalPrivileges(privileges); + } + +} // namespace + + bool RoleGraph::addPrivilegesForBuiltinRole(const RoleName& roleName, + PrivilegeVector* result) { + const bool isAdminDB = (roleName.getDB() == ADMIN_DBNAME); + + if (roleName.getRole() == BUILTIN_ROLE_READ) { + addReadOnlyDbPrivileges(result, roleName.getDB()); + } + else if (roleName.getRole() == BUILTIN_ROLE_READ_WRITE) { + addReadWriteDbPrivileges(result, roleName.getDB()); + } + else if (roleName.getRole() == BUILTIN_ROLE_USER_ADMIN) { + addUserAdminDbPrivileges(result, roleName.getDB()); + } + else if (roleName.getRole() == BUILTIN_ROLE_DB_ADMIN) { + addDbAdminDbPrivileges(result, roleName.getDB()); + } + else if (roleName.getRole() == BUILTIN_ROLE_DB_OWNER) { + addDbOwnerPrivileges(result, roleName.getDB()); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_READ_ANY_DB) { + addReadOnlyAnyDbPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_READ_WRITE_ANY_DB) { + addReadWriteAnyDbPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_USER_ADMIN_ANY_DB) { + addUserAdminAnyDbPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_DB_ADMIN_ANY_DB) { + addDbAdminAnyDbPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_CLUSTER_MONITOR) { + addClusterMonitorPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_HOST_MANAGEMENT) { + addHostManagerPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_CLUSTER_MANAGEMENT) { + addClusterManagerPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_CLUSTER_ADMIN) { + addClusterAdminPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_BACKUP) { + addBackupPrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_RESTORE) { + addRestorePrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_ROOT) { + addRootRolePrivileges(result); + } + else if (isAdminDB && roleName.getRole() == BUILTIN_ROLE_INTERNAL) { + addInternalRolePrivileges(result); + } + else { + return false; + } + return true; + } + + void RoleGraph::generateUniversalPrivileges(PrivilegeVector* privileges) { + ActionSet allActions; + allActions.addAllActions(); + privileges->push_back(Privilege(ResourcePattern::forAnyResource(), allActions)); + } + + bool RoleGraph::isBuiltinRole(const RoleName& role) { + if (!NamespaceString::validDBName(role.getDB()) || role.getDB() == "$external") { + return false; + } + + bool isAdminDB = role.getDB() == ADMIN_DBNAME; + + if (role.getRole() == BUILTIN_ROLE_READ) { + return true; + } + else if (role.getRole() == BUILTIN_ROLE_READ_WRITE) { + return true; + } + else if (role.getRole() == BUILTIN_ROLE_USER_ADMIN) { + return true; + } + else if (role.getRole() == BUILTIN_ROLE_DB_ADMIN) { + return true; + } + else if (role.getRole() == BUILTIN_ROLE_DB_OWNER) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_READ_ANY_DB) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_READ_WRITE_ANY_DB) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_USER_ADMIN_ANY_DB) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_DB_ADMIN_ANY_DB) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_CLUSTER_MONITOR) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_HOST_MANAGEMENT) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_CLUSTER_MANAGEMENT) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_CLUSTER_ADMIN) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_BACKUP) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_RESTORE) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_ROOT) { + return true; + } + else if (isAdminDB && role.getRole() == BUILTIN_ROLE_INTERNAL) { + return true; + } + return false; + } + + void RoleGraph::_createBuiltinRolesForDBIfNeeded(const std::string& dbname) { + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_READ, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_READ_WRITE, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_USER_ADMIN, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_DB_ADMIN, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_DB_OWNER, dbname)); + + if (dbname == "admin") { + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_READ_ANY_DB, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_READ_WRITE_ANY_DB, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_USER_ADMIN_ANY_DB, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_DB_ADMIN_ANY_DB, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_CLUSTER_MONITOR, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_HOST_MANAGEMENT, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_CLUSTER_MANAGEMENT, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_CLUSTER_ADMIN, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_BACKUP, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_RESTORE, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_ROOT, dbname)); + _createBuiltinRoleIfNeeded(RoleName(BUILTIN_ROLE_INTERNAL, dbname)); + } + } + + void RoleGraph::_createBuiltinRoleIfNeeded(const RoleName& role) { + if (!isBuiltinRole(role) || _roleExistsDontCreateBuiltin(role)) { + return; + } + + _createRoleDontCheckIfRoleExists(role); + PrivilegeVector privileges; + fassert(17145, addPrivilegesForBuiltinRole(role, &privileges)); + for (size_t i = 0; i < privileges.size(); ++i) { + _addPrivilegeToRoleNoChecks(role, privileges[i]); + _allPrivilegesForRole[role].push_back(privileges[i]); + } + } + +} // namespace mongo diff --git a/src/mongo/db/auth/role_graph_test.cpp b/src/mongo/db/auth/role_graph_test.cpp new file mode 100644 index 00000000000..4e99f584276 --- /dev/null +++ b/src/mongo/db/auth/role_graph_test.cpp @@ -0,0 +1,732 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * Unit tests of the RoleGraph type. + */ + +#include + +#include "mongo/db/auth/role_graph.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/sequence_util.h" + +namespace mongo { +namespace { + + // Tests adding and removing roles from other roles, the RoleNameIterator, and the + // getDirectMembers and getDirectSubordinates methods + TEST(RoleGraphTest, AddRemoveRoles) { + RoleName roleA("roleA", "dbA"); + RoleName roleB("roleB", "dbB"); + RoleName roleC("roleC", "dbC"); + RoleName roleD("readWrite", "dbD"); // built-in role + + RoleGraph graph; + ASSERT_OK(graph.createRole(roleA)); + ASSERT_OK(graph.createRole(roleB)); + ASSERT_OK(graph.createRole(roleC)); + + RoleNameIterator it; + it = graph.getDirectSubordinates(roleA); + ASSERT_FALSE(it.more()); + it = graph.getDirectMembers(roleA); + ASSERT_FALSE(it.more()); + + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + + // A -> B + it = graph.getDirectSubordinates(roleA); + ASSERT_TRUE(it.more()); + // should not advance the iterator + ASSERT_EQUALS(it.get().getFullName(), roleB.getFullName()); + ASSERT_EQUALS(it.get().getFullName(), roleB.getFullName()); + ASSERT_EQUALS(it.next().getFullName(), roleB.getFullName()); + ASSERT_FALSE(it.more()); + + it = graph.getDirectMembers(roleA); + ASSERT_FALSE(it.more()); + + it = graph.getDirectMembers(roleB); + ASSERT_EQUALS(it.next().getFullName(), roleA.getFullName()); + ASSERT_FALSE(it.more()); + + it = graph.getDirectSubordinates(roleB); + ASSERT_FALSE(it.more()); + + ASSERT_OK(graph.addRoleToRole(roleA, roleC)); + ASSERT_OK(graph.addRoleToRole(roleB, roleC)); + ASSERT_OK(graph.addRoleToRole(roleB, roleD)); + // Adding the same role twice should be a no-op, duplicate roles should be de-duped. + ASSERT_OK(graph.addRoleToRole(roleB, roleD)); + + /* + * Graph now looks like: + * A + * / \ + * v v + * B -> C + * | + * v + * D + */ + + + it = graph.getDirectSubordinates(roleA); // should be roleB and roleC, order doesn't matter + RoleName cur = it.next(); + if (cur == roleB) { + ASSERT_EQUALS(it.next().getFullName(), roleC.getFullName()); + } else if (cur == roleC) { + ASSERT_EQUALS(it.next().getFullName(), roleB.getFullName()); + } else { + FAIL(mongoutils::str::stream() << "unexpected role returned: " << cur.getFullName()); + } + ASSERT_FALSE(it.more()); + + ASSERT_OK(graph.recomputePrivilegeData()); + it = graph.getIndirectSubordinates(roleA); // should have roleB, roleC and roleD + bool hasB = false; + bool hasC = false; + bool hasD = false; + int num = 0; + while (it.more()) { + ++num; + RoleName cur = it.next(); + if (cur == roleB) { + hasB = true; + } else if (cur == roleC) { + hasC = true; + } else if (cur == roleD) { + hasD = true; + } else { + FAIL(mongoutils::str::stream() << "unexpected role returned: " << + cur.getFullName()); + } + } + ASSERT_EQUALS(3, num); + ASSERT(hasB); + ASSERT(hasC); + ASSERT(hasD); + + it = graph.getDirectSubordinates(roleB); // should be roleC and roleD, order doesn't matter + cur = it.next(); + if (cur == roleC) { + ASSERT_EQUALS(it.next().getFullName(), roleD.getFullName()); + } else if (cur == roleD) { + ASSERT_EQUALS(it.next().getFullName(), roleC.getFullName()); + } else { + FAIL(mongoutils::str::stream() << "unexpected role returned: " << cur.getFullName()); + } + ASSERT_FALSE(it.more()); + + it = graph.getDirectSubordinates(roleC); + ASSERT_FALSE(it.more()); + + it = graph.getDirectMembers(roleA); + ASSERT_FALSE(it.more()); + + it = graph.getDirectMembers(roleB); + ASSERT_EQUALS(it.next().getFullName(), roleA.getFullName()); + ASSERT_FALSE(it.more()); + + it = graph.getDirectMembers(roleC); // should be role A and role B, order doesn't matter + cur = it.next(); + if (cur == roleA) { + ASSERT_EQUALS(it.next().getFullName(), roleB.getFullName()); + } else if (cur == roleB) { + ASSERT_EQUALS(it.next().getFullName(), roleA.getFullName()); + } else { + FAIL(mongoutils::str::stream() << "unexpected role returned: " << cur.getFullName()); + } + ASSERT_FALSE(it.more()); + + // Now remove roleD from roleB and make sure graph is update correctly + ASSERT_OK(graph.removeRoleFromRole(roleB, roleD)); + + /* + * Graph now looks like: + * A + * / \ + * v v + * B -> C + */ + it = graph.getDirectSubordinates(roleB); // should be just roleC + ASSERT_EQUALS(it.next().getFullName(), roleC.getFullName()); + ASSERT_FALSE(it.more()); + + it = graph.getDirectSubordinates(roleD); // should be empty + ASSERT_FALSE(it.more()); + + + // Now delete roleB entirely and make sure that the other roles are updated properly + ASSERT_OK(graph.deleteRole(roleB)); + ASSERT_NOT_OK(graph.deleteRole(roleB)); + it = graph.getDirectSubordinates(roleA); + ASSERT_EQUALS(it.next().getFullName(), roleC.getFullName()); + ASSERT_FALSE(it.more()); + it = graph.getDirectMembers(roleC); + ASSERT_EQUALS(it.next().getFullName(), roleA.getFullName()); + ASSERT_FALSE(it.more()); + } + + const ResourcePattern collectionAFooResource(ResourcePattern::forExactNamespace( + NamespaceString("dbA.foo"))); + const ResourcePattern db1Resource(ResourcePattern::forDatabaseName("db1")); + const ResourcePattern db2Resource(ResourcePattern::forDatabaseName("db2")); + const ResourcePattern dbAResource(ResourcePattern::forDatabaseName("dbA")); + const ResourcePattern dbBResource(ResourcePattern::forDatabaseName("dbB")); + const ResourcePattern dbCResource(ResourcePattern::forDatabaseName("dbC")); + const ResourcePattern dbDResource(ResourcePattern::forDatabaseName("dbD")); + const ResourcePattern dbResource(ResourcePattern::forDatabaseName("db")); + + // Tests that adding multiple privileges on the same resource correctly collapses those to one + // privilege + TEST(RoleGraphTest, AddPrivileges) { + RoleName roleA("roleA", "dbA"); + + RoleGraph graph; + ASSERT_OK(graph.createRole(roleA)); + + // Test adding a single privilege + ActionSet actions; + actions.addAction(ActionType::find); + ASSERT_OK(graph.addPrivilegeToRole(roleA, Privilege(dbAResource, actions))); + + PrivilegeVector privileges = graph.getDirectPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(actions.toString(), privileges[0].getActions().toString()); + + // Add a privilege on a different resource + ASSERT_OK(graph.addPrivilegeToRole(roleA, Privilege(collectionAFooResource, actions))); + privileges = graph.getDirectPrivileges(roleA); + ASSERT_EQUALS(static_cast(2), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(actions.toString(), privileges[0].getActions().toString()); + ASSERT_EQUALS(collectionAFooResource, privileges[1].getResourcePattern()); + ASSERT_EQUALS(actions.toString(), privileges[1].getActions().toString()); + + + // Add different privileges on an existing resource and make sure they get de-duped + actions.removeAllActions(); + actions.addAction(ActionType::insert); + + PrivilegeVector privilegesToAdd; + privilegesToAdd.push_back(Privilege(dbAResource, actions)); + + actions.removeAllActions(); + actions.addAction(ActionType::update); + privilegesToAdd.push_back(Privilege(dbAResource, actions)); + + ASSERT_OK(graph.addPrivilegesToRole(roleA, privilegesToAdd)); + + privileges = graph.getDirectPrivileges(roleA); + ASSERT_EQUALS(static_cast(2), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_NOT_EQUALS(actions.toString(), privileges[0].getActions().toString()); + actions.addAction(ActionType::find); + actions.addAction(ActionType::insert); + ASSERT_EQUALS(actions.toString(), privileges[0].getActions().toString()); + actions.removeAction(ActionType::insert); + actions.removeAction(ActionType::update); + ASSERT_EQUALS(collectionAFooResource, privileges[1].getResourcePattern()); + ASSERT_EQUALS(actions.toString(), privileges[1].getActions().toString()); + } + + // Tests that recomputePrivilegeData correctly detects cycles in the graph. + TEST(RoleGraphTest, DetectCycles) { + RoleName roleA("roleA", "dbA"); + RoleName roleB("roleB", "dbB"); + RoleName roleC("roleC", "dbC"); + RoleName roleD("roleD", "dbD"); + + RoleGraph graph; + ASSERT_OK(graph.createRole(roleA)); + ASSERT_OK(graph.createRole(roleB)); + ASSERT_OK(graph.createRole(roleC)); + ASSERT_OK(graph.createRole(roleD)); + + // Add a role to itself + ASSERT_OK(graph.recomputePrivilegeData()); + ASSERT_OK(graph.addRoleToRole(roleA, roleA)); + ASSERT_NOT_OK(graph.recomputePrivilegeData()); + ASSERT_OK(graph.removeRoleFromRole(roleA, roleA)); + ASSERT_OK(graph.recomputePrivilegeData()); + + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_OK(graph.recomputePrivilegeData()); + ASSERT_OK(graph.addRoleToRole(roleA, roleC)); + ASSERT_OK(graph.addRoleToRole(roleB, roleC)); + ASSERT_OK(graph.recomputePrivilegeData()); + /* + * Graph now looks like: + * A + * / \ + * v v + * B -> C + */ + ASSERT_OK(graph.addRoleToRole(roleC, roleD)); + ASSERT_OK(graph.addRoleToRole(roleD, roleB)); // Add a cycle + /* + * Graph now looks like: + * A + * / \ + * v v + * B -> C + * ^ / + * \ v + * D + */ + ASSERT_NOT_OK(graph.recomputePrivilegeData()); + ASSERT_OK(graph.removeRoleFromRole(roleD, roleB)); + ASSERT_OK(graph.recomputePrivilegeData()); + } + + // Tests that recomputePrivilegeData correctly updates transitive privilege data for all roles. + TEST(RoleGraphTest, RecomputePrivilegeData) { + // We create 4 roles and give each of them a unique privilege. After that the direct + // privileges for all the roles are not touched. The only thing that is changed is the + // role membership graph, and we test how that affects the set of all transitive privileges + // for each role. + RoleName roleA("roleA", "dbA"); + RoleName roleB("roleB", "dbB"); + RoleName roleC("roleC", "dbC"); + RoleName roleD("readWrite", "dbD"); // built-in role + + ActionSet actions; + actions.addAllActions(); + + RoleGraph graph; + ASSERT_OK(graph.createRole(roleA)); + ASSERT_OK(graph.createRole(roleB)); + ASSERT_OK(graph.createRole(roleC)); + + ASSERT_OK(graph.addPrivilegeToRole(roleA, Privilege(dbAResource, actions))); + ASSERT_OK(graph.addPrivilegeToRole(roleB, Privilege(dbBResource, actions))); + ASSERT_OK(graph.addPrivilegeToRole(roleC, Privilege(dbCResource, actions))); + + ASSERT_OK(graph.recomputePrivilegeData()); + + PrivilegeVector privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + + // At this point we have all 4 roles set up, each with their own privilege, but no + // roles have been granted to each other. + + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + // Role graph: A->B + ASSERT_OK(graph.recomputePrivilegeData()); + privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(2), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(dbBResource, privileges[1].getResourcePattern()); + + // Add's roleC's privileges to roleB and make sure roleA gets them as well. + ASSERT_OK(graph.addRoleToRole(roleB, roleC)); + // Role graph: A->B->C + ASSERT_OK(graph.recomputePrivilegeData()); + privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(3), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(dbBResource, privileges[1].getResourcePattern()); + ASSERT_EQUALS(dbCResource, privileges[2].getResourcePattern()); + privileges = graph.getAllPrivileges(roleB); + ASSERT_EQUALS(static_cast(2), privileges.size()); + ASSERT_EQUALS(dbBResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(dbCResource, privileges[1].getResourcePattern()); + + // Add's roleD's privileges to roleC and make sure that roleA and roleB get them as well. + ASSERT_OK(graph.addRoleToRole(roleC, roleD)); + // Role graph: A->B->C->D + ASSERT_OK(graph.recomputePrivilegeData()); + privileges = graph.getAllPrivileges(roleA); + const size_t readWriteRolePrivilegeCount = graph.getAllPrivileges(roleD).size(); + ASSERT_EQUALS(readWriteRolePrivilegeCount + 3, privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(dbBResource, privileges[1].getResourcePattern()); + ASSERT_EQUALS(dbCResource, privileges[2].getResourcePattern()); + privileges = graph.getAllPrivileges(roleB); + ASSERT_EQUALS(readWriteRolePrivilegeCount + 2, privileges.size()); + ASSERT_EQUALS(dbBResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(dbCResource, privileges[1].getResourcePattern()); + privileges = graph.getAllPrivileges(roleC); + ASSERT_EQUALS(readWriteRolePrivilegeCount + 1, privileges.size()); + ASSERT_EQUALS(dbCResource, privileges[0].getResourcePattern()); + + // Remove roleC from roleB, make sure that roleA then loses both roleC's and roleD's + // privileges + ASSERT_OK(graph.removeRoleFromRole(roleB, roleC)); + // Role graph: A->B C->D + ASSERT_OK(graph.recomputePrivilegeData()); + privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(2), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(dbBResource, privileges[1].getResourcePattern()); + privileges = graph.getAllPrivileges(roleB); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_EQUALS(dbBResource, privileges[0].getResourcePattern()); + privileges = graph.getAllPrivileges(roleC); + ASSERT_EQUALS(readWriteRolePrivilegeCount + 1, privileges.size()); + ASSERT_EQUALS(dbCResource, privileges[0].getResourcePattern()); + privileges = graph.getAllPrivileges(roleD); + ASSERT_EQUALS(readWriteRolePrivilegeCount, privileges.size()); + + // Make sure direct privileges were untouched + privileges = graph.getDirectPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + privileges = graph.getDirectPrivileges(roleB); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_EQUALS(dbBResource, privileges[0].getResourcePattern()); + privileges = graph.getDirectPrivileges(roleC); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_EQUALS(dbCResource, privileges[0].getResourcePattern()); + privileges = graph.getDirectPrivileges(roleD); + ASSERT_EQUALS(readWriteRolePrivilegeCount, privileges.size()); + } + + // Test that if you grant 1 role to another, then remove it and change it's privileges, then + // re-grant it, the receiving role sees the new privileges and not the old ones. + TEST(RoleGraphTest, ReAddRole) { + RoleName roleA("roleA", "dbA"); + RoleName roleB("roleB", "dbB"); + RoleName roleC("roleC", "dbC"); + + ActionSet actionsA, actionsB, actionsC; + actionsA.addAction(ActionType::find); + actionsB.addAction(ActionType::insert); + actionsC.addAction(ActionType::update); + + RoleGraph graph; + ASSERT_OK(graph.createRole(roleA)); + ASSERT_OK(graph.createRole(roleB)); + ASSERT_OK(graph.createRole(roleC)); + + ASSERT_OK(graph.addPrivilegeToRole(roleA, Privilege(dbResource, actionsA))); + ASSERT_OK(graph.addPrivilegeToRole(roleB, Privilege(dbResource, actionsB))); + ASSERT_OK(graph.addPrivilegeToRole(roleC, Privilege(dbResource, actionsC))); + + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_OK(graph.addRoleToRole(roleB, roleC)); // graph: A <- B <- C + + ASSERT_OK(graph.recomputePrivilegeData()); + + // roleA should have privileges from roleB and roleC + PrivilegeVector privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::find)); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::insert)); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::update)); + + // Now remove roleB from roleA. B still is a member of C, but A no longer should have + // privileges from B or C. + ASSERT_OK(graph.removeRoleFromRole(roleA, roleB)); + ASSERT_OK(graph.recomputePrivilegeData()); + + // roleA should no longer have the privileges from roleB or roleC + privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::find)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::insert)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::update)); + + // Change the privileges that roleB grants + ASSERT_OK(graph.removeAllPrivilegesFromRole(roleB)); + ActionSet newActionsB; + newActionsB.addAction(ActionType::remove); + ASSERT_OK(graph.addPrivilegeToRole(roleB, Privilege(dbResource, newActionsB))); + + // Grant roleB back to roleA, make sure roleA has roleB's new privilege but not its old one. + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_OK(graph.recomputePrivilegeData()); + + privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::find)); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::update)); // should get roleC's actions again + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::remove)); // roleB should grant this to roleA + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::insert)); // no roles have this action anymore + + // Now delete roleB completely. A should once again lose the privileges from both B and C. + ASSERT_OK(graph.deleteRole(roleB)); + ASSERT_OK(graph.recomputePrivilegeData()); + + // roleA should no longer have the privileges from roleB or roleC + privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::find)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::update)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::remove)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::insert)); + + // Now re-create roleB and give it a new privilege, then grant it back to roleA. + // RoleA should get its new privilege but not roleC's privilege this time nor either of + // roleB's old privileges. + ASSERT_OK(graph.createRole(roleB)); + actionsB.removeAllActions(); + actionsB.addAction(ActionType::shutdown); + ASSERT_OK(graph.addPrivilegeToRole(roleB, Privilege(dbResource, actionsB))); + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_OK(graph.recomputePrivilegeData()); + + privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(1), privileges.size()); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::find)); + ASSERT_TRUE(privileges[0].getActions().contains(ActionType::shutdown)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::update)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::remove)); + ASSERT_FALSE(privileges[0].getActions().contains(ActionType::insert)); + } + + // Tests copy constructor and swap functionality. + TEST(RoleGraphTest, CopySwap) { + RoleName roleA("roleA", "dbA"); + RoleName roleB("roleB", "dbB"); + RoleName roleC("roleC", "dbC"); + + RoleGraph graph; + ASSERT_OK(graph.createRole(roleA)); + ASSERT_OK(graph.createRole(roleB)); + ASSERT_OK(graph.createRole(roleC)); + + ActionSet actions; + actions.addAction(ActionType::find); + ASSERT_OK(graph.addPrivilegeToRole(roleA, Privilege(dbAResource, actions))); + ASSERT_OK(graph.addPrivilegeToRole(roleB, Privilege(dbBResource, actions))); + ASSERT_OK(graph.addPrivilegeToRole(roleC, Privilege(dbCResource, actions))); + + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + + // Make a copy of the graph to do further modifications on. + RoleGraph tempGraph(graph); + ASSERT_OK(tempGraph.addRoleToRole(roleB, roleC)); + tempGraph.recomputePrivilegeData(); + + // Now swap the copy back with the original graph and make sure the original was updated + // properly. + swap(tempGraph, graph); + + RoleNameIterator it = graph.getDirectSubordinates(roleB); + ASSERT_TRUE(it.more()); + ASSERT_EQUALS(it.next().getFullName(), roleC.getFullName()); + ASSERT_FALSE(it.more()); + + graph.getAllPrivileges(roleA); // should have privileges from roleB *and* role C + PrivilegeVector privileges = graph.getAllPrivileges(roleA); + ASSERT_EQUALS(static_cast(3), privileges.size()); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + ASSERT_EQUALS(dbBResource, privileges[1].getResourcePattern()); + ASSERT_EQUALS(dbCResource, privileges[2].getResourcePattern()); + } + + // Tests error handling + TEST(RoleGraphTest, ErrorHandling) { + RoleName roleA("roleA", "dbA"); + RoleName roleB("roleB", "dbB"); + RoleName roleC("roleC", "dbC"); + + ActionSet actions; + actions.addAction(ActionType::find); + Privilege privilege1(db1Resource, actions); + Privilege privilege2(db2Resource, actions); + PrivilegeVector privileges; + privileges.push_back(privilege1); + privileges.push_back(privilege2); + + RoleGraph graph; + // None of the roles exist yet. + ASSERT_NOT_OK(graph.addPrivilegeToRole(roleA, privilege1)); + ASSERT_NOT_OK(graph.addPrivilegesToRole(roleA, privileges)); + ASSERT_NOT_OK(graph.removePrivilegeFromRole(roleA, privilege1)); + ASSERT_NOT_OK(graph.removePrivilegesFromRole(roleA, privileges)); + ASSERT_NOT_OK(graph.removeAllPrivilegesFromRole(roleA)); + ASSERT_NOT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_NOT_OK(graph.removeRoleFromRole(roleA, roleB)); + + // One of the roles exists + ASSERT_OK(graph.createRole(roleA)); + ASSERT_NOT_OK(graph.createRole(roleA)); // Can't create same role twice + ASSERT_NOT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_NOT_OK(graph.addRoleToRole(roleB, roleA)); + ASSERT_NOT_OK(graph.removeRoleFromRole(roleA, roleB)); + ASSERT_NOT_OK(graph.removeRoleFromRole(roleB, roleA)); + + // Should work now that both exist. + ASSERT_OK(graph.createRole(roleB)); + ASSERT_OK(graph.addRoleToRole(roleA, roleB)); + ASSERT_OK(graph.removeRoleFromRole(roleA, roleB)); + ASSERT_NOT_OK(graph.removeRoleFromRole(roleA, roleB)); // roleA isn't actually a member of roleB + + // Can't remove a privilege from a role that doesn't have it. + ASSERT_NOT_OK(graph.removePrivilegeFromRole(roleA, privilege1)); + ASSERT_OK(graph.addPrivilegeToRole(roleA, privilege1)); + ASSERT_OK(graph.removePrivilegeFromRole(roleA, privilege1)); // now should work + + // Test that removing a vector of privileges fails if *any* of the privileges are missing. + ASSERT_OK(graph.addPrivilegeToRole(roleA, privilege1)); + ASSERT_OK(graph.addPrivilegeToRole(roleA, privilege2)); + // Removing both privileges should work since it has both + ASSERT_OK(graph.removePrivilegesFromRole(roleA, privileges)); + // Now add only 1 back and this time removing both should fail + ASSERT_OK(graph.addPrivilegeToRole(roleA, privilege1)); + ASSERT_NOT_OK(graph.removePrivilegesFromRole(roleA, privileges)); + } + + + TEST(RoleGraphTest, BuiltinRoles) { + RoleName userRole("userDefined", "dbA"); + RoleName builtinRole("read", "dbA"); + + ActionSet actions; + actions.addAction(ActionType::insert); + Privilege privilege(dbAResource, actions); + + RoleGraph graph; + + ASSERT(graph.roleExists(builtinRole)); + ASSERT_NOT_OK(graph.createRole(builtinRole)); + ASSERT_NOT_OK(graph.deleteRole(builtinRole)); + ASSERT(graph.roleExists(builtinRole)); + ASSERT(!graph.roleExists(userRole)); + ASSERT_OK(graph.createRole(userRole)); + ASSERT(graph.roleExists(userRole)); + + ASSERT_NOT_OK(graph.addPrivilegeToRole(builtinRole, privilege)); + ASSERT_NOT_OK(graph.removePrivilegeFromRole(builtinRole, privilege)); + ASSERT_NOT_OK(graph.addRoleToRole(builtinRole, userRole)); + ASSERT_NOT_OK(graph.removeRoleFromRole(builtinRole, userRole)); + + ASSERT_OK(graph.addPrivilegeToRole(userRole, privilege)); + ASSERT_OK(graph.addRoleToRole(userRole, builtinRole)); + ASSERT_OK(graph.recomputePrivilegeData()); + + PrivilegeVector privileges = graph.getDirectPrivileges(userRole); + ASSERT_EQUALS(1U, privileges.size()); + ASSERT(privileges[0].getActions().equals(actions)); + ASSERT(!privileges[0].getActions().contains(ActionType::find)); + ASSERT_EQUALS(dbAResource, privileges[0].getResourcePattern()); + + privileges = graph.getAllPrivileges(userRole); + size_t i; + for (i = 0; i < privileges.size(); ++i) { + if (dbAResource == privileges[i].getResourcePattern()) + break; + } + ASSERT_NOT_EQUALS(privileges.size(), i); + ASSERT(privileges[i].getActions().isSupersetOf(actions)); + ASSERT(privileges[i].getActions().contains(ActionType::insert)); + ASSERT(privileges[i].getActions().contains(ActionType::find)); + + ASSERT_OK(graph.deleteRole(userRole)); + ASSERT(!graph.roleExists(userRole)); + } + + TEST(RoleGraphTest, BuiltinRolesOnlyOnAppropriateDatabases) { + RoleGraph graph; + ASSERT(graph.roleExists(RoleName("read", "test"))); + ASSERT(graph.roleExists(RoleName("readWrite", "test"))); + ASSERT(graph.roleExists(RoleName("userAdmin", "test"))); + ASSERT(graph.roleExists(RoleName("dbAdmin", "test"))); + ASSERT(graph.roleExists(RoleName("dbOwner", "test"))); + ASSERT(!graph.roleExists(RoleName("readAnyDatabase", "test"))); + ASSERT(!graph.roleExists(RoleName("readWriteAnyDatabase", "test"))); + ASSERT(!graph.roleExists(RoleName("userAdminAnyDatabase", "test"))); + ASSERT(!graph.roleExists(RoleName("dbAdminAnyDatabase", "test"))); + ASSERT(!graph.roleExists(RoleName("clusterAdmin", "test"))); + ASSERT(!graph.roleExists(RoleName("root", "test"))); + ASSERT(!graph.roleExists(RoleName("__system", "test"))); + ASSERT(!graph.roleExists(RoleName("MyRole", "test"))); + + ASSERT(graph.roleExists(RoleName("read", "admin"))); + ASSERT(graph.roleExists(RoleName("readWrite", "admin"))); + ASSERT(graph.roleExists(RoleName("userAdmin", "admin"))); + ASSERT(graph.roleExists(RoleName("dbAdmin", "admin"))); + ASSERT(graph.roleExists(RoleName("dbOwner", "admin"))); + ASSERT(graph.roleExists(RoleName("readAnyDatabase", "admin"))); + ASSERT(graph.roleExists(RoleName("readWriteAnyDatabase", "admin"))); + ASSERT(graph.roleExists(RoleName("userAdminAnyDatabase", "admin"))); + ASSERT(graph.roleExists(RoleName("dbAdminAnyDatabase", "admin"))); + ASSERT(graph.roleExists(RoleName("clusterAdmin", "admin"))); + ASSERT(graph.roleExists(RoleName("root", "admin"))); + ASSERT(graph.roleExists(RoleName("__system", "admin"))); + ASSERT(!graph.roleExists(RoleName("MyRole", "admin"))); + } + + TEST(RoleGraphTest, getRolesForDatabase) { + RoleGraph graph; + graph.createRole(RoleName("myRole", "test")); + // Make sure that a role on "test2" doesn't show up in the roles list for "test" + graph.createRole(RoleName("anotherRole", "test2")); + graph.createRole(RoleName("myAdminRole", "admin")); + + // Non-admin DB with no user-defined roles + RoleNameIterator it = graph.getRolesForDatabase("fakedb"); + ASSERT_EQUALS(RoleName("dbAdmin", "fakedb"), it.next()); + ASSERT_EQUALS(RoleName("dbOwner", "fakedb"), it.next()); + ASSERT_EQUALS(RoleName("read", "fakedb"), it.next()); + ASSERT_EQUALS(RoleName("readWrite", "fakedb"), it.next()); + ASSERT_EQUALS(RoleName("userAdmin", "fakedb"), it.next()); + ASSERT_FALSE(it.more()); + + // Non-admin DB with a user-defined role + it = graph.getRolesForDatabase("test"); + ASSERT_EQUALS(RoleName("dbAdmin", "test"), it.next()); + ASSERT_EQUALS(RoleName("dbOwner", "test"), it.next()); + ASSERT_EQUALS(RoleName("myRole", "test"), it.next()); + ASSERT_EQUALS(RoleName("read", "test"), it.next()); + ASSERT_EQUALS(RoleName("readWrite", "test"), it.next()); + ASSERT_EQUALS(RoleName("userAdmin", "test"), it.next()); + ASSERT_FALSE(it.more()); + + // Admin DB + it = graph.getRolesForDatabase("admin"); + ASSERT_EQUALS(RoleName("__system", "admin"), it.next()); + ASSERT_EQUALS(RoleName("backup", "admin"), it.next()); + ASSERT_EQUALS(RoleName("clusterAdmin", "admin"), it.next()); + ASSERT_EQUALS(RoleName("clusterManager", "admin"), it.next()); + ASSERT_EQUALS(RoleName("clusterMonitor", "admin"), it.next()); + ASSERT_EQUALS(RoleName("dbAdmin", "admin"), it.next()); + ASSERT_EQUALS(RoleName("dbAdminAnyDatabase", "admin"), it.next()); + ASSERT_EQUALS(RoleName("dbOwner", "admin"), it.next()); + ASSERT_EQUALS(RoleName("hostManager", "admin"), it.next()); + ASSERT_EQUALS(RoleName("myAdminRole", "admin"), it.next()); + ASSERT_EQUALS(RoleName("read", "admin"), it.next()); + ASSERT_EQUALS(RoleName("readAnyDatabase", "admin"), it.next()); + ASSERT_EQUALS(RoleName("readWrite", "admin"), it.next()); + ASSERT_EQUALS(RoleName("readWriteAnyDatabase", "admin"), it.next()); + ASSERT_EQUALS(RoleName("restore", "admin"), it.next()); + ASSERT_EQUALS(RoleName("root", "admin"), it.next()); + ASSERT_EQUALS(RoleName("userAdmin", "admin"), it.next()); + ASSERT_EQUALS(RoleName("userAdminAnyDatabase", "admin"), it.next()); + ASSERT_FALSE(it.more()); + } + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/auth/role_graph_update.cpp b/src/mongo/db/auth/role_graph_update.cpp new file mode 100644 index 00000000000..c1f0e7c834f --- /dev/null +++ b/src/mongo/db/auth/role_graph_update.cpp @@ -0,0 +1,337 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/base/status.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/bson/util/bson_extract.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/role_graph.h" +#include "mongo/db/auth/user_management_commands_parser.h" +#include "mongo/db/ops/update_driver.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + +namespace { + + /** + * Structure representing information parsed out of a role document. + */ + struct RoleInfo { + RoleName name; + std::vector roles; + PrivilegeVector privileges; + }; + + /** + * Parses the role name out of a BSON document. + */ + Status parseRoleNameFromDocument(const BSONObj& doc, RoleName* name) { + BSONElement nameElement; + BSONElement sourceElement; + Status status = bsonExtractTypedField( + doc, AuthorizationManager::ROLE_NAME_FIELD_NAME, String, &nameElement); + if (!status.isOK()) + return status; + status = bsonExtractTypedField( + doc, AuthorizationManager::ROLE_SOURCE_FIELD_NAME, String, &sourceElement); + if (!status.isOK()) + return status; + *name = RoleName(StringData(nameElement.valuestr(), nameElement.valuestrsize() - 1), + StringData(sourceElement.valuestr(), sourceElement.valuestrsize() - 1)); + return status; + } + + /** + * Checks whether the given "roleName" corresponds with the given _id field. + * In admin.system.roles, documents with role name "role@db" must have _id + * "db.role". + * + * Returns Status::OK if the two values are compatible. + */ + Status checkIdMatchesRoleName(const BSONElement& idElement, const RoleName& roleName) { + if (idElement.type() != String) { + return Status(ErrorCodes::TypeMismatch, + "Role document _id fields must be strings."); + } + StringData idField(idElement.valuestr(), idElement.valuestrsize() - 1); + size_t firstDot = idField.find('.'); + if (firstDot == std::string::npos || + idField.substr(0, firstDot) != roleName.getDB() || + idField.substr(firstDot + 1) != roleName.getRole()) { + return Status(ErrorCodes::FailedToParse, mongoutils::str::stream() << + "Role document _id fields must be encoded as the string " + "dbname.rolename. Found " << idField << " for " << + roleName.getFullName()); + } + return Status::OK(); + } + + /** + * Parses "idElement" to extract the role name, according to the "dbname.role" convention + * used for admin.system.roles documents. + */ + Status getRoleNameFromIdField(const BSONElement& idElement, RoleName* roleName) { + if (idElement.type() != String) { + return Status(ErrorCodes::TypeMismatch, + "Role document _id fields must be strings."); + } + StringData idField(idElement.valuestr(), idElement.valuestrsize() - 1); + size_t dotPos = idField.find('.'); + if (dotPos == std::string::npos) { + return Status(ErrorCodes::BadValue, + "Role document _id fields must have the form dbname.rolename"); + } + *roleName = RoleName(idField.substr(dotPos + 1), idField.substr(0, dotPos)); + return Status::OK(); + } + + /** + * Parses information about a role from a BSON document. + */ + Status parseRoleFromDocument(const BSONObj& doc, RoleInfo* role) { + BSONElement rolesElement; + Status status = parseRoleNameFromDocument(doc, &role->name); + if (!status.isOK()) + return status; + status = checkIdMatchesRoleName(doc["_id"], role->name); + if (!status.isOK()) + return status; + status = bsonExtractTypedField(doc, "roles", Array, &rolesElement); + if (!status.isOK()) + return status; + BSONForEach(singleRoleElement, rolesElement.Obj()) { + if (singleRoleElement.type() != Object) { + return Status(ErrorCodes::TypeMismatch, + "Elements of roles array must be objects."); + } + RoleName possessedRoleName; + status = parseRoleNameFromDocument(singleRoleElement.Obj(), &possessedRoleName); + if (!status.isOK()) + return status; + role->roles.push_back(possessedRoleName); + } + + BSONElement privilegesElement; + status = bsonExtractTypedField(doc, "privileges", Array, &privilegesElement); + if (!status.isOK()) + return status; + status = auth::parseAndValidatePrivilegeArray(BSONArray(privilegesElement.Obj()), + &role->privileges); + return status; + } + + /** + * Updates roleGraph for an insert-type oplog operation on admin.system.roles. + */ + Status handleOplogInsert(RoleGraph* roleGraph, const BSONObj& insertedObj) { + RoleInfo role; + Status status = parseRoleFromDocument(insertedObj, &role); + if (!status.isOK()) + return status; + status = roleGraph->replaceRole(role.name, role.roles, role.privileges); + return status; + } + + /** + * Updates roleGraph for an update-type oplog operation on admin.system.roles. + * + * Treats all updates as upserts. + */ + Status handleOplogUpdate(RoleGraph* roleGraph, + const BSONObj& updatePattern, + const BSONObj& queryPattern) { + RoleName roleToUpdate; + Status status = getRoleNameFromIdField(queryPattern["_id"], &roleToUpdate); + if (!status.isOK()) + return status; + + UpdateDriver::Options updateOptions; + UpdateDriver driver(updateOptions); + status = driver.parse(updatePattern); + if (!status.isOK()) + return status; + + mutablebson::Document roleDocument; + status = AuthorizationManager::getBSONForRole( + roleGraph, roleToUpdate, roleDocument.root()); + if (status == ErrorCodes::RoleNotFound) { + status = driver.populateDocumentWithQueryFields(queryPattern, roleDocument); + } + if (!status.isOK()) + return status; + + status = driver.update(StringData(), &roleDocument); + if (!status.isOK()) + return status; + + // Now use the updated document to totally replace the role in the graph! + RoleInfo role; + status = parseRoleFromDocument(roleDocument.getObject(), &role); + if (!status.isOK()) + return status; + status = roleGraph->replaceRole(role.name, role.roles, role.privileges); + + return status; + } + + /** + * Updates roleGraph for a delete-type oplog operation on admin.system.roles. + */ + Status handleOplogDelete( + RoleGraph* roleGraph, + const BSONObj& deletePattern) { + + RoleName roleToDelete; + Status status = getRoleNameFromIdField(deletePattern["_id"], &roleToDelete); + if (!status.isOK()) + return status; + status = roleGraph->deleteRole(roleToDelete); + if (ErrorCodes::RoleNotFound == status) { + // Double-delete can happen in oplog application. + status = Status::OK(); + } + return status; + } + + /** + * Updates roleGraph for command-type oplog operations on the admin database. + */ + Status handleOplogCommand(RoleGraph* roleGraph, const BSONObj& cmdObj) { + const NamespaceString& rolesCollectionNamespace = + AuthorizationManager::rolesCollectionNamespace; + const StringData cmdName(cmdObj.firstElement().fieldNameStringData()); + if (cmdName == "applyOps") { + // Operations applied by applyOps will be passed into RoleGraph::handleOplog() by the + // implementation of applyOps itself. + return Status::OK(); + } + if (cmdName == "create") { + return Status::OK(); + } + if (cmdName == "drop") { + if (cmdObj.firstElement().str() == rolesCollectionNamespace.coll()) { + *roleGraph = RoleGraph(); + } + return Status::OK(); + } + if (cmdName == "dropDatabase") { + *roleGraph = RoleGraph(); + return Status::OK(); + } + if (cmdName == "renameCollection") { + if (cmdObj.firstElement().str() == rolesCollectionNamespace.ns()) { + *roleGraph = RoleGraph(); + return Status::OK(); + } + if (cmdObj["to"].str() == rolesCollectionNamespace.ns()) { + *roleGraph = RoleGraph(); + return Status(ErrorCodes::OplogOperationUnsupported, + "Renaming into admin.system.roles produces inconsistent state; " + "must resynchronize role graph."); + } + return Status::OK(); + } + if (cmdName == "dropIndexes" || cmdName == "deleteIndexes") { + return Status::OK(); + } + if ((cmdName == "collMod" || cmdName == "emptyCappedCollection") && + cmdObj.firstElement().str() != rolesCollectionNamespace.coll()) { + + // We don't care about these if they're not on the roles collection. + return Status::OK(); + } + // No other commands expected. Warn. + return Status(ErrorCodes::OplogOperationUnsupported, "Unsupported oplog operation"); + } +} // namespace + + Status RoleGraph::addRoleFromDocument(const BSONObj& doc) { + RoleInfo role; + Status status = parseRoleFromDocument(doc, &role); + if (!status.isOK()) + return status; + status = replaceRole(role.name, role.roles, role.privileges); + return status; + } + + Status RoleGraph::handleLogOp( + const char* op, + const NamespaceString& ns, + const BSONObj& o, + const BSONObj* o2) { + + if (op == StringData("db", StringData::LiteralTag())) + return Status::OK(); + if (op[0] == '\0' || op[1] != '\0') { + return Status(ErrorCodes::BadValue, + mongoutils::str::stream() << "Unrecognized \"op\" field value \"" << + op << '"'); + } + + if (ns.db() != AuthorizationManager::rolesCollectionNamespace.db()) + return Status::OK(); + + if (ns.isCommand()) { + if (*op == 'c') { + return handleOplogCommand(this, o); + } + else { + return Status(ErrorCodes::BadValue, + "Non-command oplog entry on admin.$cmd namespace"); + } + } + + if (ns.coll() != AuthorizationManager::rolesCollectionNamespace.coll()) + return Status::OK(); + + switch (*op) { + case 'i': + return handleOplogInsert(this, o); + case 'u': + if (!o2) { + return Status(ErrorCodes::InternalError, + "Missing query pattern in update oplog entry."); + } + return handleOplogUpdate(this, o, *o2); + case 'd': + return handleOplogDelete(this, o); + case 'n': + return Status::OK(); + case 'c': + return Status(ErrorCodes::BadValue, + "Namespace admin.system.roles is not a valid target for commands"); + default: + return Status(ErrorCodes::BadValue, + mongoutils::str::stream() << "Unrecognized \"op\" field value \"" << + op << '"'); + } + } + +} // namespace mongo diff --git a/src/mongo/db/auth/role_name.cpp b/src/mongo/db/auth/role_name.cpp new file mode 100644 index 00000000000..00415cf7bf3 --- /dev/null +++ b/src/mongo/db/auth/role_name.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/auth/role_name.h" + +#include +#include + +#include "mongo/base/string_data.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + RoleName::RoleName(const StringData& role, const StringData& dbname) { + _fullName.resize(role.size() + dbname.size() + 1); + std::string::iterator iter = std::copy(role.rawData(), + role.rawData() + role.size(), + _fullName.begin()); + *iter = '@'; + ++iter; + iter = std::copy(dbname.rawData(), dbname.rawData() + dbname.size(), iter); + dassert(iter == _fullName.end()); + _splitPoint = role.size(); + } + + std::ostream& operator<<(std::ostream& os, const RoleName& name) { + return os << name.getFullName(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/role_name.h b/src/mongo/db/auth/role_name.h new file mode 100644 index 00000000000..63f7da816c7 --- /dev/null +++ b/src/mongo/db/auth/role_name.h @@ -0,0 +1,186 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" +#include "mongo/platform/hash_namespace.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + /** + * Representation of a name of a role in a MongoDB system. + * + * Consists of a "role name" part and a "datbase name" part. + */ + class RoleName { + public: + RoleName() : _splitPoint(0) {} + RoleName(const StringData& role, const StringData& dbname); + + /** + * Gets the name of the role excluding the "@dbname" component. + */ + StringData getRole() const { return StringData(_fullName).substr(0, _splitPoint); } + + /** + * Gets the database name part of a role name. + */ + StringData getDB() const { return StringData(_fullName).substr(_splitPoint + 1); } + + bool empty() const { return _fullName.empty(); } + + /** + * Gets the full name of a role as a string, formatted as "role@db". + * + * Allowed for keys in non-persistent data structures, such as std::map. + */ + const std::string& getFullName() const { return _fullName; } + + /** + * Stringifies the object, for logging/debugging. + */ + const std::string& toString() const { return getFullName(); } + + private: + std::string _fullName; // The full name, stored as a string. "role@db". + size_t _splitPoint; // The index of the "@" separating the role and db name parts. + }; + + static inline bool operator==(const RoleName& lhs, const RoleName& rhs) { + return lhs.getFullName() == rhs.getFullName(); + } + + static inline bool operator!=(const RoleName& lhs, const RoleName& rhs) { + return lhs.getFullName() != rhs.getFullName(); + } + + static inline bool operator<(const RoleName& lhs, const RoleName& rhs) { + if (lhs.getDB() == rhs.getDB()) { + return lhs.getRole() < rhs.getRole(); + } + return lhs.getDB() < rhs.getDB(); + } + + std::ostream& operator<<(std::ostream& os, const RoleName& name); + + + /** + * Iterator over an unspecified container of RoleName objects. + */ + class RoleNameIterator { + public: + class Impl { + MONGO_DISALLOW_COPYING(Impl); + public: + Impl() {}; + virtual ~Impl() {}; + static Impl* clone(Impl* orig) { return orig ? orig->doClone(): NULL; } + virtual bool more() const = 0; + virtual const RoleName& get() const = 0; + + virtual const RoleName& next() = 0; + + private: + virtual Impl* doClone() const = 0; + }; + + RoleNameIterator() : _impl(NULL) {} + RoleNameIterator(const RoleNameIterator& other) : _impl(Impl::clone(other._impl.get())) {} + explicit RoleNameIterator(Impl* impl) : _impl(impl) {} + + RoleNameIterator& operator=(const RoleNameIterator& other) { + _impl.reset(Impl::clone(other._impl.get())); + return *this; + } + + bool more() const { return _impl.get() && _impl->more(); } + const RoleName& get() const { return _impl->get(); } + + const RoleName& next() { return _impl->next(); } + + const RoleName& operator*() const { return get(); } + const RoleName* operator->() const { return &get(); } + + private: + boost::scoped_ptr _impl; + }; + +} // namespace mongo + +// Define hash function for RoleNames so they can be keys in std::unordered_map +MONGO_HASH_NAMESPACE_START + template <> struct hash { + size_t operator()(const mongo::RoleName& rname) const { + return hash()(rname.getFullName()); + } + }; +MONGO_HASH_NAMESPACE_END + +namespace mongo { + + template + class RoleNameContainerIteratorImpl : public RoleNameIterator::Impl { + MONGO_DISALLOW_COPYING(RoleNameContainerIteratorImpl); + public: + RoleNameContainerIteratorImpl(const ContainerIterator& begin, + const ContainerIterator& end) : + _curr(begin), _end(end) {} + virtual ~RoleNameContainerIteratorImpl() {} + virtual bool more() const { return _curr != _end; } + virtual const RoleName& next() { return *(_curr++); } + virtual const RoleName& get() const { return *_curr; } + virtual RoleNameIterator::Impl* doClone() const { + return new RoleNameContainerIteratorImpl(_curr, _end); + } + + private: + ContainerIterator _curr; + ContainerIterator _end; + }; + + template + RoleNameIterator makeRoleNameIterator(const ContainerIterator& begin, + const ContainerIterator& end) { + return RoleNameIterator( new RoleNameContainerIteratorImpl(begin, end)); + } + + template + RoleNameIterator makeRoleNameIteratorForContainer(const Container& container) { + return makeRoleNameIterator(container.begin(), container.end()); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/security_key.cpp b/src/mongo/db/auth/security_key.cpp index c5f85bee877..c3ff4c2336d 100644 --- a/src/mongo/db/auth/security_key.cpp +++ b/src/mongo/db/auth/security_key.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/db/auth/security_key.h" @@ -20,14 +32,53 @@ #include #include -#include "mongo/client/dbclientinterface.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/privilege.h" - +#include "mongo/db/auth/user.h" +#include "mongo/client/sasl_client_authenticate.h" +#include "mongo/util/password_digest.h" namespace mongo { + // not guarded by the authParams mutex never changed in + // multi-threaded operation + static bool authParamsSet = false; + // guarded by the authParams mutex + static BSONObj authParams; + static boost::mutex authParamMutex; + + bool isInternalAuthSet() { + return authParamsSet; + } + + void setInternalUserAuthParams(const BSONObj& authParamsIn) { + if (!isInternalAuthSet()) { + authParamsSet = true; + } + boost::mutex::scoped_lock lk(authParamMutex); + authParams = authParamsIn.copy(); + } + + bool authenticateInternalUser(DBClientWithCommands* conn){ + if (!isInternalAuthSet()) { + log() << "ERROR: No authentication parameters set for internal user" << endl; + return false; + } + try { + BSONObj outgoingAuthParams; + { + boost::mutex::scoped_lock lk(authParamMutex); + outgoingAuthParams = authParams.copy(); + } + conn->auth(outgoingAuthParams); + return true; + } catch(const UserException& ex) { + log() << "can't authenticate to " << conn->toString() << " as internal user, error: " + << ex.what() << endl; + return false; + } + } bool setUpSecurityKey(const string& filename) { struct stat stats; @@ -46,13 +97,6 @@ namespace mongo { } #endif - const unsigned long long fileLength = stats.st_size; - if (fileLength < 6 || fileLength > 1024) { - log() << " key file " << filename << " has length " << stats.st_size - << ", must be between 6 and 1024 chars" << endl; - return false; - } - FILE* file = fopen( filename.c_str(), "rb" ); if (!file) { log() << "error opening file: " << filename << ": " << strerror(errno) << endl; @@ -62,6 +106,7 @@ namespace mongo { string str = ""; // strip key file + const unsigned long long fileLength = stats.st_size; unsigned long long read = 0; while (read < fileLength) { char buf; @@ -90,17 +135,29 @@ namespace mongo { fclose( file ); - if (str.size() < 6) { - log() << "security key must be at least 6 characters" << endl; + const unsigned long long keyLength = str.size(); + if (keyLength < 6 || keyLength > 1024) { + log() << " security key in " << filename << " has length " << keyLength + << ", must be between 6 and 1024 chars" << endl; return false; } - LOG(1) << "security key: " << str << endl; - - // createPWDigest should really not be a member func - DBClientConnection conn; - internalSecurity.pwd = conn.createPasswordDigest(internalSecurity.user, str); - + User::CredentialData credentials; + credentials.password = mongo::createPasswordDigest( + internalSecurity.user->getName().getUser().toString(), str); + internalSecurity.user->setCredentials(credentials); + + int clusterAuthMode = serverGlobalParams.clusterAuthMode.load(); + if (clusterAuthMode == ServerGlobalParams::ClusterAuthMode_keyFile || + clusterAuthMode == ServerGlobalParams::ClusterAuthMode_sendKeyFile) { + setInternalUserAuthParams( + BSON(saslCommandMechanismFieldName << "MONGODB-CR" << + saslCommandUserDBFieldName << + internalSecurity.user->getName().getDB() << + saslCommandUserFieldName << internalSecurity.user->getName().getUser() << + saslCommandPasswordFieldName << credentials.password << + saslCommandDigestPasswordFieldName << false)); + } return true; } diff --git a/src/mongo/db/auth/security_key.h b/src/mongo/db/auth/security_key.h index e261123dcea..298a5cf0906 100644 --- a/src/mongo/db/auth/security_key.h +++ b/src/mongo/db/auth/security_key.h @@ -12,13 +12,44 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once #include +#include "mongo/client/dbclientinterface.h" + namespace mongo { + /** + * @return true if internal authentication parameters has been set up + */ + extern bool isInternalAuthSet(); + + /** + * This method initializes the internalSecurity object with authentication + * credentials to be used by authenticateInternalUser. + */ + extern void setInternalUserAuthParams(const BSONObj& authParamsIn); + + /** + * This method authenticates to another cluster member using appropriate + * authentication data + * @return true if the authentication was succesful + */ + extern bool authenticateInternalUser(DBClientWithCommands* conn); /** * This method checks the validity of filename as a security key, hashes its diff --git a/src/mongo/db/auth/user.cpp b/src/mongo/db/auth/user.cpp new file mode 100644 index 00000000000..08e11b5a2a7 --- /dev/null +++ b/src/mongo/db/auth/user.cpp @@ -0,0 +1,169 @@ +/* Copyright 2013 10gen Inc. + + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/auth/user.h" + +#include + +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/platform/atomic_word.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/sequence_util.h" + +namespace mongo { + + User::User(const UserName& name) : + _name(name), + _schemaVersion(AuthorizationManager::schemaVersion26Final), + _refCount(0), + _isValid(1) {} + + User::~User() { + dassert(_refCount == 0); + } + + const UserName& User::getName() const { + return _name; + } + + RoleNameIterator User::getRoles() const { + return makeRoleNameIteratorForContainer(_roles); + } + + RoleNameIterator User::getIndirectRoles() const { + return makeRoleNameIteratorForContainer(_indirectRoles); + } + + bool User::hasRole(const RoleName& roleName) const { + return _roles.count(roleName); + } + + const User::CredentialData& User::getCredentials() const { + return _credentials; + } + + bool User::isValid() const { + return _isValid.loadRelaxed() == 1; + } + + uint32_t User::getRefCount() const { + return _refCount; + } + + const ActionSet User::getActionsForResource(const ResourcePattern& resource) const { + unordered_map::const_iterator it = _privileges.find(resource); + if (it == _privileges.end()) { + return ActionSet(); + } + return it->second.getActions(); + } + + User* User::clone() const { + std::auto_ptr result(new User(_name)); + result->_privileges = _privileges; + result->_roles = _roles; + result->_probedDatabases = _probedDatabases; + result->_credentials = _credentials; + result->_schemaVersion = _schemaVersion; + return result.release(); + } + + void User::setCredentials(const CredentialData& credentials) { + _credentials = credentials; + } + + void User::setRoles(RoleNameIterator roles) { + _roles.clear(); + while (roles.more()) { + _roles.insert(roles.next()); + } + } + + void User::setIndirectRoles(RoleNameIterator indirectRoles) { + _indirectRoles.clear(); + while (indirectRoles.more()) { + _indirectRoles.push_back(indirectRoles.next()); + } + } + + void User::setPrivileges(const PrivilegeVector& privileges) { + _privileges.clear(); + for (size_t i = 0; i < privileges.size(); ++i) { + const Privilege& privilege = privileges[i]; + _privileges[privilege.getResourcePattern()] = privilege; + } + } + + void User::addRole(const RoleName& roleName) { + _roles.insert(roleName); + } + + void User::addRoles(const std::vector& roles) { + for (std::vector::const_iterator it = roles.begin(); it != roles.end(); ++it) { + addRole(*it); + } + } + + void User::addPrivilege(const Privilege& privilegeToAdd) { + ResourcePrivilegeMap::iterator it = _privileges.find(privilegeToAdd.getResourcePattern()); + if (it == _privileges.end()) { + // No privilege exists yet for this resource + _privileges.insert(std::make_pair(privilegeToAdd.getResourcePattern(), privilegeToAdd)); + } else { + dassert(it->first == privilegeToAdd.getResourcePattern()); + it->second.addActions(privilegeToAdd.getActions()); + } + } + + void User::addPrivileges(const PrivilegeVector& privileges) { + for (PrivilegeVector::const_iterator it = privileges.begin(); + it != privileges.end(); ++it) { + addPrivilege(*it); + } + } + + void User::setSchemaVersion1() { + _schemaVersion = AuthorizationManager::schemaVersion24; + } + + void User::markProbedV1(const StringData& dbname) { + dassert(_schemaVersion == AuthorizationManager::schemaVersion24); + if (!hasProbedV1(dbname)) + _probedDatabases.push_back(dbname.toString()); + } + + bool User::hasProbedV1(const StringData& dbname) const { + dassert(_schemaVersion == AuthorizationManager::schemaVersion24); + return sequenceContains(_probedDatabases, dbname); + } + + void User::invalidate() { + _isValid.store(0); + } + + void User::incrementRefCount() { + ++_refCount; + } + + void User::decrementRefCount() { + dassert(_refCount > 0); + --_refCount; + } +} // namespace mongo diff --git a/src/mongo/db/auth/user.h b/src/mongo/db/auth/user.h new file mode 100644 index 00000000000..786d0737914 --- /dev/null +++ b/src/mongo/db/auth/user.h @@ -0,0 +1,237 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/platform/atomic_word.h" +#include "mongo/platform/unordered_map.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + /** + * Represents a MongoDB user. Stores information about the user necessary for access control + * checks and authentications, such as what privileges this user has, as well as what roles + * the user belongs to. + * + * Every User object is owned by an AuthorizationManager. The AuthorizationManager is the only + * one that should construct, modify, or delete a User object. All other consumers of User must + * use only the const methods. The AuthorizationManager is responsible for maintaining the + * reference count on all User objects it gives out and must not mutate any User objects with + * a non-zero reference count (except to call invalidate()). Any consumer of a User object + * should check isInvalidated() before using it, and if it has been invalidated, it should + * return the object to the AuthorizationManager and fetch a new User object instance for this + * user from the AuthorizationManager. + */ + class User { + MONGO_DISALLOW_COPYING(User); + public: + struct CredentialData { + std::string password; + bool isExternal; + }; + + typedef unordered_map ResourcePrivilegeMap; + + explicit User(const UserName& name); + ~User(); + + /** + * Returns the user name for this user. + */ + const UserName& getName() const; + + /** + * Returns an iterator over the names of the user's direct roles + */ + RoleNameIterator getRoles() const; + + /** + * Returns an iterator over the names of the user's indirect roles + */ + RoleNameIterator getIndirectRoles() const; + + /** + * Returns true if this user is a member of the given role. + */ + bool hasRole(const RoleName& roleName) const; + + /** + * Returns a reference to the information about the user's privileges. + */ + const ResourcePrivilegeMap& getPrivileges() const { return _privileges; } + + /** + * Returns the CredentialData for this user. + */ + const CredentialData& getCredentials() const; + + /** + * Gets the set of actions this user is allowed to perform on the given resource. + */ + const ActionSet getActionsForResource(const ResourcePattern& resource) const; + + /** + * Gets the schema version of user documents used to build this user. See comment on + * _schemaVersion field, below. + */ + int getSchemaVersion() const { return _schemaVersion; } + + /** + * Returns true if this user object, generated from V1-schema user documents, + * has been probed for privileges on database "dbname", according to the V1 + * implicit privilge acquisition rules. + */ + bool hasProbedV1(const StringData& dbname) const; + + /** + * Returns true if this copy of information about this user is still valid. If this returns + * false, this object should no longer be used and should be returned to the + * AuthorizationManager and a new User object for this user should be requested. + */ + bool isValid() const; + + /** + * This returns the reference count for this User. The AuthorizationManager should be the + * only caller of this. + */ + uint32_t getRefCount() const; + + /** + * Clones this user into a new, valid User object with refcount of 0. + */ + User* clone() const; + + // Mutators below. Mutation functions should *only* be called by the AuthorizationManager + + /** + * Sets this user's authentication credentials. + */ + void setCredentials(const CredentialData& credentials); + + /** + * Replaces any existing user role membership information with the roles from "roles". + */ + void setRoles(RoleNameIterator roles); + + /** + * Replaces any existing indirect user role membership information with the roles from + * "indirectRoles". + */ + void setIndirectRoles(RoleNameIterator indirectRoles); + + /** + * Replaces any existing user privilege information with "privileges". + */ + void setPrivileges(const PrivilegeVector& privileges); + + /** + * Adds the given role name to the list of roles of which this user is a member. + */ + void addRole(const RoleName& role); + + /** + * Adds the given role names to the list of roles that this user belongs to. + */ + void addRoles(const std::vector& roles); + + /** + * Adds the given privilege to the list of privileges this user is authorized for. + */ + void addPrivilege(const Privilege& privilege); + + /** + * Adds the given privileges to the list of privileges this user is authorized for. + */ + void addPrivileges(const PrivilegeVector& privileges); + + /** + * Sets the schema version of documents used for building this user to 1, for V1 and V0 + * documents. The default value is 2, for V2 documents. + */ + void setSchemaVersion1(); + + /** + * Marks that this user object, generated from V1-schema user documents, + * has been probed for privileges on database "dbname", according to the V1 + * implicit privilge acquisition rules. + */ + void markProbedV1(const StringData& dbname); + + /** + * Marks this instance of the User object as invalid, most likely because information about + * the user has been updated and needs to be reloaded from the AuthorizationManager. + * + * This method should *only* be called by the AuthorizationManager. + */ + void invalidate(); + + /** + * Increments the reference count for this User object, which records how many threads have + * a reference to it. + * + * This method should *only* be called by the AuthorizationManager. + */ + void incrementRefCount(); + + /** + * Decrements the reference count for this User object, which records how many threads have + * a reference to it. Once the reference count goes to zero, the AuthorizationManager is + * allowed to destroy this instance. + * + * This method should *only* be called by the AuthorizationManager. + */ + void decrementRefCount(); + + private: + + UserName _name; + + // Maps resource name to privilege on that resource + ResourcePrivilegeMap _privileges; + + // Roles the user has privileges from + unordered_set _roles; + + // List of databases already probed for privilege information for this user. Only + // meaningful for V2.4-schema users. + std::vector _probedDatabases; + + // Roles that the user indirectly has privileges from, due to role inheritance. + std::vector _indirectRoles; + + // Credential information. + CredentialData _credentials; + + // Schema version of user documents used to build this user. Valid values are + // AuthorizationManager::schemaVersion24 and schemaVersion26Final. + int _schemaVersion; + + // _refCount and _isInvalidated are modified exclusively by the AuthorizationManager + // _isInvalidated can be read by any consumer of User, but _refCount can only be + // meaningfully read by the AuthorizationManager, as _refCount is guarded by the AM's _lock + uint32_t _refCount; + AtomicUInt32 _isValid; // Using as a boolean + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/user_cache_invalidator_job.cpp b/src/mongo/db/auth/user_cache_invalidator_job.cpp new file mode 100644 index 00000000000..8b3f2a56047 --- /dev/null +++ b/src/mongo/db/auth/user_cache_invalidator_job.cpp @@ -0,0 +1,141 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/pch.h" + +#include "mongo/db/auth/user_cache_invalidator_job.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/base/status_with.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/server_parameters.h" +#include "mongo/s/config.h" +#include "mongo/util/background.h" +#include "mongo/util/log.h" + +namespace mongo { +namespace { + + // How often to check with the config servers whether authorization information has changed. + int userCacheInvalidationIntervalSecs = 30; // 30 second default + + class ExportedInvalidationIntervalParameter : public ExportedServerParameter { + public: + ExportedInvalidationIntervalParameter() : + ExportedServerParameter(ServerParameterSet::getGlobal(), + "userCacheInvalidationIntervalSecs", + &userCacheInvalidationIntervalSecs, + true, + true) {} + + virtual Status validate( const int& potentialNewValue ) + { + if (potentialNewValue < 1 || potentialNewValue > 86400) { + return Status(ErrorCodes::BadValue, + "userCacheInvalidationIntervalSecs must be between 1 " + "and 86400 (24 hours)"); + } + return Status::OK(); + } + } exportedIntervalParam; + + StatusWith getCurrentCacheGeneration() { + try { + ConnectionString config = configServer.getConnectionString(); + ScopedDbConnection conn(config.toString(), 30); + + BSONObj result; + conn->runCommand("admin", BSON("_getUserCacheGeneration" << 1), result); + conn.done(); + + Status status = Command::getStatusFromCommandResult(result); + if (!status.isOK()) { + return StatusWith(status); + } + + return StatusWith(result["cacheGeneration"].OID()); + } catch (const DBException& e) { + return StatusWith(e.toStatus()); + } catch (const std::exception& e) { + return StatusWith(ErrorCodes::UnknownError, e.what()); + } + } + +} // namespace + + UserCacheInvalidator::UserCacheInvalidator(AuthorizationManager* authzManager) : + _authzManager(authzManager) { + + StatusWith currentGeneration = getCurrentCacheGeneration(); + if (currentGeneration.isOK()) { + _previousCacheGeneration = currentGeneration.getValue(); + return; + } + + if (currentGeneration.getStatus().code() == ErrorCodes::CommandNotFound) { + warning() << "_getUserCacheGeneration command not found while fetching initial user " + "cache generation from the config server(s). This most likely means you are " + "running an outdated version of mongod on the config servers"; + } else { + warning() << "An error occurred while fetching initial user cache generation from " + "config servers: " << currentGeneration.getStatus(); + } + _previousCacheGeneration = OID(); + } + + void UserCacheInvalidator::run() { + Client::initThread("UserCacheInvalidatorThread"); + + while (true) { + sleepsecs(userCacheInvalidationIntervalSecs); + if (inShutdown()) { + break; + } + + StatusWith currentGeneration = getCurrentCacheGeneration(); + if (!currentGeneration.isOK()) { + if (currentGeneration.getStatus().code() == ErrorCodes::CommandNotFound) { + warning() << "_getUserCacheGeneration command not found on config server(s), " + "this most likely means you are running an outdated version of mongod " + "on the config servers" << std::endl; + } else { + warning() << "An error occurred while fetching current user cache generation " + "to check if user cache needs invalidation: " << + currentGeneration.getStatus() << std::endl; + } + // When in doubt, invalidate the cache + _authzManager->invalidateUserCache(); + continue; + } + + if (currentGeneration.getValue() != _previousCacheGeneration) { + log() << "User cache generation changed from " << _previousCacheGeneration << + " to " << currentGeneration.getValue() << "; invalidating user cache" << + std::endl; + _authzManager->invalidateUserCache(); + _previousCacheGeneration = currentGeneration.getValue(); + } + } + } + + std::string UserCacheInvalidator::name() const { + return "UserCacheInvalidatorThread"; + } + +} // namespace mongo diff --git a/src/mongo/db/auth/user_cache_invalidator_job.h b/src/mongo/db/auth/user_cache_invalidator_job.h new file mode 100644 index 00000000000..5dc9403272d --- /dev/null +++ b/src/mongo/db/auth/user_cache_invalidator_job.h @@ -0,0 +1,44 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/bson/oid.h" +#include "mongo/util/background.h" + +#include + +namespace mongo { + + class AuthorizationManager; + + /** + * Background job that runs only in mongos and periodically checks in with the config servers + * to determine whether any authorization information has changed, and if so causes the + * AuthorizationManager to throw out its in-memory cache of User objects (which contains the + * users' credentials, roles, privileges, etc). + */ + class UserCacheInvalidator : public BackgroundJob { + public: + explicit UserCacheInvalidator(AuthorizationManager* authzManager); + + protected: + virtual std::string name() const; + virtual void run(); + + private: + AuthorizationManager* _authzManager; + OID _previousCacheGeneration; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/user_document_parser.cpp b/src/mongo/db/auth/user_document_parser.cpp new file mode 100644 index 00000000000..3cdaa98d087 --- /dev/null +++ b/src/mongo/db/auth/user_document_parser.cpp @@ -0,0 +1,505 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/user_document_parser.h" + +#include + +#include "mongo/base/init.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + +namespace { + const std::string ADMIN_DBNAME = "admin"; + + const std::string ROLES_FIELD_NAME = "roles"; + const std::string PRIVILEGES_FIELD_NAME = "inheritedPrivileges"; + const std::string INHERITED_ROLES_FIELD_NAME = "inheritedRoles"; + const std::string OTHER_DB_ROLES_FIELD_NAME = "otherDBRoles"; + const std::string READONLY_FIELD_NAME = "readOnly"; + const std::string CREDENTIALS_FIELD_NAME = "credentials"; + const std::string ROLE_NAME_FIELD_NAME = "role"; + const std::string ROLE_SOURCE_FIELD_NAME = "db"; + const std::string MONGODB_CR_CREDENTIAL_FIELD_NAME = "MONGODB-CR"; + const std::string MONGODB_EXTERNAL_CREDENTIAL_FIELD_NAME = "external"; + + inline Status _badValue(const char* reason, int location) { + return Status(ErrorCodes::BadValue, reason, location); + } + + inline Status _badValue(const std::string& reason, int location) { + return Status(ErrorCodes::BadValue, reason, location); + } + + inline StringData makeStringDataFromBSONElement(const BSONElement& element) { + return StringData(element.valuestr(), element.valuestrsize() - 1); + } + + Status _checkV1RolesArray(const BSONElement& rolesElement) { + if (rolesElement.type() != Array) { + return _badValue("Role fields must be an array when present in system.users entries", + 0); + } + for (BSONObjIterator iter(rolesElement.embeddedObject()); iter.more(); iter.next()) { + BSONElement element = *iter; + if (element.type() != String || makeStringDataFromBSONElement(element).empty()) { + return _badValue("Roles must be non-empty strings.", 0); + } + } + return Status::OK(); + } +} // namespace + + std::string V1UserDocumentParser::extractUserNameFromUserDocument( + const BSONObj& doc) const { + return doc[AuthorizationManager::V1_USER_NAME_FIELD_NAME].str(); + } + + Status V1UserDocumentParser::initializeUserCredentialsFromUserDocument( + User* user, const BSONObj& privDoc) const { + User::CredentialData credentials; + if (privDoc.hasField(AuthorizationManager::PASSWORD_FIELD_NAME)) { + credentials.password = privDoc[AuthorizationManager::PASSWORD_FIELD_NAME].String(); + credentials.isExternal = false; + } + else if (privDoc.hasField(AuthorizationManager::V1_USER_SOURCE_FIELD_NAME)) { + std::string userSource = privDoc[AuthorizationManager::V1_USER_SOURCE_FIELD_NAME].String(); + if (userSource != "$external") { + return Status(ErrorCodes::UnsupportedFormat, + "Cannot extract credentials from user documents without a password " + "and with userSource != \"$external\""); + } else { + credentials.isExternal = true; + } + } else { + return Status(ErrorCodes::UnsupportedFormat, + "Invalid user document: must have one of \"pwd\" and \"userSource\""); + } + + user->setCredentials(credentials); + return Status::OK(); + } + + static void _initializeUserRolesFromV0UserDocument( + User* user, const BSONObj& privDoc, const StringData& dbname) { + bool readOnly = privDoc["readOnly"].trueValue(); + if (dbname == "admin") { + if (readOnly) { + user->addRole(RoleName(RoleGraph::BUILTIN_ROLE_V0_ADMIN_READ, "admin")); + } else { + user->addRole(RoleName(RoleGraph::BUILTIN_ROLE_V0_ADMIN_READ_WRITE, "admin")); + } + } else { + if (readOnly) { + user->addRole(RoleName(RoleGraph::BUILTIN_ROLE_V0_READ, dbname)); + } else { + user->addRole(RoleName(RoleGraph::BUILTIN_ROLE_V0_READ_WRITE, dbname)); + } + } + } + + Status _initializeUserRolesFromV1RolesArray(User* user, + const BSONElement& rolesElement, + const StringData& dbname) { + static const char privilegesTypeMismatchMessage[] = + "Roles in V1 user documents must be enumerated in an array of strings."; + + if (rolesElement.type() != Array) + return Status(ErrorCodes::TypeMismatch, privilegesTypeMismatchMessage); + + for (BSONObjIterator iter(rolesElement.embeddedObject()); iter.more(); iter.next()) { + BSONElement roleElement = *iter; + if (roleElement.type() != String) + return Status(ErrorCodes::TypeMismatch, privilegesTypeMismatchMessage); + + user->addRole(RoleName(roleElement.String(), dbname)); + } + return Status::OK(); + } + + static Status _initializeUserRolesFromV1UserDocument( + User* user, const BSONObj& privDoc, const StringData& dbname) { + + if (!privDoc[READONLY_FIELD_NAME].eoo()) { + return Status(ErrorCodes::UnsupportedFormat, + "User documents may not contain both \"readonly\" and " + "\"roles\" fields"); + } + + Status status = _initializeUserRolesFromV1RolesArray(user, + privDoc[ROLES_FIELD_NAME], + dbname); + if (!status.isOK()) { + return status; + } + + // If "dbname" is the admin database, handle the otherDBPrivileges field, which + // grants privileges on databases other than "dbname". + BSONElement otherDbPrivileges = privDoc[OTHER_DB_ROLES_FIELD_NAME]; + if (dbname == ADMIN_DBNAME) { + switch (otherDbPrivileges.type()) { + case EOO: + break; + case Object: { + for (BSONObjIterator iter(otherDbPrivileges.embeddedObject()); + iter.more(); iter.next()) { + + BSONElement rolesElement = *iter; + status = _initializeUserRolesFromV1RolesArray(user, + rolesElement, + rolesElement.fieldName()); + if (!status.isOK()) + return status; + } + break; + } + default: + return Status(ErrorCodes::TypeMismatch, + "Field \"otherDBRoles\" must be an object, if present."); + } + } + else if (!otherDbPrivileges.eoo()) { + return Status(ErrorCodes::UnsupportedFormat, + "Only the admin database may contain a field called \"otherDBRoles\""); + } + + return Status::OK(); + } + + Status V1UserDocumentParser::initializeUserRolesFromUserDocument( + User* user, const BSONObj& privDoc, const StringData& dbname) const { + if (!privDoc.hasField("roles")) { + _initializeUserRolesFromV0UserDocument(user, privDoc, dbname); + } else { + return _initializeUserRolesFromV1UserDocument(user, privDoc, dbname); + } + return Status::OK(); + } + + + Status _checkV2RolesArray(const BSONElement& rolesElement) { + if (rolesElement.eoo()) { + return _badValue("User document needs 'roles' field to be provided", 0); + } + if (rolesElement.type() != Array) { + return _badValue("'roles' field must be an array", 0); + } + for (BSONObjIterator iter(rolesElement.embeddedObject()); iter.more(); iter.next()) { + if ((*iter).type() != Object) { + return _badValue("Elements in 'roles' array must objects", 0); + } + Status status = V2UserDocumentParser::checkValidRoleObject((*iter).Obj()); + if (!status.isOK()) + return status; + } + return Status::OK(); + } + + Status V2UserDocumentParser::checkValidUserDocument(const BSONObj& doc) const { + BSONElement userElement = doc[AuthorizationManager::USER_NAME_FIELD_NAME]; + BSONElement userDBElement = doc[AuthorizationManager::USER_DB_FIELD_NAME]; + BSONElement credentialsElement = doc[CREDENTIALS_FIELD_NAME]; + BSONElement rolesElement = doc[ROLES_FIELD_NAME]; + + // Validate the "user" element. + if (userElement.type() != String) + return _badValue("User document needs 'user' field to be a string", 0); + if (makeStringDataFromBSONElement(userElement).empty()) + return _badValue("User document needs 'user' field to be non-empty", 0); + + // Validate the "db" element + if (userDBElement.type() != String || + makeStringDataFromBSONElement(userDBElement).empty()) { + return _badValue("User document needs 'db' field to be a non-empty string", 0); + } + StringData userDBStr = makeStringDataFromBSONElement(userDBElement); + if (!NamespaceString::validDBName(userDBStr) && userDBStr != "$external") { + return _badValue(mongoutils::str::stream() << "'" << userDBStr << + "' is not a valid value for the db field.", + 0); + } + + // Validate the "credentials" element + if (credentialsElement.eoo()) { + return _badValue("User document needs 'credentials' object", + 0); + } + if (credentialsElement.type() != Object) { + return _badValue("User document needs 'credentials' field to be an object", 0); + } + + BSONObj credentialsObj = credentialsElement.Obj(); + if (credentialsObj.isEmpty()) { + return _badValue("User document needs 'credentials' field to be a non-empty object", + 0); + } + if (userDBStr == "$external") { + BSONElement externalElement = credentialsObj[MONGODB_EXTERNAL_CREDENTIAL_FIELD_NAME]; + if (externalElement.eoo() || externalElement.type() != Bool || + !externalElement.Bool()) { + return _badValue("User documents for users defined on '$external' must have " + "'credentials' field set to {external: true}", 0); + } + } else { + BSONElement MongoCRElement = credentialsObj[MONGODB_CR_CREDENTIAL_FIELD_NAME]; + if (MongoCRElement.eoo()) { + return _badValue("User document must provide MONGODB-CR credential to all " + "non-external users", 0); + } + if (MongoCRElement.type() != String || + makeStringDataFromBSONElement(MongoCRElement).empty()) { + return _badValue("MONGODB-CR credential must to be a non-empty string, if present", + 0); + } + } + + // Validate the "roles" element. + Status status = _checkV2RolesArray(rolesElement); + if (!status.isOK()) + return status; + + return Status::OK(); + } + + std::string V2UserDocumentParser::extractUserNameFromUserDocument( + const BSONObj& doc) const { + return doc[AuthorizationManager::USER_NAME_FIELD_NAME].str(); + } + + Status V2UserDocumentParser::initializeUserCredentialsFromUserDocument( + User* user, const BSONObj& privDoc) const { + User::CredentialData credentials; + std::string userDB = privDoc[AuthorizationManager::USER_DB_FIELD_NAME].String(); + BSONElement credentialsElement = privDoc[CREDENTIALS_FIELD_NAME]; + if (!credentialsElement.eoo()) { + if (credentialsElement.type() != Object) { + return Status(ErrorCodes::UnsupportedFormat, + "'credentials' field in user documents must be an object"); + } + if (userDB == "$external") { + BSONElement externalCredentialElement = + credentialsElement.Obj()[MONGODB_EXTERNAL_CREDENTIAL_FIELD_NAME]; + if (!externalCredentialElement.eoo()) { + if (externalCredentialElement.type() != Bool || + !externalCredentialElement.Bool()) { + return Status(ErrorCodes::UnsupportedFormat, + "'external' field in credentials object must be set to true"); + } else { + credentials.isExternal = true; + } + } else { + return Status(ErrorCodes::UnsupportedFormat, + "User documents defined on '$external' must provide set " + "credentials to {external:true}"); + } + } else { + BSONElement mongoCRCredentialElement = + credentialsElement.Obj()[MONGODB_CR_CREDENTIAL_FIELD_NAME]; + if (!mongoCRCredentialElement.eoo()) { + if (mongoCRCredentialElement.type() != String || + makeStringDataFromBSONElement(mongoCRCredentialElement).empty()) { + return Status(ErrorCodes::UnsupportedFormat, + "MONGODB-CR credentials must be non-empty strings"); + } else { + credentials.isExternal = false; + credentials.password = mongoCRCredentialElement.String(); + } + } else { + return Status(ErrorCodes::UnsupportedFormat, + "User documents must provide credentials for MONGODB-CR" + " authentication"); + } + } + } else { + return Status(ErrorCodes::UnsupportedFormat, + "Cannot extract credentials from user documents without a " + "'credentials' field"); + } + + user->setCredentials(credentials); + return Status::OK(); + } + + static Status _extractRoleDocumentElements( + const BSONObj& roleObject, + BSONElement* roleNameElement, + BSONElement* roleSourceElement) { + + *roleNameElement = roleObject[ROLE_NAME_FIELD_NAME]; + *roleSourceElement = roleObject[ROLE_SOURCE_FIELD_NAME]; + + if (roleNameElement->type() != String || + makeStringDataFromBSONElement(*roleNameElement).empty()) { + return Status(ErrorCodes::UnsupportedFormat, + "Role names must be non-empty strings"); + } + if (roleSourceElement->type() != String || + makeStringDataFromBSONElement(*roleSourceElement).empty()) { + return Status(ErrorCodes::UnsupportedFormat, "Role db must be non-empty strings"); + } + + return Status::OK(); + } + + Status V2UserDocumentParser::checkValidRoleObject(const BSONObj& roleObject) { + BSONElement roleNameElement; + BSONElement roleSourceElement; + return _extractRoleDocumentElements( + roleObject, + &roleNameElement, + &roleSourceElement); + } + + Status V2UserDocumentParser::parseRoleName(const BSONObj& roleObject, RoleName* result) { + BSONElement roleNameElement; + BSONElement roleSourceElement; + Status status = _extractRoleDocumentElements( + roleObject, + &roleNameElement, + &roleSourceElement); + if (!status.isOK()) + return status; + *result = RoleName(roleNameElement.str(), roleSourceElement.str()); + return status; + } + + Status V2UserDocumentParser::parseRoleVector(const BSONArray& rolesArray, + std::vector* result) { + std::vector roles; + for (BSONObjIterator it(rolesArray); it.more(); it.next()) { + if ((*it).type() != Object) { + return Status(ErrorCodes::TypeMismatch, "Roles must be objects."); + } + RoleName role; + Status status = parseRoleName((*it).Obj(), &role); + if (!status.isOK()) + return status; + roles.push_back(role); + } + std::swap(*result, roles); + return Status::OK(); + } + + Status V2UserDocumentParser::initializeUserRolesFromUserDocument( + const BSONObj& privDoc, User* user) const { + + BSONElement rolesElement = privDoc[ROLES_FIELD_NAME]; + + if (rolesElement.type() != Array) { + return Status(ErrorCodes::UnsupportedFormat, + "User document needs 'roles' field to be an array"); + } + + std::vector roles; + for (BSONObjIterator it(rolesElement.Obj()); it.more(); it.next()) { + if ((*it).type() != Object) { + return Status(ErrorCodes::UnsupportedFormat, + "User document needs values in 'roles' array to be a sub-documents"); + } + BSONObj roleObject = (*it).Obj(); + + RoleName role; + Status status = parseRoleName(roleObject, &role); + if (!status.isOK()) { + return status; + } + roles.push_back(role); + } + user->setRoles(makeRoleNameIteratorForContainer(roles)); + return Status::OK(); + } + + Status V2UserDocumentParser::initializeUserIndirectRolesFromUserDocument( + const BSONObj& privDoc, User* user) const { + + BSONElement indirectRolesElement = privDoc[INHERITED_ROLES_FIELD_NAME]; + + if (indirectRolesElement.type() != Array) { + return Status(ErrorCodes::UnsupportedFormat, + "User document needs 'inheritedRoles' field to be an array"); + } + + std::vector indirectRoles; + for (BSONObjIterator it(indirectRolesElement.Obj()); it.more(); it.next()) { + if ((*it).type() != Object) { + return Status(ErrorCodes::UnsupportedFormat, + "User document needs values in 'inheritedRoles'" + " array to be a sub-documents"); + } + BSONObj indirectRoleObject = (*it).Obj(); + + RoleName indirectRole; + Status status = parseRoleName(indirectRoleObject, &indirectRole); + if (!status.isOK()) { + return status; + } + indirectRoles.push_back(indirectRole); + } + user->setIndirectRoles(makeRoleNameIteratorForContainer(indirectRoles)); + return Status::OK(); + } + + Status V2UserDocumentParser::initializeUserPrivilegesFromUserDocument(const BSONObj& doc, + User* user) const { + BSONElement privilegesElement = doc[PRIVILEGES_FIELD_NAME]; + if (privilegesElement.eoo()) + return Status::OK(); + if (privilegesElement.type() != Array) { + return Status(ErrorCodes::UnsupportedFormat, + "User document 'inheritedPrivileges' element must be Array if present."); + } + PrivilegeVector privileges; + std::string errmsg; + for (BSONObjIterator it(privilegesElement.Obj()); it.more(); it.next()) { + if ((*it).type() != Object) { + warning() << "Wrong type of element in inheritedPrivileges array for " << + user->getName() << ": " << *it; + continue; + } + Privilege privilege; + ParsedPrivilege pp; + if (!pp.parseBSON((*it).Obj(), &errmsg) || + !ParsedPrivilege::parsedPrivilegeToPrivilege(pp, &privilege, &errmsg)) { + + warning() << "Could not parse privilege element in user document for " << + user->getName() << ": " << errmsg; + continue; + } + privileges.push_back(privilege); + } + user->setPrivileges(privileges); + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/user_document_parser.h b/src/mongo/db/auth/user_document_parser.h new file mode 100644 index 00000000000..3be1ae98812 --- /dev/null +++ b/src/mongo/db/auth/user_document_parser.h @@ -0,0 +1,76 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class V1UserDocumentParser { + MONGO_DISALLOW_COPYING(V1UserDocumentParser); + public: + V1UserDocumentParser() {} + std::string extractUserNameFromUserDocument(const BSONObj& doc) const; + + Status initializeUserCredentialsFromUserDocument(User* user, + const BSONObj& privDoc) const; + + Status initializeUserRolesFromUserDocument( + User* user, const BSONObj& privDoc, const StringData& dbname) const; + }; + + class V2UserDocumentParser { + MONGO_DISALLOW_COPYING(V2UserDocumentParser); + public: + V2UserDocumentParser() {} + Status checkValidUserDocument(const BSONObj& doc) const; + + /** + * Returns Status::OK() iff the given BSONObj describes a valid element from a roles array. + */ + static Status checkValidRoleObject(const BSONObj& roleObject); + + static Status parseRoleName(const BSONObj& roleObject, RoleName* result); + + static Status parseRoleVector(const BSONArray& rolesArray, std::vector* result); + + std::string extractUserNameFromUserDocument(const BSONObj& doc) const; + + Status initializeUserCredentialsFromUserDocument(User* user, const BSONObj& privDoc) const; + + Status initializeUserRolesFromUserDocument(const BSONObj& doc, User* user) const; + Status initializeUserIndirectRolesFromUserDocument(const BSONObj& doc, User* user) const; + Status initializeUserPrivilegesFromUserDocument(const BSONObj& doc, User* user) const; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/user_document_parser_test.cpp b/src/mongo/db/auth/user_document_parser_test.cpp new file mode 100644 index 00000000000..f4a67373170 --- /dev/null +++ b/src/mongo/db/auth/user_document_parser_test.cpp @@ -0,0 +1,388 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests of the UserDocumentParser type. + */ + +#include "mongo/base/status.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user_document_parser.h" +#include "mongo/db/jsobj.h" +#include "mongo/unittest/unittest.h" + +#define ASSERT_NULL(EXPR) ASSERT_FALSE(EXPR) +#define ASSERT_NON_NULL(EXPR) ASSERT_TRUE(EXPR) + +namespace mongo { +namespace { + + class V1UserDocumentParsing : public ::mongo::unittest::Test { + public: + V1UserDocumentParsing() {} + + scoped_ptr user; + scoped_ptr adminUser; + V1UserDocumentParser v1parser; + + void setUp() { + resetUsers(); + } + + void resetUsers() { + user.reset(new User(UserName("spencer", "test"))); + adminUser.reset(new User(UserName("admin", "admin"))); + } + }; + + TEST_F(V1UserDocumentParsing, testParsingV0UserDocuments) { + BSONObj readWrite = BSON("user" << "spencer" << "pwd" << "passwordHash"); + BSONObj readOnly = BSON("user" << "spencer" << "pwd" << "passwordHash" << + "readOnly" << true); + BSONObj readWriteAdmin = BSON("user" << "admin" << "pwd" << "passwordHash"); + BSONObj readOnlyAdmin = BSON("user" << "admin" << "pwd" << "passwordHash" << + "readOnly" << true); + + ASSERT_OK(v1parser.initializeUserRolesFromUserDocument( + user.get(), readOnly, "test")); + RoleNameIterator roles = user->getRoles(); + ASSERT_EQUALS(RoleName("read", "test"), roles.next()); + ASSERT_FALSE(roles.more()); + + resetUsers(); + ASSERT_OK(v1parser.initializeUserRolesFromUserDocument( + user.get(), readWrite, "test")); + roles = user->getRoles(); + ASSERT_EQUALS(RoleName("dbOwner", "test"), roles.next()); + ASSERT_FALSE(roles.more()); + + resetUsers(); + ASSERT_OK(v1parser.initializeUserRolesFromUserDocument( + adminUser.get(), readOnlyAdmin, "admin")); + roles = adminUser->getRoles(); + ASSERT_EQUALS(RoleName("readAnyDatabase", "admin"), roles.next()); + ASSERT_FALSE(roles.more()); + + resetUsers(); + ASSERT_OK(v1parser.initializeUserRolesFromUserDocument( + adminUser.get(), readWriteAdmin, "admin")); + roles = adminUser->getRoles(); + ASSERT_EQUALS(RoleName("root", "admin"), roles.next()); + ASSERT_FALSE(roles.more()); + } + + TEST_F(V1UserDocumentParsing, VerifyRolesFieldMustBeAnArray) { + ASSERT_NOT_OK(v1parser.initializeUserRolesFromUserDocument( + user.get(), + BSON("user" << "spencer" << "pwd" << "" << "roles" << "read"), + "test")); + ASSERT_FALSE(user->getRoles().more()); + } + + TEST_F(V1UserDocumentParsing, VerifySemanticallyInvalidRolesStillParse) { + ASSERT_OK(v1parser.initializeUserRolesFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "pwd" << "" << + "roles" << BSON_ARRAY("read" << "frim")), + "test")); + RoleNameIterator roles = user->getRoles(); + RoleName role = roles.next(); + if (role == RoleName("read", "test")) { + ASSERT_EQUALS(RoleName("frim", "test"), roles.next()); + } else { + ASSERT_EQUALS(RoleName("frim", "test"), role); + ASSERT_EQUALS(RoleName("read", "test"), roles.next()); + } + ASSERT_FALSE(roles.more()); + } + + TEST_F(V1UserDocumentParsing, VerifyOtherDBRolesMustBeAnObjectOfArraysOfStrings) { + ASSERT_NOT_OK(v1parser.initializeUserRolesFromUserDocument( + adminUser.get(), + BSON("user" << "admin" << + "pwd" << "" << + "roles" << BSON_ARRAY("read") << + "otherDBRoles" << BSON_ARRAY("read")), + "admin")); + + ASSERT_NOT_OK(v1parser.initializeUserRolesFromUserDocument( + adminUser.get(), + BSON("user" << "admin" << + "pwd" << "" << + "roles" << BSON_ARRAY("read") << + "otherDBRoles" << BSON("test2" << "read")), + "admin")); + } + + TEST_F(V1UserDocumentParsing, VerifyCannotGrantPrivilegesOnOtherDatabasesNormally) { + // Cannot grant roles on other databases, except from admin database. + ASSERT_NOT_OK(v1parser.initializeUserRolesFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "pwd" << "" << + "roles" << BSONArrayBuilder().arr() << + "otherDBRoles" << BSON("test2" << BSON_ARRAY("read"))), + "test")); + ASSERT_FALSE(user->getRoles().more()); + } + + TEST_F(V1UserDocumentParsing, GrantUserAdminOnTestViaAdmin) { + // Grant userAdmin on test via admin. + ASSERT_OK(v1parser.initializeUserRolesFromUserDocument( + adminUser.get(), + BSON("user" << "admin" << + "pwd" << "" << + "roles" << BSONArrayBuilder().arr() << + "otherDBRoles" << BSON("test" << BSON_ARRAY("userAdmin"))), + "admin")); + RoleNameIterator roles = adminUser->getRoles(); + ASSERT_EQUALS(RoleName("userAdmin", "test"), roles.next()); + ASSERT_FALSE(roles.more()); + } + + TEST_F(V1UserDocumentParsing, MixedV0V1UserDocumentsAreInvalid) { + // Try to mix fields from V0 and V1 user documents and make sure it fails. + ASSERT_NOT_OK(v1parser.initializeUserRolesFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "pwd" << "passwordHash" << + "readOnly" << false << + "roles" << BSON_ARRAY("read")), + "test")); + ASSERT_FALSE(user->getRoles().more()); + } + + class V2UserDocumentParsing : public ::mongo::unittest::Test { + public: + V2UserDocumentParsing() {} + + scoped_ptr user; + scoped_ptr adminUser; + V2UserDocumentParser v2parser; + + void setUp() { + user.reset(new User(UserName("spencer", "test"))); + adminUser.reset(new User(UserName("admin", "admin"))); + } + }; + + + TEST_F(V2UserDocumentParsing, V2DocumentValidation) { + BSONArray emptyArray = BSONArrayBuilder().arr(); + + // V1 documents don't work + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << "pwd" << "a" << + "roles" << BSON_ARRAY("read")))); + + // Need name field + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << emptyArray))); + + // Need source field + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << emptyArray))); + + // Need credentials field + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "roles" << emptyArray))); + + // Need roles field + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a")))); + + // Empty roles arrays are OK + ASSERT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << emptyArray))); + + // Need credentials of {external: true} if user's db is $external + ASSERT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "$external" << + "credentials" << BSON("external" << true) << + "roles" << emptyArray))); + + // Roles must be objects + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY("read")))); + + // Role needs name + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("db" << "dbA"))))); + + // Role needs source + ASSERT_NOT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "roleA"))))); + + + // Basic valid user document + ASSERT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "roleA" << + "db" << "dbA"))))); + + // Multiple roles OK + ASSERT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "roles" << BSON_ARRAY(BSON("role" << "roleA" << + "db" << "dbA") << + BSON("role" << "roleB" << + "db" << "dbB"))))); + + // Optional extraData field OK + ASSERT_OK(v2parser.checkValidUserDocument( + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a") << + "extraData" << BSON("foo" << "bar") << + "roles" << BSON_ARRAY(BSON("role" << "roleA" << + "db" << "dbA"))))); + } + + TEST_F(V2UserDocumentParsing, V2CredentialExtraction) { + // Old "pwd" field not valid + ASSERT_NOT_OK(v2parser.initializeUserCredentialsFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "db" << "test" << + "pwd" << ""))); + + // Credentials must be provided + ASSERT_NOT_OK(v2parser.initializeUserCredentialsFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "db" << "test"))); + + // Credentials must be object + ASSERT_NOT_OK(v2parser.initializeUserCredentialsFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << "a"))); + + // Must specify credentials for MONGODB-CR + ASSERT_NOT_OK(v2parser.initializeUserCredentialsFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("foo" << "bar")))); + + // Make sure extracting valid credentials works + ASSERT_OK(v2parser.initializeUserCredentialsFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "db" << "test" << + "credentials" << BSON("MONGODB-CR" << "a")))); + ASSERT(user->getCredentials().password == "a"); + ASSERT(!user->getCredentials().isExternal); + + // Credentials are {external:true if users's db is $external + ASSERT_OK(v2parser.initializeUserCredentialsFromUserDocument( + user.get(), + BSON("user" << "spencer" << + "db" << "$external" << + "credentials" << BSON("external" << true)))); + ASSERT(user->getCredentials().password.empty()); + ASSERT(user->getCredentials().isExternal); + + } + + TEST_F(V2UserDocumentParsing, V2RoleExtraction) { + // "roles" field must be provided + ASSERT_NOT_OK(v2parser.initializeUserRolesFromUserDocument( + BSON("user" << "spencer"), + user.get())); + + // V1-style roles arrays no longer work + ASSERT_NOT_OK(v2parser.initializeUserRolesFromUserDocument( + BSON("user" << "spencer" << + "roles" << BSON_ARRAY("read")), + user.get())); + + // Roles must have "db" field + ASSERT_NOT_OK(v2parser.initializeUserRolesFromUserDocument( + BSON("user" << "spencer" << + "roles" << BSON_ARRAY(BSONObj())), + user.get())); + + ASSERT_NOT_OK(v2parser.initializeUserRolesFromUserDocument( + BSON("user" << "spencer" << + "roles" << BSON_ARRAY(BSON("role" << "roleA"))), + user.get())); + + ASSERT_NOT_OK(v2parser.initializeUserRolesFromUserDocument( + BSON("user" << "spencer" << + "roles" << BSON_ARRAY(BSON("user" << "roleA" << + "db" << "dbA"))), + user.get())); + + // Valid role names are extracted successfully + ASSERT_OK(v2parser.initializeUserRolesFromUserDocument( + BSON("user" << "spencer" << + "roles" << BSON_ARRAY(BSON("role" << "roleA" << + "db" << "dbA"))), + user.get())); + RoleNameIterator roles = user->getRoles(); + ASSERT_EQUALS(RoleName("roleA", "dbA"), roles.next()); + ASSERT_FALSE(roles.more()); + + // Multiple roles OK + ASSERT_OK(v2parser.initializeUserRolesFromUserDocument( + BSON("user" << "spencer" << + "roles" << BSON_ARRAY(BSON("role" << "roleA" << + "db" << "dbA") << + BSON("role" << "roleB" << + "db" << "dbB"))), + user.get())); + roles = user->getRoles(); + RoleName role = roles.next(); + if (role == RoleName("roleA", "dbA")) { + ASSERT_EQUALS(RoleName("roleB", "dbB"), roles.next()); + } else { + ASSERT_EQUALS(RoleName("roleB", "dbB"), role); + ASSERT_EQUALS(RoleName("roleA", "dbA"), roles.next()); + } + ASSERT_FALSE(roles.more()); + } + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/auth/user_management_commands_parser.cpp b/src/mongo/db/auth/user_management_commands_parser.cpp new file mode 100644 index 00000000000..ea2c0fd262c --- /dev/null +++ b/src/mongo/db/auth/user_management_commands_parser.cpp @@ -0,0 +1,749 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/user_management_commands_parser.h" + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/bson/util/bson_extract.h" +#include "mongo/client/auth_helpers.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/privilege_parser.h" +#include "mongo/db/auth/user_document_parser.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/unordered_set.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/password_digest.h" + +namespace mongo { +namespace auth { + + /** + * Writes into *writeConcern a BSONObj describing the parameters to getLastError to use for + * the write confirmation. + */ + Status _extractWriteConcern(const BSONObj& cmdObj, BSONObj* writeConcern) { + BSONElement writeConcernElement; + Status status = bsonExtractTypedField(cmdObj, "writeConcern", Object, &writeConcernElement); + if (!status.isOK()) { + if (status.code() == ErrorCodes::NoSuchKey) { + *writeConcern = BSONObj(); + return Status::OK(); + } + return status; + } + *writeConcern = writeConcernElement.Obj().getOwned();; + return Status::OK(); + } + + Status _checkNoExtraFields(const BSONObj& cmdObj, + const StringData& cmdName, + const unordered_set& validFieldNames) { + // Iterate through all fields in command object and make sure there are no unexpected + // ones. + for (BSONObjIterator iter(cmdObj); iter.more(); iter.next()) { + StringData fieldName = (*iter).fieldNameStringData(); + if (!validFieldNames.count(fieldName.toString())) { + return Status(ErrorCodes::BadValue, + mongoutils::str::stream() << "\"" << fieldName << "\" is not " + "a valid argument to " << cmdName); + } + } + return Status::OK(); + } + + // Extracts a UserName or RoleName object from a BSONElement. + template + Status _parseNameFromBSONElement(const BSONElement& element, + const StringData& dbname, + const StringData& nameFieldName, + const StringData& sourceFieldName, + Name* parsedName) { + if (element.type() == String) { + *parsedName = Name(element.String(), dbname); + } + else if (element.type() == Object) { + BSONObj obj = element.Obj(); + + std::string name; + std::string source; + Status status = bsonExtractStringField(obj, nameFieldName, &name); + if (!status.isOK()) { + return status; + } + status = bsonExtractStringField(obj, sourceFieldName, &source); + if (!status.isOK()) { + return status; + } + + *parsedName = Name(name, source); + } + else { + return Status(ErrorCodes::BadValue, + "User and role names must be either strings or objects"); + } + return Status::OK(); + } + + // Extracts UserName or RoleName objects from a BSONArray of role/user names. + template + Status _parseNamesFromBSONArray(const BSONArray& array, + const StringData& dbname, + const StringData& nameFieldName, + const StringData& sourceFieldName, + std::vector* parsedNames) { + for (BSONObjIterator it(array); it.more(); it.next()) { + BSONElement element = *it; + Name name; + Status status = _parseNameFromBSONElement(element, + dbname, + nameFieldName, + sourceFieldName, + &name); + if (!status.isOK()) { + return status; + } + parsedNames->push_back(name); + } + return Status::OK(); + } + + Status parseUserNamesFromBSONArray(const BSONArray& usersArray, + const StringData& dbname, + std::vector* parsedUserNames) { + return _parseNamesFromBSONArray(usersArray, + dbname, + AuthorizationManager::USER_NAME_FIELD_NAME, + AuthorizationManager::USER_DB_FIELD_NAME, + parsedUserNames); + } + + Status parseRoleNamesFromBSONArray(const BSONArray& rolesArray, + const StringData& dbname, + std::vector* parsedRoleNames) { + return _parseNamesFromBSONArray(rolesArray, + dbname, + AuthorizationManager::ROLE_NAME_FIELD_NAME, + AuthorizationManager::ROLE_SOURCE_FIELD_NAME, + parsedRoleNames); + } + + Status parseRolePossessionManipulationCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + std::string* parsedName, + vector* parsedRoleNames, + BSONObj* parsedWriteConcern) { + unordered_set validFieldNames; + validFieldNames.insert(cmdName.toString()); + validFieldNames.insert("roles"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, cmdName, validFieldNames); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, parsedWriteConcern); + if (!status.isOK()) { + return status; + } + + status = bsonExtractStringField(cmdObj, cmdName, parsedName); + if (!status.isOK()) { + return status; + } + + BSONElement rolesElement; + status = bsonExtractTypedField(cmdObj, "roles", Array, &rolesElement); + if (!status.isOK()) { + return status; + } + + status = parseRoleNamesFromBSONArray(BSONArray(rolesElement.Obj()), + dbname, + parsedRoleNames); + if (!status.isOK()) { + return status; + } + + if (!parsedRoleNames->size()) { + return Status(ErrorCodes::BadValue, + mongoutils::str::stream() << cmdName << " command requires a non-empty " + "\"roles\" array"); + } + return Status::OK(); + } + + Status parseCreateOrUpdateUserCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + CreateOrUpdateUserArgs* parsedArgs) { + unordered_set validFieldNames; + validFieldNames.insert(cmdName.toString()); + validFieldNames.insert("customData"); + validFieldNames.insert("digestPassword"); + validFieldNames.insert("pwd"); + validFieldNames.insert("roles"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, cmdName, validFieldNames); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, &parsedArgs->writeConcern); + if (!status.isOK()) { + return status; + } + + BSONObjBuilder userObjBuilder; + + // Parse user name + std::string userName; + status = bsonExtractStringField(cmdObj, cmdName, &userName); + if (!status.isOK()) { + return status; + } + + parsedArgs->userName = UserName(userName, dbname); + + // Parse password + if (cmdObj.hasField("pwd")) { + std::string password; + status = bsonExtractStringField(cmdObj, "pwd", &password); + if (!status.isOK()) { + return status; + } + if (password.empty()) { + return Status(ErrorCodes::BadValue, "User passwords must not be empty"); + } + + bool digestPassword; // True if the server should digest the password + status = bsonExtractBooleanFieldWithDefault(cmdObj, + "digestPassword", + true, + &digestPassword); + if (!status.isOK()) { + return status; + } + + if (digestPassword) { + parsedArgs->hashedPassword = mongo::createPasswordDigest( + userName, password); + } else { + parsedArgs->hashedPassword = password; + } + parsedArgs->hasHashedPassword = true; + } + + // Parse custom data + if (cmdObj.hasField("customData")) { + BSONElement element; + status = bsonExtractTypedField(cmdObj, "customData", Object, &element); + if (!status.isOK()) { + return status; + } + parsedArgs->customData = element.Obj(); + parsedArgs->hasCustomData = true; + } + + // Parse roles + if (cmdObj.hasField("roles")) { + BSONElement rolesElement; + status = bsonExtractTypedField(cmdObj, "roles", Array, &rolesElement); + if (!status.isOK()) { + return status; + } + status = parseRoleNamesFromBSONArray(BSONArray(rolesElement.Obj()), + dbname, + &parsedArgs->roles); + if (!status.isOK()) { + return status; + } + parsedArgs->hasRoles = true; + } + + return Status::OK(); + } + + Status parseAndValidateDropUserCommand(const BSONObj& cmdObj, + const std::string& dbname, + UserName* parsedUserName, + BSONObj* parsedWriteConcern) { + unordered_set validFieldNames; + validFieldNames.insert("dropUser"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, "dropUser", validFieldNames); + if (!status.isOK()) { + return status; + } + + std::string user; + status = bsonExtractStringField(cmdObj, "dropUser", &user); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, parsedWriteConcern); + if (!status.isOK()) { + return status; + } + + *parsedUserName = UserName(user, dbname); + return Status::OK(); + } + + Status parseAndValidateDropAllUsersFromDatabaseCommand(const BSONObj& cmdObj, + const std::string& dbname, + BSONObj* parsedWriteConcern) { + unordered_set validFieldNames; + validFieldNames.insert("dropAllUsersFromDatabase"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, "dropAllUsersFromDatabase", validFieldNames); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, parsedWriteConcern); + if (!status.isOK()) { + return status; + } + + return Status::OK(); + } + + Status parseUsersInfoCommand(const BSONObj& cmdObj, + const StringData& dbname, + UsersInfoArgs* parsedArgs) { + unordered_set validFieldNames; + validFieldNames.insert("usersInfo"); + validFieldNames.insert("showPrivileges"); + validFieldNames.insert("showCredentials"); + + Status status = _checkNoExtraFields(cmdObj, "usersInfo", validFieldNames); + if (!status.isOK()) { + return status; + } + + if (cmdObj["usersInfo"].numberInt() == 1) { + parsedArgs->allForDB = true; + } else if (cmdObj["usersInfo"].type() == Array) { + status = parseUserNamesFromBSONArray(BSONArray(cmdObj["usersInfo"].Obj()), + dbname, + &parsedArgs->userNames); + if (!status.isOK()) { + return status; + } + } else { + UserName name; + status = _parseNameFromBSONElement(cmdObj["usersInfo"], + dbname, + AuthorizationManager::USER_NAME_FIELD_NAME, + AuthorizationManager::USER_DB_FIELD_NAME, + &name); + if (!status.isOK()) { + return status; + } + parsedArgs->userNames.push_back(name); + } + + status = bsonExtractBooleanFieldWithDefault(cmdObj, + "showPrivileges", + false, + &parsedArgs->showPrivileges); + if (!status.isOK()) { + return status; + } + status = bsonExtractBooleanFieldWithDefault(cmdObj, + "showCredentials", + false, + &parsedArgs->showCredentials); + if (!status.isOK()) { + return status; + } + + return Status::OK(); + } + + Status parseRolesInfoCommand(const BSONObj& cmdObj, + const StringData& dbname, + RolesInfoArgs* parsedArgs) { + unordered_set validFieldNames; + validFieldNames.insert("rolesInfo"); + validFieldNames.insert("showPrivileges"); + validFieldNames.insert("showBuiltinRoles"); + + Status status = _checkNoExtraFields(cmdObj, "rolesInfo", validFieldNames); + if (!status.isOK()) { + return status; + } + + if (cmdObj["rolesInfo"].numberInt() == 1) { + parsedArgs->allForDB = true; + } else if (cmdObj["rolesInfo"].type() == Array) { + status = parseRoleNamesFromBSONArray(BSONArray(cmdObj["rolesInfo"].Obj()), + dbname, + &parsedArgs->roleNames); + if (!status.isOK()) { + return status; + } + } else { + RoleName name; + status = _parseNameFromBSONElement(cmdObj["rolesInfo"], + dbname, + AuthorizationManager::ROLE_NAME_FIELD_NAME, + AuthorizationManager::ROLE_SOURCE_FIELD_NAME, + &name); + if (!status.isOK()) { + return status; + } + parsedArgs->roleNames.push_back(name); + } + + status = bsonExtractBooleanFieldWithDefault(cmdObj, + "showPrivileges", + false, + &parsedArgs->showPrivileges); + if (!status.isOK()) { + return status; + } + + status = bsonExtractBooleanFieldWithDefault(cmdObj, + "showBuiltinRoles", + false, + &parsedArgs->showBuiltinRoles); + if (!status.isOK()) { + return status; + } + + return Status::OK(); + } + + /* + * Validates that the given privilege BSONArray is valid. + * If parsedPrivileges is not NULL, adds to it the privileges parsed out of the input BSONArray. + */ + Status parseAndValidatePrivilegeArray(const BSONArray& privileges, + PrivilegeVector* parsedPrivileges) { + for (BSONObjIterator it(privileges); it.more(); it.next()) { + BSONElement element = *it; + if (element.type() != Object) { + return Status(ErrorCodes::FailedToParse, + "Elements in privilege arrays must be objects"); + } + + ParsedPrivilege parsedPrivilege; + std::string errmsg; + if (!parsedPrivilege.parseBSON(element.Obj(), &errmsg)) { + return Status(ErrorCodes::FailedToParse, errmsg); + } + if (!parsedPrivilege.isValid(&errmsg)) { + return Status(ErrorCodes::FailedToParse, errmsg); + } + + Privilege privilege; + if (!ParsedPrivilege::parsedPrivilegeToPrivilege(parsedPrivilege, &privilege, &errmsg)) { + return Status(ErrorCodes::FailedToParse, errmsg); + } + + parsedPrivileges->push_back(privilege); + } + return Status::OK(); + } + + Status parseCreateOrUpdateRoleCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + CreateOrUpdateRoleArgs* parsedArgs) { + unordered_set validFieldNames; + validFieldNames.insert(cmdName.toString()); + validFieldNames.insert("privileges"); + validFieldNames.insert("roles"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, cmdName, validFieldNames); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, &parsedArgs->writeConcern); + if (!status.isOK()) { + return status; + } + + std::string roleName; + status = bsonExtractStringField(cmdObj, cmdName, &roleName); + if (!status.isOK()) { + return status; + } + parsedArgs->roleName = RoleName(roleName, dbname); + + // Parse privileges + if (cmdObj.hasField("privileges")) { + BSONElement privilegesElement; + status = bsonExtractTypedField(cmdObj, "privileges", Array, &privilegesElement); + if (!status.isOK()) { + return status; + } + status = parseAndValidatePrivilegeArray(BSONArray(privilegesElement.Obj()), + &parsedArgs->privileges); + if (!status.isOK()) { + return status; + } + parsedArgs->hasPrivileges = true; + } + + // Parse roles + if (cmdObj.hasField("roles")) { + BSONElement rolesElement; + status = bsonExtractTypedField(cmdObj, "roles", Array, &rolesElement); + if (!status.isOK()) { + return status; + } + status = parseRoleNamesFromBSONArray(BSONArray(rolesElement.Obj()), + dbname, + &parsedArgs->roles); + if (!status.isOK()) { + return status; + } + parsedArgs->hasRoles = true; + } + return Status::OK(); + } + + Status parseAndValidateRolePrivilegeManipulationCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + RoleName* parsedRoleName, + PrivilegeVector* parsedPrivileges, + BSONObj* parsedWriteConcern) { + unordered_set validFieldNames; + validFieldNames.insert(cmdName.toString()); + validFieldNames.insert("privileges"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, cmdName, validFieldNames); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, parsedWriteConcern); + if (!status.isOK()) { + return status; + } + + BSONObjBuilder roleObjBuilder; + + // Parse role name + std::string roleName; + status = bsonExtractStringField(cmdObj, cmdName, &roleName); + if (!status.isOK()) { + return status; + } + *parsedRoleName = RoleName(roleName, dbname); + + // Parse privileges + BSONElement privilegesElement; + status = bsonExtractTypedField(cmdObj, "privileges", Array, &privilegesElement); + if (!status.isOK()) { + return status; + } + status = parseAndValidatePrivilegeArray(BSONArray(privilegesElement.Obj()), + parsedPrivileges); + if (!status.isOK()) { + return status; + } + if (!parsedPrivileges->size()) { + return Status(ErrorCodes::BadValue, + mongoutils::str::stream() << cmdName << " command requires a non-empty " + "\"privileges\" array"); + } + + return Status::OK(); + } + + Status parseDropRoleCommand(const BSONObj& cmdObj, + const std::string& dbname, + RoleName* parsedRoleName, + BSONObj* parsedWriteConcern) { + unordered_set validFieldNames; + validFieldNames.insert("dropRole"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, "dropRole", validFieldNames); + if (!status.isOK()) { + return status; + } + + std::string user; + status = bsonExtractStringField(cmdObj, "dropRole", &user); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, parsedWriteConcern); + if (!status.isOK()) { + return status; + } + + *parsedRoleName = RoleName(user, dbname); + return Status::OK(); + } + + Status parseDropAllRolesFromDatabaseCommand(const BSONObj& cmdObj, + const std::string& dbname, + BSONObj* parsedWriteConcern) { + unordered_set validFieldNames; + validFieldNames.insert("dropAllRolesFromDatabase"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, "dropAllRolesFromDatabase", validFieldNames); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, parsedWriteConcern); + if (!status.isOK()) { + return status; + } + + return Status::OK(); + } + + Status parseMergeAuthzCollectionsCommand(const BSONObj& cmdObj, + MergeAuthzCollectionsArgs* parsedArgs) { + unordered_set validFieldNames; + validFieldNames.insert("_mergeAuthzCollections"); + validFieldNames.insert("tempUsersCollection"); + validFieldNames.insert("tempRolesCollection"); + validFieldNames.insert("db"); + validFieldNames.insert("drop"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, "_mergeAuthzCollections", validFieldNames); + if (!status.isOK()) { + return status; + } + + status = _extractWriteConcern(cmdObj, &parsedArgs->writeConcern); + if (!status.isOK()) { + return status; + } + + status = bsonExtractStringFieldWithDefault(cmdObj, + "tempUsersCollection", + "", + &parsedArgs->usersCollName); + if (!status.isOK()) { + return status; + } + + status = bsonExtractStringFieldWithDefault(cmdObj, + "tempRolesCollection", + "", + &parsedArgs->rolesCollName); + if (!status.isOK()) { + return status; + } + + status = bsonExtractStringField(cmdObj, "db", &parsedArgs->db); + if (!status.isOK()) { + if (status == ErrorCodes::NoSuchKey) { + return Status(ErrorCodes::OutdatedClient, + "Missing \"db\" field for _mergeAuthzCollections command. This is " + "most likely due to running an outdated (pre-2.6.4) version of " + "mongorestore."); + } + return status; + } + + status = bsonExtractBooleanFieldWithDefault(cmdObj, + "drop", + false, + &parsedArgs->drop); + if (!status.isOK()) { + return status; + } + + return Status::OK(); + } + + Status parseAuthSchemaUpgradeStepCommand(const BSONObj& cmdObj, + const std::string& dbname, + int* maxSteps, + bool* shouldUpgradeShards, + BSONObj* parsedWriteConcern) { + static const int minUpgradeSteps = 1; + static const int maxUpgradeSteps = 2; + + unordered_set validFieldNames; + validFieldNames.insert("authSchemaUpgrade"); + validFieldNames.insert("maxSteps"); + validFieldNames.insert("upgradeShards"); + validFieldNames.insert("writeConcern"); + + Status status = _checkNoExtraFields(cmdObj, "authSchemaUpgrade", validFieldNames); + if (!status.isOK()) { + return status; + } + + status = bsonExtractBooleanFieldWithDefault( + cmdObj, "upgradeShards", true, shouldUpgradeShards); + if (!status.isOK()) { + return status; + } + + long long steps; + status = bsonExtractIntegerFieldWithDefault(cmdObj, "maxSteps", maxUpgradeSteps, &steps); + if (!status.isOK()) + return status; + if (steps < minUpgradeSteps || steps > maxUpgradeSteps) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Legal values for \"maxSteps\" are at least " << minUpgradeSteps << + " and no more than " << maxUpgradeSteps << "; found " << steps); + } + *maxSteps = static_cast(steps); + + status = _extractWriteConcern(cmdObj, parsedWriteConcern); + if (!status.isOK()) { + return status; + } + + return Status::OK(); + } + +} // namespace auth +} // namespace mongo diff --git a/src/mongo/db/auth/user_management_commands_parser.h b/src/mongo/db/auth/user_management_commands_parser.h new file mode 100644 index 00000000000..3d87cbfd9d0 --- /dev/null +++ b/src/mongo/db/auth/user_management_commands_parser.h @@ -0,0 +1,242 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/base/disallow_copying.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/jsobj.h" + +namespace mongo { +namespace auth { + + struct CreateOrUpdateUserArgs { + UserName userName; + bool hasHashedPassword; + std::string hashedPassword; + bool hasCustomData; + BSONObj customData; + bool hasRoles; + std::vector roles; + BSONObj writeConcern; + CreateOrUpdateUserArgs() : + hasHashedPassword(false), hasCustomData(false), hasRoles(false) {} + }; + + /** + * Takes a command object describing an invocation of the "createUser" or "updateUser" commands + * (which command it is is specified in "cmdName") on the database "dbname", and parses out all + * the arguments into the "parsedArgs" output param. + */ + Status parseCreateOrUpdateUserCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + CreateOrUpdateUserArgs* parsedArgs); + + /** + * Takes a command object describing an invocation of one of "grantRolesToUser", + * "revokeRolesFromUser", "grantDelegateRolesToUser", "revokeDelegateRolesFromUser", + * "grantRolesToRole", and "revokeRolesFromRoles" (which command it is is specified in the + * "cmdName" argument), and parses out (into the parsedName out param) the user/role name of + * the user/roles being modified, the roles being granted or revoked, and the write concern to + * use. + */ + Status parseRolePossessionManipulationCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + std::string* parsedName, + vector* parsedRoleNames, + BSONObj* parsedWriteConcern); + + /** + * Takes a command object describing an invocation of the "dropUser" command and parses out + * the UserName of the user to be removed and the writeConcern. + * Also validates the input and returns a non-ok Status if there is anything wrong. + */ + Status parseAndValidateDropUserCommand(const BSONObj& cmdObj, + const std::string& dbname, + UserName* parsedUserName, + BSONObj* parsedWriteConcern); + + /** + * Takes a command object describing an invocation of the "dropAllUsersFromDatabase" command and + * parses out the write concern. + * Also validates the input and returns a non-ok Status if there is anything wrong. + */ + Status parseAndValidateDropAllUsersFromDatabaseCommand(const BSONObj& cmdObj, + const std::string& dbname, + BSONObj* parsedWriteConcern); + + struct UsersInfoArgs { + std::vector userNames; + bool allForDB; + bool showPrivileges; + bool showCredentials; + UsersInfoArgs() : allForDB(false), showPrivileges(false), showCredentials(false) {} + }; + + /** + * Takes a command object describing an invocation of the "usersInfo" command and parses out + * all the arguments into the "parsedArgs" output param. + */ + Status parseUsersInfoCommand(const BSONObj& cmdObj, + const StringData& dbname, + UsersInfoArgs* parsedArgs); + + struct RolesInfoArgs { + std::vector roleNames; + bool allForDB; + bool showPrivileges; + bool showBuiltinRoles; + RolesInfoArgs() : allForDB(false), showPrivileges(false), showBuiltinRoles(false) {} + }; + + /** + * Takes a command object describing an invocation of the "rolesInfo" command and parses out + * the arguments into the "parsedArgs" output param. + */ + Status parseRolesInfoCommand(const BSONObj& cmdObj, + const StringData& dbname, + RolesInfoArgs* parsedArgs); + + struct CreateOrUpdateRoleArgs { + RoleName roleName; + bool hasRoles; + std::vector roles; + bool hasPrivileges; + PrivilegeVector privileges; + BSONObj writeConcern; + CreateOrUpdateRoleArgs() : hasRoles(false), hasPrivileges(false) {} + }; + + /** + * Takes a command object describing an invocation of the "createRole" or "updateRole" commands + * (which command it is is specified in "cmdName") on the database "dbname", and parses out all + * the arguments into the "parsedArgs" output param. + */ + Status parseCreateOrUpdateRoleCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + CreateOrUpdateRoleArgs* parsedArgs); + + /** + * Takes a command object describing an invocation of the "grantPrivilegesToRole" or + * "revokePrivilegesFromRole" commands, and parses out the role name of the + * role being modified, the privileges being granted or revoked, and the write concern to use. + */ + Status parseAndValidateRolePrivilegeManipulationCommands(const BSONObj& cmdObj, + const StringData& cmdName, + const std::string& dbname, + RoleName* parsedRoleName, + PrivilegeVector* parsedPrivileges, + BSONObj* parsedWriteConcern); + + /** + * Takes a command object describing an invocation of the "dropRole" command and parses out + * the RoleName of the role to be removed and the writeConcern. + */ + Status parseDropRoleCommand(const BSONObj& cmdObj, + const std::string& dbname, + RoleName* parsedRoleName, + BSONObj* parsedWriteConcern); + + /** + * Takes a command object describing an invocation of the "dropAllRolesFromDatabase" command and + * parses out the write concern. + */ + Status parseDropAllRolesFromDatabaseCommand(const BSONObj& cmdObj, + const std::string& dbname, + BSONObj* parsedWriteConcern); + + /** + * Takes a command object describing an invocation of the "authSchemaUpgrade" command and + * parses out the write concern, maximum steps to take and whether or not shard servers should + * also be upgraded, in the sharded deployment case. + */ + Status parseAuthSchemaUpgradeStepCommand(const BSONObj& cmdObj, + const std::string& dbname, + int* maxSteps, + bool* shouldUpgradeShards, + BSONObj* parsedWriteConcern); + /** + * Parses the privileges described in "privileges" into a vector of Privilege objects. + * Returns Status::OK() upon successfully parsing all the elements of "privileges". + */ + Status parseAndValidatePrivilegeArray(const BSONArray& privileges, + PrivilegeVector* parsedPrivileges); + + /** + * Takes a BSONArray of name,db pair documents, parses that array and returns (via the + * output param parsedRoleNames) a list of the role names in the input array. + * Performs syntactic validation of "rolesArray", only. + */ + Status parseRoleNamesFromBSONArray(const BSONArray& rolesArray, + const StringData& dbname, + std::vector* parsedRoleNames); + + /** + * Takes a BSONArray of name,db pair documents, parses that array and returns (via the + * output param parsedUserNames) a list of the usernames in the input array. + * Performs syntactic validation of "usersArray", only. + */ + Status parseUserNamesFromBSONArray(const BSONArray& usersArray, + const StringData& dbname, + std::vector* parsedUserNames); + + + struct MergeAuthzCollectionsArgs { + std::string usersCollName; + std::string rolesCollName; + std::string db; + bool drop; + BSONObj writeConcern; + MergeAuthzCollectionsArgs() : drop(false) {} + }; + + /** + * Takes a command object describing an invocation of the "_mergeAuthzCollections" command and + * parses out the name of the temporary collections to use for user and role data, whether or + * not to drop the existing users/roles, the database if this is a for a db-specific restore, + * and the writeConcern. + * Returns ErrorCodes::OutdatedClient if the "db" field is missing, as that likely indicates + * the command was sent by an outdated (pre 2.6.4) version of mongorestore. + * Returns other codes indicating missing or incorrectly typed fields. + */ + Status parseMergeAuthzCollectionsCommand(const BSONObj& cmdObj, + MergeAuthzCollectionsArgs* parsedArgs); + +} // namespace auth +} // namespace mongo diff --git a/src/mongo/db/auth/user_name.cpp b/src/mongo/db/auth/user_name.cpp new file mode 100644 index 00000000000..db3d25a69ec --- /dev/null +++ b/src/mongo/db/auth/user_name.cpp @@ -0,0 +1,42 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/auth/user_name.h" + +#include +#include + +#include "mongo/base/string_data.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + UserName::UserName(const StringData& user, const StringData& dbname) { + _fullName.resize(user.size() + dbname.size() + 1); + std::string::iterator iter = std::copy(user.rawData(), + user.rawData() + user.size(), + _fullName.begin()); + *iter = '@'; + ++iter; + iter = std::copy(dbname.rawData(), dbname.rawData() + dbname.size(), iter); + dassert(iter == _fullName.end()); + _splitPoint = user.size(); + } + + std::ostream& operator<<(std::ostream& os, const UserName& name) { + return os << name.getFullName(); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/user_name.h b/src/mongo/db/auth/user_name.h new file mode 100644 index 00000000000..9104fded846 --- /dev/null +++ b/src/mongo/db/auth/user_name.h @@ -0,0 +1,150 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" + +namespace mongo { + + /** + * Representation of a name of a principal (authenticatable user) in a MongoDB system. + * + * Consists of a "user name" part, and a "database name" part. + */ + class UserName { + public: + UserName() : _splitPoint(0) {} + UserName(const StringData& user, const StringData& dbname); + + /** + * Gets the user part of a UserName. + */ + StringData getUser() const { return StringData(_fullName).substr(0, _splitPoint); } + + /** + * Gets the database name part of a UserName. + */ + StringData getDB() const { return StringData(_fullName).substr(_splitPoint + 1); } + + /** + * Gets the full unique name of a user as a string, formatted as "user@db". + */ + const std::string& getFullName() const { return _fullName; } + + /** + * Stringifies the object, for logging/debugging. + */ + std::string toString() const { return getFullName(); } + + private: + std::string _fullName; // The full name, stored as a string. "user@db". + size_t _splitPoint; // The index of the "@" separating the user and db name parts. + }; + + static inline bool operator==(const UserName& lhs, const UserName& rhs) { + return lhs.getFullName() == rhs.getFullName(); + } + + static inline bool operator!=(const UserName& lhs, const UserName& rhs) { + return lhs.getFullName() != rhs.getFullName(); + } + + static inline bool operator<(const UserName& lhs, const UserName& rhs) { + return lhs.getFullName() < rhs.getFullName(); + } + + std::ostream& operator<<(std::ostream& os, const UserName& name); + + /** + * Iterator over an unspecified container of UserName objects. + */ + class UserNameIterator { + public: + class Impl { + MONGO_DISALLOW_COPYING(Impl); + public: + Impl() {}; + virtual ~Impl() {}; + static Impl* clone(Impl* orig) { return orig ? orig->doClone(): NULL; } + virtual bool more() const = 0; + virtual const UserName& get() const = 0; + + virtual const UserName& next() = 0; + + private: + virtual Impl* doClone() const = 0; + }; + + UserNameIterator() : _impl(NULL) {} + UserNameIterator(const UserNameIterator& other) : _impl(Impl::clone(other._impl.get())) {} + explicit UserNameIterator(Impl* impl) : _impl(impl) {} + + UserNameIterator& operator=(const UserNameIterator& other) { + _impl.reset(Impl::clone(other._impl.get())); + return *this; + } + + bool more() const { return _impl.get() && _impl->more(); } + const UserName& get() const { return _impl->get(); } + + const UserName& next() { return _impl->next(); } + + const UserName& operator*() const { return get(); } + const UserName* operator->() const { return &get(); } + + private: + boost::scoped_ptr _impl; + }; + + + template + class UserNameContainerIteratorImpl : public UserNameIterator::Impl { + MONGO_DISALLOW_COPYING(UserNameContainerIteratorImpl); + public: + UserNameContainerIteratorImpl(const ContainerIterator& begin, + const ContainerIterator& end) : + _curr(begin), _end(end) {} + virtual ~UserNameContainerIteratorImpl() {} + virtual bool more() const { return _curr != _end; } + virtual const UserName& next() { return *(_curr++); } + virtual const UserName& get() const { return *_curr; } + virtual UserNameIterator::Impl* doClone() const { + return new UserNameContainerIteratorImpl(_curr, _end); + } + + private: + ContainerIterator _curr; + ContainerIterator _end; + }; + + template + UserNameIterator makeUserNameIterator(const ContainerIterator& begin, + const ContainerIterator& end) { + return UserNameIterator( new UserNameContainerIteratorImpl(begin, end)); + } + + template + UserNameIterator makeUserNameIteratorForContainer(const Container& container) { + return makeUserNameIterator(container.begin(), container.end()); + } + +} // namespace mongo diff --git a/src/mongo/db/auth/user_name_hash.h b/src/mongo/db/auth/user_name_hash.h new file mode 100644 index 00000000000..fdae32d4efb --- /dev/null +++ b/src/mongo/db/auth/user_name_hash.h @@ -0,0 +1,31 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "mongo/db/auth/user_name.h" +#include "mongo/platform/hash_namespace.h" + + +// Define hash function for UserNames so they can be keys in std::unordered_map +MONGO_HASH_NAMESPACE_START + template <> struct hash { + size_t operator()(const mongo::UserName& pname) const { + return hash()(pname.getFullName()); + } + }; +MONGO_HASH_NAMESPACE_END diff --git a/src/mongo/db/auth/user_set.cpp b/src/mongo/db/auth/user_set.cpp new file mode 100644 index 00000000000..34c1318ab9d --- /dev/null +++ b/src/mongo/db/auth/user_set.cpp @@ -0,0 +1,117 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/auth/user_set.h" + +#include +#include + +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user.h" + +namespace mongo { + +namespace { + class UserSetNameIteratorImpl : public UserNameIterator::Impl { + MONGO_DISALLOW_COPYING(UserSetNameIteratorImpl); + public: + UserSetNameIteratorImpl(const UserSet::iterator& begin, + const UserSet::iterator& end) : + _curr(begin), _end(end) {} + virtual ~UserSetNameIteratorImpl() {} + virtual bool more() const { return _curr != _end; } + virtual const UserName& next() { return (*(_curr++))->getName(); } + virtual const UserName& get() const { return (*_curr)->getName(); } + virtual UserNameIterator::Impl* doClone() const { + return new UserSetNameIteratorImpl(_curr, _end); + } + + private: + UserSet::iterator _curr; + UserSet::iterator _end; + }; +} // namespace + + UserSet::UserSet() : _users(), _usersEnd(_users.end()) {} + UserSet::~UserSet() {} + + User* UserSet::add(User* user) { + for (mutable_iterator it = mbegin(); it != mend(); ++it) { + User* current = *it; + if (current->getName().getDB() == user->getName().getDB()) { + // There can be only one user per database. + *it = user; + return current; + } + } + if (_usersEnd == _users.end()) { + _users.push_back(user); + _usersEnd = _users.end(); + } + else { + *_usersEnd = user; + ++_usersEnd; + } + return NULL; + } + + User* UserSet::removeByDBName(const StringData& dbname) { + for (iterator it = begin(); it != end(); ++it) { + User* current = *it; + if (current->getName().getDB() == dbname) { + return removeAt(it); + } + } + return NULL; + } + + User* UserSet::replaceAt(iterator it, User* replacement) { + size_t offset = it - begin(); + User* old = _users[offset]; + _users[offset] = replacement; + return old; + } + + User* UserSet::removeAt(iterator it) { + size_t offset = it - begin(); + User* old = _users[offset]; + --_usersEnd; + _users[offset] = *_usersEnd; + *_usersEnd = NULL; + return old; + } + + User* UserSet::lookup(const UserName& name) const { + User* user = lookupByDBName(name.getDB()); + if (user && user->getName() == name) { + return user; + } + return NULL; + } + + User* UserSet::lookupByDBName(const StringData& dbname) const { + for (iterator it = begin(); it != end(); ++it) { + User* current = *it; + if (current->getName().getDB() == dbname) { + return current; + } + } + return NULL; + } + + UserNameIterator UserSet::getNames() const { + return UserNameIterator(new UserSetNameIteratorImpl(begin(), end())); + } +} // namespace mongo diff --git a/src/mongo/db/auth/user_set.h b/src/mongo/db/auth/user_set.h new file mode 100644 index 00000000000..37053157559 --- /dev/null +++ b/src/mongo/db/auth/user_set.h @@ -0,0 +1,107 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_name.h" + + +namespace mongo { + + /** + * A collection of authenticated users. + * This class does not do any locking/synchronization, the consumer will be responsible for + * synchronizing access. + */ + class UserSet { + MONGO_DISALLOW_COPYING(UserSet); + public: + typedef std::vector::const_iterator iterator; + + UserSet(); + ~UserSet(); + + /** + * Adds a User to the UserSet. + * + * The UserSet does not take ownership of the User. + * + * As there can only be one user per database in the UserSet, if a User already exists for + * the new User's database, the old user will be removed from the set and returned. It is + * the caller's responsibility to then release that user. If no user already exists for the + * new user's database, returns NULL. + * + * Invalidates any outstanding iterators or NameIterators. + */ + User* add(User* user); + + /** + * Replaces the user at "it" with "replacement." Does not take ownership of the User. + * Returns a pointer to the old user referenced by "it". Does _not_ invalidate "iterator" + * instances. + */ + User* replaceAt(iterator it, User* replacement); + + /** + * Removes the user at "it", and returns a pointer to it. After this call, "it" remains + * valid. It will either equal "end()", or refer to some user between the values of "it" + * and "end()" before this call was made. + */ + User* removeAt(iterator it); + + /** + * Removes the User whose authentication credentials came from dbname, and returns that + * user. It is the caller's responsibility to then release that user back to the + * authorizationManger. If no user exists for the given database, returns NULL; + */ + User* removeByDBName(const StringData& dbname); + + // Returns the User with the given name, or NULL if not found. + // Ownership of the returned User remains with the UserSet. The pointer + // returned is only guaranteed to remain valid until the next non-const method is called + // on the UserSet. + User* lookup(const UserName& name) const; + + // Gets the user whose authentication credentials came from dbname, or NULL if none + // exist. There should be at most one such user. + User* lookupByDBName(const StringData& dbname) const; + + // Gets an iterator over the names of the users stored in the set. The iterator is + // valid until the next non-const method is called on the UserSet. + UserNameIterator getNames() const; + + iterator begin() const { return _users.begin(); } + iterator end() const { return _usersEnd; } + + private: + typedef std::vector::iterator mutable_iterator; + + mutable_iterator mbegin() { return _users.begin(); } + mutable_iterator mend() { return _usersEnd; } + + // The UserSet maintains ownership of the Users in it, and is responsible for + // returning them to the AuthorizationManager when done with them. + std::vector _users; + std::vector::iterator _usersEnd; + }; + +} // namespace mongo diff --git a/src/mongo/db/auth/user_set_test.cpp b/src/mongo/db/auth/user_set_test.cpp new file mode 100644 index 00000000000..5063f727412 --- /dev/null +++ b/src/mongo/db/auth/user_set_test.cpp @@ -0,0 +1,99 @@ +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests of the UserSet type. + */ + +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/user_set.h" +#include "mongo/unittest/unittest.h" + +#define ASSERT_NULL(EXPR) ASSERT_FALSE((EXPR)) + +namespace mongo { +namespace { + + TEST(UserSetTest, BasicTest) { + UserSet set; + + User* p1 = new User(UserName("Bob", "test")); + User* p2 = new User(UserName("George", "test")); + User* p3 = new User(UserName("Bob", "test2")); + + ASSERT_NULL(set.lookup(UserName("Bob", "test"))); + ASSERT_NULL(set.lookup(UserName("George", "test"))); + ASSERT_NULL(set.lookup(UserName("Bob", "test2"))); + ASSERT_NULL(set.lookupByDBName("test")); + ASSERT_NULL(set.lookupByDBName("test2")); + + ASSERT_NULL(set.add(p1)); + + ASSERT_EQUALS(p1, set.lookup(UserName("Bob", "test"))); + ASSERT_EQUALS(p1, set.lookupByDBName("test")); + ASSERT_NULL(set.lookup(UserName("George", "test"))); + ASSERT_NULL(set.lookup(UserName("Bob", "test2"))); + ASSERT_NULL(set.lookupByDBName("test2")); + + // This should not replace the existing user "Bob" because they are different databases + ASSERT_NULL(set.add(p3)); + + ASSERT_EQUALS(p1, set.lookup(UserName("Bob", "test"))); + ASSERT_EQUALS(p1, set.lookupByDBName("test")); + ASSERT_NULL(set.lookup(UserName("George", "test"))); + ASSERT_EQUALS(p3, set.lookup(UserName("Bob", "test2"))); + ASSERT_EQUALS(p3, set.lookupByDBName("test2")); + + User* replaced = set.add(p2); // This should replace Bob since they're on the same database + + ASSERT_EQUALS(replaced, p1); + ASSERT_NULL(set.lookup(UserName("Bob", "test"))); + ASSERT_EQUALS(p2, set.lookup(UserName("George", "test"))); + ASSERT_EQUALS(p2, set.lookupByDBName("test")); + ASSERT_EQUALS(p3, set.lookup(UserName("Bob", "test2"))); + ASSERT_EQUALS(p3, set.lookupByDBName("test2")); + + User* removed = set.removeByDBName("test"); + + ASSERT_EQUALS(removed, p2); + ASSERT_NULL(set.lookup(UserName("Bob", "test"))); + ASSERT_NULL(set.lookup(UserName("George", "test"))); + ASSERT_NULL(set.lookupByDBName("test")); + ASSERT_EQUALS(p3, set.lookup(UserName("Bob", "test2"))); + ASSERT_EQUALS(p3, set.lookupByDBName("test2")); + + UserNameIterator iter = set.getNames(); + ASSERT_TRUE(iter.more()); + ASSERT_EQUALS(iter.next(), UserName("Bob", "test2")); + ASSERT_FALSE(iter.more()); + } + + TEST(UserSetTest, IterateNames) { + UserSet pset; + UserNameIterator iter = pset.getNames(); + ASSERT(!iter.more()); + + ASSERT_NULL(pset.add(new User(UserName("bob", "test")))); + + iter = pset.getNames(); + ASSERT(iter.more()); + ASSERT_EQUALS(*iter, UserName("bob", "test")); + ASSERT_EQUALS(iter.next(), UserName("bob", "test")); + ASSERT(!iter.more()); + } + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/background.cpp b/src/mongo/db/background.cpp new file mode 100644 index 00000000000..eb9860ba405 --- /dev/null +++ b/src/mongo/db/background.cpp @@ -0,0 +1,179 @@ +// background.cpp + +/** +* Copyright (C) 2010 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/platform/basic.h" + +#include "mongo/db/background.h" + +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/map_util.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/string_map.h" + +namespace mongo { + +namespace { + + class BgInfo { + MONGO_DISALLOW_COPYING(BgInfo); + public: + BgInfo() : _opsInProgCount(0) {} + + void recordBegin(); + int recordEnd(); + void awaitNoBgOps(boost::unique_lock& lk); + + int getOpsInProgCount() const { return _opsInProgCount; } + + private: + int _opsInProgCount; + boost::condition_variable _noOpsInProg; + }; + + typedef StringMap > BgInfoMap; + typedef BgInfoMap::const_iterator BgInfoMapIterator; + + boost::mutex m; + BgInfoMap dbsInProg; + BgInfoMap nsInProg; + + void BgInfo::recordBegin() { + ++_opsInProgCount; + } + + int BgInfo::recordEnd() { + dassert(_opsInProgCount > 0); + --_opsInProgCount; + if (0 == _opsInProgCount) { + _noOpsInProg.notify_all(); + } + return _opsInProgCount; + } + + void BgInfo::awaitNoBgOps(boost::unique_lock& lk) { + while (_opsInProgCount > 0) + _noOpsInProg.wait(lk); + } + + void recordBeginAndInsert(BgInfoMap* bgiMap, const StringData& key) { + boost::shared_ptr& bgInfo = bgiMap->get(key); + if (!bgInfo) + bgInfo.reset(new BgInfo); + bgInfo->recordBegin(); + } + + void recordEndAndRemove(BgInfoMap* bgiMap, const StringData& key) { + BgInfoMapIterator iter = bgiMap->find(key); + fassert(17431, iter != bgiMap->end()); + if (0 == iter->second->recordEnd()) { + bgiMap->erase(iter); + } + } + + void awaitNoBgOps( + boost::unique_lock& lk, + BgInfoMap* bgiMap, + const StringData& key) { + + boost::shared_ptr bgInfo = mapFindWithDefault( + *bgiMap, key, boost::shared_ptr()); + if (!bgInfo) + return; + bgInfo->awaitNoBgOps(lk); + } + +} // namespace + bool BackgroundOperation::inProgForDb(const StringData& db) { + boost::mutex::scoped_lock lk(m); + return dbsInProg.find(db) != dbsInProg.end(); + } + + bool BackgroundOperation::inProgForNs(const StringData& ns) { + boost::mutex::scoped_lock lk(m); + return nsInProg.find(ns) != nsInProg.end(); + } + + void BackgroundOperation::assertNoBgOpInProgForDb(const StringData& db) { + uassert(ErrorCodes::BackgroundOperationInProgressForDatabase, mongoutils::str::stream() << + "cannot perform operation: a background operation is currently running for " + "database " << db, + !inProgForDb(db)); + } + + void BackgroundOperation::assertNoBgOpInProgForNs(const StringData& ns) { + uassert(ErrorCodes::BackgroundOperationInProgressForNamespace, mongoutils::str::stream() << + "cannot perform operation: a background operation is currently running for " + "collection" << ns, + !inProgForNs(ns)); + } + + void BackgroundOperation::awaitNoBgOpInProgForDb(const StringData& db) { + boost::unique_lock lk(m); + awaitNoBgOps(lk, &dbsInProg, db); + } + + void BackgroundOperation::awaitNoBgOpInProgForNs(const StringData& ns) { + boost::unique_lock lk(m); + awaitNoBgOps(lk, &nsInProg, ns); + } + + BackgroundOperation::BackgroundOperation(const StringData& ns) : _ns(ns) { + boost::mutex::scoped_lock lk(m); + recordBeginAndInsert(&dbsInProg, _ns.db()); + recordBeginAndInsert(&nsInProg, _ns.ns()); + } + + BackgroundOperation::~BackgroundOperation() { + boost::mutex::scoped_lock lk(m); + recordEndAndRemove(&dbsInProg, _ns.db()); + recordEndAndRemove(&nsInProg, _ns.ns()); + } + + void BackgroundOperation::dump(std::ostream& ss) { + boost::mutex::scoped_lock lk(m); + if( nsInProg.size() ) { + ss << "\nBackground Jobs in Progress\n"; + for( BgInfoMapIterator i = nsInProg.begin(); i != nsInProg.end(); ++i ) + ss << " " << i->first << '\n'; + } + for( BgInfoMapIterator i = dbsInProg.begin(); i != dbsInProg.end(); ++i ) { + if( i->second->getOpsInProgCount() ) + ss << "database " << i->first << ": " << i->second->getOpsInProgCount() << '\n'; + } + } + +} // namespace mongo + diff --git a/src/mongo/db/background.h b/src/mongo/db/background.h index 0b76abf3a8e..3db4767f95a 100644 --- a/src/mongo/db/background.h +++ b/src/mongo/db/background.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* background.h @@ -21,7 +33,13 @@ #pragma once -#include "mongo/db/namespacestring.h" +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" +#include "mongo/db/namespace_string.h" namespace mongo { @@ -35,24 +53,24 @@ namespace mongo { It's assumed this is not for super-high RPS things, so we don't do anything special in the implementation here to be fast. */ - class BackgroundOperation : public boost::noncopyable { + class BackgroundOperation { + MONGO_DISALLOW_COPYING(BackgroundOperation); public: - static bool inProgForDb(const char *db); - static bool inProgForNs(const char *ns); - static void assertNoBgOpInProgForDb(const char *db); - static void assertNoBgOpInProgForNs(const char *ns); - static void dump(stringstream&); + static bool inProgForDb(const StringData& db); + static bool inProgForNs(const StringData& ns); + static void assertNoBgOpInProgForDb(const StringData& db); + static void assertNoBgOpInProgForNs(const StringData& ns); + static void awaitNoBgOpInProgForDb(const StringData& db); + static void awaitNoBgOpInProgForNs(const StringData& ns); + static void dump(std::ostream&); /* check for in progress before instantiating */ - BackgroundOperation(const char *ns); + BackgroundOperation(const StringData& ns); virtual ~BackgroundOperation(); private: NamespaceString _ns; - static map dbsInProg; - static set nsInProg; - static SimpleMutex m; }; } // namespace mongo diff --git a/src/mongo/db/btree.cpp b/src/mongo/db/btree.cpp deleted file mode 100644 index f669bdb142c..00000000000 --- a/src/mongo/db/btree.cpp +++ /dev/null @@ -1,1982 +0,0 @@ -// btree.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/btree.h" -#include "mongo/db/btree_stats.h" -#include "mongo/db/btreebuilder.h" -#include "mongo/db/client.h" -#include "mongo/db/clientcursor.h" -#include "mongo/db/curop-inl.h" -#include "mongo/db/db.h" -#include "mongo/db/dbhelpers.h" -#include "mongo/db/dur_commitjob.h" -#include "mongo/db/index_insertion_continuation.h" -#include "mongo/db/json.h" -#include "mongo/db/kill_current_op.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/stats/counters.h" -#include "mongo/server.h" -#include "mongo/util/startup_test.h" - -namespace mongo { - - BOOST_STATIC_ASSERT( Record::HeaderSize == 16 ); - BOOST_STATIC_ASSERT( Record::HeaderSize + BtreeData_V1::BucketSize == 8192 ); - - NOINLINE_DECL void checkFailed(unsigned line) { - static time_t last; - if( time(0) - last >= 10 ) { - msgasserted(15898, str::stream() << "error in index possibly corruption consider repairing " << line); - } - } - - /** data check. like assert, but gives a reasonable error message to the user. */ -#define check(expr) if(!(expr) ) { checkFailed(__LINE__); } - -#define VERIFYTHISLOC dassert( thisLoc.btree() == this ); - - template< class Loc > - __KeyNode & __KeyNode::writing() const { - return *getDur().writing( const_cast< __KeyNode * >( this ) ); - } - - // BucketBasics::lowWaterMark() - // - // We define this value as the maximum number of bytes such that, if we have - // fewer than this many bytes, we must be able to either merge with or receive - // keys from any neighboring node. If our utilization goes below this value we - // know we can bring up the utilization with a simple operation. Ignoring the - // 90/10 split policy which is sometimes employed and our 'unused' nodes, this - // is a lower bound on bucket utilization for non root buckets. - // - // Note that the exact value here depends on the implementation of - // rebalancedSeparatorPos(). The conditions for lowWaterMark - 1 are as - // follows: We know we cannot merge with the neighbor, so the total data size - // for us, the neighbor, and the separator must be at least - // BtreeBucket::bodySize() + 1. We must be able to accept one key of any - // allowed size, so our size plus storage for that additional key must be - // <= BtreeBucket::bodySize() / 2. This way, with the extra key we'll have a - // new bucket data size < half the total data size and by the implementation - // of rebalancedSeparatorPos() the key must be added. - - static const int split_debug = 0; - static const int insert_debug = 0; - - /** - * this error is ok/benign when doing a background indexing -- that logic in pdfile checks explicitly - * for the 10287 error code. - */ - static void alreadyInIndex() { - // we don't use massert() here as that does logging and this is 'benign' - see catches in _indexRecord() - throw MsgAssertionException(10287, "btree: key+recloc already in index"); - } - - /* BucketBasics --------------------------------------------------- */ - - template< class V > - void BucketBasics::assertWritable() { - if( cmdLine.dur ) - dur::assertAlreadyDeclared(this, V::BucketSize); - } - - template< class V > - string BtreeBucket::bucketSummary() const { - stringstream ss; - ss << " Bucket info:" << endl; - ss << " n: " << this->n << endl; - ss << " parent: " << this->parent.toString() << endl; - ss << " nextChild: " << this->nextChild.toString() << endl; - ss << " flags:" << this->flags << endl; - ss << " emptySize: " << this->emptySize << " topSize: " << this->topSize << endl; - return ss.str(); - } - - template< class V > - int BucketBasics::Size() const { - return V::BucketSize; - } - - template< class V > - void BucketBasics::_shape(int level, stringstream& ss) const { - for ( int i = 0; i < level; i++ ) ss << ' '; - ss << "*[" << this->n << "]\n"; - for ( int i = 0; i < this->n; i++ ) { - if ( !k(i).prevChildBucket.isNull() ) { - DiskLoc ll = k(i).prevChildBucket; - ll.btree()->_shape(level+1,ss); - } - } - if ( !this->nextChild.isNull() ) { - DiskLoc ll = this->nextChild; - ll.btree()->_shape(level+1,ss); - } - } - - int bt_fv=0; - int bt_dmp=0; - - template< class V > - void BtreeBucket::dumpTree(const DiskLoc &thisLoc, const BSONObj &order) const { - bt_dmp=1; - fullValidate(thisLoc, order); - bt_dmp=0; - } - - template< class V > - long long BtreeBucket::fullValidate(const DiskLoc& thisLoc, const BSONObj &order, long long *unusedCount, bool strict, unsigned depth) const { - killCurrentOp.checkForInterrupt(); - this->assertValid(order, true); - - if ( bt_dmp ) { - _log() << thisLoc.toString() << ' '; - ((BtreeBucket *) this)->dump(depth); - } - - // keycount - long long kc = 0; - - for ( int i = 0; i < this->n; i++ ) { - const _KeyNode& kn = this->k(i); - - if ( kn.isUsed() ) { - kc++; - } - else { - if ( unusedCount ) { - ++( *unusedCount ); - } - } - if ( !kn.prevChildBucket.isNull() ) { - DiskLoc left = kn.prevChildBucket; - const BtreeBucket *b = left.btree(); - if ( strict ) { - verify( b->parent == thisLoc ); - } - else { - wassert( b->parent == thisLoc ); - } - kc += b->fullValidate(kn.prevChildBucket, order, unusedCount, strict, depth+1); - } - } - if ( !this->nextChild.isNull() ) { - DiskLoc ll = this->nextChild; - const BtreeBucket *b = ll.btree(); - if ( strict ) { - verify( b->parent == thisLoc ); - } - else { - wassert( b->parent == thisLoc ); - } - kc += b->fullValidate(this->nextChild, order, unusedCount, strict, depth+1); - } - - return kc; - } - - int nDumped = 0; - - template< class V > - void BucketBasics::assertValid(const Ordering &order, bool force) const { - if ( !debug && !force ) - return; - { - int foo = this->n; - wassert( foo >= 0 && this->n < Size() ); - foo = this->emptySize; - wassert( foo >= 0 && this->emptySize < V::BucketSize ); - wassert( this->topSize >= this->n && this->topSize <= V::BucketSize ); - } - - // this is very slow so don't do often - { - static int _k; - if( ++_k % 128 ) - return; - } - - DEV { - // slow: - for ( int i = 0; i < this->n-1; i++ ) { - Key k1 = keyNode(i).key; - Key k2 = keyNode(i+1).key; - int z = k1.woCompare(k2, order); //OK - if ( z > 0 ) { - out() << "ERROR: btree key order corrupt. Keys:" << endl; - if ( ++nDumped < 5 ) { - for ( int j = 0; j < this->n; j++ ) { - out() << " " << keyNode(j).key.toString() << endl; - } - ((BtreeBucket *) this)->dump(); - } - wassert(false); - break; - } - else if ( z == 0 ) { - if ( !(k(i).recordLoc < k(i+1).recordLoc) ) { - out() << "ERROR: btree key order corrupt (recordloc's wrong):" << endl; - out() << " k(" << i << ")" << keyNode(i).key.toString() << " RL:" << k(i).recordLoc.toString() << endl; - out() << " k(" << i+1 << ")" << keyNode(i+1).key.toString() << " RL:" << k(i+1).recordLoc.toString() << endl; - wassert( k(i).recordLoc < k(i+1).recordLoc ); - } - } - } - } - else { - //faster: - if ( this->n > 1 ) { - Key k1 = keyNode(0).key; - Key k2 = keyNode(this->n-1).key; - int z = k1.woCompare(k2, order); - //wassert( z <= 0 ); - if ( z > 0 ) { - problem() << "btree keys out of order" << '\n'; - ONCE { - ((BtreeBucket *) this)->dump(); - } - verify(false); - } - } - } - } - - template< class V > - inline void BucketBasics::markUnused(int keypos) { - verify( keypos >= 0 && keypos < this->n ); - k(keypos).setUnused(); - } - - template< class V > - inline int BucketBasics::totalDataSize() const { - return (int) (Size() - (this->data-(char*)this)); - } - - template< class V > - void BucketBasics::init() { - this->_init(); - this->parent.Null(); - this->nextChild.Null(); - this->flags = Packed; - this->n = 0; - this->emptySize = totalDataSize(); - this->topSize = 0; - } - - /** see _alloc */ - template< class V > - inline void BucketBasics::_unalloc(int bytes) { - this->topSize -= bytes; - this->emptySize += bytes; - } - - /** - * we allocate space from the end of the buffer for data. - * the keynodes grow from the front. - */ - template< class V > - inline int BucketBasics::_alloc(int bytes) { - verify( this->emptySize >= bytes ); - this->topSize += bytes; - this->emptySize -= bytes; - int ofs = totalDataSize() - this->topSize; - verify( ofs > 0 ); - return ofs; - } - - template< class V > - void BucketBasics::_delKeyAtPos(int keypos, bool mayEmpty) { - // TODO This should be keypos < n - verify( keypos >= 0 && keypos <= this->n ); - verify( childForPos(keypos).isNull() ); - // TODO audit cases where nextChild is null - verify( ( mayEmpty && this->n > 0 ) || this->n > 1 || this->nextChild.isNull() ); - this->emptySize += sizeof(_KeyNode); - this->n--; - for ( int j = keypos; j < this->n; j++ ) - k(j) = k(j+1); - setNotPacked(); - } - - /** - * pull rightmost key from the bucket. this version requires its right child to be null so it - * does not bother returning that value. - */ - template< class V > - void BucketBasics::popBack(DiskLoc& recLoc, Key &key) { - massert( 10282 , "n==0 in btree popBack()", this->n > 0 ); - verify( k(this->n-1).isUsed() ); // no unused skipping in this function at this point - btreebuilder doesn't require that - KeyNode kn = keyNode(this->n-1); - recLoc = kn.recordLoc; - key.assign(kn.key); - int keysize = kn.key.dataSize(); - - massert( 10283 , "rchild not null in btree popBack()", this->nextChild.isNull()); - - // weirdly, we also put the rightmost down pointer in nextchild, even when bucket isn't full. - this->nextChild = kn.prevChildBucket; - - this->n--; - // This is risky because the key we are returning points to this unalloc'ed memory, - // and we are assuming that the last key points to the last allocated - // bson region. - this->emptySize += sizeof(_KeyNode); - _unalloc(keysize); - } - - /** add a key. must be > all existing. be careful to set next ptr right. */ - template< class V > - bool BucketBasics::_pushBack(const DiskLoc recordLoc, const Key& key, const Ordering &order, const DiskLoc prevChild) { - int bytesNeeded = key.dataSize() + sizeof(_KeyNode); - if ( bytesNeeded > this->emptySize ) - return false; - verify( bytesNeeded <= this->emptySize ); - if( this->n ) { - const KeyNode klast = keyNode(this->n-1); - if( klast.key.woCompare(key, order) > 0 ) { - log() << "btree bucket corrupt? consider reindexing or running validate command" << endl; - log() << " klast: " << keyNode(this->n-1).key.toString() << endl; - log() << " key: " << key.toString() << endl; - DEV klast.key.woCompare(key, order); - verify(false); - } - } - this->emptySize -= sizeof(_KeyNode); - _KeyNode& kn = k(this->n++); - kn.prevChildBucket = prevChild; - kn.recordLoc = recordLoc; - kn.setKeyDataOfs( (short) _alloc(key.dataSize()) ); - short ofs = kn.keyDataOfs(); - char *p = dataAt(ofs); - memcpy(p, key.data(), key.dataSize()); - - return true; - } - - /* durability note - we do separate intent declarations herein. arguably one could just declare - the whole bucket given we do group commits. this is something we could investigate - later as to what is faster under what situations. - */ - /** insert a key in a bucket with no complexity -- no splits required - @return false if a split is required. - */ - template< class V > - bool BucketBasics::basicInsert(const DiskLoc thisLoc, int &keypos, const DiskLoc recordLoc, const Key& key, const Ordering &order) const { - check( this->n < 1024 ); - check( keypos >= 0 && keypos <= this->n ); - int bytesNeeded = key.dataSize() + sizeof(_KeyNode); - if ( bytesNeeded > this->emptySize ) { - _pack(thisLoc, order, keypos); - if ( bytesNeeded > this->emptySize ) - return false; - } - - BucketBasics *b; - { - const char *p = (const char *) &k(keypos); - const char *q = (const char *) &k(this->n+1); - // declare that we will write to [k(keypos),k(n)] - // todo: this writes a medium amount to the journal. we may want to add a verb "shift" to the redo log so - // we can log a very small amount. - b = (BucketBasics*) getDur().writingAtOffset((void *) this, p-(char*)this, q-p); - - // e.g. n==3, keypos==2 - // 1 4 9 - // -> - // 1 4 _ 9 - for ( int j = this->n; j > keypos; j-- ) // make room - b->k(j) = b->k(j-1); - } - - getDur().declareWriteIntent(&b->emptySize, sizeof(this->emptySize)+sizeof(this->topSize)+sizeof(this->n)); - b->emptySize -= sizeof(_KeyNode); - b->n++; - - // This _KeyNode was marked for writing above. - _KeyNode& kn = b->k(keypos); - kn.prevChildBucket.Null(); - kn.recordLoc = recordLoc; - kn.setKeyDataOfs((short) b->_alloc(key.dataSize()) ); - char *p = b->dataAt(kn.keyDataOfs()); - getDur().declareWriteIntent(p, key.dataSize()); - memcpy(p, key.data(), key.dataSize()); - return true; - } - - /** - * With this implementation, refPos == 0 disregards effect of refPos. - * index > 0 prevents creation of an empty bucket. - */ - template< class V > - bool BucketBasics::mayDropKey( int index, int refPos ) const { - return index > 0 && ( index != refPos ) && k( index ).isUnused() && k( index ).prevChildBucket.isNull(); - } - - template< class V > - int BucketBasics::packedDataSize( int refPos ) const { - if ( this->flags & Packed ) { - return V::BucketSize - this->emptySize - headerSize(); - } - int size = 0; - for( int j = 0; j < this->n; ++j ) { - if ( mayDropKey( j, refPos ) ) { - continue; - } - size += keyNode( j ).key.dataSize() + sizeof( _KeyNode ); - } - return size; - } - - /** - * when we delete things we just leave empty space until the node is - * full and then we repack it. - */ - template< class V > - void BucketBasics::_pack(const DiskLoc thisLoc, const Ordering &order, int &refPos) const { - if ( this->flags & Packed ) - return; - - VERIFYTHISLOC - - /** TODO perhaps this can be optimized. for example if packing does no write, we can skip intent decl. - an empirical approach is probably best than just adding new code : perhaps the bucket would need - declaration anyway within the group commit interval, in which case we would just be adding - code and complexity without benefit. - */ - thisLoc.btreemod()->_packReadyForMod(order, refPos); - } - - /** version when write intent already declared */ - template< class V > - void BucketBasics::_packReadyForMod( const Ordering &order, int &refPos ) { - assertWritable(); - - if ( this->flags & Packed ) - return; - - int tdz = totalDataSize(); - char temp[V::BucketSize]; - int ofs = tdz; - this->topSize = 0; - int i = 0; - for ( int j = 0; j < this->n; j++ ) { - if( mayDropKey( j, refPos ) ) { - continue; // key is unused and has no children - drop it - } - if( i != j ) { - if ( refPos == j ) { - refPos = i; // i < j so j will never be refPos again - } - k( i ) = k( j ); - } - short ofsold = k(i).keyDataOfs(); - int sz = keyNode(i).key.dataSize(); - ofs -= sz; - this->topSize += sz; - memcpy(temp+ofs, dataAt(ofsold), sz); - k(i).setKeyDataOfsSavingUse( ofs ); - ++i; - } - if ( refPos == this->n ) { - refPos = i; - } - this->n = i; - int dataUsed = tdz - ofs; - memcpy(this->data + ofs, temp + ofs, dataUsed); - - // assertWritable(); - // TEMP TEST getDur().declareWriteIntent(this, sizeof(*this)); - - this->emptySize = tdz - dataUsed - this->n * sizeof(_KeyNode); - { - int foo = this->emptySize; - verify( foo >= 0 ); - } - - setPacked(); - - assertValid( order ); - } - - template< class V > - inline void BucketBasics::truncateTo(int N, const Ordering &order, int &refPos) { - verify( Lock::somethingWriteLocked() ); - assertWritable(); - this->n = N; - setNotPacked(); - _packReadyForMod( order, refPos ); - } - - /** - * In the standard btree algorithm, we would split based on the - * existing keys _and_ the new key. But that's more work to - * implement, so we split the existing keys and then add the new key. - * - * There are several published heuristic algorithms for doing splits, - * but basically what you want are (1) even balancing between the two - * sides and (2) a small split key so the parent can have a larger - * branching factor. - * - * We just have a simple algorithm right now: if a key includes the - * halfway point (or 10% way point) in terms of bytes, split on that key; - * otherwise split on the key immediately to the left of the halfway - * point (or 10% point). - * - * This function is expected to be called on a packed bucket. - */ - template< class V > - int BucketBasics::splitPos( int keypos ) const { - verify( this->n > 2 ); - int split = 0; - int rightSize = 0; - // when splitting a btree node, if the new key is greater than all the other keys, we should not do an even split, but a 90/10 split. - // see SERVER-983 - // TODO I think we only want to do the 90% split on the rhs node of the tree. - int rightSizeLimit = ( this->topSize + sizeof( _KeyNode ) * this->n ) / ( keypos == this->n ? 10 : 2 ); - for( int i = this->n - 1; i > -1; --i ) { - rightSize += keyNode( i ).key.dataSize() + sizeof( _KeyNode ); - if ( rightSize > rightSizeLimit ) { - split = i; - break; - } - } - // safeguards - we must not create an empty bucket - if ( split < 1 ) { - split = 1; - } - else if ( split > this->n - 2 ) { - split = this->n - 2; - } - - return split; - } - - template< class V > - void BucketBasics::reserveKeysFront( int nAdd ) { - verify( this->emptySize >= int( sizeof( _KeyNode ) * nAdd ) ); - this->emptySize -= sizeof( _KeyNode ) * nAdd; - for( int i = this->n - 1; i > -1; --i ) { - k( i + nAdd ) = k( i ); - } - this->n += nAdd; - } - - template< class V > - void BucketBasics::setKey( int i, const DiskLoc recordLoc, const Key &key, const DiskLoc prevChildBucket ) { - _KeyNode &kn = k( i ); - kn.recordLoc = recordLoc; - kn.prevChildBucket = prevChildBucket; - short ofs = (short) _alloc( key.dataSize() ); - kn.setKeyDataOfs( ofs ); - char *p = dataAt( ofs ); - memcpy( p, key.data(), key.dataSize() ); - } - - template< class V > - void BucketBasics::dropFront( int nDrop, const Ordering &order, int &refpos ) { - for( int i = nDrop; i < this->n; ++i ) { - k( i - nDrop ) = k( i ); - } - this->n -= nDrop; - setNotPacked(); - _packReadyForMod( order, refpos ); - } - - /* - BtreeBucket --------------------------------------------------- */ - - /** @return largest key in the subtree. */ - template< class V > - void BtreeBucket::findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey) { - DiskLoc loc = thisLoc; - while ( 1 ) { - const BtreeBucket *b = loc.btree(); - if ( !b->nextChild.isNull() ) { - loc = b->nextChild; - continue; - } - - verify(b->n>0); - largestLoc = loc; - largestKey = b->n-1; - - break; - } - } - - /** - * NOTE Currently the Ordering implementation assumes a compound index will - * not have more keys than an unsigned variable has bits. The same - * assumption is used in the implementation below with respect to the 'mask' - * variable. - * - * @param l a regular bsonobj - * @param rBegin composed partly of an existing bsonobj, and the remaining keys are taken from a vector of elements that frequently changes - * - * see - * jstests/index_check6.js - * https://jira.mongodb.org/browse/SERVER-371 - */ - /* static */ - template< class V > - int BtreeBucket::customBSONCmp( const BSONObj &l, const BSONObj &rBegin, int rBeginLen, bool rSup, const vector< const BSONElement * > &rEnd, const vector< bool > &rEndInclusive, const Ordering &o, int direction ) { - BSONObjIterator ll( l ); - BSONObjIterator rr( rBegin ); - vector< const BSONElement * >::const_iterator rr2 = rEnd.begin(); - vector< bool >::const_iterator inc = rEndInclusive.begin(); - unsigned mask = 1; - for( int i = 0; i < rBeginLen; ++i, mask <<= 1 ) { - BSONElement lll = ll.next(); - BSONElement rrr = rr.next(); - ++rr2; - ++inc; - - int x = lll.woCompare( rrr, false ); - if ( o.descending( mask ) ) - x = -x; - if ( x != 0 ) - return x; - } - if ( rSup ) { - return -direction; - } - for( ; ll.more(); mask <<= 1 ) { - BSONElement lll = ll.next(); - BSONElement rrr = **rr2; - ++rr2; - int x = lll.woCompare( rrr, false ); - if ( o.descending( mask ) ) - x = -x; - if ( x != 0 ) - return x; - if ( !*inc ) { - return -direction; - } - ++inc; - } - return 0; - } - - template< class V > - bool BtreeBucket::exists(const IndexDetails& idx, const DiskLoc &thisLoc, const Key& key, const Ordering& order) const { - int pos; - bool found; - DiskLoc b = locate(idx, thisLoc, key, order, pos, found, minDiskLoc); - - // skip unused keys - while ( 1 ) { - if( b.isNull() ) - break; - const BtreeBucket *bucket = b.btree(); - const _KeyNode& kn = bucket->k(pos); - if ( kn.isUsed() ) - return bucket->keyAt(pos).woEqual(key); - b = bucket->advance(b, pos, 1, "BtreeBucket::exists"); - } - return false; - } - - template< class V > - bool BtreeBucket::wouldCreateDup( - const IndexDetails& idx, const DiskLoc &thisLoc, - const Key& key, const Ordering& order, - const DiskLoc &self) const { - int pos; - bool found; - DiskLoc b = locate(idx, thisLoc, key, order, pos, found, minDiskLoc); - - while ( !b.isNull() ) { - // we skip unused keys - const BtreeBucket *bucket = b.btree(); - const _KeyNode& kn = bucket->k(pos); - if ( kn.isUsed() ) { - if( bucket->keyAt(pos).woEqual(key) ) - return kn.recordLoc != self; - break; - } - b = bucket->advance(b, pos, 1, "BtreeBucket::dupCheck"); - } - - return false; - } - - template< class V > - string BtreeBucket::dupKeyError( const IndexDetails& idx , const Key& key ) { - stringstream ss; - ss << "E11000 duplicate key error "; - ss << "index: " << idx.indexNamespace() << " "; - ss << "dup key: " << key.toString(); - return ss.str(); - } - - /** - * Find a key withing this btree bucket. - * - * When duplicate keys are allowed, we use the DiskLoc of the record as if it were part of the - * key. That assures that even when there are many duplicates (e.g., 1 million) for a key, - * our performance is still good. - * - * assertIfDup: if the key exists (ignoring the recordLoc), uassert - * - * pos: for existing keys k0...kn-1. - * returns # it goes BEFORE. so key[pos-1] < key < key[pos] - * returns n if it goes after the last existing key. - * note result might be an Unused location! - */ - - bool guessIncreasing = false; - template< class V > - bool BtreeBucket::find(const IndexDetails& idx, const Key& key, const DiskLoc &rl, - const Ordering &order, int& pos, bool assertIfDup) const { - Loc recordLoc; - recordLoc = rl; - globalIndexCounters->btree( reinterpret_cast(this) ); - - // binary search for this key - bool dupsChecked = false; - int l=0; - int h=this->n-1; - int m = (l+h)/2; - if( guessIncreasing ) { - m = h; - } - while ( l <= h ) { - KeyNode M = this->keyNode(m); - int x = key.woCompare(M.key, order); - if ( x == 0 ) { - if( assertIfDup ) { - if( k(m).isUnused() ) { - // ok that key is there if unused. but we need to check that there aren't other - // entries for the key then. as it is very rare that we get here, we don't put any - // coding effort in here to make this particularly fast - if( !dupsChecked ) { - dupsChecked = true; - if( idx.head.btree()->exists(idx, idx.head, key, order) ) { - if( idx.head.btree()->wouldCreateDup(idx, idx.head, key, order, recordLoc) ) - uasserted( ASSERT_ID_DUPKEY , dupKeyError( idx , key ) ); - else - alreadyInIndex(); - } - } - } - else { - if( M.recordLoc == recordLoc ) - alreadyInIndex(); - uasserted( ASSERT_ID_DUPKEY , dupKeyError( idx , key ) ); - } - } - - // dup keys allowed. use recordLoc as if it is part of the key - Loc unusedRL = M.recordLoc; - unusedRL.GETOFS() &= ~1; // so we can test equality without the used bit messing us up - x = recordLoc.compare(unusedRL); - } - if ( x < 0 ) // key < M.key - h = m-1; - else if ( x > 0 ) - l = m+1; - else { - // found it. - pos = m; - return true; - } - m = (l+h)/2; - } - // not found - pos = l; - if ( pos != this->n ) { - Key keyatpos = keyNode(pos).key; - wassert( key.woCompare(keyatpos, order) <= 0 ); - if ( pos > 0 ) { - if( !( keyNode(pos-1).key.woCompare(key, order) <= 0 ) ) { - DEV { - log() << key.toString() << endl; - log() << keyNode(pos-1).key.toString() << endl; - } - wassert(false); - } - } - } - - return false; - } - - template< class V > - void BtreeBucket::delBucket(const DiskLoc thisLoc, const IndexDetails& id) { - ClientCursor::informAboutToDeleteBucket(thisLoc); // slow... - verify( !isHead() ); - - DiskLoc ll = this->parent; - const BtreeBucket *p = ll.btree(); - int parentIdx = indexInParent( thisLoc ); - p->childForPos( parentIdx ).writing().Null(); - deallocBucket( thisLoc, id ); - } - - template< class V > - void BtreeBucket::deallocBucket(const DiskLoc thisLoc, const IndexDetails &id) { -#if 0 - // as a temporary defensive measure, we zap the whole bucket, AND don't truly delete - // it (meaning it is ineligible for reuse). - memset(this, 0, Size()); -#else - // Mark the bucket as deallocated, see SERVER-4575. - this->n = this->INVALID_N_SENTINEL; - // defensive: - this->parent.Null(); - string ns = id.indexNamespace(); - theDataFileMgr._deleteRecord(nsdetails(ns), ns.c_str(), thisLoc.rec(), thisLoc); -#endif - } - - /** note: may delete the entire bucket! this invalid upon return sometimes. */ - template< class V > - void BtreeBucket::delKeyAtPos( const DiskLoc thisLoc, IndexDetails& id, int p, const Ordering &order) { - verify(this->n>0); - DiskLoc left = this->childForPos(p); - - if ( this->n == 1 ) { - if ( left.isNull() && this->nextChild.isNull() ) { - this->_delKeyAtPos(p); - if ( isHead() ) { - // we don't delete the top bucket ever - } - else { - if ( !mayBalanceWithNeighbors( thisLoc, id, order ) ) { - // An empty bucket is only allowed as a transient state. If - // there are no neighbors to balance with, we delete ourself. - // This condition is only expected in legacy btrees. - delBucket(thisLoc, id); - } - } - return; - } - deleteInternalKey( thisLoc, p, id, order ); - return; - } - - if ( left.isNull() ) { - this->_delKeyAtPos(p); - mayBalanceWithNeighbors( thisLoc, id, order ); - } - else { - deleteInternalKey( thisLoc, p, id, order ); - } - } - - /** - * This function replaces the specified key (k) by either the prev or next - * key in the btree (k'). We require that k have either a left or right - * child. If k has a left child, we set k' to the prev key of k, which must - * be a leaf present in the left child. If k does not have a left child, we - * set k' to the next key of k, which must be a leaf present in the right - * child. When we replace k with k', we copy k' over k (which may cause a - * split) and then remove k' from its original location. Because k' is - * stored in a descendent of k, replacing k by k' will not modify the - * storage location of the original k', and we can easily remove k' from - * its original location. - * - * This function is only needed in cases where k has a left or right child; - * in other cases a simpler key removal implementation is possible. - * - * NOTE on noncompliant BtreeBuilder btrees: - * It is possible (though likely rare) for btrees created by BtreeBuilder to - * have k' that is not a leaf, see SERVER-2732. These cases are handled in - * the same manner as described in the "legacy btree structures" note below. - * - * NOTE on legacy btree structures: - * In legacy btrees, k' can be a nonleaf. In such a case we 'delete' k by - * marking it as an unused node rather than replacing it with k'. Also, k' - * may be a leaf but marked as an unused node. In such a case we replace - * k by k', preserving the key's unused marking. This function is only - * expected to mark a key as unused when handling a legacy btree. - */ - template< class V > - void BtreeBucket::deleteInternalKey( const DiskLoc thisLoc, int keypos, IndexDetails &id, const Ordering &order ) { - DiskLoc lchild = this->childForPos( keypos ); - DiskLoc rchild = this->childForPos( keypos + 1 ); - verify( !lchild.isNull() || !rchild.isNull() ); - int advanceDirection = lchild.isNull() ? 1 : -1; - int advanceKeyOfs = keypos; - DiskLoc advanceLoc = advance( thisLoc, advanceKeyOfs, advanceDirection, __FUNCTION__ ); - // advanceLoc must be a descentant of thisLoc, because thisLoc has a - // child in the proper direction and all descendants of thisLoc must be - // nonempty because they are not the root. - - if ( !advanceLoc.btree()->childForPos( advanceKeyOfs ).isNull() || - !advanceLoc.btree()->childForPos( advanceKeyOfs + 1 ).isNull() ) { - // only expected with legacy btrees, see note above - this->markUnused( keypos ); - return; - } - - KeyNode kn = advanceLoc.btree()->keyNode( advanceKeyOfs ); - // Because advanceLoc is a descendant of thisLoc, updating thisLoc will - // not affect packing or keys of advanceLoc and kn will be stable - // during the following setInternalKey() - setInternalKey( thisLoc, keypos, kn.recordLoc, kn.key, order, this->childForPos( keypos ), this->childForPos( keypos + 1 ), id ); - advanceLoc.btreemod()->delKeyAtPos( advanceLoc, id, advanceKeyOfs, order ); - } - -//#define BTREE(loc) (static_cast(loc).btree()) -#define BTREE(loc) (loc.template btree()) -//#define BTREEMOD(loc) (static_cast(loc).btreemod()) -#define BTREEMOD(loc) (loc.template btreemod()) - - template< class V > - void BtreeBucket::replaceWithNextChild( const DiskLoc thisLoc, IndexDetails &id ) { - verify( this->n == 0 && !this->nextChild.isNull() ); - if ( this->parent.isNull() ) { - verify( id.head == thisLoc ); - id.head.writing() = this->nextChild; - } - else { - DiskLoc ll = this->parent; - ll.btree()->childForPos( indexInParent( thisLoc ) ).writing() = this->nextChild; - } - BTREE(this->nextChild)->parent.writing() = this->parent; - ClientCursor::informAboutToDeleteBucket( thisLoc ); - deallocBucket( thisLoc, id ); - } - - template< class V > - bool BtreeBucket::canMergeChildren( const DiskLoc &thisLoc, int leftIndex ) const { - verify( leftIndex >= 0 && leftIndex < this->n ); - DiskLoc leftNodeLoc = this->childForPos( leftIndex ); - DiskLoc rightNodeLoc = this->childForPos( leftIndex + 1 ); - if ( leftNodeLoc.isNull() || rightNodeLoc.isNull() ) { - // TODO if this situation is possible in long term implementation, maybe we should compact somehow anyway - return false; - } - int pos = 0; - { - const BtreeBucket *l = leftNodeLoc.btree(); - const BtreeBucket *r = rightNodeLoc.btree(); - if ( ( this->headerSize() + l->packedDataSize( pos ) + r->packedDataSize( pos ) + keyNode( leftIndex ).key.dataSize() + sizeof(_KeyNode) > unsigned( V::BucketSize ) ) ) { - return false; - } - } - return true; - } - - /** - * This implementation must respect the meaning and value of lowWaterMark. - * Also see comments in splitPos(). - */ - template< class V > - int BtreeBucket::rebalancedSeparatorPos( const DiskLoc &thisLoc, int leftIndex ) const { - int split = -1; - int rightSize = 0; - const BtreeBucket *l = BTREE(this->childForPos( leftIndex )); - const BtreeBucket *r = BTREE(this->childForPos( leftIndex + 1 )); - - int KNS = sizeof( _KeyNode ); - int rightSizeLimit = ( l->topSize + l->n * KNS + keyNode( leftIndex ).key.dataSize() + KNS + r->topSize + r->n * KNS ) / 2; - // This constraint should be ensured by only calling this function - // if we go below the low water mark. - verify( rightSizeLimit < BtreeBucket::bodySize() ); - for( int i = r->n - 1; i > -1; --i ) { - rightSize += r->keyNode( i ).key.dataSize() + KNS; - if ( rightSize > rightSizeLimit ) { - split = l->n + 1 + i; - break; - } - } - if ( split == -1 ) { - rightSize += keyNode( leftIndex ).key.dataSize() + KNS; - if ( rightSize > rightSizeLimit ) { - split = l->n; - } - } - if ( split == -1 ) { - for( int i = l->n - 1; i > -1; --i ) { - rightSize += l->keyNode( i ).key.dataSize() + KNS; - if ( rightSize > rightSizeLimit ) { - split = i; - break; - } - } - } - // safeguards - we must not create an empty bucket - if ( split < 1 ) { - split = 1; - } - else if ( split > l->n + 1 + r->n - 2 ) { - split = l->n + 1 + r->n - 2; - } - - return split; - } - - template< class V > - void BtreeBucket::doMergeChildren( const DiskLoc thisLoc, int leftIndex, IndexDetails &id, const Ordering &order ) { - DiskLoc leftNodeLoc = this->childForPos( leftIndex ); - DiskLoc rightNodeLoc = this->childForPos( leftIndex + 1 ); - BtreeBucket *l = leftNodeLoc.btreemod(); - BtreeBucket *r = rightNodeLoc.btreemod(); - int pos = 0; - l->_packReadyForMod( order, pos ); - r->_packReadyForMod( order, pos ); // pack r in case there are droppable keys - - // We know the additional keys below will fit in l because canMergeChildren() - // must be true. - int oldLNum = l->n; - { - KeyNode kn = keyNode( leftIndex ); - l->pushBack( kn.recordLoc, kn.key, order, l->nextChild ); // left child's right child becomes old parent key's left child - } - for( int i = 0; i < r->n; ++i ) { - KeyNode kn = r->keyNode( i ); - l->pushBack( kn.recordLoc, kn.key, order, kn.prevChildBucket ); - } - l->nextChild = r->nextChild; - l->fixParentPtrs( leftNodeLoc, oldLNum ); - r->delBucket( rightNodeLoc, id ); - this->childForPos( leftIndex + 1 ) = leftNodeLoc; - this->childForPos( leftIndex ) = DiskLoc(); - this->_delKeyAtPos( leftIndex, true ); - if ( this->n == 0 ) { - // will trash this and thisLoc - // TODO To ensure all leaves are of equal height, we should ensure - // this is only called on the root. - replaceWithNextChild( thisLoc, id ); - } - else { - // balance recursively - maybe we should do this even when n == 0? - mayBalanceWithNeighbors( thisLoc, id, order ); - } - } - - template< class V > - int BtreeBucket::indexInParent( const DiskLoc &thisLoc ) const { - verify( !this->parent.isNull() ); - const BtreeBucket *p = BTREE(this->parent); - if ( p->nextChild == thisLoc ) { - return p->n; - } - else { - for( int i = 0; i < p->n; ++i ) { - if ( p->k( i ).prevChildBucket == thisLoc ) { - return i; - } - } - } - out() << "ERROR: can't find ref to child bucket.\n"; - out() << "child: " << thisLoc << "\n"; - dump(); - out() << "Parent: " << this->parent << "\n"; - p->dump(); - verify(false); - return -1; // just to compile - } - - template< class V > - bool BtreeBucket::tryBalanceChildren( const DiskLoc thisLoc, int leftIndex, IndexDetails &id, const Ordering &order ) const { - // If we can merge, then we must merge rather than balance to preserve - // bucket utilization constraints. - if ( canMergeChildren( thisLoc, leftIndex ) ) { - return false; - } - thisLoc.btreemod()->doBalanceChildren( thisLoc, leftIndex, id, order ); - return true; - } - - template< class V > - void BtreeBucket::doBalanceLeftToRight( const DiskLoc thisLoc, int leftIndex, int split, - BtreeBucket *l, const DiskLoc lchild, - BtreeBucket *r, const DiskLoc rchild, - IndexDetails &id, const Ordering &order ) { - // TODO maybe do some audits the same way pushBack() does? - // As a precondition, rchild + the old separator are <= half a body size, - // and lchild is at most completely full. Based on the value of split, - // rchild will get <= half of the total bytes which is at most 75% - // of a full body. So rchild will have room for the following keys: - int rAdd = l->n - split; - r->reserveKeysFront( rAdd ); - for( int i = split + 1, j = 0; i < l->n; ++i, ++j ) { - KeyNode kn = l->keyNode( i ); - r->setKey( j, kn.recordLoc, kn.key, kn.prevChildBucket ); - } - { - KeyNode kn = keyNode( leftIndex ); - r->setKey( rAdd - 1, kn.recordLoc, kn.key, l->nextChild ); // left child's right child becomes old parent key's left child - } - r->fixParentPtrs( rchild, 0, rAdd - 1 ); - { - KeyNode kn = l->keyNode( split ); - l->nextChild = kn.prevChildBucket; - // Because lchild is a descendant of thisLoc, updating thisLoc will - // not affect packing or keys of lchild and kn will be stable - // during the following setInternalKey() - setInternalKey( thisLoc, leftIndex, kn.recordLoc, kn.key, order, lchild, rchild, id ); - } - int zeropos = 0; - // lchild and rchild cannot be merged, so there must be >0 (actually more) - // keys to the left of split. - l->truncateTo( split, order, zeropos ); - } - - template< class V > - void BtreeBucket::doBalanceRightToLeft( const DiskLoc thisLoc, int leftIndex, int split, - BtreeBucket *l, const DiskLoc lchild, - BtreeBucket *r, const DiskLoc rchild, - IndexDetails &id, const Ordering &order ) { - // As a precondition, lchild + the old separator are <= half a body size, - // and rchild is at most completely full. Based on the value of split, - // lchild will get less than half of the total bytes which is at most 75% - // of a full body. So lchild will have room for the following keys: - int lN = l->n; - { - KeyNode kn = keyNode( leftIndex ); - l->pushBack( kn.recordLoc, kn.key, order, l->nextChild ); // left child's right child becomes old parent key's left child - } - for( int i = 0; i < split - lN - 1; ++i ) { - KeyNode kn = r->keyNode( i ); - l->pushBack( kn.recordLoc, kn.key, order, kn.prevChildBucket ); - } - { - KeyNode kn = r->keyNode( split - lN - 1 ); - l->nextChild = kn.prevChildBucket; - // Child lN was lchild's old nextChild, and don't need to fix that one. - l->fixParentPtrs( lchild, lN + 1, l->n ); - // Because rchild is a descendant of thisLoc, updating thisLoc will - // not affect packing or keys of rchild and kn will be stable - // during the following setInternalKey() - setInternalKey( thisLoc, leftIndex, kn.recordLoc, kn.key, order, lchild, rchild, id ); - } - int zeropos = 0; - // lchild and rchild cannot be merged, so there must be >0 (actually more) - // keys to the right of split. - r->dropFront( split - lN, order, zeropos ); - } - - template< class V > - void BtreeBucket::doBalanceChildren( const DiskLoc thisLoc, int leftIndex, IndexDetails &id, const Ordering &order ) { - DiskLoc lchild = this->childForPos( leftIndex ); - DiskLoc rchild = this->childForPos( leftIndex + 1 ); - int zeropos = 0; - BtreeBucket *l = lchild.btreemod(); - l->_packReadyForMod( order, zeropos ); - BtreeBucket *r = rchild.btreemod(); - r->_packReadyForMod( order, zeropos ); - int split = rebalancedSeparatorPos( thisLoc, leftIndex ); - - // By definition, if we are below the low water mark and cannot merge - // then we must actively balance. - verify( split != l->n ); - if ( split < l->n ) { - doBalanceLeftToRight( thisLoc, leftIndex, split, l, lchild, r, rchild, id, order ); - } - else { - doBalanceRightToLeft( thisLoc, leftIndex, split, l, lchild, r, rchild, id, order ); - } - } - - template< class V > - bool BtreeBucket::mayBalanceWithNeighbors( const DiskLoc thisLoc, IndexDetails &id, const Ordering &order ) const { - if ( this->parent.isNull() ) { // we are root, there are no neighbors - return false; - } - - if ( this->packedDataSize( 0 ) >= this->lowWaterMark() ) { - return false; - } - - const BtreeBucket *p = BTREE(this->parent); - int parentIdx = indexInParent( thisLoc ); - - // TODO will missing neighbor case be possible long term? Should we try to merge/balance somehow in that case if so? - bool mayBalanceRight = ( ( parentIdx < p->n ) && !p->childForPos( parentIdx + 1 ).isNull() ); - bool mayBalanceLeft = ( ( parentIdx > 0 ) && !p->childForPos( parentIdx - 1 ).isNull() ); - - // Balance if possible on one side - we merge only if absolutely necessary - // to preserve btree bucket utilization constraints since that's a more - // heavy duty operation (especially if we must re-split later). - if ( mayBalanceRight && - p->tryBalanceChildren( this->parent, parentIdx, id, order ) ) { - return true; - } - if ( mayBalanceLeft && - p->tryBalanceChildren( this->parent, parentIdx - 1, id, order ) ) { - return true; - } - - BtreeBucket *pm = BTREEMOD(this->parent); - if ( mayBalanceRight ) { - pm->doMergeChildren( this->parent, parentIdx, id, order ); - return true; - } - else if ( mayBalanceLeft ) { - pm->doMergeChildren( this->parent, parentIdx - 1, id, order ); - return true; - } - - return false; - } - - /** remove a key from the index */ - template< class V > - bool BtreeBucket::unindex(const DiskLoc thisLoc, IndexDetails& id, const BSONObj& key, const DiskLoc recordLoc ) const { - int pos; - bool found; - const Ordering ord = Ordering::make(id.keyPattern()); - DiskLoc loc = locate(id, thisLoc, key, ord, pos, found, recordLoc, 1); - if ( found ) { - if ( key.objsize() > this->KeyMax ) { - OCCASIONALLY problem() << "unindex: key too large to index but was found for " << id.indexNamespace() << " reIndex suggested" << endl; - } - loc.btreemod()->delKeyAtPos(loc, id, pos, ord); - return true; - } - return false; - } - - template< class V > - inline void BtreeBucket::fix(const DiskLoc thisLoc, const DiskLoc child) { - if ( !child.isNull() ) { - if ( insert_debug ) - out() << " fix " << child.toString() << ".parent=" << thisLoc.toString() << endl; - child.btree()->parent.writing() = thisLoc; - } - } - - /** - * This can cause a lot of additional page writes when we assign buckets to - * different parents. Maybe get rid of parent ptrs? - */ - template< class V > - void BtreeBucket::fixParentPtrs(const DiskLoc thisLoc, int firstIndex, int lastIndex) const { - VERIFYTHISLOC - if ( lastIndex == -1 ) { - lastIndex = this->n; - } - for ( int i = firstIndex; i <= lastIndex; i++ ) { - fix(thisLoc, this->childForPos(i)); - } - } - - template< class V > - void BtreeBucket::setInternalKey( const DiskLoc thisLoc, int keypos, - const DiskLoc recordLoc, const Key &key, const Ordering &order, - const DiskLoc lchild, const DiskLoc rchild, IndexDetails &idx ) { - this->childForPos( keypos ).Null(); - - // This may leave the bucket empty (n == 0) which is ok only as a - // transient state. In the instant case, the implementation of - // insertHere behaves correctly when n == 0 and as a side effect - // increments n. - this->_delKeyAtPos( keypos, true ); - - // Ensure we do not orphan neighbor's old child. - verify( this->childForPos( keypos ) == rchild ); - - // Just set temporarily - required to pass validation in insertHere() - this->childForPos( keypos ) = lchild; - - insertHere( thisLoc, keypos, recordLoc, key, order, lchild, rchild, idx ); - } - - /** - * insert a key in this bucket, splitting if necessary. - * @keypos - where to insert the key in range 0..n. 0=make leftmost, n=make rightmost. - * NOTE this function may free some data, and as a result the value passed for keypos may - * be invalid after calling insertHere() - * - * Some of the write intent signaling below relies on the implementation of - * the optimized write intent code in basicInsert(). - */ - template< class V > - void BtreeBucket::insertHere( const DiskLoc thisLoc, int keypos, - const DiskLoc recordLoc, const Key& key, const Ordering& order, - const DiskLoc lchild, const DiskLoc rchild, IndexDetails& idx) const { - if ( insert_debug ) - out() << " " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' ' - << lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl; - - if ( !this->basicInsert(thisLoc, keypos, recordLoc, key, order) ) { - // If basicInsert() fails, the bucket will be packed as required by split(). - thisLoc.btreemod()->split(thisLoc, keypos, recordLoc, key, order, lchild, rchild, idx); - return; - } - - { - const _KeyNode *_kn = &k(keypos); - _KeyNode *kn = (_KeyNode *) getDur().alreadyDeclared((_KeyNode*) _kn); // already declared intent in basicInsert() - if ( keypos+1 == this->n ) { // last key - if ( this->nextChild != lchild ) { - out() << "ERROR nextChild != lchild" << endl; - out() << " thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl; - out() << " keyPos: " << keypos << " n:" << this->n << endl; - out() << " nextChild: " << this->nextChild.toString() << " lchild: " << lchild.toString() << endl; - out() << " recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl; - out() << " key: " << key.toString() << endl; - dump(); - verify(false); - } - kn->prevChildBucket = this->nextChild; - verify( kn->prevChildBucket == lchild ); - this->nextChild.writing() = rchild; - if ( !rchild.isNull() ) - BTREE(rchild)->parent.writing() = thisLoc; - } - else { - kn->prevChildBucket = lchild; - if ( k(keypos+1).prevChildBucket != lchild ) { - out() << "ERROR k(keypos+1).prevChildBucket != lchild" << endl; - out() << " thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl; - out() << " keyPos: " << keypos << " n:" << this->n << endl; - out() << " k(keypos+1).pcb: " << k(keypos+1).prevChildBucket.toString() << " lchild: " << lchild.toString() << endl; - out() << " recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl; - out() << " key: " << key.toString() << endl; - dump(); - verify(false); - } - const Loc *pc = &k(keypos+1).prevChildBucket; - *getDur().alreadyDeclared( const_cast(pc) ) = rchild; // declared in basicInsert() - if ( !rchild.isNull() ) - rchild.btree()->parent.writing() = thisLoc; - } - return; - } - } - - template< class V > - void BtreeBucket::split(const DiskLoc thisLoc, int keypos, const DiskLoc recordLoc, const Key& key, const Ordering& order, const DiskLoc lchild, const DiskLoc rchild, IndexDetails& idx) { - this->assertWritable(); - - if ( split_debug ) - out() << " " << thisLoc.toString() << ".split" << endl; - - int split = this->splitPos( keypos ); - DiskLoc rLoc = addBucket(idx); - BtreeBucket *r = rLoc.btreemod(); - if ( split_debug ) - out() << " split:" << split << ' ' << keyNode(split).key.toString() << " n:" << this->n << endl; - for ( int i = split+1; i < this->n; i++ ) { - KeyNode kn = keyNode(i); - r->pushBack(kn.recordLoc, kn.key, order, kn.prevChildBucket); - } - r->nextChild = this->nextChild; - r->assertValid( order ); - - if ( split_debug ) - out() << " new rLoc:" << rLoc.toString() << endl; - r = 0; - rLoc.btree()->fixParentPtrs(rLoc); - - { - KeyNode splitkey = keyNode(split); - this->nextChild = splitkey.prevChildBucket; // splitkey key gets promoted, its children will be thisLoc (l) and rLoc (r) - if ( split_debug ) { - out() << " splitkey key:" << splitkey.key.toString() << endl; - } - - // Because thisLoc is a descendant of parent, updating parent will - // not affect packing or keys of thisLoc and splitkey will be stable - // during the following: - - // promote splitkey to a parent this->node - if ( this->parent.isNull() ) { - // make a new parent if we were the root - DiskLoc L = addBucket(idx); - BtreeBucket *p = L.btreemod(); - p->pushBack(splitkey.recordLoc, splitkey.key, order, thisLoc); - p->nextChild = rLoc; - p->assertValid( order ); - this->parent = idx.head.writing() = L; - if ( split_debug ) - out() << " we were root, making new root:" << hex << this->parent.getOfs() << dec << endl; - rLoc.btree()->parent.writing() = this->parent; - } - else { - // set this before calling _insert - if it splits it will do fixParent() logic and change the value. - rLoc.btree()->parent.writing() = this->parent; - if ( split_debug ) - out() << " promoting splitkey key " << splitkey.key.toString() << endl; - BTREE(this->parent)->_insert(this->parent, splitkey.recordLoc, splitkey.key, order, /*dupsallowed*/true, thisLoc, rLoc, idx); - } - } - - int newpos = keypos; - // note this may trash splitkey.key. thus we had to promote it before finishing up here. - this->truncateTo(split, order, newpos); - - // add our this->new key, there is room this->now - { - if ( keypos <= split ) { - if ( split_debug ) - out() << " keypos=0); - BTREE(rLoc)->insertHere(rLoc, kp, recordLoc, key, order, lchild, rchild, idx); - } - } - - if ( split_debug ) - out() << " split end " << hex << thisLoc.getOfs() << dec << endl; - } - - /** start a new index off, empty */ - template< class V > - DiskLoc BtreeBucket::addBucket(const IndexDetails& id) { - string ns = id.indexNamespace(); - DiskLoc loc = theDataFileMgr.insert(ns.c_str(), 0, V::BucketSize, false, true); - BtreeBucket *b = BTREEMOD(loc); - b->init(); - return loc; - } - - template< class V > - const DiskLoc BtreeBucket::getHead(const DiskLoc& thisLoc) const { - DiskLoc p = thisLoc; - while ( !BTREE(p)->isHead() ) - p = BTREE(p)->parent; - return p; - } - - template< class V > - DiskLoc BtreeBucket::advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) const { - if ( keyOfs < 0 || keyOfs >= this->n ) { - out() << "ASSERT failure BtreeBucket::advance, caller: " << caller << endl; - out() << " thisLoc: " << thisLoc.toString() << endl; - out() << " keyOfs: " << keyOfs << " n:" << this->n << " direction: " << direction << endl; - out() << bucketSummary() << endl; - verify(false); - } - int adj = direction < 0 ? 1 : 0; - int ko = keyOfs + direction; - DiskLoc nextDown = this->childForPos(ko+adj); - if ( !nextDown.isNull() ) { - while ( 1 ) { - keyOfs = direction>0 ? 0 : BTREE(nextDown)->n - 1; - DiskLoc loc = BTREE(nextDown)->childForPos(keyOfs + adj); - if ( loc.isNull() ) - break; - nextDown = loc; - } - return nextDown; - } - - if ( ko < this->n && ko >= 0 ) { - keyOfs = ko; - return thisLoc; - } - - // end of bucket. traverse back up. - DiskLoc childLoc = thisLoc; - DiskLoc ancestor = this->parent; - while ( 1 ) { - if ( ancestor.isNull() ) - break; - const BtreeBucket *an = BTREE(ancestor); - for ( int i = 0; i < an->n; i++ ) { - if ( an->childForPos(i+adj) == childLoc ) { - keyOfs = i; - return ancestor; - } - } - verify( direction<0 || an->nextChild == childLoc ); - // parent exhausted also, keep going up - childLoc = ancestor; - ancestor = an->parent; - } - - return DiskLoc(); - } - - template< class V > - DiskLoc BtreeBucket::locate(const IndexDetails& idx, const DiskLoc& thisLoc, const BSONObj& key, const Ordering &order, int& pos, bool& found, const DiskLoc &recordLoc, int direction) const { - KeyOwned k(key); - return locate(idx, thisLoc, k, order, pos, found, recordLoc, direction); - } - - template< class V > - DiskLoc BtreeBucket::locate(const IndexDetails& idx, const DiskLoc& thisLoc, const Key& key, const Ordering &order, int& pos, bool& found, const DiskLoc &recordLoc, int direction) const { - int p; - found = find(idx, key, recordLoc, order, p, /*assertIfDup*/ false); - if ( found ) { - pos = p; - return thisLoc; - } - - DiskLoc child = this->childForPos(p); - - if ( !child.isNull() ) { - DiskLoc l = BTREE(child)->locate(idx, child, key, order, pos, found, recordLoc, direction); - if ( !l.isNull() ) - return l; - } - - pos = p; - if ( direction < 0 ) - return --pos == -1 ? DiskLoc() /*theend*/ : thisLoc; - else - return pos == this->n ? DiskLoc() /*theend*/ : thisLoc; - } - - template< class V > - bool BtreeBucket::customFind( int l, int h, const BSONObj &keyBegin, int keyBeginLen, bool afterKey, const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive, const Ordering &order, int direction, DiskLoc &thisLoc, int &keyOfs, pair< DiskLoc, int > &bestParent ) { - const BtreeBucket * bucket = BTREE(thisLoc); - while( 1 ) { - if ( l + 1 == h ) { - keyOfs = ( direction > 0 ) ? h : l; - DiskLoc next = bucket->k( h ).prevChildBucket; - if ( !next.isNull() ) { - bestParent = make_pair( thisLoc, keyOfs ); - thisLoc = next; - return true; - } - else { - return false; - } - } - int m = l + ( h - l ) / 2; - int cmp = customBSONCmp( bucket->keyNode( m ).key.toBson(), keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction ); - if ( cmp < 0 ) { - l = m; - } - else if ( cmp > 0 ) { - h = m; - } - else { - if ( direction < 0 ) { - l = m; - } - else { - h = m; - } - } - } - } - - /** - * find smallest/biggest value greater-equal/less-equal than specified - * starting thisLoc + keyOfs will be strictly less than/strictly greater than keyBegin/keyBeginLen/keyEnd - * All the direction checks below allowed me to refactor the code, but possibly separate forward and reverse implementations would be more efficient - */ - template< class V > - void BtreeBucket::advanceTo(DiskLoc &thisLoc, int &keyOfs, const BSONObj &keyBegin, int keyBeginLen, bool afterKey, const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive, const Ordering &order, int direction ) const { - int l,h; - bool dontGoUp; - if ( direction > 0 ) { - l = keyOfs; - h = this->n - 1; - dontGoUp = ( customBSONCmp( keyNode( h ).key.toBson(), keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction ) >= 0 ); - } - else { - l = 0; - h = keyOfs; - dontGoUp = ( customBSONCmp( keyNode( l ).key.toBson(), keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction ) <= 0 ); - } - pair< DiskLoc, int > bestParent; - if ( dontGoUp ) { - // this comparison result assures h > l - if ( !customFind( l, h, keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction, thisLoc, keyOfs, bestParent ) ) { - return; - } - } - else { - // go up parents until rightmost/leftmost node is >=/<= target or at top - while( !BTREE(thisLoc)->parent.isNull() ) { - thisLoc = BTREE(thisLoc)->parent; - if ( direction > 0 ) { - if ( customBSONCmp( BTREE(thisLoc)->keyNode( BTREE(thisLoc)->n - 1 ).key.toBson(), keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction ) >= 0 ) { - break; - } - } - else { - if ( customBSONCmp( BTREE(thisLoc)->keyNode( 0 ).key.toBson(), keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction ) <= 0 ) { - break; - } - } - } - } - customLocate( thisLoc, keyOfs, keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction, bestParent ); - } - - /** @param thisLoc in/out param. perhaps thisLoc isn't the best name given that. - Ut is used by advanceTo, which skips - from one key to another key without necessarily checking all the keys - between them in the btree (it can skip to different btree buckets). - The advanceTo function can get called a lot, and for different targets - we want to advance to, don't want to create a bson obj in a new - buffer each time we call that function. The - customLocate function necessary for advanceTo, and does the same thing - as normal locate function but takes basically the same arguments - as advanceTo. - */ - template< class V > - void BtreeBucket::customLocate(DiskLoc &locInOut, int &keyOfs, const BSONObj &keyBegin, int keyBeginLen, bool afterKey, - const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive, - const Ordering &order, int direction, pair< DiskLoc, int > &bestParent ) { - dassert( direction == 1 || direction == -1 ); - const BtreeBucket *bucket = BTREE(locInOut); - if ( bucket->n == 0 ) { - locInOut = DiskLoc(); - return; - } - // go down until find smallest/biggest >=/<= target - while( 1 ) { - int l = 0; - int h = bucket->n - 1; - - // +direction: 0, -direction: h - int z = (1-direction)/2*h; - - // leftmost/rightmost key may possibly be >=/<= search key - int res = customBSONCmp( bucket->keyNode( z ).key.toBson(), keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction ); - bool firstCheck = direction*res >= 0; - - if ( firstCheck ) { - DiskLoc next; - keyOfs = z; - if ( direction > 0 ) { - dassert( z == 0 ); - next = bucket->k( 0 ).prevChildBucket; - } - else { - next = bucket->nextChild; - } - if ( !next.isNull() ) { - bestParent = pair< DiskLoc, int >( locInOut, keyOfs ); - locInOut = next; - bucket = BTREE(locInOut); - continue; - } - else { - return; - } - } - - res = customBSONCmp( bucket->keyNode( h-z ).key.toBson(), keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction ); - bool secondCheck = direction*res < 0; - - if ( secondCheck ) { - DiskLoc next; - if ( direction > 0 ) { - next = bucket->nextChild; - } - else { - next = bucket->k( 0 ).prevChildBucket; - } - if ( next.isNull() ) { - // if bestParent is null, we've hit the end and locInOut gets set to DiskLoc() - locInOut = bestParent.first; - keyOfs = bestParent.second; - return; - } - else { - locInOut = next; - bucket = BTREE(locInOut); - continue; - } - } - - if ( !customFind( l, h, keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction, locInOut, keyOfs, bestParent ) ) { - return; - } - bucket = BTREE(locInOut); - } - } - - /** @thisLoc disk location of *this */ - template< class V > - void BtreeBucket::insertStepOne(DiskLoc thisLoc, - IndexInsertionContinuationImpl& c, - bool dupsAllowed) const { - dassert( c.key.dataSize() <= this->KeyMax ); - verify( c.key.dataSize() > 0 ); - - int pos; - bool found = find(c.idx, c.key, c.recordLoc, c.order, pos, !dupsAllowed); - - if ( found ) { - const _KeyNode& kn = k(pos); - if ( kn.isUnused() ) { - LOG(4) << "btree _insert: reusing unused key" << endl; - c.b = this; - c.pos = pos; - c.op = IndexInsertionContinuation::SetUsed; - return; - } - - DEV { - log() << "_insert(): key already exists in index (ok for background:true)\n"; - log() << " " << c.idx.indexNamespace() << " thisLoc:" << thisLoc.toString() << '\n'; - log() << " " << c.key.toString() << '\n'; - log() << " " << "recordLoc:" << c.recordLoc.toString() << " pos:" << pos << endl; - log() << " old l r: " << this->childForPos(pos).toString() << ' ' << this->childForPos(pos+1).toString() << endl; - } - alreadyInIndex(); - } - - Loc ch = this->childForPos(pos); - DiskLoc child = ch; - - if ( child.isNull() ) { - // A this->new key will be inserted at the same tree height as an adjacent existing key. - c.bLoc = thisLoc; - c.b = this; - c.pos = pos; - c.op = IndexInsertionContinuation::InsertHere; - return; - } - - child.btree()->insertStepOne(child, c, dupsAllowed); - } - - /** @thisLoc disk location of *this */ - template< class V > - int BtreeBucket::_insert(const DiskLoc thisLoc, const DiskLoc recordLoc, - const Key& key, const Ordering &order, bool dupsAllowed, - const DiskLoc lChild, const DiskLoc rChild, IndexDetails& idx) const { - if ( key.dataSize() > this->KeyMax ) { - problem() << "ERROR: key too large len:" << key.dataSize() << " max:" << this->KeyMax << ' ' << key.dataSize() << ' ' << idx.indexNamespace() << endl; - return 2; - } - verify( key.dataSize() > 0 ); - - int pos; - bool found = find(idx, key, recordLoc, order, pos, !dupsAllowed); - if ( insert_debug ) { - out() << " " << thisLoc.toString() << '.' << "_insert " << - key.toString() << '/' << recordLoc.toString() << - " l:" << lChild.toString() << " r:" << rChild.toString() << endl; - out() << " found:" << found << " pos:" << pos << " n:" << this->n << endl; - } - - if ( found ) { - const _KeyNode& kn = k(pos); - if ( kn.isUnused() ) { - LOG(4) << "btree _insert: reusing unused key" << endl; - massert( 10285 , "_insert: reuse key but lchild is not null", lChild.isNull()); - massert( 10286 , "_insert: reuse key but rchild is not null", rChild.isNull()); - kn.writing().setUsed(); - return 0; - } - - DEV { - log() << "_insert(): key already exists in index (ok for background:true)\n"; - log() << " " << idx.indexNamespace() << " thisLoc:" << thisLoc.toString() << '\n'; - log() << " " << key.toString() << '\n'; - log() << " " << "recordLoc:" << recordLoc.toString() << " pos:" << pos << endl; - log() << " old l r: " << this->childForPos(pos).toString() << ' ' << this->childForPos(pos+1).toString() << endl; - log() << " new l r: " << lChild.toString() << ' ' << rChild.toString() << endl; - } - alreadyInIndex(); - } - - DEBUGGING out() << "TEMP: key: " << key.toString() << endl; - Loc ch = this->childForPos(pos); - DiskLoc child = ch; - if ( insert_debug ) - out() << " getChild(" << pos << "): " << child.toString() << endl; - // In current usage, rChild isNull() for a new key and false when we are - // promoting a split key. These are the only two cases where _insert() - // is called currently. - if ( child.isNull() || !rChild.isNull() ) { - // A new key will be inserted at the same tree height as an adjacent existing key. - insertHere(thisLoc, pos, recordLoc, key, order, lChild, rChild, idx); - return 0; - } - - return child.btree()->_insert(child, recordLoc, key, order, dupsAllowed, /*lchild*/DiskLoc(), /*rchild*/DiskLoc(), idx); - } - - template< class V > - void BtreeBucket::dump(unsigned depth) const { - string indent = string(depth, ' '); - _log() << "BUCKET n:" << this->n; - _log() << " parent:" << hex << this->parent.getOfs() << dec; - for ( int i = 0; i < this->n; i++ ) { - _log() << '\n' << indent; - KeyNode k = keyNode(i); - string ks = k.key.toString(); - _log() << " " << hex << k.prevChildBucket.getOfs() << '\n'; - _log() << indent << " " << i << ' ' << ks.substr(0, 30) << " Loc:" << k.recordLoc.toString() << dec; - if ( this->k(i).isUnused() ) - _log() << " UNUSED"; - } - _log() << "\n" << indent << " " << hex << this->nextChild.getOfs() << dec << endl; - } - - template< class V > - void BtreeBucket::twoStepInsert(DiskLoc thisLoc, IndexInsertionContinuationImpl &c, - bool dupsAllowed) const - { - - if ( c.key.dataSize() > this->KeyMax ) { - problem() << "ERROR: key too large len:" << c.key.dataSize() << " max:" << this->KeyMax << ' ' << c.key.dataSize() << ' ' << c.idx.indexNamespace() << endl; - return; // op=Nothing - } - insertStepOne(thisLoc, c, dupsAllowed); - } - - /** todo: meaning of return code unclear clean up */ - template< class V > - int BtreeBucket::bt_insert(const DiskLoc thisLoc, const DiskLoc recordLoc, - const BSONObj& _key, const Ordering &order, bool dupsAllowed, - IndexDetails& idx, bool toplevel) const - { - guessIncreasing = _key.firstElementType() == jstOID && idx.isIdIndex(); - KeyOwned key(_key); - - dassert(toplevel); - if ( toplevel ) { - if ( key.dataSize() > this->KeyMax ) { - problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace() << ' ' << key.dataSize() << ' ' << key.toString() << endl; - return 3; - } - } - - int x; - try { - x = _insert(thisLoc, recordLoc, key, order, dupsAllowed, DiskLoc(), DiskLoc(), idx); - this->assertValid( order ); - } - catch( ... ) { - guessIncreasing = false; - throw; - } - guessIncreasing = false; - return x; - } - - template< class V > - void BtreeBucket::shape(stringstream& ss) const { - this->_shape(0, ss); - } - - template< class V > - int BtreeBucket::getKeyMax() { - return V::KeyMax; - } - - template< class V > - DiskLoc BtreeBucket::findSingle( const IndexDetails& indexdetails , const DiskLoc& thisLoc, const BSONObj& key ) const { - int pos; - bool found; - // TODO: is it really ok here that the order is a default? - // for findById() use, yes. for checkNoIndexConflicts, no? - Ordering o = Ordering::make(BSONObj()); - DiskLoc bucket = locate( indexdetails , indexdetails.head , key , o , pos , found , minDiskLoc ); - if ( bucket.isNull() ) - return bucket; - - const BtreeBucket *b = bucket.btree(); - while ( 1 ) { - const _KeyNode& knraw = b->k(pos); - if ( knraw.isUsed() ) - break; - bucket = b->advance( bucket , pos , 1 , "findSingle" ); - if ( bucket.isNull() ) - return bucket; - b = bucket.btree(); - } - KeyNode kn = b->keyNode( pos ); - if ( KeyOwned(key).woCompare( kn.key, o ) != 0 ) - return DiskLoc(); - return kn.recordLoc; - } - -} // namespace mongo - -#include "db.h" -#include "dbhelpers.h" - -namespace mongo { - - template< class V > - void BtreeBucket::a_test(IndexDetails& id) { - BtreeBucket *b = id.head.btreemod(); - - // record locs for testing - DiskLoc A(1, 20); - DiskLoc B(1, 30); - DiskLoc C(1, 40); - - DiskLoc rl; - BSONObj key = fromjson("{x:9}"); - BSONObj orderObj = fromjson("{}"); - Ordering order = Ordering::make(orderObj); - - b->bt_insert(id.head, A, key, order, true, id); - A.GETOFS() += 2; - b->bt_insert(id.head, A, key, order, true, id); - A.GETOFS() += 2; - b->bt_insert(id.head, A, key, order, true, id); - A.GETOFS() += 2; - b->bt_insert(id.head, A, key, order, true, id); - A.GETOFS() += 2; - verify( b->k(0).isUsed() ); -// b->k(0).setUnused(); - b->k(1).setUnused(); - b->k(2).setUnused(); - b->k(3).setUnused(); - - b->dumpTree(id.head, orderObj); - - /* b->bt_insert(id.head, B, key, order, false, id); - b->k(1).setUnused(); - b->dumpTree(id.head, order); - b->bt_insert(id.head, A, key, order, false, id); - b->dumpTree(id.head, order); - */ - - // this should assert. does it? (it might "accidentally" though, not asserting proves a problem, asserting proves nothing) - b->bt_insert(id.head, C, key, order, false, id); - - // b->dumpTree(id.head, order); - } - - template class BucketBasics; - template class BucketBasics; - template class BtreeBucket; - template class BtreeBucket; - template struct __KeyNode; - template struct __KeyNode; - - struct BTUnitTest : public StartupTest { - void run() { - DiskLoc big(0xf12312, 0x70001234); - DiskLoc56Bit bigl; - { - bigl = big; - verify( big == bigl ); - DiskLoc e = bigl; - verify( big == e ); - } - { - DiskLoc d; - verify( d.isNull() ); - DiskLoc56Bit l; - l = d; - verify( l.isNull() ); - d = l; - verify( d.isNull() ); - verify( l < bigl ); - } - } - } btunittest; - - - IndexInsertionContinuation::~IndexInsertionContinuation() {} -} diff --git a/src/mongo/db/btree.h b/src/mongo/db/btree.h deleted file mode 100644 index 32e0e239ea3..00000000000 --- a/src/mongo/db/btree.h +++ /dev/null @@ -1,1026 +0,0 @@ -// btree.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/pch.h" - -#include "mongo/db/diskloc.h" -#include "mongo/db/dur.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/key.h" - -namespace mongo { - - /** - * Our btree implementation generally follows the standard btree algorithm, - * which is described in many places. The nodes of our btree are referred to - * as buckets below. These buckets are of size BucketSize and their body is - * an ordered array of pairs, where disk loc is the disk - * location of a document and bson key is a projection of this document into - * the schema of the index for this btree. Ordering is determined on the - * basis of bson key first and then disk loc in case of a tie. All bson keys - * for a btree have identical schemas with empty string field names and may - * not have an objsize() exceeding KeyMax. The btree's buckets are - * themselves organized into an ordered tree. Although there are exceptions, - * generally buckets with n keys have n+1 children and the body of a bucket is - * at least lowWaterMark bytes. A more strictly enforced requirement is that - * a non root bucket must have at least one key except in certain transient - * states. - * - * Our btrees support the following primary read operations: finding a - * specified key; iterating from a starting key to the next or previous - * ordered key; and skipping from a starting key to another specified key - * without checking every intermediate key. The primary write operations - * are insertion and deletion of keys. Insertion may trigger a bucket split - * if necessary to avoid bucket overflow. In such a case, subsequent splits - * will occur recursively as necessary. Deletion may trigger a bucket - * rebalance, in which a size deficient bucket is filled with keys from an - * adjacent bucket. In this case, splitting may potentially occur in the - * parent. Deletion may alternatively trigger a merge, in which the keys - * from two buckets and a key from their shared parent are combined into the - * same bucket. In such a case, rebalancing or merging may proceed - * recursively from the parent. - * - * While the btree data format has been relatively constant over time, btrees - * initially created by versions of mongo earlier than the current version - * may embody different properties than freshly created btrees (while - * following the same data format). These older btrees are referred to - * below as legacy btrees. - */ - - const int OldBucketSize = 8192; - -#pragma pack(1) - template< class Version > class BucketBasics; - - /** - * This is the fixed width data component for storage of a key within a - * bucket. It contains an offset pointer to the variable width bson - * data component. A _KeyNode may be 'unused', please see below. - */ - template< class Loc > - struct __KeyNode { - /** Signals that we are writing this _KeyNode and casts away const */ - __KeyNode & writing() const; - /** - * The 'left' child bucket of this key. If this is the i-th key, it - * points to the i index child bucket. - */ - Loc prevChildBucket; - /** The location of the record associated with this key. */ - Loc recordLoc; - short keyDataOfs() const { return (short) _kdo; } - - /** Offset within current bucket of the variable width bson key for this _KeyNode. */ - unsigned short _kdo; - void setKeyDataOfs(short s) { - _kdo = s; - verify(s>=0); - } - /** Seems to be redundant. */ - void setKeyDataOfsSavingUse(short s) { - _kdo = s; - verify(s>=0); - } - /** - * Unused keys are not returned by read operations. Keys may be marked - * as unused in cases where it is difficult to delete them while - * maintaining the constraints required of a btree. - * - * Setting ofs to odd is the sentinel for unused, as real recordLoc's - * are always even numbers. Note we need to keep its value basically - * the same as we use the recordLoc as part of the key in the index - * (to handle duplicate keys efficiently). - * - * Flagging keys as unused is a feature that is being phased out in favor - * of deleting the keys outright. The current btree implementation is - * not expected to mark a key as unused in a non legacy btree. - */ - void setUnused() { - recordLoc.GETOFS() |= 1; - } - void setUsed() { recordLoc.GETOFS() &= ~1; } - int isUnused() const { - return recordLoc.getOfs() & 1; - } - int isUsed() const { - return !isUnused(); - } - }; - - /** - * This structure represents header data for a btree bucket. An object of - * this type is typically allocated inside of a buffer of size BucketSize, - * resulting in a full bucket with an appropriate header. - * - * The body of a btree bucket contains an array of _KeyNode objects starting - * from its lowest indexed bytes and growing to higher indexed bytes. The - * body also contains variable width bson keys, which are allocated from the - * highest indexed bytes toward lower indexed bytes. - * - * |hhhh|kkkkkkk--------bbbbbbbbbbbuuubbbuubbb| - * h = header data - * k = KeyNode data - * - = empty space - * b = bson key data - * u = unused (old) bson key data, that may be garbage collected - */ - class BtreeData_V0 { - protected: - /** Parent bucket of this bucket, which isNull() for the root bucket. */ - DiskLoc parent; - /** Given that there are n keys, this is the n index child. */ - DiskLoc nextChild; - /** can be reused, value is 8192 in current pdfile version Apr2010 */ - unsigned short _wasSize; - /** zero */ - unsigned short _reserved1; - int flags; - - void _init() { - _reserved1 = 0; - _wasSize = BucketSize; - reserved = 0; - } - - /** basicInsert() assumes the next three members are consecutive and in this order: */ - - /** Size of the empty region. */ - int emptySize; - /** Size used for bson storage, including storage of old keys. */ - int topSize; - /* Number of keys in the bucket. */ - int n; - - int reserved; - /* Beginning of the bucket's body */ - char data[4]; - - public: - typedef __KeyNode _KeyNode; - typedef DiskLoc Loc; - typedef KeyBson Key; - typedef KeyBson KeyOwned; - enum { BucketSize = 8192 }; - - // largest key size we allow. note we very much need to support bigger keys (somehow) in the future. - static const int KeyMax = OldBucketSize / 10; - // A sentinel value sometimes used to identify a deallocated bucket. - static const int INVALID_N_SENTINEL = -1; - }; - - // a a a ofs ofs ofs ofs - class DiskLoc56Bit { - int ofs; - unsigned char _a[3]; - unsigned long long Z() const { - // endian - return *((unsigned long long*)this) & 0x00ffffffffffffffULL; - } - enum { - // first bit of offsets used in _KeyNode we don't use -1 here. - OurNullOfs = -2 - }; - public: - template< class V > - const BtreeBucket * btree() const { - return DiskLoc(*this).btree(); - } - template< class V > - BtreeBucket * btreemod() const { - return DiskLoc(*this).btreemod(); - } - operator const DiskLoc() const { - // endian - if( isNull() ) return DiskLoc(); - unsigned a = *((unsigned *) (_a-1)); - return DiskLoc(a >> 8, ofs); - } - int& GETOFS() { return ofs; } - int getOfs() const { return ofs; } - bool operator<(const DiskLoc56Bit& rhs) const { - // the orderering of dup keys in btrees isn't too critical, but we'd like to put items that are - // close together on disk close together in the tree, so we do want the file # to be the most significant - // bytes - return Z() < rhs.Z(); - } - int compare(const DiskLoc56Bit& rhs) const { - unsigned long long a = Z(); - unsigned long long b = rhs.Z(); - if( a < b ) return -1; - return a == b ? 0 : 1; - } - bool operator==(const DiskLoc56Bit& rhs) const { return Z() == rhs.Z(); } - bool operator!=(const DiskLoc56Bit& rhs) const { return Z() != rhs.Z(); } - bool operator==(const DiskLoc& rhs) const { - return DiskLoc(*this) == rhs; - } - bool operator!=(const DiskLoc& rhs) const { return !(*this==rhs); } - bool isNull() const { return ofs < 0; } - void Null() { - ofs = OurNullOfs; - _a[0] = _a[1] = _a[2] = 0; - } - string toString() const { return DiskLoc(*this).toString(); } - void operator=(const DiskLoc& loc) { - ofs = loc.getOfs(); - int la = loc.a(); - verify( la <= 0xffffff ); // must fit in 3 bytes - if( la < 0 ) { - if ( la != -1 ) { - log() << "btree diskloc isn't negative 1: " << la << endl; - verify ( la == -1 ); - } - la = 0; - ofs = OurNullOfs; - } - memcpy(_a, &la, 3); // endian - } - DiskLoc56Bit& writing() const { - return *((DiskLoc56Bit*) getDur().writingPtr((void*)this, 7)); - } - }; - - class BtreeData_V1 { - public: - typedef DiskLoc56Bit Loc; - //typedef DiskLoc Loc; - typedef __KeyNode _KeyNode; - typedef KeyV1 Key; - typedef KeyV1Owned KeyOwned; - enum { BucketSize = 8192-16 }; // leave room for Record header - // largest key size we allow. note we very much need to support bigger keys (somehow) in the future. - static const int KeyMax = 1024; - // A sentinel value sometimes used to identify a deallocated bucket. - static const unsigned short INVALID_N_SENTINEL = 0xffff; - protected: - /** Parent bucket of this bucket, which isNull() for the root bucket. */ - Loc parent; - /** Given that there are n keys, this is the n index child. */ - Loc nextChild; - - unsigned short flags; - - /** basicInsert() assumes the next three members are consecutive and in this order: */ - - /** Size of the empty region. */ - unsigned short emptySize; - /** Size used for bson storage, including storage of old keys. */ - unsigned short topSize; - /* Number of keys in the bucket. */ - unsigned short n; - - /* Beginning of the bucket's body */ - char data[4]; - - void _init() { } - }; - - typedef BtreeData_V0 V0; - typedef BtreeData_V1 V1; - - /** - * This class adds functionality to BtreeData for managing a single bucket. - * The following policies are used in an attempt to encourage simplicity: - * - * Const member functions of this class are those which may be called on - * an object for which writing has not been signaled. Non const member - * functions may only be called on objects for which writing has been - * signaled. Note that currently some const functions write to the - * underlying memory representation of this bucket using optimized methods - * to signal write operations. - * - * DiskLoc parameters that may shadow references within the btree should - * be passed by value rather than by reference to non const member - * functions or to const member functions which may perform writes. This way - * a callee need not worry that write operations will change or invalidate - * its arguments. - * - * The current policy for dealing with bson arguments is the opposite of - * what is described above for DiskLoc arguments. We do not want to copy - * bson into memory as an intermediate step for btree changes, and if bson - * is to be moved it must be copied to the new location before the old - * location is invalidated. Care should be taken in cases where that invalid - * memory may be implicitly referenced by function arguments. - * - * A number of functions below require a thisLoc argument, which must be the - * disk location of the bucket mapped to 'this'. - */ - template< class Version > - class BucketBasics : public Version { - public: - template friend class BtreeBuilder; - typedef typename Version::Key Key; - typedef typename Version::_KeyNode _KeyNode; - typedef typename Version::Loc Loc; - - int getN() const { return this->n; } - - /** - * This is an in memory wrapper for a _KeyNode, and not itself part of btree - * storage. This object and its BSONObj 'key' will become invalid if the - * _KeyNode data that generated it is moved within the btree. In general, - * a KeyNode should not be expected to be valid after a write. - */ - class KeyNode { - public: - KeyNode(const BucketBasics& bb, const _KeyNode &k); - const Loc& prevChildBucket; - const Loc& recordLoc; - /* Points to the bson key storage for a _KeyNode */ - Key key; - }; - friend class KeyNode; - - /** Assert write intent declared for this bucket already. */ - void assertWritable(); - - void assertValid(const Ordering &order, bool force = false) const; - void assertValid(const BSONObj &orderObj, bool force = false) const { return assertValid(Ordering::make(orderObj),force); } - - /** - * @return KeyNode for key at index i. The KeyNode will become invalid - * if the key is moved or reassigned, or if the node is packed. In general - * a KeyNode should not be expected to be valid after a write. - */ - const KeyNode keyNode(int i) const { - if ( i >= this->n ) { - massert( 13000 , (string)"invalid keyNode: " + BSON( "i" << i << "n" << this->n ).jsonString() , i < this->n ); - } - return KeyNode(*this, k(i)); - } - - static int headerSize() { - const BucketBasics *d = 0; - return (char*)&(d->data) - (char*)&(d->parent); - } - static int bodySize() { return Version::BucketSize - headerSize(); } - static int lowWaterMark() { return bodySize() / 2 - Version::KeyMax - sizeof( _KeyNode ) + 1; } // see comment in btree.cpp - - // for testing - int nKeys() const { return this->n; } - const DiskLoc getNextChild() const { return this->nextChild; } - - // for tree inspection and statistical analysis - // NOTE: topSize and emptySize have different types in BtreeData_V0 and BtreeData_V1 - - /** Size used for bson storage, including storage of old keys. */ - unsigned int getTopSize() const { return static_cast(this->topSize); } - /** Size of the empty region. */ - unsigned int getEmptySize() const { return static_cast(this->emptySize); } - - protected: - char * dataAt(short ofs) { return this->data + ofs; } - - /** Initialize the header for a new node. */ - void init(); - - /** - * Preconditions: - * - 0 <= keypos <= n - * - If key is inserted at position keypos, the bucket's keys will still be - * in order. - * Postconditions: - * - If key can fit in the bucket, the bucket may be packed and keypos - * may be decreased to reflect deletion of earlier indexed keys during - * packing, the key will be inserted at the updated keypos index with - * a null prevChildBucket, the subsequent keys shifted to the right, - * and the function will return true. - * - If key cannot fit in the bucket, the bucket will be packed and - * the function will return false. - * Although this function is marked const, it modifies the underlying - * btree representation through an optimized write intent mechanism. - */ - bool basicInsert(const DiskLoc thisLoc, int &keypos, const DiskLoc recordLoc, const Key& key, const Ordering &order) const; - - /** - * Preconditions: - * - key / recordLoc are > all existing keys - * - The keys in prevChild and their descendants are between all existing - * keys and 'key'. - * Postconditions: - * - If there is space for key without packing, it is inserted as the - * last key with specified prevChild and true is returned. - * Importantly, nextChild is not updated! - * - Otherwise false is returned and there is no change. - */ - bool _pushBack(const DiskLoc recordLoc, const Key& key, const Ordering &order, const DiskLoc prevChild); - void pushBack(const DiskLoc recordLoc, const Key& key, const Ordering &order, const DiskLoc prevChild) { - bool ok = _pushBack( recordLoc , key , order , prevChild ); - verify(ok); - } - - /** - * This is a special purpose function used by BtreeBuilder. The - * interface is quite dangerous if you're not careful. The bson key - * returned here points to bucket memory that has been invalidated but - * not yet reclaimed. - * - * TODO Maybe this could be replaced with two functions, one which - * returns the last key without deleting it and another which simply - * deletes the last key. Then the caller would have enough control to - * ensure proper memory integrity. - * - * Preconditions: - * - bucket is not empty - * - last key of bucket is used (not unused) - * - nextChild isNull() - * - _unalloc will work correctly as used - see code - * Postconditions: - * - The last key of the bucket is removed, and its key and recLoc are - * returned. As mentioned above, the key points to unallocated memory. - */ - void popBack(DiskLoc& recLoc, Key &key); - - /** - * Preconditions: - * - 0 <= keypos < n - * - there is no child bucket at keypos - * - n > 1 - * - if mayEmpty == false or nextChild.isNull(), n > 0 - * Postconditions: - * - The key at keypos is removed, and remaining keys are shifted over. - * - The bucket becomes unpacked. - * - if mayEmpty is true and nextChild.isNull(), the bucket may have no keys. - */ - void _delKeyAtPos(int keypos, bool mayEmpty = false); - - /* !Packed means there is deleted fragment space within the bucket. - We "repack" when we run out of space before considering the node - to be full. - */ - enum Flags { Packed=1 }; - - /** n == 0 is ok */ - const Loc& childForPos(int p) const { return p == this->n ? this->nextChild : k(p).prevChildBucket; } - Loc& childForPos(int p) { return p == this->n ? this->nextChild : k(p).prevChildBucket; } - - /** Same as bodySize(). */ - int totalDataSize() const; - /** - * @return true when a key may be dropped by pack() - * @param index index of the key that may be dropped - * @param refPos index of a particular key of interest, which must not - * be dropped; = 0 to safely ignore - */ - bool mayDropKey( int index, int refPos ) const; - - /** - * Pack the bucket to reclaim space from invalidated memory. - * @refPos is an index in the bucket which may be updated if we - * delete keys from the bucket - * This function may cast away const and perform a write. - * Preconditions: none - * Postconditions: - * - Bucket will be packed - * - Some unused nodes may be dropped, but not ones at index 0 or refPos - * - Some used nodes may be moved - * - If refPos is the index of an existing key, it will be updated to that - * key's new index if the key is moved. - */ - void _pack(const DiskLoc thisLoc, const Ordering &order, int &refPos) const; - /** Pack when already writable */ - void _packReadyForMod(const Ordering &order, int &refPos); - - /** @return the size the bucket's body would have if we were to call pack() */ - int packedDataSize( int refPos ) const; - void setNotPacked() { this->flags &= ~Packed; } - void setPacked() { this->flags |= Packed; } - /** - * Preconditions: 'bytes' is <= emptySize - * Postconditions: A buffer of size 'bytes' is allocated on the top side, - * and its offset is returned. - */ - int _alloc(int bytes); - /** - * This function can be used to deallocate the lowest byte index bson - * buffer in the top region, which in some but not all cases is for the - * n - 1 index key. This function only works correctly in certain - * special cases, please be careful. - * Preconditions: 'bytes' <= topSize - * Postconditions: The top region is decreased - */ - void _unalloc(int bytes); - /** - * Preconditions: 'N' <= n - * Postconditions: - * - All keys after the N index key are dropped. - * - Then bucket is packed, without dropping refPos if < refPos N. - */ - void truncateTo(int N, const Ordering &order, int &refPos); - /** - * Preconditions: - * - 'nDrop' < n - * - for now, refPos should be zero. - * Postconditions: - * - All keys before the nDrop index key are dropped. - * - The bucket is packed. - */ - void dropFront(int nDrop, const Ordering &order, int &refPos); - /** - * Preconditions: 0 <= keypos < n - * Postconditions: keypos indexed key is marked unused. - */ - void markUnused(int keypos); - - /** - * BtreeBuilder uses the parent var as a temp place to maintain a linked list chain. - * we use tempNext() when we do that to be less confusing. (one might have written a union in C) - */ - DiskLoc tempNext() const { return this->parent; } - void setTempNext(DiskLoc l) { this->parent = l; } - - void _shape(int level, stringstream&) const; - int Size() const; - - /** @return i-indexed _KeyNode, without bounds checking */ - public: - const _KeyNode& k(int i) const { return ((const _KeyNode*)this->data)[i]; } - _KeyNode& _k(int i) { return ((_KeyNode*)this->data)[i]; } - protected: - _KeyNode& k(int i) { return ((_KeyNode*)this->data)[i]; } - - /** - * Preconditions: 'this' is packed - * @return the key index to be promoted on split - * @param keypos The requested index of a key to insert, which may affect - * the choice of split position. - */ - int splitPos( int keypos ) const; - - /** - * Preconditions: nAdd * sizeof( _KeyNode ) <= emptySize - * Postconditions: - * - Increases indexes of existing _KeyNode objects by nAdd, reserving - * space for additional _KeyNode objects at front. - * - Does not initialize ofs values for the bson data of these - * _KeyNode objects. - */ - void reserveKeysFront( int nAdd ); - - /** - * Preconditions: - * - 0 <= i < n - * - The bson 'key' must fit in the bucket without packing. - * - If 'key' and 'prevChildBucket' are set at index i, the btree - * ordering properties will be maintained. - * Postconditions: - * - The specified key is set at index i, replacing the existing - * _KeyNode data and without shifting any other _KeyNode objects. - */ - void setKey( int i, const DiskLoc recordLoc, const Key& key, const DiskLoc prevChildBucket ); - }; - - class IndexDetails; - class IndexInsertionContinuation; - template< class V> - struct IndexInsertionContinuationImpl; - - /** - * This class adds functionality for manipulating buckets that are assembled - * in a tree. The requirements for const and non const functions and - * arguments are generally the same as in BtreeBucket. Because this class - * deals with tree structure, some functions that are marked const may - * trigger modification of another node in the btree or potentially of the - * current node. In such cases, the function's implementation explicitly - * casts away const when indicating an intent to write to the durability - * layer. The DiskLocs provided to such functions should be passed by - * value if they shadow pointers within the btree. - * - * To clarify enforcement of referential integrity in this implementation, - * we use the following pattern when deleting data we have a persistent - * pointer to. The pointer is cleared or removed explicitly, then the data - * it pointed to is cleaned up with a helper function. - * - * TODO It might make sense to put some of these functions in a class - * representing a full btree instead of a single btree bucket. That would - * allow us to use the const qualifier in a manner more consistent with - * standard usage. Right now the interface is for both a node and a tree, - * so assignment of const is sometimes nonideal. - * - * TODO There are several cases in which the 'this' pointer is invalidated - * as a result of deallocation. A separate class representing a btree would - * alleviate some fragile cases where the implementation must currently - * behave correctly if the 'this' pointer is suddenly invalidated by a - * callee. - */ - template< class V > - class BtreeBucket : public BucketBasics { - friend class BtreeCursor; - friend struct IndexInsertionContinuationImpl; - public: - // make compiler happy: - typedef typename V::Key Key; - typedef typename V::KeyOwned KeyOwned; - typedef typename BucketBasics::KeyNode KeyNode; - typedef typename BucketBasics::_KeyNode _KeyNode; - typedef typename BucketBasics::Loc Loc; - const _KeyNode& k(int i) const { return static_cast< const BucketBasics * >(this)->k(i); } - protected: - _KeyNode& k(int i) { return static_cast< BucketBasics * >(this)->_k(i); } - public: - const KeyNode keyNode(int i) const { return static_cast< const BucketBasics * >(this)->keyNode(i); } - - bool isHead() const { return this->parent.isNull(); } - void dumpTree(const DiskLoc &thisLoc, const BSONObj &order) const; - long long fullValidate(const DiskLoc& thisLoc, const BSONObj &order, long long *unusedCount = 0, bool strict = false, unsigned depth=0) const; /* traverses everything */ - - bool isUsed( int i ) const { return this->k(i).isUsed(); } - string bucketSummary() const; - void dump(unsigned depth=0) const; - - /** - * @return true if key exists in index - * - * @order - indicates order of keys in the index. this is basically the index's key pattern, e.g.: - * BSONObj order = ((IndexDetails&)idx).keyPattern(); - * likewise below in bt_insert() etc. - */ - private: - bool exists(const IndexDetails& idx, const DiskLoc &thisLoc, const Key& key, const Ordering& order) const; - public: - - /** - * @param self - Don't complain about ourself already being in the index case. - * @return true = There is a duplicate used key. - */ - bool wouldCreateDup( - const IndexDetails& idx, const DiskLoc &thisLoc, - const Key& key, const Ordering& order, - const DiskLoc &self) const; - - /** - * Preconditions: none - * Postconditions: @return a new bucket allocated from pdfile storage - * and init()-ed. This bucket is suitable to for use as a new root - * or any other new node in the tree. - */ - static DiskLoc addBucket(const IndexDetails&); - - /** - * Preconditions: none - * Postconditions: - * - Some header values in this bucket are cleared, and the bucket is - * deallocated from pdfile storage. - * - The memory at thisLoc is invalidated, and 'this' is invalidated. - */ - void deallocBucket(const DiskLoc thisLoc, const IndexDetails &id); - - /** - * Preconditions: - * - 'key' has a valid schema for this index. - * - All other paramenters are valid and consistent with this index if applicable. - * Postconditions: - * - If key is bigger than KeyMax, @return 2 or 3 and no change. - * - If key / recordLoc exist in the btree as an unused key, set them - * as used and @return 0 - * - If key / recordLoc exist in the btree as a used key, @throw - * exception 10287 and no change. - * - If key / recordLoc do not exist in the btree, they are inserted - * and @return 0. The root of the btree may be changed, so - * 'this'/thisLoc may no longer be the root upon return. - */ - int bt_insert(const DiskLoc thisLoc, const DiskLoc recordLoc, - const BSONObj& key, const Ordering &order, bool dupsAllowed, - IndexDetails& idx, bool toplevel = true) const; - - /** does the insert in two steps - can then use an upgradable lock for step 1, which - is the part which may have page faults. also that step is most of the computational work. - */ - void twoStepInsert(DiskLoc thisLoc, IndexInsertionContinuationImpl &c, bool dupsAllowed) const; - - /** - * Preconditions: - * - 'key' has a valid schema for this index, and may have objsize() > KeyMax. - * Postconditions: - * - If key / recordLoc are in the btree, they are removed (possibly - * by being marked as an unused key), @return true, and potentially - * invalidate 'this' / thisLoc and change the head. - * - If key / recordLoc are not in the btree, @return false and do nothing. - */ - bool unindex(const DiskLoc thisLoc, IndexDetails& id, const BSONObj& key, const DiskLoc recordLoc) const; - - /** - * locate may return an "unused" key that is just a marker. so be careful. - * looks for a key:recordloc pair. - * - * @found - returns true if exact match found. note you can get back a position - * result even if found is false. - */ - DiskLoc locate(const IndexDetails &idx , const DiskLoc& thisLoc, const BSONObj& key, const Ordering &order, - int& pos, bool& found, const DiskLoc &recordLoc, int direction=1) const; - DiskLoc locate(const IndexDetails &idx , const DiskLoc& thisLoc, const Key& key, const Ordering &order, - int& pos, bool& found, const DiskLoc &recordLoc, int direction=1) const; - - /** - * find the first instance of the key - * does not handle dups - * WARNING: findSingle may not be compound index safe. this may need to change. see notes in - * findSingle code. - * @return the record location of the first match - */ - DiskLoc findSingle( const IndexDetails &indexdetails , const DiskLoc& thisLoc, const BSONObj& key ) const; - - /** - * Advance to next or previous key in the index. - * @param direction to advance. - */ - DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) const; - - /** Advance in specified direction to the specified key */ - void advanceTo(DiskLoc &thisLoc, int &keyOfs, const BSONObj &keyBegin, int keyBeginLen, bool afterKey, const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive, const Ordering &order, int direction ) const; - - /** Locate a key with fields comprised of a combination of keyBegin fields and keyEnd fields. */ - static void customLocate(DiskLoc &locInOut, int &keyOfs, const BSONObj &keyBegin, int keyBeginLen, bool afterKey, const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive, const Ordering &order, int direction, pair< DiskLoc, int > &bestParent ) ; - - /** @return head of the btree by traversing from current bucket. */ - const DiskLoc getHead(const DiskLoc& thisLoc) const; - - /** get tree shape */ - void shape(stringstream&) const; - - static void a_test(IndexDetails&); - - static int getKeyMax(); - - protected: - /** - * Preconditions: - * - 0 <= firstIndex <= n - * - -1 <= lastIndex <= n ( -1 is equivalent to n ) - * Postconditions: - * - Any children at indexes firstIndex through lastIndex (inclusive) - * will have their parent pointers set to thisLoc. - */ - void fixParentPtrs(const DiskLoc thisLoc, int firstIndex = 0, int lastIndex = -1) const; - - /** - * Preconditions: - * - thisLoc is not the btree head. - * - n == 0 is ok - * Postconditions: - * - All cursors pointing to this bucket will be updated. - * - This bucket's parent's child pointer is set to null. - * - This bucket is deallocated from pdfile storage. - * - 'this' and thisLoc are invalidated. - */ - void delBucket(const DiskLoc thisLoc, const IndexDetails&); - - /** - * Preconditions: 0 <= p < n - * Postconditions: - * - The key at index p is removed from the btree. - * - 'this' and thisLoc may be invalidated. - * - The tree head may change. - */ - void delKeyAtPos(const DiskLoc thisLoc, IndexDetails& id, int p, const Ordering &order); - - /** - * Preconditions: - * - n == 0 is ok - * Postconditions: - * - If thisLoc is head, or if its body has at least lowWaterMark bytes, - * return false and do nothing. - * - Otherwise, if thisLoc has left or right neighbors, either balance - * or merge with them and return true. Also, 'this' and thisLoc may - * be invalidated and the tree head may change. - */ - bool mayBalanceWithNeighbors(const DiskLoc thisLoc, IndexDetails &id, const Ordering &order) const; - - /** - * Preconditions: - * - 0 <= leftIndex < n - * - The child at leftIndex or the child at leftIndex + 1 contains - * fewer than lowWaterMark bytes. - * Postconditions: - * - If the child bucket at leftIndex can merge with the child index - * at leftIndex + 1, do nothing and return false. - * - Otherwise, balance keys between the leftIndex child and the - * leftIndex + 1 child, return true, and possibly change the tree head. - */ - bool tryBalanceChildren( const DiskLoc thisLoc, int leftIndex, IndexDetails &id, const Ordering &order ) const; - - /** - * Preconditions: - * - All preconditions of tryBalanceChildren. - * - The leftIndex child and leftIndex + 1 child cannot be merged. - * Postconditions: - * - Keys are moved between the leftIndex child and the leftIndex + 1 - * child such that neither child has fewer than lowWaterMark bytes. - * The tree head may change. - */ - void doBalanceChildren( const DiskLoc thisLoc, int leftIndex, IndexDetails &id, const Ordering &order ); - - /** - * Preconditions: - * - All preconditions of doBalanceChildren - * - The leftIndex and leftIndex + 1 children are packed. - * - The leftIndex + 1 child has fewer than lowWaterMark bytes. - * - split returned by rebalancedSeparatorPos() - * Postconditions: - * - The key in lchild at index split is set as thisLoc's key at index - * leftIndex, which may trigger a split and change the tree head. - * The previous key in thisLoc at index leftIndex and all keys with - * indexes greater than split in lchild are moved to rchild. - */ - void doBalanceLeftToRight( const DiskLoc thisLoc, int leftIndex, int split, - BtreeBucket *l, const DiskLoc lchild, - BtreeBucket *r, const DiskLoc rchild, - IndexDetails &id, const Ordering &order ); - /** - * Preconditions: - * - All preconditions of doBalanceChildren - * - The leftIndex and leftIndex + 1 children are packed. - * - The leftIndex child has fewer than lowWaterMark bytes. - * - split returned by rebalancedSeparatorPos() - * Postconditions: - * - The key in rchild at index split - l->n - 1 is set as thisLoc's key - * at index leftIndex, which may trigger a split and change the tree - * head. The previous key in thisLoc at index leftIndex and all keys - * with indexes less than split - l->n - 1 in rchild are moved to - * lchild. - */ - void doBalanceRightToLeft( const DiskLoc thisLoc, int leftIndex, int split, - BtreeBucket *l, const DiskLoc lchild, - BtreeBucket *r, const DiskLoc rchild, - IndexDetails &id, const Ordering &order ); - - /** - * Preconditions: - * - 0 <= leftIndex < n - * - this->canMergeChildren( thisLoc, leftIndex ) == true - * Postconditions: - * - All of the above mentioned keys will be placed in the left child. - * - The tree may be updated recursively, resulting in 'this' and - * thisLoc being invalidated and the tree head being changed. - */ - void doMergeChildren( const DiskLoc thisLoc, int leftIndex, IndexDetails &id, const Ordering &order); - - /** - * Preconditions: - * - n == 0 - * - !nextChild.isNull() - * Postconditions: - * - 'this' and thisLoc are deallocated (and invalidated), any cursors - * to them are updated, and the tree head may change. - * - nextChild replaces thisLoc in the btree structure. - */ - void replaceWithNextChild( const DiskLoc thisLoc, IndexDetails &id ); - - /** - * @return true iff the leftIndex and leftIndex + 1 children both exist, - * and if their body sizes when packed and the thisLoc key at leftIndex - * would fit in a single bucket body. - */ - bool canMergeChildren( const DiskLoc &thisLoc, int leftIndex ) const; - - /** - * Preconditions: - * - leftIndex and leftIndex + 1 children are packed - * - leftIndex or leftIndex + 1 child is below lowWaterMark - * @return index of the rebalanced separator; the index value is - * determined as if we had a bucket with body - * .push( ).concat( ) - * and called splitPos( 0 ) on it. - */ - int rebalancedSeparatorPos( const DiskLoc &thisLoc, int leftIndex ) const; - - /** - * Preconditions: thisLoc has a parent - * @return parent's index of thisLoc. - */ - int indexInParent( const DiskLoc &thisLoc ) const; - - public: - Key keyAt(int i) const { - if( i >= this->n ) - return Key(); - return Key(this->data + k(i).keyDataOfs()); - } - protected: - - /** - * Preconditions: - * - This bucket is packed. - * - Cannot add a key of size KeyMax to this bucket. - * - 0 <= keypos <= n is the position of a new key that will be inserted - * - lchild is equal to the existing child at index keypos. - * Postconditions: - * - The thisLoc bucket is split into two packed buckets, possibly - * invalidating the initial position of keypos, with a split key - * promoted to the parent. The new key key/recordLoc will be inserted - * into one of the split buckets, and lchild/rchild set appropriately. - * Splitting may occur recursively, possibly changing the tree head. - */ - void split(const DiskLoc thisLoc, int keypos, - const DiskLoc recordLoc, const Key& key, - const Ordering& order, const DiskLoc lchild, const DiskLoc rchild, IndexDetails& idx); - - /** - * Preconditions: - * - 0 <= keypos <= n - * - If key / recordLoc are inserted at position keypos, with provided - * lchild and rchild, the btree ordering requirements will be - * maintained. - * - lchild is equal to the existing child at index keypos. - * - n == 0 is ok. - * Postconditions: - * - The key / recordLoc are inserted at position keypos, and the - * bucket is split if necessary, which may change the tree head. - * - The bucket may be packed or split, invalidating the specified value - * of keypos. - * This function will always modify thisLoc, but it's marked const because - * it commonly relies on the specialized writ]e intent mechanism of basicInsert(). - */ - void insertHere(const DiskLoc thisLoc, int keypos, - const DiskLoc recordLoc, const Key& key, const Ordering &order, - const DiskLoc lchild, const DiskLoc rchild, IndexDetails &idx) const; - - /** bt_insert() is basically just a wrapper around this. */ - int _insert(const DiskLoc thisLoc, const DiskLoc recordLoc, - const Key& key, const Ordering &order, bool dupsAllowed, - const DiskLoc lChild, const DiskLoc rChild, IndexDetails &idx) const; - - void insertStepOne( - DiskLoc thisLoc, IndexInsertionContinuationImpl& c, bool dupsAllowed) const; - - bool find(const IndexDetails& idx, const Key& key, const DiskLoc &recordLoc, const Ordering &order, int& pos, bool assertIfDup) const; - static bool customFind( int l, int h, const BSONObj &keyBegin, int keyBeginLen, bool afterKey, const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive, const Ordering &order, int direction, DiskLoc &thisLoc, int &keyOfs, pair< DiskLoc, int > &bestParent ) ; - static void findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey); - static int customBSONCmp( const BSONObj &l, const BSONObj &rBegin, int rBeginLen, bool rSup, const vector< const BSONElement * > &rEnd, const vector< bool > &rEndInclusive, const Ordering &o, int direction ); - - /** If child is non null, set its parent to thisLoc */ - static void fix(const DiskLoc thisLoc, const DiskLoc child); - - /** - * Preconditions: - * - 0 <= keypos < n - * - If the specified key and recordLoc are placed in keypos of thisLoc, - * and lchild and rchild are set, the btree ordering properties will - * be maintained. - * - rchild == childForPos( keypos + 1 ) - * - childForPos( keypos ) is referenced elsewhere if nonnull. - * Postconditions: - * - The key at keypos will be replaced with the specified key and - * lchild, potentially splitting this bucket and changing the tree - * head. - * - childForPos( keypos ) will be orphaned. - */ - void setInternalKey( const DiskLoc thisLoc, int keypos, - const DiskLoc recordLoc, const Key &key, const Ordering &order, - const DiskLoc lchild, const DiskLoc rchild, IndexDetails &idx); - - /** - * Preconditions: - * - 0 <= keypos < n - * - The keypos or keypos+1 indexed child is non null. - * Postconditions: - * - The specified key is deleted by replacing it with another key if - * possible. This replacement may cause a split and change the tree - * head. The replacement key will be deleted from its original - * location, potentially causing merges and splits that may invalidate - * 'this' and thisLoc and change the tree head. - * - If the key cannot be replaced, it will be marked as unused. This - * is only expected in legacy btrees. - */ - void deleteInternalKey( const DiskLoc thisLoc, int keypos, IndexDetails &id, const Ordering &order ); - public: - /** simply builds and returns a dup key error message string */ - static string dupKeyError( const IndexDetails& idx , const Key& key ); - }; -#pragma pack() - - /** - * give us a writable version of the btree bucket (declares write intent). - * note it is likely more efficient to declare write intent on something smaller when you can. - */ - template< class V > - BtreeBucket * DiskLoc::btreemod() const { - verify( _a != -1 ); - BtreeBucket *b = const_cast< BtreeBucket * >( btree() ); - return static_cast< BtreeBucket* >( getDur().writingPtr( b, V::BucketSize ) ); - } - - template< class V > - BucketBasics::KeyNode::KeyNode(const BucketBasics& bb, const _KeyNode &k) : - prevChildBucket(k.prevChildBucket), - recordLoc(k.recordLoc), key(bb.data+k.keyDataOfs()) - { } - -} // namespace mongo; diff --git a/src/mongo/db/btree_stats.cpp b/src/mongo/db/btree_stats.cpp deleted file mode 100644 index 0ec80b77646..00000000000 --- a/src/mongo/db/btree_stats.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// btree_stats.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/base/init.h" -#include "mongo/db/btree_stats.h" - -namespace mongo { - - - IndexCounters::IndexCounters() - : ServerStatusSection( "indexCounters" ) { - - _memSupported = ProcessInfo().blockCheckSupported(); - - _btreeMemHits = 0; - _btreeMemMisses = 0; - _btreeAccesses = 0; - - - _maxAllowed = ( numeric_limits< long long >::max() ) / 2; - _resets = 0; - } - - IndexCounters::~IndexCounters(){ - } - - BSONObj IndexCounters::generateSection(const BSONElement& configElement) const { - if ( ! _memSupported ) { - return BSON( "note" << "not supported on this platform" ); - } - - BSONObjBuilder bb; - bb.appendNumber( "accesses" , _btreeAccesses ); - bb.appendNumber( "hits" , _btreeMemHits ); - bb.appendNumber( "misses" , _btreeMemMisses ); - - bb.append( "resets" , _resets ); - - bb.append( "missRatio" , (_btreeAccesses ? (_btreeMemMisses / (double)_btreeAccesses) : 0) ); - - return bb.obj(); - } - - void IndexCounters::_roll() { - _btreeAccesses = 0; - _btreeMemMisses = 0; - _btreeMemHits = 0; - _resets++; - } - - IndexCounters* globalIndexCounters = NULL; - - MONGO_INITIALIZER_WITH_PREREQUISITES(BtreeIndexCountersBlockSupported, - ("SystemInfo"))(InitializerContext* cx) { - if (globalIndexCounters == NULL) { - globalIndexCounters = new IndexCounters(); - } - return Status::OK(); - } - -} diff --git a/src/mongo/db/btree_stats.h b/src/mongo/db/btree_stats.h deleted file mode 100644 index 9d9e638668c..00000000000 --- a/src/mongo/db/btree_stats.h +++ /dev/null @@ -1,74 +0,0 @@ -// btree_stats.h - -/** -* Copyright (C) 2008-2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - - -#pragma once - -#include "mongo/db/commands/server_status.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/record.h" -#include "mongo/util/processinfo.h" - -namespace mongo { - - - class IndexCounters : public ServerStatusSection { - public: - IndexCounters(); - virtual ~IndexCounters(); - - virtual bool includeByDefault() const { return true; } - - virtual BSONObj generateSection(const BSONElement& configElement) const; - - - // used without a mutex intentionally (can race) - void btree( const char* node ) { - if ( ! _memSupported ) - return; - btree( Record::likelyInPhysicalMemory( node ) ); - } - - void btree( bool memHit ) { - if ( memHit ) - _btreeMemHits++; - else - _btreeMemMisses++; - if ( _btreeAccesses++ > _maxAllowed ) - _roll(); - - } - void btreeHit() { _btreeMemHits++; _btreeAccesses++; } - void btreeMiss() { _btreeMemMisses++; _btreeAccesses++; } - - private: - - void _roll(); - - bool _memSupported; - - int _resets; - long long _maxAllowed; - - long long _btreeMemMisses; - long long _btreeMemHits; - long long _btreeAccesses; - }; - - extern IndexCounters* globalIndexCounters; -} diff --git a/src/mongo/db/btreebuilder.cpp b/src/mongo/db/btreebuilder.cpp deleted file mode 100644 index 26c8e27c4b4..00000000000 --- a/src/mongo/db/btreebuilder.cpp +++ /dev/null @@ -1,168 +0,0 @@ -// btreebuilder.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" -#include "db.h" -#include "btree.h" -#include "pdfile.h" -#include "json.h" -#include "clientcursor.h" -#include "client.h" -#include "dbhelpers.h" -#include "curop-inl.h" -#include "stats/counters.h" -#include "dur_commitjob.h" -#include "btreebuilder.h" -#include "mongo/db/kill_current_op.h" - -namespace mongo { - - /* --- BtreeBuilder --- */ - - template - BtreeBuilder::BtreeBuilder(bool _dupsAllowed, IndexDetails& _idx) : - dupsAllowed(_dupsAllowed), - idx(_idx), - n(0), - order( idx.keyPattern() ), - ordering( Ordering::make(idx.keyPattern()) ) { - first = cur = BtreeBucket::addBucket(idx); - b = cur.btreemod(); - committed = false; - } - - template - void BtreeBuilder::newBucket() { - DiskLoc L = BtreeBucket::addBucket(idx); - b->setTempNext(L); - cur = L; - b = cur.btreemod(); - } - - template - void BtreeBuilder::mayCommitProgressDurably() { - if ( getDur().commitIfNeeded() ) { - b = cur.btreemod(); - } - } - - template - void BtreeBuilder::addKey(BSONObj& _key, DiskLoc loc) { - - auto_ptr< KeyOwned > key( new KeyOwned(_key) ); - if ( key->dataSize() > BtreeBucket::KeyMax ) { - problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace() - << ' ' << key->dataSize() << ' ' << key->toString() << endl; - return; - } - - if( !dupsAllowed ) { - if( n > 0 ) { - int cmp = keyLast->woCompare(*key, ordering); - massert( 10288 , "bad key order in BtreeBuilder - server internal error", cmp <= 0 ); - if( cmp == 0 ) { - //if( !dupsAllowed ) - uasserted( ASSERT_ID_DUPKEY , BtreeBucket::dupKeyError( idx , *keyLast ) ); - } - } - } - - if ( ! b->_pushBack(loc, *key, ordering, DiskLoc()) ) { - // bucket was full - newBucket(); - b->pushBack(loc, *key, ordering, DiskLoc()); - } - keyLast = key; - n++; - mayCommitProgressDurably(); - } - - template - void BtreeBuilder::buildNextLevel(DiskLoc loc, bool mayInterrupt) { - int levels = 1; - while( 1 ) { - if( loc.btree()->tempNext().isNull() ) { - // only 1 bucket at this level. we are done. - getDur().writingDiskLoc(idx.head) = loc; - break; - } - levels++; - - DiskLoc upLoc = BtreeBucket::addBucket(idx); - DiskLoc upStart = upLoc; - BtreeBucket *up = upLoc.btreemod(); - - DiskLoc xloc = loc; - while( !xloc.isNull() ) { - killCurrentOp.checkForInterrupt( !mayInterrupt ); - - if ( getDur().commitIfNeeded() ) { - b = cur.btreemod(); - up = upLoc.btreemod(); - } - - BtreeBucket *x = xloc.btreemod(); - Key k; - DiskLoc r; - x->popBack(r,k); - bool keepX = ( x->n != 0 ); - DiskLoc keepLoc = keepX ? xloc : x->nextChild; - - if ( ! up->_pushBack(r, k, ordering, keepLoc) ) { - // current bucket full - DiskLoc n = BtreeBucket::addBucket(idx); - up->setTempNext(n); - upLoc = n; - up = upLoc.btreemod(); - up->pushBack(r, k, ordering, keepLoc); - } - - DiskLoc nextLoc = x->tempNext(); // get next in chain at current level - if ( keepX ) { - x->parent = upLoc; - } - else { - if ( !x->nextChild.isNull() ) { - DiskLoc ll = x->nextChild; - ll.btreemod()->parent = upLoc; - //(x->nextChild.btreemod())->parent = upLoc; - } - x->deallocBucket( xloc, idx ); - } - xloc = nextLoc; - } - - loc = upStart; - mayCommitProgressDurably(); - } - - if( levels > 1 ) - LOG(2) << "btree levels: " << levels << endl; - } - - /** when all addKeys are done, we then build the higher levels of the tree */ - template - void BtreeBuilder::commit(bool mayInterrupt) { - buildNextLevel(first, mayInterrupt); - committed = true; - } - - template class BtreeBuilder; - template class BtreeBuilder; - -} diff --git a/src/mongo/db/btreebuilder.h b/src/mongo/db/btreebuilder.h deleted file mode 100644 index a671d3895df..00000000000 --- a/src/mongo/db/btreebuilder.h +++ /dev/null @@ -1,67 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "btree.h" - -namespace mongo { - - /** - * build btree from the bottom up - */ - template< class V > - class BtreeBuilder { - typedef typename V::KeyOwned KeyOwned; - typedef typename V::Key Key; - - bool dupsAllowed; - IndexDetails& idx; - /** Number of keys added to btree. */ - unsigned long long n; - /** Last key passed to addKey(). */ - auto_ptr< typename V::KeyOwned > keyLast; - BSONObj order; - Ordering ordering; - /** true iff commit() completed successfully. */ - bool committed; - - DiskLoc cur, first; - BtreeBucket *b; - - void newBucket(); - void buildNextLevel(DiskLoc loc, bool mayInterrupt); - void mayCommitProgressDurably(); - - public: - BtreeBuilder(bool _dupsAllowed, IndexDetails& _idx); - - /** - * Preconditions: 'key' is > or >= last key passed to this function (depends on _dupsAllowed) - * Postconditions: 'key' is added to intermediate storage. - */ - void addKey(BSONObj& key, DiskLoc loc); - - /** - * commit work. if not called, destructor will clean up partially completed work - * (in case exception has happened). - */ - void commit(bool mayInterrupt); - - unsigned long long getn() { return n; } - }; - -} diff --git a/src/mongo/db/btreecursor.cpp b/src/mongo/db/btreecursor.cpp deleted file mode 100644 index 47c91d3a14e..00000000000 --- a/src/mongo/db/btreecursor.cpp +++ /dev/null @@ -1,425 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/btreecursor.h" - -#include "mongo/db/btree.h" -#include "mongo/db/curop-inl.h" -#include "mongo/db/kill_current_op.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/queryutil.h" - -namespace mongo { - - template< class V > - class BtreeCursorImpl : public BtreeCursor { - public: - typedef typename BucketBasics::KeyNode KeyNode; - typedef typename V::Key Key; - typedef typename V::_KeyNode _KeyNode; - - BtreeCursorImpl( NamespaceDetails* a , int b, const IndexDetails& c ) - : BtreeCursor(a,b,c){ - } - - void init(const BSONObj &d, const BSONObj &e, bool f, int g) { - BtreeCursor::init(d,e,f,g); - } - - void init( const shared_ptr< FieldRangeVector >& bounds, int singleIntervalLimit, int direction ) { - BtreeCursor::init(bounds,singleIntervalLimit,direction ); - pair< DiskLoc, int > noBestParent; - indexDetails.head.btree()->customLocate( bucket, keyOfs, startKey, 0, false, _boundsIterator->cmp(), _boundsIterator->inc(), _ordering, direction, noBestParent ); - skipAndCheck(); - dassert( _dups.size() == 0 ); - } - - - virtual DiskLoc currLoc() { - if( bucket.isNull() ) return DiskLoc(); - return currKeyNode().recordLoc; - } - - virtual BSONObj keyAt(int ofs) const { - verify( !bucket.isNull() ); - const BtreeBucket *b = bucket.btree(); - int n = b->getN(); - if( n == b->INVALID_N_SENTINEL ) { - throw UserException(15850, "keyAt bucket deleted"); - } - dassert( n >= 0 && n < 10000 ); - return ofs >= n ? BSONObj() : b->keyNode(ofs).key.toBson(); - } - - virtual BSONObj currKey() const { - verify( !bucket.isNull() ); - return bucket.btree()->keyNode(keyOfs).key.toBson(); - } - - virtual bool curKeyHasChild() { - return !currKeyNode().prevChildBucket.isNull(); - } - - bool skipUnusedKeys() { - int u = 0; - while ( 1 ) { - if ( !ok() ) - break; - const _KeyNode& kn = keyNode(keyOfs); - if ( kn.isUsed() ) - break; - bucket = _advance(bucket, keyOfs, _direction, "skipUnusedKeys"); - u++; - //don't include unused keys in nscanned - //++_nscanned; - } - if ( u > 10 ) - OCCASIONALLY log() << "btree unused skipped:" << u << endl; - return u; - } - - /* Since the last noteLocation(), our key may have moved around, and that old cached - information may thus be stale and wrong (although often it is right). We check - that here; if we have moved, we have to search back for where we were at. - - i.e., after operations on the index, the BtreeCursor's cached location info may - be invalid. This function ensures validity, so you should call it before using - the cursor if other writers have used the database since the last noteLocation - call. - */ - void checkLocation() { - if ( eof() ) - return; - - _multikey = d->isMultikey(idxNo); - - if ( keyOfs >= 0 ) { - verify( !keyAtKeyOfs.isEmpty() ); - - try { - // Note keyAt() returns an empty BSONObj if keyOfs is now out of range, - // which is possible as keys may have been deleted. - int x = 0; - while( 1 ) { - // if ( b->keyAt(keyOfs).woEqual(keyAtKeyOfs) && - // b->k(keyOfs).recordLoc == locAtKeyOfs ) { - if ( keyAt(keyOfs).binaryEqual(keyAtKeyOfs) ) { - const _KeyNode& kn = keyNode(keyOfs); - if( kn.recordLoc == locAtKeyOfs ) { - if ( !kn.isUsed() ) { - // we were deleted but still exist as an unused - // marker key. advance. - skipUnusedKeys(); - } - return; - } - } - - // we check one key earlier too, in case a key was just deleted. this is - // important so that multi updates are reasonably fast. - if( keyOfs == 0 || x++ ) - break; - keyOfs--; - } - } - catch(UserException& e) { - if( e.getCode() != 15850 ) - throw; - // hack: fall through if bucket was just deleted. should only happen under deleteObjects() - DEV log() << "debug info: bucket was deleted" << endl; - } - } - - /* normally we don't get to here. when we do, old position is no longer - valid and we must refind where we left off (which is expensive) - */ - - /* TODO: Switch to keep indexdetails and do idx.head! */ - bucket = _locate(keyAtKeyOfs, locAtKeyOfs); - RARELY log() << "key seems to have moved in the index, refinding. " << bucket.toString() << endl; - if ( ! bucket.isNull() ) - skipUnusedKeys(); - - } - - protected: - virtual void _advanceTo(DiskLoc &thisLoc, int &keyOfs, const BSONObj &keyBegin, int keyBeginLen, bool afterKey, const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive, const Ordering &order, int direction ) { - thisLoc.btree()->advanceTo(thisLoc, keyOfs, keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, order, direction); - } - virtual DiskLoc _advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) { - return thisLoc.btree()->advance(thisLoc, keyOfs, direction, caller); - } - virtual void _audit() { - out() << "BtreeCursor(). dumping head bucket" << endl; - indexDetails.head.btree()->dump(); - } - virtual DiskLoc _locate(const BSONObj& key, const DiskLoc& loc) { - bool found; - return indexDetails.head.btree()-> - locate(indexDetails, indexDetails.head, key, _ordering, keyOfs, found, loc, _direction); - } - - const _KeyNode& keyNode(int keyOfs) const { - return bucket.btree()->k(keyOfs); - } - - private: - const KeyNode currKeyNode() const { - verify( !bucket.isNull() ); - const BtreeBucket *b = bucket.btree(); - return b->keyNode(keyOfs); - } - }; - - template class BtreeCursorImpl; - template class BtreeCursorImpl; - - BtreeCursor* BtreeCursor::make( NamespaceDetails * nsd , int idxNo , const IndexDetails& indexDetails ) { - int v = indexDetails.version(); - - if( v == 1 ) - return new BtreeCursorImpl( nsd , idxNo , indexDetails ); - - if( v == 0 ) - return new BtreeCursorImpl( nsd , idxNo , indexDetails ); - - dassert( IndexDetails::isASupportedIndexVersionNumber(v) ); - uasserted(14800, str::stream() << "unsupported index version " << v); - return 0; // not reachable - } - - BtreeCursor* BtreeCursor::make( NamespaceDetails* namespaceDetails, - const IndexDetails& id, - const BSONObj& startKey, - const BSONObj& endKey, - bool endKeyInclusive, - int direction ) { - auto_ptr c( make( namespaceDetails, namespaceDetails->idxNo( id ), id ) ); - c->init(startKey,endKey,endKeyInclusive,direction); - c->initWithoutIndependentFieldRanges(); - dassert( c->_dups.size() == 0 ); - return c.release(); - } - - BtreeCursor* BtreeCursor::make( NamespaceDetails* namespaceDetails, - const IndexDetails& id, - const shared_ptr& bounds, - int singleIntervalLimit, - int direction ) - { - auto_ptr c( make( namespaceDetails, namespaceDetails->idxNo( id ), id ) ); - c->init(bounds,singleIntervalLimit,direction); - return c.release(); - } - - BtreeCursor::BtreeCursor( NamespaceDetails* nsd, int theIndexNo, const IndexDetails& id ) : - d( nsd ), - idxNo( theIndexNo ), - indexDetails( id ), - _ordering( Ordering::make( BSONObj() ) ), - _boundsMustMatch( true ), - _nscanned() { - } - - void BtreeCursor::_finishConstructorInit() { - _multikey = d->isMultikey( idxNo ); - _order = indexDetails.keyPattern(); - _ordering = Ordering::make( _order ); - } - - void BtreeCursor::init( const BSONObj& sk, const BSONObj& ek, bool endKeyInclusive, int direction ) { - _finishConstructorInit(); - startKey = sk; - endKey = ek; - _endKeyInclusive = endKeyInclusive; - _direction = direction; - _independentFieldRanges = false; - audit(); - } - - void BtreeCursor::init( const shared_ptr< FieldRangeVector > &bounds, int singleIntervalLimit, int direction ) { - _finishConstructorInit(); - _bounds = bounds; - verify( _bounds ); - _direction = direction; - _endKeyInclusive = true; - _boundsIterator.reset( new FieldRangeVectorIterator( *_bounds , singleIntervalLimit ) ); - _independentFieldRanges = true; - audit(); - startKey = _bounds->startKey(); - _boundsIterator->advance( startKey ); // handles initialization - _boundsIterator->prepDive(); - bucket = indexDetails.head; - keyOfs = 0; - } - - /** Properly destroy forward declared class members. */ - BtreeCursor::~BtreeCursor() {} - - void BtreeCursor::audit() { - dassert( d->idxNo((IndexDetails&) indexDetails) == idxNo ); - } - - void BtreeCursor::initWithoutIndependentFieldRanges() { - if ( indexDetails.getSpec().getType() ) { - startKey = indexDetails.getSpec().getType()->fixKey( startKey ); - endKey = indexDetails.getSpec().getType()->fixKey( endKey ); - } - bucket = _locate(startKey, _direction > 0 ? minDiskLoc : maxDiskLoc); - if ( ok() ) { - _nscanned = 1; - } - skipUnusedKeys(); - checkEnd(); - } - - void BtreeCursor::skipAndCheck() { - long long startNscanned = _nscanned; - skipUnusedKeys(); - while( 1 ) { - if ( !skipOutOfRangeKeysAndCheckEnd() ) { - break; - } - do { - // If nscanned is increased by more than 20 before a matching key is found, abort - // skipping through the btree to find a matching key. This iteration cutoff - // prevents unbounded internal iteration within BtreeCursor::init() and - // BtreeCursor::advance() (the callers of skipAndCheck()). See SERVER-3448. - if ( _nscanned > startNscanned + 20 ) { - skipUnusedKeys(); - // If iteration is aborted before a key matching _bounds is identified, the - // cursor may be left pointing at a key that is not within bounds - // (_bounds->matchesKey( currKey() ) may be false). Set _boundsMustMatch to - // false accordingly. - _boundsMustMatch = false; - return; - } - } while( skipOutOfRangeKeysAndCheckEnd() ); - if ( !skipUnusedKeys() ) { - break; - } - } - } - - bool BtreeCursor::skipOutOfRangeKeysAndCheckEnd() { - if ( !ok() ) { - return false; - } - int ret = _boundsIterator->advance( currKey() ); - if ( ret == -2 ) { - bucket = DiskLoc(); - return false; - } - else if ( ret == -1 ) { - ++_nscanned; - return false; - } - ++_nscanned; - advanceTo( currKey(), ret, _boundsIterator->after(), _boundsIterator->cmp(), _boundsIterator->inc() ); - return true; - } - - // Return a value in the set {-1, 0, 1} to represent the sign of parameter i. - int sgn( int i ) { - if ( i == 0 ) - return 0; - return i > 0 ? 1 : -1; - } - - // Check if the current key is beyond endKey. - void BtreeCursor::checkEnd() { - if ( bucket.isNull() ) - return; - if ( !endKey.isEmpty() ) { - int cmp = sgn( endKey.woCompare( currKey(), _order ) ); - if ( ( cmp != 0 && cmp != _direction ) || - ( cmp == 0 && !_endKeyInclusive ) ) - bucket = DiskLoc(); - } - } - - void BtreeCursor::advanceTo( const BSONObj &keyBegin, int keyBeginLen, bool afterKey, const vector< const BSONElement * > &keyEnd, const vector< bool > &keyEndInclusive) { - _advanceTo( bucket, keyOfs, keyBegin, keyBeginLen, afterKey, keyEnd, keyEndInclusive, _ordering, _direction ); - } - - bool BtreeCursor::advance() { - // Reset this flag at the start of a new iteration. - _boundsMustMatch = true; - - killCurrentOp.checkForInterrupt(); - if ( bucket.isNull() ) - return false; - - bucket = _advance(bucket, keyOfs, _direction, "BtreeCursor::advance"); - - if ( !_independentFieldRanges ) { - skipUnusedKeys(); - checkEnd(); - if ( ok() ) { - ++_nscanned; - } - } - else { - skipAndCheck(); - } - return ok(); - } - - void BtreeCursor::noteLocation() { - if ( !eof() ) { - BSONObj o = currKey().getOwned(); - keyAtKeyOfs = o; - locAtKeyOfs = currLoc(); - } - } - - string BtreeCursor::toString() { - string s = string("BtreeCursor ") + indexDetails.indexName(); - if ( _direction < 0 ) s += " reverse"; - if ( _bounds.get() && _bounds->size() > 1 ) s += " multi"; - return s; - } - - BSONObj BtreeCursor::prettyIndexBounds() const { - if ( !_independentFieldRanges ) { - return BSON( "start" << prettyKey( startKey ) << "end" << prettyKey( endKey ) ); - } - else { - return _bounds->obj(); - } - } - - bool BtreeCursor::currentMatches( MatchDetails* details ) { - // If currKey() might not match the specified _bounds, check whether or not it does. - if ( !_boundsMustMatch && _bounds && !_bounds->matchesKey( currKey() ) ) { - // If the key does not match _bounds, it does not match the query. - return false; - } - // Forward to the base class implementation, which may utilize a Matcher. - return Cursor::currentMatches( details ); - } - - /* ----------------------------------------------------------------------------- */ - - struct BtreeCursorUnitTest { - BtreeCursorUnitTest() { - verify( minDiskLoc.compare(maxDiskLoc) < 0 ); - } - } btut; - -} // namespace mongo diff --git a/src/mongo/db/btreecursor.h b/src/mongo/db/btreecursor.h deleted file mode 100644 index 977453fc817..00000000000 --- a/src/mongo/db/btreecursor.h +++ /dev/null @@ -1,252 +0,0 @@ -/** - * Copyright (C) 2008 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "mongo/db/cursor.h" -#include "mongo/db/diskloc.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/namespace_details.h" - -namespace mongo { - - class FieldRangeVector; - class FieldRangeVectorIterator; - - /** - * A Cursor class for btree iteration. - * - * A BtreeCursor iterates over all keys of a specified btree that are within the provided - * btree bounds, using the specified direction of traversal. - * - * Notes on specifying btree bounds: - * - * A BtreeCursor may be initialized with the 'startKey' and 'endKey' bounds of an interval of - * keys within a btree. A BtreeCursor may alternatively be initialized with a FieldRangeVector - * describing a list of intervals for every field of the btree index. - * - * Notes on inheritance: - * - * BtreeCursor is an abstract class. Btrees come in different versions, with different storage - * formats. The versions are v0 and v1 as of this writing. Member functions implemented in - * BtreeCursor are storage format agnostic. Classes derived from BtreeCursor implement the - * format specific bits. - * - * Notes on the yielding implementation: - * - * When an operation using a BtreeCursor yields the database mutex that locks the btree data - * structure, the btree may be modified. When the operation regains the database mutex, the - * BtreeCursor can relocate its position in the modified btree and continue iteration from that - * point. - * - * Before the database mutex is yielded, a BtreeCursor records its position (noteLoc()). A - * recorded btree position consists of a btree bucket, bucket key offset, and unique btree key. - * - * After the database mutex is regained, a BtreeCursor relocates its position (checkLoc()). To - * relocate a unique btree key, a BtreeCursor first checks the btree key at its recorded btree - * bucket and bucket key offset. If the key at that location does not match the recorded btree - * key, and a preceding offset also fails to match, the recorded key (or the next existing key - * following it) is located in the btree using binary search. If the recorded btree bucket is - * invalidated, the initial recorded bucket check is not attempted (see SERVER-4575). - */ - class BtreeCursor : public Cursor { - public: - virtual ~BtreeCursor(); - - /** Makes an appropriate subclass depending on the index version. */ - - static BtreeCursor* make( NamespaceDetails* namespaceDetails, - const IndexDetails& id, - const BSONObj& startKey, - const BSONObj& endKey, - bool endKeyInclusive, - int direction ); - - static BtreeCursor* make( NamespaceDetails* namespaceDetails, - const IndexDetails& id, - const shared_ptr& bounds, - int singleIntervalLimit, - int direction ); - - virtual bool ok() { return !bucket.isNull(); } - virtual bool advance(); - virtual void noteLocation(); // updates keyAtKeyOfs... - virtual void checkLocation() = 0; - virtual bool supportGetMore() { return true; } - virtual bool supportYields() { return true; } - - /** - * used for multikey index traversal to avoid sending back dups. see Matcher::matches(). - * if a multikey index traversal: - * if loc has already been sent, returns true. - * otherwise, marks loc as sent. - * @return false if the loc has not been seen - */ - virtual bool getsetdup(DiskLoc loc) { - if( _multikey ) { - pair::iterator, bool> p = _dups.insert(loc); - return !p.second; - } - return false; - } - - virtual bool modifiedKeys() const { return _multikey; } - virtual bool isMultiKey() const { return _multikey; } - - /** returns BSONObj() if ofs is out of range */ - virtual BSONObj keyAt(int ofs) const = 0; - - virtual BSONObj currKey() const = 0; - virtual BSONObj indexKeyPattern() { return _order; } - - virtual void aboutToDeleteBucket(const DiskLoc& b) { - if ( bucket == b ) - keyOfs = -1; - } - - virtual DiskLoc currLoc() = 0; - virtual DiskLoc refLoc() { return currLoc(); } - virtual Record* _current() { return currLoc().rec(); } - virtual BSONObj current() { return BSONObj::make(_current()); } - virtual string toString(); - - BSONObj prettyKey( const BSONObj& key ) const { - return key.replaceFieldNames( indexDetails.keyPattern() ).clientReadable(); - } - - virtual BSONObj prettyIndexBounds() const; - - virtual CoveredIndexMatcher* matcher() const { return _matcher.get(); } - - virtual bool currentMatches( MatchDetails* details = 0 ); - - virtual void setMatcher( shared_ptr matcher ) { _matcher = matcher; } - - virtual const Projection::KeyOnly* keyFieldsOnly() const { return _keyFieldsOnly.get(); } - - virtual void setKeyFieldsOnly( const shared_ptr& keyFieldsOnly ) { - _keyFieldsOnly = keyFieldsOnly; - } - - virtual long long nscanned() { return _nscanned; } - - /** for debugging only */ - const DiskLoc getBucket() const { return bucket; } - int getKeyOfs() const { return keyOfs; } - - // just for unit tests - virtual bool curKeyHasChild() = 0; - - protected: - BtreeCursor( NamespaceDetails* nsd, int theIndexNo, const IndexDetails& idxDetails ); - - virtual void init( const BSONObj& startKey, - const BSONObj& endKey, - bool endKeyInclusive, - int direction ); - - virtual void init( const shared_ptr& bounds, - int singleIntervalLimit, - int direction ); - - /** - * Our btrees may (rarely) have "unused" keys when items are deleted. - * Skip past them. - */ - virtual bool skipUnusedKeys() = 0; - - bool skipOutOfRangeKeysAndCheckEnd(); - - /** - * Attempt to locate the next btree key matching _bounds. This may mean advancing to the - * next successive key in the btree, or skipping to a new position in the btree. If an - * internal iteration cutoff is reached before a matching key is found, then the search for - * a matching key will be aborted, leaving the cursor pointing at a key that is not within - * bounds. - */ - void skipAndCheck(); - - void checkEnd(); - - /** selective audits on construction */ - void audit(); - - virtual void _audit() = 0; - - virtual DiskLoc _locate(const BSONObj& key, const DiskLoc& loc) = 0; - - virtual DiskLoc _advance(const DiskLoc& thisLoc, - int& keyOfs, - int direction, - const char* caller) = 0; - - virtual void _advanceTo(DiskLoc& thisLoc, - int& keyOfs, - const BSONObj& keyBegin, - int keyBeginLen, - bool afterKey, - const vector& keyEnd, - const vector& keyEndInclusive, - const Ordering& order, - int direction ) = 0; - - /** set initial bucket */ - void initWithoutIndependentFieldRanges(); - - /** if afterKey is true, we want the first key with values of the keyBegin fields greater than keyBegin */ - void advanceTo( const BSONObj& keyBegin, - int keyBeginLen, - bool afterKey, - const vector& keyEnd, - const vector& keyEndInclusive ); - - // these are set in the construtor - NamespaceDetails* const d; - const int idxNo; - const IndexDetails& indexDetails; - - // these are all set in init() - set _dups; - BSONObj startKey; - BSONObj endKey; - bool _endKeyInclusive; - bool _multikey; // this must be updated every getmore batch in case someone added a multikey - BSONObj _order; // this is the same as indexDetails.keyPattern() - Ordering _ordering; - DiskLoc bucket; - int keyOfs; - int _direction; // 1=fwd,-1=reverse - BSONObj keyAtKeyOfs; // so we can tell if things moved around on us between the query and the getMore call - DiskLoc locAtKeyOfs; - shared_ptr _bounds; - auto_ptr _boundsIterator; - bool _boundsMustMatch; // If iteration is aborted before a key matching _bounds is - // identified, the cursor may be left pointing at a key that is not - // within bounds (_bounds->matchesKey( currKey() ) may be false). - // _boundsMustMatch will be set to false accordingly. - shared_ptr _matcher; - shared_ptr _keyFieldsOnly; - bool _independentFieldRanges; - long long _nscanned; - - private: - void _finishConstructorInit(); - static BtreeCursor* make( NamespaceDetails* nsd, - int idxNo, - const IndexDetails& indexDetails ); - }; - -} // namespace mongo diff --git a/src/mongo/db/btreeposition.cpp b/src/mongo/db/btreeposition.cpp deleted file mode 100644 index b34fd1b9334..00000000000 --- a/src/mongo/db/btreeposition.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "mongo/db/btreeposition.h" - -#include "mongo/db/btree.h" -#include "mongo/db/index.h" -#include "mongo/db/pdfile.h" - -namespace mongo { - - std::ostream& operator<<( std::ostream& stream, const BtreeKeyLocation& loc ) { - return stream << BSON( "bucket" << loc.bucket.toString() << - "index" << loc.pos ).jsonString(); - } - - LogicalBtreePosition::LogicalBtreePosition( const IndexDetails& indexDetails, - Ordering ordering, - const BtreeKeyLocation& initialLocation ) : - _indexDetails( &indexDetails ), - _ordering( ordering ), - _initialLocation( initialLocation ), - _initialLocationValid() { - fassert( 16494, _indexDetails->version() == 1 ); - } - - void LogicalBtreePosition::init() { - if ( _initialLocation.bucket.isNull() ) { - // Abort if the initial location is not a valid bucket. - return; - } - - // Store the key and record referenced at the supplied initial location. - BucketBasics::KeyNode keyNode = - _initialLocation.bucket.btree()->keyNode( _initialLocation.pos ); - _key = keyNode.key.toBson().getOwned(); - _record = keyNode.recordLoc; - _initialLocationValid = true; - } - - BtreeKeyLocation LogicalBtreePosition::currentLocation() const { - if ( _initialLocation.bucket.isNull() ) { - // Abort if the initial location is not a valid bucket. - return BtreeKeyLocation(); - } - - // If the initial location has not been invalidated ... - if ( _initialLocationValid ) { - - const BtreeBucket* bucket = _initialLocation.bucket.btree(); - if ( // ... and the bucket was not marked as invalid ... - bucket->getN() != bucket->INVALID_N_SENTINEL && - // ... and the initial location index is valid for the bucket ... - _initialLocation.pos < bucket->getN() ) { - - BucketBasics::KeyNode keyNode = bucket->keyNode( _initialLocation.pos ); - if ( // ... and the record location equals the initial record location ... - keyNode.recordLoc == _record && - // ... and the key equals the initial key ... - keyNode.key.toBson().binaryEqual( _key ) ) { - // ... then the initial location is the current location, so return it. - return _initialLocation; - } - } - } - - // Relocate the key and record location retrieved from _initialLocation. - BtreeKeyLocation ret; - bool found; - ret.bucket = _indexDetails->head.btree()->locate( *_indexDetails, - _indexDetails->head, - _key, - _ordering, - ret.pos, - found, - _record, - 1 // Forward direction means the next - // ordered key will be returned if - // the requested key is missing. - ); - return ret; - } - -} // namespace mongo diff --git a/src/mongo/db/btreeposition.h b/src/mongo/db/btreeposition.h deleted file mode 100644 index 5499dc43662..00000000000 --- a/src/mongo/db/btreeposition.h +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "mongo/db/diskloc.h" -#include "mongo/db/jsobj.h" -#include "mongo/platform/cstdint.h" - -namespace mongo { - - class IndexDetails; - - /** - * Physical location of a key within a btree. Comprised of the DiskLoc address of a btree - * bucket and the index of a key within that bucket. - */ - struct BtreeKeyLocation { - - BtreeKeyLocation() : - pos() { - } - - BtreeKeyLocation( DiskLoc initialBucket, int32_t initialPos ) : - bucket( initialBucket ), - pos( initialPos ) { - } - - bool operator==( const BtreeKeyLocation& other ) const { - return bucket == other.bucket && pos == other.pos; - } - - DiskLoc bucket; // Bucket within btree. - int32_t pos; // Index within bucket. - }; - - std::ostream& operator<<( std::ostream& stream, const BtreeKeyLocation& loc ); - - /** - * Logical btree position independent of the physical structure of a btree. This is used to - * track a position within a btree while the structure of the btree is changing. - * - * For example, a btree containing keys 'a', 'b', and 'c' might have all three keys in one - * bucket or alternatively 'b' within a left child of 'c'. The same LogicalBtreePosition can - * represent the position of 'b' in both cases and can retrieve the physical BtreeKeyLocation of - * 'b' in each case. If the btree is changed so that it lacks a 'b' key, the position will - * reference the lowest key greater than 'b'. This is desirable behavior when the logical btree - * position is used to implement a forward direction iterator. - * - * The class is seeded with a BtreeKeyLocation identifying a btree key. This initial physical - * location is cached in order to quickly check if the physical location corresponding to the - * logical position is unchanged and can be returned as is. - * - * NOTE Only supports V1 indexes. - */ - class LogicalBtreePosition { - public: - - /** - * Create a position with the @param 'indexDetails', @param 'ordering', and initial key - * location @param 'initialLocation' specified. - * @fasserts if 'indexDetails' is not a V1 index. - */ - LogicalBtreePosition( const IndexDetails& indexDetails, - Ordering ordering, - const BtreeKeyLocation& initialLocation ); - - /** Initialize the position by reading the key at the supplied initial location. */ - void init(); - - /** - * Invalidate the supplied initial location. This may be called when bucket containing the - * supplied location is deleted. - */ - void invalidateInitialLocation() { _initialLocationValid = false; } - - /** - * Retrieve the current physical location in the btree corresponding to this logical - * position. - */ - BtreeKeyLocation currentLocation() const; - - private: - const IndexDetails* _indexDetails; - Ordering _ordering; - BtreeKeyLocation _initialLocation; - bool _initialLocationValid; - BSONObj _key; - DiskLoc _record; - }; - -} // namespace mongo diff --git a/src/mongo/db/cap.cpp b/src/mongo/db/cap.cpp deleted file mode 100644 index 9989b6cfbc1..00000000000 --- a/src/mongo/db/cap.cpp +++ /dev/null @@ -1,494 +0,0 @@ -// @file cap.cpp capped collection related -// the "old" version (<= v1.6) - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" -#include "pdfile.h" -#include "db.h" -#include "../util/mmap.h" -#include "../util/hashtab.h" -#include "../scripting/engine.h" -#include -#include -#include "json.h" -#include "clientcursor.h" -#include "index_update.h" - -/* - capped collection layout - - d's below won't exist if things align perfectly: - - extent1 -> extent2 -> extent3 - ------------------- ----------------------- --------------------- - d r r r r r r r r d d r r r r d r r r r r d d r r r r r r r r r d - ^ ^ - oldest newest - - ^cappedFirstDeletedInCurExtent() - ^cappedLastDelRecLastExtent() - ^cappedListOfAllDeletedRecords() -*/ - -//#define DDD(x) log() << "cap.cpp debug:" << x << endl; -#define DDD(x) - -namespace mongo { - - /* combine adjacent deleted records *for the current extent* of the capped collection - - this is O(n^2) but we call it for capped tables where typically n==1 or 2! - (or 3...there will be a little unused sliver at the end of the extent.) - */ - void NamespaceDetails::compact() { - DDD( "NamespaceDetails::compact enter" ); - verify( isCapped() ); - - vector drecs; - - // Pull out capExtent's DRs from deletedList - DiskLoc i = cappedFirstDeletedInCurExtent(); - for (; !i.isNull() && inCapExtent( i ); i = i.drec()->nextDeleted() ) { - DDD( "\t" << i ); - drecs.push_back( i ); - } - - getDur().writingDiskLoc( cappedFirstDeletedInCurExtent() ) = i; - - std::sort( drecs.begin(), drecs.end() ); - DDD( "\t drecs.size(): " << drecs.size() ); - - vector::const_iterator j = drecs.begin(); - verify( j != drecs.end() ); - DiskLoc a = *j; - while ( 1 ) { - j++; - if ( j == drecs.end() ) { - DDD( "\t compact adddelrec" ); - addDeletedRec(a.drec(), a); - break; - } - DiskLoc b = *j; - while ( a.a() == b.a() && a.getOfs() + a.drec()->lengthWithHeaders() == b.getOfs() ) { - // a & b are adjacent. merge. - getDur().writingInt( a.drec()->lengthWithHeaders() ) += b.drec()->lengthWithHeaders(); - j++; - if ( j == drecs.end() ) { - DDD( "\t compact adddelrec2" ); - addDeletedRec(a.drec(), a); - return; - } - b = *j; - } - DDD( "\t compact adddelrec3" ); - addDeletedRec(a.drec(), a); - a = b; - } - - } - - DiskLoc &NamespaceDetails::cappedFirstDeletedInCurExtent() { - if ( cappedLastDelRecLastExtent().isNull() ) - return cappedListOfAllDeletedRecords(); - else - return cappedLastDelRecLastExtent().drec()->nextDeleted(); - } - - void NamespaceDetails::cappedCheckMigrate() { - // migrate old NamespaceDetails format - verify( isCapped() ); - if ( capExtent.a() == 0 && capExtent.getOfs() == 0 ) { - //capFirstNewRecord = DiskLoc(); - capFirstNewRecord.writing().setInvalid(); - // put all the DeletedRecords in cappedListOfAllDeletedRecords() - for ( int i = 1; i < Buckets; ++i ) { - DiskLoc first = deletedList[ i ]; - if ( first.isNull() ) - continue; - DiskLoc last = first; - for (; !last.drec()->nextDeleted().isNull(); last = last.drec()->nextDeleted() ); - last.drec()->nextDeleted().writing() = cappedListOfAllDeletedRecords(); - cappedListOfAllDeletedRecords().writing() = first; - deletedList[i].writing() = DiskLoc(); - } - // NOTE cappedLastDelRecLastExtent() set to DiskLoc() in above - - // Last, in case we're killed before getting here - capExtent.writing() = firstExtent; - } - } - - bool NamespaceDetails::inCapExtent( const DiskLoc &dl ) const { - verify( !dl.isNull() ); - // We could have a rec or drec, doesn't matter. - bool res = dl.drec()->myExtentLoc(dl) == capExtent; - DEV { - // old implementation. this check is temp to test works the same. new impl should be a little faster. - verify( res == (dl.drec()->myExtent( dl ) == capExtent.ext()) ); - } - return res; - } - - bool NamespaceDetails::nextIsInCapExtent( const DiskLoc &dl ) const { - verify( !dl.isNull() ); - DiskLoc next = dl.drec()->nextDeleted(); - if ( next.isNull() ) - return false; - return inCapExtent( next ); - } - - void NamespaceDetails::advanceCapExtent( const char *ns ) { - // We want cappedLastDelRecLastExtent() to be the last DeletedRecord of the prev cap extent - // (or DiskLoc() if new capExtent == firstExtent) - if ( capExtent == lastExtent ) - getDur().writingDiskLoc( cappedLastDelRecLastExtent() ) = DiskLoc(); - else { - DiskLoc i = cappedFirstDeletedInCurExtent(); - for (; !i.isNull() && nextIsInCapExtent( i ); i = i.drec()->nextDeleted() ); - getDur().writingDiskLoc( cappedLastDelRecLastExtent() ) = i; - } - - getDur().writingDiskLoc( capExtent ) = theCapExtent()->xnext.isNull() ? firstExtent : theCapExtent()->xnext; - - /* this isn't true if a collection has been renamed...that is ok just used for diagnostics */ - //dassert( theCapExtent()->ns == ns ); - - theCapExtent()->assertOk(); - getDur().writingDiskLoc( capFirstNewRecord ) = DiskLoc(); - } - - DiskLoc NamespaceDetails::__capAlloc( int len ) { - DiskLoc prev = cappedLastDelRecLastExtent(); - DiskLoc i = cappedFirstDeletedInCurExtent(); - DiskLoc ret; - for (; !i.isNull() && inCapExtent( i ); prev = i, i = i.drec()->nextDeleted() ) { - // We need to keep at least one DR per extent in cappedListOfAllDeletedRecords(), - // so make sure there's space to create a DR at the end. - if ( i.drec()->lengthWithHeaders() >= len + 24 ) { - ret = i; - break; - } - } - - /* unlink ourself from the deleted list */ - if ( !ret.isNull() ) { - if ( prev.isNull() ) - cappedListOfAllDeletedRecords().writing() = ret.drec()->nextDeleted(); - else - prev.drec()->nextDeleted().writing() = ret.drec()->nextDeleted(); - ret.drec()->nextDeleted().writing().setInvalid(); // defensive. - verify( ret.drec()->extentOfs() < ret.getOfs() ); - } - - return ret; - } - - DiskLoc NamespaceDetails::cappedAlloc(const char *ns, int len) { - - if ( len > theCapExtent()->length ) { - // the extent check is a way to try and improve performance - uassert( 16328 , str::stream() << "document is larger than capped size " - << len << " > " << storageSize() , len <= storageSize() ); - } - - // signal done allocating new extents. - if ( !cappedLastDelRecLastExtent().isValid() ) - getDur().writingDiskLoc( cappedLastDelRecLastExtent() ) = DiskLoc(); - - verify( len < 400000000 ); - int passes = 0; - int maxPasses = ( len / 30 ) + 2; // 30 is about the smallest entry that could go in the oplog - if ( maxPasses < 5000 ) { - // this is for bacwards safety since 5000 was the old value - maxPasses = 5000; - } - DiskLoc loc; - - // delete records until we have room and the max # objects limit achieved. - - /* this fails on a rename -- that is ok but must keep commented out */ - //verify( theCapExtent()->ns == ns ); - - theCapExtent()->assertOk(); - DiskLoc firstEmptyExtent; - while ( 1 ) { - if ( stats.nrecords < maxCappedDocs() ) { - loc = __capAlloc( len ); - if ( !loc.isNull() ) - break; - } - - // If on first iteration through extents, don't delete anything. - if ( !capFirstNewRecord.isValid() ) { - advanceCapExtent( ns ); - - if ( capExtent != firstExtent ) - capFirstNewRecord.writing().setInvalid(); - // else signal done with first iteration through extents. - continue; - } - - if ( !capFirstNewRecord.isNull() && - theCapExtent()->firstRecord == capFirstNewRecord ) { - // We've deleted all records that were allocated on the previous - // iteration through this extent. - advanceCapExtent( ns ); - continue; - } - - if ( theCapExtent()->firstRecord.isNull() ) { - if ( firstEmptyExtent.isNull() ) - firstEmptyExtent = capExtent; - advanceCapExtent( ns ); - if ( firstEmptyExtent == capExtent ) { - maybeComplain( ns, len ); - return DiskLoc(); - } - continue; - } - - DiskLoc fr = theCapExtent()->firstRecord; - theDataFileMgr.deleteRecord(this, ns, fr.rec(), fr, true); // ZZZZZZZZZZZZ - compact(); - if( ++passes > maxPasses ) { - StringBuilder sb; - sb << "passes >= maxPasses in NamespaceDetails::cappedAlloc: ns: " << ns - << ", len: " << len - << ", maxPasses: " << maxPasses - << ", _maxDocsInCapped: " << _maxDocsInCapped - << ", nrecords: " << stats.nrecords - << ", datasize: " << stats.datasize; - msgasserted(10345, sb.str()); - } - } - - // Remember first record allocated on this iteration through capExtent. - if ( capFirstNewRecord.isValid() && capFirstNewRecord.isNull() ) - getDur().writingDiskLoc(capFirstNewRecord) = loc; - - return loc; - } - - void NamespaceDetails::dumpExtents() { - cout << "dumpExtents:" << endl; - for ( DiskLoc i = firstExtent; !i.isNull(); i = i.ext()->xnext ) { - Extent *e = i.ext(); - stringstream ss; - e->dump(ss); - cout << ss.str() << endl; - } - } - - void NamespaceDetails::cappedDumpDelInfo() { - cout << "dl[0]: " << deletedList[0].toString() << endl; - for( DiskLoc z = deletedList[0]; !z.isNull(); z = z.drec()->nextDeleted() ) { - cout << " drec:" << z.toString() << " dreclen:" << hex << z.drec()->lengthWithHeaders() << - " ext:" << z.drec()->myExtent(z)->myLoc.toString() << endl; - } - cout << "dl[1]: " << deletedList[1].toString() << endl; - } - - void NamespaceDetails::cappedTruncateLastDelUpdate() { - if ( capExtent == firstExtent ) { - // Only one extent of the collection is in use, so there - // is no deleted record in a previous extent, so nullify - // cappedLastDelRecLastExtent(). - cappedLastDelRecLastExtent().writing() = DiskLoc(); - } - else { - // Scan through all deleted records in the collection - // until the last deleted record for the extent prior - // to the new capExtent is found. Then set - // cappedLastDelRecLastExtent() to that deleted record. - DiskLoc i = cappedListOfAllDeletedRecords(); - for( ; - !i.drec()->nextDeleted().isNull() && - !inCapExtent( i.drec()->nextDeleted() ); - i = i.drec()->nextDeleted() ); - // In our capped storage model, every extent must have at least one - // deleted record. Here we check that 'i' is not the last deleted - // record. (We expect that there will be deleted records in the new - // capExtent as well.) - verify( !i.drec()->nextDeleted().isNull() ); - cappedLastDelRecLastExtent().writing() = i; - } - } - - void NamespaceDetails::cappedTruncateAfter(const char *ns, DiskLoc end, bool inclusive) { - DEV verify( this == nsdetails(ns) ); - verify( cappedLastDelRecLastExtent().isValid() ); - - // We iteratively remove the newest document until the newest document - // is 'end', then we remove 'end' if requested. - bool foundLast = false; - while( 1 ) { - if ( foundLast ) { - // 'end' has been found and removed, so break. - break; - } - getDur().commitIfNeeded(); - // 'curr' will point to the newest document in the collection. - DiskLoc curr = theCapExtent()->lastRecord; - verify( !curr.isNull() ); - if ( curr == end ) { - if ( inclusive ) { - // 'end' has been found, so break next iteration. - foundLast = true; - } - else { - // 'end' has been found, so break. - break; - } - } - - // TODO The algorithm used in this function cannot generate an - // empty collection, but we could call emptyCappedCollection() in - // this case instead of asserting. - uassert( 13415, "emptying the collection is not allowed", stats.nrecords > 1 ); - - // Delete the newest record, and coalesce the new deleted - // record with existing deleted records. - theDataFileMgr.deleteRecord(this, ns, curr.rec(), curr, true); - compact(); - - // This is the case where we have not yet had to remove any - // documents to make room for other documents, and we are allocating - // documents from free space in fresh extents instead of reusing - // space from familiar extents. - if ( !capLooped() ) { - - // We just removed the last record from the 'capExtent', and - // the 'capExtent' can't be empty, so we set 'capExtent' to - // capExtent's prev extent. - if ( theCapExtent()->lastRecord.isNull() ) { - verify( !theCapExtent()->xprev.isNull() ); - // NOTE Because we didn't delete the last document, and - // capLooped() is false, capExtent is not the first extent - // so xprev will be nonnull. - capExtent.writing() = theCapExtent()->xprev; - theCapExtent()->assertOk(); - - // update cappedLastDelRecLastExtent() - cappedTruncateLastDelUpdate(); - } - continue; - } - - // This is the case where capLooped() is true, and we just deleted - // from capExtent, and we just deleted capFirstNewRecord, which was - // the last record on the fresh side of capExtent. - // NOTE In this comparison, curr and potentially capFirstNewRecord - // may point to invalid data, but we can still compare the - // references themselves. - if ( curr == capFirstNewRecord ) { - - // Set 'capExtent' to the first nonempty extent prior to the - // initial capExtent. There must be such an extent because we - // have not deleted the last document in the collection. It is - // possible that all extents other than the capExtent are empty. - // In this case we will keep the initial capExtent and specify - // that all records contained within are on the fresh rather than - // stale side of the extent. - DiskLoc newCapExtent = capExtent; - do { - // Find the previous extent, looping if necessary. - newCapExtent = ( newCapExtent == firstExtent ) ? lastExtent : newCapExtent.ext()->xprev; - newCapExtent.ext()->assertOk(); - } - while ( newCapExtent.ext()->firstRecord.isNull() ); - capExtent.writing() = newCapExtent; - - // Place all documents in the new capExtent on the fresh side - // of the capExtent by setting capFirstNewRecord to the first - // document in the new capExtent. - capFirstNewRecord.writing() = theCapExtent()->firstRecord; - - // update cappedLastDelRecLastExtent() - cappedTruncateLastDelUpdate(); - } - } - } - - void NamespaceDetails::emptyCappedCollection( const char *ns ) { - DEV verify( this == nsdetails(ns) ); - massert( 13424, "collection must be capped", isCapped() ); - massert( 13425, "background index build in progress", !indexBuildsInProgress ); - - vector indexes = Helpers::findAll( Namespace( ns ).getSisterNS( "system.indexes" ) , BSON( "ns" << ns ) ); - for ( unsigned i=0; icappedLastDelRecLastExtent() = DiskLoc(); - t->cappedListOfAllDeletedRecords() = DiskLoc(); - - // preserve firstExtent/lastExtent - t->capExtent = firstExtent; - t->stats.datasize = stats.nrecords = 0; - // lastExtentSize preserve - // nIndexes preserve 0 - // capped preserve true - // max preserve - t->_paddingFactor = 1.0; - t->_systemFlags = 0; - t->capFirstNewRecord = DiskLoc(); - t->capFirstNewRecord.setInvalid(); - t->cappedLastDelRecLastExtent().setInvalid(); - // dataFileVersion preserve - // indexFileVersion preserve - t->multiKeyIndexBits = 0; - t->reservedA = 0; - t->extraOffset = 0; - // indexBuildInProgress preserve 0 - memset(t->reserved, 0, sizeof(t->reserved)); - - // Reset all existing extents and recreate the deleted list. - for( DiskLoc ext = firstExtent; !ext.isNull(); ext = ext.ext()->xnext ) { - DiskLoc prev = ext.ext()->xprev; - DiskLoc next = ext.ext()->xnext; - DiskLoc empty = ext.ext()->reuse( ns, true ); - ext.ext()->xprev.writing() = prev; - ext.ext()->xnext.writing() = next; - addDeletedRec( empty.drec(), empty ); - } - - for ( unsigned i=0; i. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/collection.h" + +#include "mongo/base/counter.h" +#include "mongo/base/owned_pointer_map.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/commands/server_status.h" +#include "mongo/db/curop.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/index_create.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/storage/extent.h" +#include "mongo/db/storage/extent_manager.h" +#include "mongo/db/structure/collection_iterator.h" + +#include "mongo/db/pdfile.h" // XXX-ERH +#include "mongo/db/auth/user_document_parser.h" // XXX-ANDY + +namespace mongo { + + std::string CompactOptions::toString() const { + std::stringstream ss; + ss << "paddingMode: "; + switch ( paddingMode ) { + case NONE: + ss << "NONE"; + break; + case PRESERVE: + ss << "PRESERVE"; + break; + case MANUAL: + ss << "MANUAL (" << paddingBytes << " + ( doc * " << paddingFactor <<") )"; + } + + ss << " validateDocuments: " << validateDocuments; + + return ss.str(); + } + + // ---- + + Collection::Collection( const StringData& fullNS, + NamespaceDetails* details, + Database* database ) + : _ns( fullNS ), + _infoCache( this ), + _indexCatalog( this, details ), + _cursorCache( fullNS ) { + _details = details; + _database = database; + + if ( details->isCapped() ) { + _recordStore.reset( new CappedRecordStoreV1( this, + _ns.ns(), + details, + &database->getExtentManager(), + _ns.coll() == "system.indexes" ) ); + } + else { + _recordStore.reset( new SimpleRecordStoreV1( _ns.ns(), + details, + &database->getExtentManager(), + _ns.coll() == "system.indexes" ) ); + } + _magic = 1357924; + _indexCatalog.init(); + } + + Collection::~Collection() { + verify( ok() ); + _magic = 0; + } + + bool Collection::requiresIdIndex() const { + + if ( _ns.ns().find( '$' ) != string::npos ) { + // no indexes on indexes + return false; + } + + if ( _ns == _database->_namespacesName || + _ns == _database->_indexesName || + _ns == _database->_profileName ) { + return false; + } + + if ( _ns.db() == "local" ) { + if ( _ns.coll().startsWith( "oplog." ) ) + return false; + } + + if ( !_ns.isSystem() ) { + // non system collections definitely have an _id index + return true; + } + + + return true; + } + + CollectionIterator* Collection::getIterator( const DiskLoc& start, bool tailable, + const CollectionScanParams::Direction& dir) const { + verify( ok() ); + if ( _details->isCapped() ) + return new CappedIterator( this, start, tailable, dir ); + return new FlatIterator( this, start, dir ); + } + + int64_t Collection::countTableScan( const MatchExpression* expression ) { + scoped_ptr iterator( getIterator( DiskLoc(), + false, + CollectionScanParams::FORWARD ) ); + int64_t count = 0; + while ( !iterator->isEOF() ) { + DiskLoc loc = iterator->getNext(); + BSONObj obj = docFor( loc ); + if ( expression->matchesBSON( obj ) ) + count++; + } + + return count; + } + + BSONObj Collection::docFor( const DiskLoc& loc ) { + Record* rec = getExtentManager()->recordFor( loc ); + return BSONObj::make( rec->accessed() ); + } + + StatusWith Collection::insertDocument( const DocWriter* doc, bool enforceQuota ) { + verify( _indexCatalog.numIndexesTotal() == 0 ); // eventually can implement, just not done + + StatusWith loc = _recordStore->insertRecord( doc, + enforceQuota ? largestFileNumberInQuota() : 0 ); + if ( !loc.isOK() ) + return loc; + + return StatusWith( loc ); + } + + StatusWith Collection::insertDocument( const BSONObj& docToInsert, + bool enforceQuota, + const PregeneratedKeys* preGen ) { + if ( _indexCatalog.findIdIndex() ) { + if ( docToInsert["_id"].eoo() ) { + return StatusWith( ErrorCodes::InternalError, + str::stream() << "Collection::insertDocument got " + "document without _id for ns:" << _ns.ns() ); + } + } + + if ( _details->isCapped() ) { + // TOOD: old god not done + Status ret = _indexCatalog.checkNoIndexConflicts( docToInsert, preGen ); + if ( !ret.isOK() ) + return StatusWith( ret ); + } + + StatusWith status = _insertDocument( docToInsert, enforceQuota, preGen, false ); + if ( status.isOK() ) { + _details->paddingFits(); + } + + return status; + } + + StatusWith Collection::insertDocument( const BSONObj& doc, + MultiIndexBlock& indexBlock ) { + StatusWith loc = _recordStore->insertRecord( doc.objdata(), + doc.objsize(), + 0 ); + + if ( !loc.isOK() ) + return loc; + + InsertDeleteOptions indexOptions; + indexOptions.logIfError = false; + indexOptions.dupsAllowed = true; // in repair we should be doing no checking + + Status status = indexBlock.insert( doc, loc.getValue(), indexOptions ); + if ( !status.isOK() ) + return StatusWith( status ); + + return loc; + } + + + StatusWith Collection::_insertDocument( const BSONObj& docToInsert, + bool enforceQuota, + const PregeneratedKeys* preGen, + bool ignoreKeyTooLong ) { + + // TODO: for now, capped logic lives inside NamespaceDetails, which is hidden + // under the RecordStore, this feels broken since that should be a + // collection access method probably + + if ( preGen ) { + _indexCatalog.touch( preGen ); + } + + StatusWith loc = _recordStore->insertRecord( docToInsert.objdata(), + docToInsert.objsize(), + enforceQuota ? largestFileNumberInQuota() : 0 ); + if ( !loc.isOK() ) + return loc; + + _infoCache.notifyOfWriteOp(); + + try { + _indexCatalog.indexRecord( docToInsert, loc.getValue(), preGen, ignoreKeyTooLong ); + } + catch ( AssertionException& e ) { + if ( _details->isCapped() ) { + return StatusWith( ErrorCodes::InternalError, + str::stream() << "unexpected index insertion failure on" + << " capped collection" << e.toString() + << " - collection and its index will not match" ); + } + + // indexRecord takes care of rolling back indexes + // so we just have to delete the main storage + _recordStore->deleteRecord( loc.getValue() ); + return StatusWith( e.toStatus( "insertDocument" ) ); + } + + return loc; + } + + void Collection::deleteDocument( const DiskLoc& loc, bool cappedOK, bool noWarn, + BSONObj* deletedId ) { + if ( _details->isCapped() && !cappedOK ) { + log() << "failing remove on a capped ns " << _ns << endl; + uasserted( 10089, "cannot remove from a capped collection" ); + return; + } + + BSONObj doc = docFor( loc ); + + if ( deletedId ) { + BSONElement e = doc["_id"]; + if ( e.type() ) { + *deletedId = e.wrap(); + } + } + + /* check if any cursors point to us. if so, advance them. */ + _cursorCache.invalidateDocument(loc, INVALIDATION_DELETION); + + _indexCatalog.unindexRecord( doc, loc, noWarn); + + _recordStore->deleteRecord( loc ); + + _infoCache.notifyOfWriteOp(); + } + + Counter64 moveCounter; + ServerStatusMetricField moveCounterDisplay( "record.moves", &moveCounter ); + + StatusWith Collection::updateDocument( const DiskLoc& oldLocation, + const BSONObj& objNew, + bool enforceQuota, + OpDebug* debug ) { + + Record* oldRecord = getExtentManager()->recordFor( oldLocation ); + BSONObj objOld = BSONObj::make( oldRecord ); + + if ( objOld.hasElement( "_id" ) ) { + BSONElement oldId = objOld["_id"]; + BSONElement newId = objNew["_id"]; + if ( oldId != newId ) + return StatusWith( ErrorCodes::InternalError, + "in Collection::updateDocument _id mismatch", + 13596 ); + } + + /* duplicate key check. we descend the btree twice - once for this check, and once for the actual inserts, further + below. that is suboptimal, but it's pretty complicated to do it the other way without rollbacks... + */ + OwnedPointerMap updateTickets; + IndexCatalog::IndexIterator ii = _indexCatalog.getIndexIterator( true ); + while ( ii.more() ) { + IndexDescriptor* descriptor = ii.next(); + IndexAccessMethod* iam = _indexCatalog.getIndex( descriptor ); + + InsertDeleteOptions options; + options.logIfError = false; + options.dupsAllowed = + !(KeyPattern::isIdKeyPattern(descriptor->keyPattern()) || descriptor->unique()) + || ignoreUniqueIndex(descriptor); + UpdateTicket* updateTicket = new UpdateTicket(); + updateTickets.mutableMap()[descriptor] = updateTicket; + Status ret = iam->validateUpdate(objOld, objNew, oldLocation, options, updateTicket ); + if ( !ret.isOK() ) { + return StatusWith( ret ); + } + } + + if ( oldRecord->netLength() < objNew.objsize() ) { + // doesn't fit, have to move to new location + + if ( _details->isCapped() ) + return StatusWith( ErrorCodes::InternalError, + "failing update: objects in a capped ns cannot grow", + 10003 ); + + moveCounter.increment(); + _details->paddingTooSmall(); + + // unindex old record, don't delete + // this way, if inserting new doc fails, we can re-index this one + _cursorCache.invalidateDocument(oldLocation, INVALIDATION_DELETION); + _indexCatalog.unindexRecord( objOld, oldLocation, true ); + + if ( debug ) { + if (debug->nmoved == -1) // default of -1 rather than 0 + debug->nmoved = 1; + else + debug->nmoved += 1; + } + + StatusWith loc = _insertDocument( objNew, enforceQuota, NULL, true ); + + if ( loc.isOK() ) { + // insert successful, now lets deallocate the old location + // remember its already unindexed + _recordStore->deleteRecord( oldLocation ); + } + else { + // new doc insert failed, so lets re-index the old document and location + _indexCatalog.indexRecord( objOld, oldLocation, NULL, true ); + } + + return loc; + } + + _infoCache.notifyOfWriteOp(); + _details->paddingFits(); + + if ( debug ) + debug->keyUpdates = 0; + + ii = _indexCatalog.getIndexIterator( true ); + while ( ii.more() ) { + IndexDescriptor* descriptor = ii.next(); + IndexAccessMethod* iam = _indexCatalog.getIndex( descriptor ); + + int64_t updatedKeys; + Status ret = iam->update(*updateTickets.mutableMap()[descriptor], &updatedKeys); + if ( !ret.isOK() ) + return StatusWith( ret ); + if ( debug ) + debug->keyUpdates += updatedKeys; + } + + // Broadcast the mutation so that query results stay correct. + _cursorCache.invalidateDocument(oldLocation, INVALIDATION_MUTATION); + + // update in place + int sz = objNew.objsize(); + memcpy(getDur().writingPtr(oldRecord->data(), sz), objNew.objdata(), sz); + + return StatusWith( oldLocation ); + } + + int64_t Collection::storageSize( int* numExtents, BSONArrayBuilder* extentInfo ) const { + if ( _details->firstExtent().isNull() ) { + if ( numExtents ) + *numExtents = 0; + return 0; + } + + Extent* e = getExtentManager()->getExtent( _details->firstExtent() ); + + long long total = 0; + int n = 0; + while ( e ) { + total += e->length; + n++; + + if ( extentInfo ) { + extentInfo->append( BSON( "len" << e->length << "loc: " << e->myLoc.toBSONObj() ) ); + } + + e = getExtentManager()->getNextExtent( e ); + } + + if ( numExtents ) + *numExtents = n; + + return total; + } + + ExtentManager* Collection::getExtentManager() { + verify( ok() ); + return &_database->getExtentManager(); + } + + const ExtentManager* Collection::getExtentManager() const { + verify( ok() ); + return &_database->getExtentManager(); + } + + Extent* Collection::increaseStorageSize( int size, bool enforceQuota ) { + return getExtentManager()->increaseStorageSize( _ns, + _details, + size, + enforceQuota ? largestFileNumberInQuota() : 0 ); + } + + int Collection::largestFileNumberInQuota() const { + if ( !storageGlobalParams.quota ) + return 0; + + if ( _ns.db() == "local" ) + return 0; + + if ( _ns.isSpecial() ) + return 0; + + return storageGlobalParams.quotaFiles; + } + + bool Collection::isCapped() const { + return _details->isCapped(); + } + + uint64_t Collection::numRecords() const { + return _details->numRecords(); + } + + uint64_t Collection::dataSize() const { + return _details->dataSize(); + } + +} diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h new file mode 100644 index 00000000000..4b8e23805d8 --- /dev/null +++ b/src/mongo/db/catalog/collection.h @@ -0,0 +1,251 @@ +// collection.h + +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/string_data.h" +#include "mongo/db/catalog/collection_cursor_cache.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/collection_scan_common.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/structure/record_store.h" +#include "mongo/db/catalog/collection_info_cache.h" +#include "mongo/platform/cstdint.h" + +namespace mongo { + + class Database; + class ExtentManager; + class NamespaceDetails; + class IndexCatalog; + class MultiIndexBlock; + + class CollectionIterator; + class FlatIterator; + class CappedIterator; + + class OpDebug; + + class DocWriter { + public: + virtual ~DocWriter() {} + virtual void writeDocument( char* buf ) const = 0; + virtual size_t documentSize() const = 0; + virtual bool addPadding() const { return true; } + }; + + struct CompactOptions { + + CompactOptions() { + paddingMode = NONE; + validateDocuments = true; + paddingFactor = 1; + paddingBytes = 0; + } + + // padding + enum PaddingMode { + PRESERVE, NONE, MANUAL + } paddingMode; + + // only used if _paddingMode == MANUAL + double paddingFactor; // what to multiple document size by + int paddingBytes; // what to add to ducment size after multiplication + unsigned computeRecordSize( unsigned recordSize ) const { + recordSize = static_cast( paddingFactor * recordSize ); + recordSize += paddingBytes; + return recordSize; + } + + // other + bool validateDocuments; + + std::string toString() const; + }; + + struct CompactStats { + CompactStats() { + corruptDocuments = 0; + } + + long long corruptDocuments; + }; + + /** + * this is NOT safe through a yield right now + * not sure if it will be, or what yet + */ + class Collection { + public: + Collection( const StringData& fullNS, + NamespaceDetails* details, + Database* database ); + + ~Collection(); + + bool ok() const { return _magic == 1357924; } + + NamespaceDetails* details() { return _details; } // TODO: remove + const NamespaceDetails* details() const { return _details; } + + CollectionInfoCache* infoCache() { return &_infoCache; } + const CollectionInfoCache* infoCache() const { return &_infoCache; } + + const NamespaceString& ns() const { return _ns; } + + const IndexCatalog* getIndexCatalog() const { return &_indexCatalog; } + IndexCatalog* getIndexCatalog() { return &_indexCatalog; } + + CollectionCursorCache* cursorCache() const { return &_cursorCache; } + + bool requiresIdIndex() const; + + BSONObj docFor( const DiskLoc& loc ); + + // ---- things that should move to a CollectionAccessMethod like thing + /** + * canonical to get all would be + * getIterator( DiskLoc(), false, CollectionScanParams::FORWARD ) + */ + CollectionIterator* getIterator( const DiskLoc& start, bool tailable, + const CollectionScanParams::Direction& dir) const; + + + /** + * does a table scan to do a count + * this should only be used at a very low level + * does no yielding, indexes, etc... + */ + int64_t countTableScan( const MatchExpression* expression ); + + void deleteDocument( const DiskLoc& loc, + bool cappedOK = false, + bool noWarn = false, + BSONObj* deletedId = 0 ); + + /** + * this does NOT modify the doc before inserting + * i.e. will not add an _id field for documents that are missing it + */ + StatusWith insertDocument( const BSONObj& doc, bool enforceQuota, + const PregeneratedKeys* preGen = NULL ); + + StatusWith insertDocument( const DocWriter* doc, bool enforceQuota ); + + StatusWith insertDocument( const BSONObj& doc, MultiIndexBlock& indexBlock ); + + /** + * updates the document @ oldLocation with newDoc + * if the document fits in the old space, it is put there + * if not, it is moved + * @return the post update location of the doc (may or may not be the same as oldLocation) + */ + StatusWith updateDocument( const DiskLoc& oldLocation, + const BSONObj& newDoc, + bool enforceQuota, + OpDebug* debug ); + + int64_t storageSize( int* numExtents = NULL, BSONArrayBuilder* extentInfo = NULL ) const; + + // ----------- + + StatusWith compact( const CompactOptions* options ); + + // ----------- + + + // this is temporary, moving up from DB for now + // this will add a new extent the collection + // the new extent will be returned + // it will have been added to the linked list already + Extent* increaseStorageSize( int size, bool enforceQuota ); + + // + // Stats + // + + bool isCapped() const; + + uint64_t numRecords() const; + + uint64_t dataSize() const; + + int averageObjectSize() const { + uint64_t n = numRecords(); + if ( n == 0 ) + return 5; + return static_cast( dataSize() / n ); + } + + private: + /** + * same semantics as insertDocument, but doesn't do: + * - some user error checks + * - adjust padding + */ + StatusWith _insertDocument( const BSONObj& doc, + bool enforceQuota, + const PregeneratedKeys* preGen, + bool ignoreUniqueIndex ); + + void _compactExtent(const DiskLoc diskloc, int extentNumber, + MultiIndexBlock& indexesToInsertTo, + const CompactOptions* compactOptions, CompactStats* stats ); + + // @return 0 for inf., otherwise a number of files + int largestFileNumberInQuota() const; + + ExtentManager* getExtentManager(); + const ExtentManager* getExtentManager() const; + + int _magic; + + NamespaceString _ns; + NamespaceDetails* _details; + Database* _database; + scoped_ptr _recordStore; + CollectionInfoCache _infoCache; + IndexCatalog _indexCatalog; + + // this is mutable because read only users of the Collection class + // use it keep state. This seems valid as const correctness of Collection + // should be about the data. + mutable CollectionCursorCache _cursorCache; + + friend class Database; + friend class FlatIterator; + friend class CappedIterator; + friend class IndexCatalog; + }; + +} diff --git a/src/mongo/db/catalog/collection_cursor_cache.cpp b/src/mongo/db/catalog/collection_cursor_cache.cpp new file mode 100644 index 00000000000..7cbceebfa8b --- /dev/null +++ b/src/mongo/db/catalog/collection_cursor_cache.cpp @@ -0,0 +1,496 @@ +// collection_cursor_cache.h + +/** +* Copyright (C) 2013 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/collection_cursor_cache.h" + +#include "mongo/db/audit.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_holder.h" +#include "mongo/db/client.h" +#include "mongo/db/query/runner.h" +#include "mongo/platform/random.h" +#include "mongo/util/startup_test.h" + +namespace mongo { + + namespace { + unsigned idFromCursorId( CursorId id ) { + uint64_t x = static_cast(id); + x = x >> 32; + return static_cast( x ); + } + + CursorId cursorIdFromParts( unsigned collection, + unsigned cursor ) { + CursorId x = static_cast( collection ) << 32; + x |= cursor; + return x; + } + + class IdWorkTest : public StartupTest { + public: + void _run( unsigned a, unsigned b) { + CursorId x = cursorIdFromParts( a, b ); + invariant( a == idFromCursorId( x ) ); + CursorId y = cursorIdFromParts( a, b + 1 ); + invariant( x != y ); + } + + void run() { + _run( 123, 456 ); + _run( 0xdeadbeef, 0xcafecafe ); + _run( 0, 0 ); + _run( 99999999, 999 ); + _run( 0xFFFFFFFF, 1 ); + _run( 0xFFFFFFFF, 0 ); + _run( 0xFFFFFFFF, 0xFFFFFFFF ); + } + } idWorkTest; + } + + class GlobalCursorIdCache { + public: + + GlobalCursorIdCache(); + ~GlobalCursorIdCache(); + + /** + * this gets called when a CollectionCursorCache gets created + * @return the id the CollectionCursorCache should use when generating + * cursor ids + */ + unsigned created( const std::string& ns ); + + /** + * called by CollectionCursorCache when its going away + */ + void destroyed( unsigned id, const std::string& ns ); + + /** + * works globally + */ + bool eraseCursor( CursorId id, bool checkAuth ); + + void appendStats( BSONObjBuilder& builder ); + + std::size_t timeoutCursors( unsigned millisSinceLastCall ); + + int64_t nextSeed(); + + private: + SimpleMutex _mutex; + + typedef unordered_map Map; + Map _idToNS; + unsigned _nextId; + + SecureRandom* _secureRandom; + } _globalCursorIdCache; + + GlobalCursorIdCache::GlobalCursorIdCache() + : _mutex( "GlobalCursorIdCache" ), + _nextId( 0 ), + _secureRandom( NULL ) { + } + + GlobalCursorIdCache::~GlobalCursorIdCache() { + // we're just going to leak everything, as it doesn't matter + } + + int64_t GlobalCursorIdCache::nextSeed() { + SimpleMutex::scoped_lock lk( _mutex ); + if ( !_secureRandom ) + _secureRandom = SecureRandom::create(); + return _secureRandom->nextInt64(); + } + + unsigned GlobalCursorIdCache::created( const std::string& ns ) { + static const unsigned MAX_IDS = 1000 * 1000 * 1000; + + SimpleMutex::scoped_lock lk( _mutex ); + + fassert( 17359, _idToNS.size() < MAX_IDS ); + + for ( unsigned i = 0; i <= MAX_IDS; i++ ) { + unsigned id = ++_nextId; + if ( id == 0 ) + continue; + if ( _idToNS.count( id ) > 0 ) + continue; + _idToNS[id] = ns; + return id; + } + + invariant( false ); + } + + void GlobalCursorIdCache::destroyed( unsigned id, const std::string& ns ) { + SimpleMutex::scoped_lock lk( _mutex ); + invariant( ns == _idToNS[id] ); + _idToNS.erase( id ); + } + + bool GlobalCursorIdCache::eraseCursor(CursorId id, bool checkAuth) { + string ns; + { + SimpleMutex::scoped_lock lk( _mutex ); + unsigned nsid = idFromCursorId( id ); + Map::const_iterator it = _idToNS.find( nsid ); + if ( it == _idToNS.end() ) { + return false; + } + ns = it->second; + } + + NamespaceString nss( ns ); + + if ( checkAuth ) { + AuthorizationSession* as = cc().getAuthorizationSession(); + bool isAuthorized = as->isAuthorizedForActionsOnNamespace(nss, + ActionType::killCursors); + if ( !isAuthorized ) { + audit::logKillCursorsAuthzCheck( currentClient.get(), + nss, + id, + ErrorCodes::Unauthorized ); + return false; + } + } + + Lock::DBRead lock( ns ); + Database* db = dbHolder().get( ns, storageGlobalParams.dbpath ); + if ( !db ) + return false; + Client::Context context( ns, db ); + Collection* collection = db->getCollection( ns ); + if ( !collection ) { + if ( checkAuth ) + audit::logKillCursorsAuthzCheck( currentClient.get(), + nss, + id, + ErrorCodes::CursorNotFound ); + return false; + } + + return collection->cursorCache()->eraseCursor( id, checkAuth ); + } + + std::size_t GlobalCursorIdCache::timeoutCursors( unsigned millisSinceLastCall ) { + vector todo; + { + SimpleMutex::scoped_lock lk( _mutex ); + for ( Map::const_iterator i = _idToNS.begin(); i != _idToNS.end(); ++i ) + todo.push_back( i->second ); + } + + size_t totalTimedOut = 0; + + for ( unsigned i = 0; i < todo.size(); i++ ) { + const string& ns = todo[i]; + Lock::DBRead lock( ns ); + Database* db = dbHolder().get( ns, storageGlobalParams.dbpath ); + if ( !db ) + continue; + Client::Context context( ns, db ); + Collection* collection = db->getCollection( ns ); + if ( collection == NULL ) { + continue; + } + + totalTimedOut += collection->cursorCache()->timeoutCursors( millisSinceLastCall ); + + } + + return totalTimedOut; + } + + // --- + + std::size_t CollectionCursorCache::timeoutCursorsGlobal( unsigned millisSinceLastCall ) { + return _globalCursorIdCache.timeoutCursors( millisSinceLastCall ); + } + + int CollectionCursorCache::eraseCursorGlobalIfAuthorized(int n, const long long* ids) { + int numDeleted = 0; + for ( int i = 0; i < n; i++ ) { + if ( eraseCursorGlobalIfAuthorized( ids[i] ) ) + numDeleted++; + if ( inShutdown() ) + break; + } + return numDeleted; + } + bool CollectionCursorCache::eraseCursorGlobalIfAuthorized(CursorId id) { + return _globalCursorIdCache.eraseCursor( id, true ); + } + bool CollectionCursorCache::eraseCursorGlobal( CursorId id ) { + return _globalCursorIdCache.eraseCursor( id, false ); + } + + + // -------------------------- + + + CollectionCursorCache::CollectionCursorCache( const StringData& ns ) + : _nss( ns ), + _mutex( "CollectionCursorCache" ) { + _collectionCacheRuntimeId = _globalCursorIdCache.created( _nss.ns() ); + _random.reset( new PseudoRandom( _globalCursorIdCache.nextSeed() ) ); + } + + CollectionCursorCache::~CollectionCursorCache() { + invalidateAll( true ); + _globalCursorIdCache.destroyed( _collectionCacheRuntimeId, _nss.ns() ); + } + + void CollectionCursorCache::invalidateAll( bool collectionGoingAway ) { + SimpleMutex::scoped_lock lk( _mutex ); + + for ( RunnerSet::iterator it = _nonCachedRunners.begin(); + it != _nonCachedRunners.end(); + ++it ) { + + // we kill the runner, but it deletes itself + Runner* runner = *it; + runner->kill(); + invariant( runner->collection() == NULL ); + } + _nonCachedRunners.clear(); + + if ( collectionGoingAway ) { + // we're going to wipe out the world + for ( CursorMap::const_iterator i = _cursors.begin(); i != _cursors.end(); ++i ) { + ClientCursor* cc = i->second; + + cc->kill(); + + invariant( cc->getRunner() == NULL || cc->getRunner()->collection() == NULL ); + + // If there is a pinValue >= 100, somebody is actively using the CC and we do + // not delete it. Instead we notify the holder that we killed it. The holder + // will then delete the CC. + // pinvalue is <100, so there is nobody actively holding the CC. We can + // safely delete it as nobody is holding the CC. + + if (cc->pinValue() < 100) { + delete cc; + } + } + } + else { + CursorMap newMap; + + // collection will still be around, just all Runners are invalid + for ( CursorMap::const_iterator i = _cursors.begin(); i != _cursors.end(); ++i ) { + ClientCursor* cc = i->second; + + // Note that a valid ClientCursor state is "no cursor no runner." This is because + // the set of active cursor IDs in ClientCursor is used as representation of query + // state. See sharding_block.h. TODO(greg,hk): Move this out. + if (NULL == cc->getRunner() ) { + newMap.insert( *i ); + continue; + } + + if (cc->pinValue() >= 100 || cc->isAggCursor) { + // Pinned cursors need to stay alive, so we leave them around. Aggregation + // cursors also can stay alive (since they don't have their lifetime bound to + // the underlying collection). However, if they have an associated runner, we + // need to kill it, because it's now invalid. + if ( cc->getRunner() ) + cc->getRunner()->kill(); + newMap.insert( *i ); + } + else { + cc->kill(); + delete cc; + } + + } + + _cursors = newMap; + } + } + + void CollectionCursorCache::invalidateDocument( const DiskLoc& dl, + InvalidationType type ) { + SimpleMutex::scoped_lock lk( _mutex ); + + for ( RunnerSet::iterator it = _nonCachedRunners.begin(); + it != _nonCachedRunners.end(); + ++it ) { + + Runner* runner = *it; + runner->invalidate(dl, type); + } + + for ( CursorMap::const_iterator i = _cursors.begin(); i != _cursors.end(); ++i ) { + Runner* runner = i->second->getRunner(); + if ( runner ) { + runner->invalidate(dl, type); + } + } + } + + std::size_t CollectionCursorCache::timeoutCursors( unsigned millisSinceLastCall ) { + SimpleMutex::scoped_lock lk( _mutex ); + + vector toDelete; + + for ( CursorMap::const_iterator i = _cursors.begin(); i != _cursors.end(); ++i ) { + ClientCursor* cc = i->second; + if ( cc->shouldTimeout( millisSinceLastCall ) ) + toDelete.push_back( cc ); + } + + for ( vector::const_iterator i = toDelete.begin(); + i != toDelete.end(); ++i ) { + ClientCursor* cc = *i; + _deregisterCursor_inlock( cc ); + cc->kill(); + delete cc; + } + + return toDelete.size(); + } + + void CollectionCursorCache::registerRunner( Runner* runner ) { + SimpleMutex::scoped_lock lk( _mutex ); + const std::pair result = _nonCachedRunners.insert(runner); + invariant(result.second); // make sure this was inserted + } + + void CollectionCursorCache::deregisterRunner( Runner* runner ) { + SimpleMutex::scoped_lock lk( _mutex ); + _nonCachedRunners.erase( runner ); + } + + ClientCursor* CollectionCursorCache::find( CursorId id, bool pin ) { + SimpleMutex::scoped_lock lk( _mutex ); + CursorMap::const_iterator it = _cursors.find( id ); + if ( it == _cursors.end() ) + return NULL; + + ClientCursor* cursor = it->second; + if ( pin ) { + uassert( 12051, + "clientcursor already in use? driver problem?", + cursor->_pinValue < 100 ); + cursor->_pinValue += 100; + } + + return cursor; + } + + void CollectionCursorCache::unpin( ClientCursor* cursor ) { + SimpleMutex::scoped_lock lk( _mutex ); + + invariant( cursor->_pinValue >= 100 ); + cursor->_pinValue -= 100; + } + + void CollectionCursorCache::getCursorIds( std::set* openCursors ) { + SimpleMutex::scoped_lock lk( _mutex ); + + for ( CursorMap::const_iterator i = _cursors.begin(); i != _cursors.end(); ++i ) { + ClientCursor* cc = i->second; + openCursors->insert( cc->cursorid() ); + } + } + + size_t CollectionCursorCache::numCursors(){ + SimpleMutex::scoped_lock lk( _mutex ); + return _cursors.size(); + } + + CursorId CollectionCursorCache::_allocateCursorId_inlock() { + for ( int i = 0; i < 10000; i++ ) { + unsigned mypart = static_cast( _random->nextInt32() ); + CursorId id = cursorIdFromParts( _collectionCacheRuntimeId, mypart ); + if ( _cursors.count( id ) == 0 ) + return id; + } + fassertFailed( 17360 ); + } + + CursorId CollectionCursorCache::registerCursor( ClientCursor* cc ) { + invariant( cc ); + SimpleMutex::scoped_lock lk( _mutex ); + CursorId id = _allocateCursorId_inlock(); + _cursors[id] = cc; + return id; + } + + void CollectionCursorCache::deregisterCursor( ClientCursor* cc ) { + SimpleMutex::scoped_lock lk( _mutex ); + _deregisterCursor_inlock( cc ); + } + + bool CollectionCursorCache::eraseCursor( CursorId id, bool checkAuth ) { + + SimpleMutex::scoped_lock lk( _mutex ); + + CursorMap::iterator it = _cursors.find( id ); + if ( it == _cursors.end() ) { + if ( checkAuth ) + audit::logKillCursorsAuthzCheck( currentClient.get(), + _nss, + id, + ErrorCodes::CursorNotFound ); + return false; + } + + ClientCursor* cursor = it->second; + + if ( checkAuth ) + audit::logKillCursorsAuthzCheck( currentClient.get(), + _nss, + id, + ErrorCodes::OK ); + + massert( 16089, + str::stream() << "Cannot kill active cursor " << id, + cursor->pinValue() < 100 ); + + cursor->kill(); + _deregisterCursor_inlock( cursor ); + delete cursor; + return true; + } + + void CollectionCursorCache::_deregisterCursor_inlock( ClientCursor* cc ) { + invariant( cc ); + CursorId id = cc->cursorid(); + _cursors.erase( id ); + } + +} diff --git a/src/mongo/db/catalog/collection_cursor_cache.h b/src/mongo/db/catalog/collection_cursor_cache.h new file mode 100644 index 00000000000..503acdf5c40 --- /dev/null +++ b/src/mongo/db/catalog/collection_cursor_cache.h @@ -0,0 +1,139 @@ +// collection_cursor_cache.h + +/** +* Copyright (C) 2013 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/clientcursor.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/invalidation_type.h" +#include "mongo/db/namespace_string.h" +#include "mongo/platform/unordered_set.h" +#include "mongo/util/concurrency/mutex.h" + +namespace mongo { + + class PseudoRandom; + class Runner; + + class CollectionCursorCache { + public: + CollectionCursorCache( const StringData& ns ); + + /** + * will kill() all Runner instances it has + */ + ~CollectionCursorCache(); + + // ----------------- + + /** + * @param collectionGoingAway should be tru if the Collection instance is going away + * this could be because the db is being closed, or the + * collection/db is being dropped. + */ + void invalidateAll( bool collectionGoingAway ); + + /** + * Broadcast a document invalidation to all relevant Runner(s). invalidateDocument must + * called *before* the provided DiskLoc is about to be deleted or mutated. + */ + void invalidateDocument( const DiskLoc& dl, + InvalidationType type ); + + /* + * timesout cursors that have been idle for too long + * note: must have a readlock on the collection + * @return number timed out + */ + std::size_t timeoutCursors( unsigned millisSinceLastCall ); + + // ----------------- + + /** + * Register a runner so that it can be notified of deletion/invalidation during yields. + * Must be called before a runner yields. If a runner is cached (inside a ClientCursor) it + * MUST NOT be registered; the two are mutually exclusive. + */ + void registerRunner(Runner* runner); + + /** + * Remove a runner from the runner registry. + */ + void deregisterRunner(Runner* runner); + + // ----------------- + + CursorId registerCursor( ClientCursor* cc ); + void deregisterCursor( ClientCursor* cc ); + + bool eraseCursor( CursorId id, bool checkAuth ); + + void getCursorIds( std::set* openCursors ); + std::size_t numCursors(); + + /** + * @param pin - if true, will try to pin cursor + * if pinned already, will assert + * otherwise will pin + */ + ClientCursor* find( CursorId id, bool pin ); + + void unpin( ClientCursor* cursor ); + + // ---------------------- + + static int eraseCursorGlobalIfAuthorized( int n, const long long* ids ); + static bool eraseCursorGlobalIfAuthorized( CursorId id ); + + static bool eraseCursorGlobal( CursorId id ); + + /** + * @return number timed out + */ + static std::size_t timeoutCursorsGlobal( unsigned millisSinceLastCall ); + + private: + CursorId _allocateCursorId_inlock(); + void _deregisterCursor_inlock( ClientCursor* cc ); + + NamespaceString _nss; + unsigned _collectionCacheRuntimeId; + scoped_ptr _random; + + SimpleMutex _mutex; + + typedef unordered_set RunnerSet; + RunnerSet _nonCachedRunners; + + typedef std::map CursorMap; + CursorMap _cursors; + }; + +} diff --git a/src/mongo/db/catalog/collection_info_cache.cpp b/src/mongo/db/catalog/collection_info_cache.cpp new file mode 100644 index 00000000000..cf11069e7fc --- /dev/null +++ b/src/mongo/db/catalog/collection_info_cache.cpp @@ -0,0 +1,131 @@ +// collection_info_cache.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/collection_info_cache.h" + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/index_pregen.h" +#include "mongo/db/d_concurrency.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/structure/catalog/namespace_details-inl.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/util/debug_util.h" + +#include "mongo/db/structure/catalog/index_details.h" // XXX +#include "mongo/db/pdfile.h" // XXX + +namespace mongo { + + CollectionInfoCache::CollectionInfoCache( Collection* collection ) + : _collection( collection ), + _keysComputed( false ), + _planCache(new PlanCache(collection->ns().ns())), + _querySettings(new QuerySettings()) { } + + void CollectionInfoCache::reset() { + Lock::assertWriteLocked( _collection->ns().ns() ); + LOG(1) << _collection->ns().ns() << ": clearing plan cache - collection info cache reset"; + clearQueryCache(); + _keysComputed = false; + // query settings is not affected by info cache reset. + // index filters should persist throughout life of collection + + GeneratorHolder::getInstance()->reset( _collection ); + } + + void CollectionInfoCache::computeIndexKeys() { + DEV Lock::assertWriteLocked( _collection->ns().ns() ); + + _indexedPaths.clear(); + + IndexCatalog::IndexIterator i = _collection->getIndexCatalog()->getIndexIterator(true); + while (i.more()) { + IndexDescriptor* descriptor = i.next(); + + if (descriptor->getAccessMethodName() != IndexNames::TEXT) { + BSONObj key = descriptor->keyPattern(); + BSONObjIterator j(key); + while (j.more()) { + BSONElement e = j.next(); + _indexedPaths.addPath(e.fieldName()); + } + } + else { + fts::FTSSpec ftsSpec(descriptor->infoObj()); + + if (ftsSpec.wildcard()) { + _indexedPaths.allPathsIndexed(); + } + else { + for (size_t i = 0; i < ftsSpec.numExtraBefore(); ++i) { + _indexedPaths.addPath(ftsSpec.extraBefore(i)); + } + for (fts::Weights::const_iterator it = ftsSpec.weights().begin(); + it != ftsSpec.weights().end(); + ++it) { + _indexedPaths.addPath(it->first); + } + for (size_t i = 0; i < ftsSpec.numExtraAfter(); ++i) { + _indexedPaths.addPath(ftsSpec.extraAfter(i)); + } + // Any update to a path containing "language" as a component could change the + // language of a subdocument. Add the override field as a path component. + _indexedPaths.addPathComponent(ftsSpec.languageOverrideField()); + } + } + } + + _keysComputed = true; + + } + + void CollectionInfoCache::notifyOfWriteOp() { + if (NULL != _planCache.get()) { + _planCache->notifyOfWriteOp(); + } + } + + void CollectionInfoCache::clearQueryCache() { + if (NULL != _planCache.get()) { + _planCache->clear(); + } + } + + PlanCache* CollectionInfoCache::getPlanCache() const { + return _planCache.get(); + } + + QuerySettings* CollectionInfoCache::getQuerySettings() const { + return _querySettings.get(); + } + +} diff --git a/src/mongo/db/catalog/collection_info_cache.h b/src/mongo/db/catalog/collection_info_cache.h new file mode 100644 index 00000000000..83e98a71b0f --- /dev/null +++ b/src/mongo/db/catalog/collection_info_cache.h @@ -0,0 +1,112 @@ +// collection_info_cache.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/query_settings.h" +#include "mongo/db/update_index_data.h" + +namespace mongo { + + class Collection; + + /** + * this is for storing things that you want to cache about a single collection + * life cycle is managed for you from inside Collection + */ + class CollectionInfoCache { + public: + + CollectionInfoCache( Collection* collection ); + + /* + * resets entire cache state + */ + void reset(); + + // + // New Query Execution + // + + /** + * Get the PlanCache for this collection. + */ + PlanCache* getPlanCache() const; + + /** + * Get the QuerySettings for this collection. + */ + QuerySettings* getQuerySettings() const; + + // ------------------- + + /* get set of index keys for this namespace. handy to quickly check if a given + field is indexed (Note it might be a secondary component of a compound index.) + */ + const UpdateIndexData& indexKeys() { + if ( !_keysComputed ) + computeIndexKeys(); + return _indexedPaths; + } + + // --------------------- + + /** + * Called when an index is added to this collection. + */ + void addedIndex() { reset(); } + + void clearQueryCache(); + + /* you must notify the cache if you are doing writes, as query plan utility will change */ + void notifyOfWriteOp(); + + private: + + Collection* _collection; // not owned + + // --- index keys cache + bool _keysComputed; + UpdateIndexData _indexedPaths; + + // A cache for query plans. + boost::scoped_ptr _planCache; + + // Query settings. + // Includes index filters. + boost::scoped_ptr _querySettings; + + void computeIndexKeys(); + }; + +} // namespace mongo diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp new file mode 100644 index 00000000000..c4f4bcd272b --- /dev/null +++ b/src/mongo/db/catalog/database.cpp @@ -0,0 +1,848 @@ +// database.cpp + +/** +* Copyright (C) 2008 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/pch.h" + +#include "mongo/db/catalog/database.h" + +#include +#include + +#include "mongo/db/audit.h" +#include "mongo/db/auth/auth_index_d.h" +#include "mongo/db/background.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/catalog/database_holder.h" +#include "mongo/db/dbhelpers.h" +#include "mongo/db/structure/catalog/index_details.h" +#include "mongo/db/instance.h" +#include "mongo/db/introspect.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/ops/delete.h" +#include "mongo/db/server_parameters.h" +#include "mongo/db/storage_options.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + MONGO_EXPORT_SERVER_PARAMETER(newCollectionsUsePowerOf2Sizes, bool, true); + + Status CollectionOptions::parse( const BSONObj& options ) { + reset(); + + // During parsing, ignore some validation errors in order to accept options objects that + // were valid in previous versions of the server. SERVER-13737. + BSONObjIterator i( options ); + while ( i.more() ) { + BSONElement e = i.next(); + StringData fieldName = e.fieldName(); + + if ( fieldName == "capped" ) { + capped = e.trueValue(); + } + else if ( fieldName == "size" ) { + if ( !e.isNumber() ) { + // Ignoring for backwards compatibility. + continue; + } + cappedSize = e.numberLong(); + if ( cappedSize < 0 ) + return Status( ErrorCodes::BadValue, "size has to be >= 0" ); + cappedSize += 0xff; + cappedSize &= 0xffffffffffffff00LL; + if ( cappedSize < Extent::minSize() ) + cappedSize = Extent::minSize(); + } + else if ( fieldName == "max" ) { + if ( !options["capped"].trueValue() || !e.isNumber() ) { + // Ignoring for backwards compatibility. + continue; + } + cappedMaxDocs = e.numberLong(); + if ( !NamespaceDetails::validMaxCappedDocs( &cappedMaxDocs ) ) + return Status( ErrorCodes::BadValue, + "max in a capped collection has to be < 2^31 or not set" ); + } + else if ( fieldName == "$nExtents" ) { + if ( e.type() == Array ) { + BSONObjIterator j( e.Obj() ); + while ( j.more() ) { + BSONElement inner = j.next(); + initialExtentSizes.push_back( inner.numberInt() ); + } + } + else { + initialNumExtents = e.numberLong(); + } + } + else if ( fieldName == "autoIndexId" ) { + if ( e.trueValue() ) + autoIndexId = YES; + else + autoIndexId = NO; + } + else if ( fieldName == "flags" ) { + flags = e.numberInt(); + flagsSet = true; + } + else if ( fieldName == "temp" ) { + temp = e.trueValue(); + } + } + + return Status::OK(); + } + + BSONObj CollectionOptions::toBSON() const { + BSONObjBuilder b; + if ( capped ) { + b.appendBool( "capped", true ); + if ( cappedSize ) + b.appendNumber( "size", cappedSize ); + if ( cappedMaxDocs ) + b.appendNumber( "max", cappedMaxDocs ); + } + + if ( initialNumExtents ) + b.appendNumber( "$nExtents", initialNumExtents ); + if ( !initialExtentSizes.empty() ) + b.append( "$nExtents", initialExtentSizes ); + + if ( autoIndexId != DEFAULT ) + b.appendBool( "autoIndexId", autoIndexId == YES ); + + if ( flagsSet ) + b.append( "flags", flags ); + + if ( temp ) + b.appendBool( "temp", true ); + + return b.obj(); + } + + void massertNamespaceNotIndex( const StringData& ns, const StringData& caller ) { + massert( 17320, + str::stream() << "cannot do " << caller + << " on namespace with a $ in it: " << ns, + NamespaceString::normal( ns ) ); + } + + Database::~Database() { + verify( Lock::isW() ); + _magic = 0; + + for ( CollectionMap::const_iterator i = _collections.begin(); i != _collections.end(); ++i ) + delete i->second; + } + + Status Database::validateDBName( const StringData& dbname ) { + + if ( dbname.size() <= 0 ) + return Status( ErrorCodes::BadValue, "db name is empty" ); + + if ( dbname.size() >= 64 ) + return Status( ErrorCodes::BadValue, "db name is too long" ); + + if ( dbname.find( '.' ) != string::npos ) + return Status( ErrorCodes::BadValue, "db name cannot contain a ." ); + + if ( dbname.find( ' ' ) != string::npos ) + return Status( ErrorCodes::BadValue, "db name cannot contain a space" ); + +#ifdef _WIN32 + static const char* windowsReservedNames[] = { + "con", "prn", "aux", "nul", + "com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8", "com9", + "lpt1", "lpt2", "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9" + }; + + string lower( dbname.toString() ); + std::transform( lower.begin(), lower.end(), lower.begin(), ::tolower ); + for ( size_t i = 0; i < (sizeof(windowsReservedNames) / sizeof(char*)); ++i ) { + if ( lower == windowsReservedNames[i] ) { + stringstream errorString; + errorString << "db name \"" << dbname.toString() << "\" is a reserved name"; + return Status( ErrorCodes::BadValue, errorString.str() ); + } + } +#endif + + return Status::OK(); + } + + Database::Database(const char *nm, bool& newDb, const string& path ) + : _name(nm), _path(path), + _namespaceIndex( _path, _name ), + _extentManager(_name, _path, storageGlobalParams.directoryperdb), + _profileName(_name + ".system.profile"), + _namespacesName(_name + ".system.namespaces"), + _indexesName(_name + ".system.indexes"), + _collectionLock( "Database::_collectionLock" ) + { + Status status = validateDBName( _name ); + if ( !status.isOK() ) { + warning() << "tried to open invalid db: " << _name << endl; + uasserted( 10028, status.toString() ); + } + + try { + newDb = _namespaceIndex.exists(); + _profile = serverGlobalParams.defaultProfile; + checkDuplicateUncasedNames(true); + + // If already exists, open. Otherwise behave as if empty until + // there's a write, then open. + if (!newDb) { + _namespaceIndex.init(); + openAllFiles(); + + // upgrade freelist + string oldFreeList = _name + ".$freelist"; + NamespaceDetails* details = _namespaceIndex.details( oldFreeList ); + if ( details ) { + if ( !details->firstExtent().isNull() ) { + _extentManager.freeExtents(details->firstExtent(), + details->lastExtent()); + } + _namespaceIndex.kill_ns( oldFreeList ); + } + } + _magic = 781231; + } + catch(std::exception& e) { + log() << "warning database " << path << " " << nm << " could not be opened" << endl; + DBException* dbe = dynamic_cast(&e); + if ( dbe != 0 ) { + log() << "DBException " << dbe->getCode() << ": " << e.what() << endl; + } + else { + log() << e.what() << endl; + } + _extentManager.reset(); + throw; + } + } + + void Database::checkDuplicateUncasedNames(bool inholderlock) const { + string duplicate = duplicateUncasedName(inholderlock, _name, _path ); + if ( !duplicate.empty() ) { + stringstream ss; + ss << "db already exists with different case already have: [" << duplicate + << "] trying to create [" << _name << "]"; + uasserted( DatabaseDifferCaseCode , ss.str() ); + } + } + + /*static*/ + string Database::duplicateUncasedName( bool inholderlock, const string &name, const string &path, set< string > *duplicates ) { + Lock::assertAtLeastReadLocked(name); + + if ( duplicates ) { + duplicates->clear(); + } + + vector others; + getDatabaseNames( others , path ); + + set allShortNames; + dbHolder().getAllShortNames( allShortNames ); + + others.insert( others.end(), allShortNames.begin(), allShortNames.end() ); + + for ( unsigned i=0; iinsert( others[i] ); + } else { + return others[i]; + } + } + if ( duplicates ) { + return duplicates->empty() ? "" : *duplicates->begin(); + } + return ""; + } + + // todo : we stop once a datafile dne. + // if one datafile were missing we should keep going for + // repair purposes yet we do not. + void Database::openAllFiles() { + verify(this); + Status s = _extentManager.init(); + if ( !s.isOK() ) { + msgasserted( 16966, str::stream() << "_extentManager.init failed: " << s.toString() ); + } + } + + void Database::clearTmpCollections() { + + Lock::assertWriteLocked( _name ); + Client::Context ctx( _name ); + + string systemNamespaces = _name + ".system.namespaces"; + + // Note: we build up a toDelete vector rather than dropping the collection inside the loop + // to avoid modifying the system.namespaces collection while iterating over it since that + // would corrupt the cursor. + vector toDelete; + auto_ptr runner(InternalPlanner::collectionScan(systemNamespaces)); + BSONObj nsObj; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&nsObj, NULL))) { + BSONElement e = nsObj.getFieldDotted( "options.temp" ); + if ( !e.trueValue() ) + continue; + + string ns = nsObj["name"].String(); + + // Do not attempt to drop indexes + if ( !NamespaceString::normal(ns.c_str()) ) + continue; + + toDelete.push_back(ns); + } + + if (Runner::RUNNER_EOF != state) { + warning() << "Internal error while reading collection " << systemNamespaces << endl; + } + + for (size_t i=0; i < toDelete.size(); i++) { + BSONObj info; + // using DBDirectClient to ensure this ends up in opLog + bool ok = DBDirectClient().dropCollection(toDelete[i], &info); + if (!ok) + warning() << "could not drop temp collection '" << toDelete[i] << "': " << info; + } + } + + bool Database::setProfilingLevel( int newLevel , string& errmsg ) { + if ( _profile == newLevel ) + return true; + + if ( newLevel < 0 || newLevel > 2 ) { + errmsg = "profiling level has to be >=0 and <= 2"; + return false; + } + + if ( newLevel == 0 ) { + _profile = 0; + return true; + } + + verify( cc().database() == this ); + + if (!getOrCreateProfileCollection(this, true, &errmsg)) + return false; + + _profile = newLevel; + return true; + } + + Status Database::dropCollection( const StringData& fullns ) { + LOG(1) << "dropCollection: " << fullns << endl; + massertNamespaceNotIndex( fullns, "dropCollection" ); + + Collection* collection = getCollection( fullns ); + if ( !collection ) { + // collection doesn't exist + return Status::OK(); + } + + { + NamespaceString s( fullns ); + verify( s.db() == _name ); + + if( s.isSystem() ) { + if( s.coll() == "system.profile" ) { + if ( _profile != 0 ) + return Status( ErrorCodes::IllegalOperation, + "turn off profiling before dropping system.profile collection" ); + } + else { + return Status( ErrorCodes::IllegalOperation, "can't drop system ns" ); + } + } + } + + BackgroundOperation::assertNoBgOpInProgForNs( fullns ); + + audit::logDropCollection( currentClient.get(), fullns ); + + GeneratorHolder::getInstance()->dropped( fullns.toString() ); + + try { + Status s = collection->getIndexCatalog()->dropAllIndexes( true ); + if ( !s.isOK() ) { + warning() << "could not drop collection, trying to drop indexes" + << fullns << " because of " << s.toString(); + return s; + } + } + catch( DBException& e ) { + stringstream ss; + ss << "drop: dropIndexes for collection failed. cause: " << e.what(); + ss << ". See http://dochub.mongodb.org/core/data-recovery"; + warning() << ss.str() << endl; + return Status( ErrorCodes::InternalError, ss.str() ); + } + + verify( collection->_details->getTotalIndexCount() == 0 ); + LOG(1) << "\t dropIndexes done" << endl; + + Top::global.collectionDropped( fullns ); + + Status s = _dropNS( fullns ); + + _clearCollectionCache( fullns ); // we want to do this always + + GeneratorHolder::getInstance()->dropped( fullns.toString() ); + + if ( !s.isOK() ) + return s; + + DEV { + // check all index collection entries are gone + string nstocheck = fullns.toString() + ".$"; + scoped_lock lk( _collectionLock ); + for ( CollectionMap::const_iterator i = _collections.begin(); + i != _collections.end(); + ++i ) { + string temp = i->first; + if ( temp.find( nstocheck ) != 0 ) + continue; + log() << "after drop, bad cache entries for: " + << fullns << " have " << temp; + verify(0); + } + } + + return Status::OK(); + } + + void Database::_clearCollectionCache( const StringData& fullns ) { + scoped_lock lk( _collectionLock ); + _clearCollectionCache_inlock( fullns ); + } + + void Database::_clearCollectionCache_inlock( const StringData& fullns ) { + verify( _name == nsToDatabaseSubstring( fullns ) ); + CollectionMap::const_iterator it = _collections.find( fullns.toString() ); + if ( it == _collections.end() ) + return; + + delete it->second; // this also deletes all cursors + runners + _collections.erase( it ); + } + + Collection* Database::getCollection( const StringData& ns ) { + verify( _name == nsToDatabaseSubstring( ns ) ); + + scoped_lock lk( _collectionLock ); + + CollectionMap::const_iterator it = _collections.find( ns ); + if ( it != _collections.end() ) { + if ( it->second ) { + DEV { + NamespaceDetails* details = _namespaceIndex.details( ns ); + if ( details != it->second->_details ) { + log() << "about to crash for mismatch on ns: " << ns + << " current: " << (void*)details + << " cached: " << (void*)it->second->_details; + } + verify( details == it->second->_details ); + } + return it->second; + } + } + + NamespaceDetails* details = _namespaceIndex.details( ns ); + if ( !details ) { + return NULL; + } + + Collection* c = new Collection( ns, details, this ); + _collections[ns] = c; + return c; + } + + + + Status Database::renameCollection( const StringData& fromNS, const StringData& toNS, + bool stayTemp ) { + + // move data namespace + Status s = _renameSingleNamespace( fromNS, toNS, stayTemp ); + if ( !s.isOK() ) + return s; + + NamespaceDetails* details = _namespaceIndex.details( toNS ); + verify( details ); + + audit::logRenameCollection( currentClient.get(), fromNS, toNS ); + + // move index namespaces + BSONObj oldIndexSpec; + while( Helpers::findOne( _indexesName, BSON( "ns" << fromNS ), oldIndexSpec ) ) { + oldIndexSpec = oldIndexSpec.getOwned(); + + BSONObj newIndexSpec; + { + BSONObjBuilder b; + BSONObjIterator i( oldIndexSpec ); + while( i.more() ) { + BSONElement e = i.next(); + if ( strcmp( e.fieldName(), "ns" ) != 0 ) + b.append( e ); + else + b << "ns" << toNS; + } + newIndexSpec = b.obj(); + } + + StatusWith newIndexSpecLoc = + getCollection( _indexesName )->insertDocument( newIndexSpec, false ); + if ( !newIndexSpecLoc.isOK() ) + return newIndexSpecLoc.getStatus(); + + int indexI = details->_catalogFindIndexByName( oldIndexSpec.getStringField( "name" ) ); + IndexDetails &indexDetails = details->idx(indexI); + string oldIndexNs = indexDetails.indexNamespace(); + indexDetails.info = newIndexSpecLoc.getValue(); + string newIndexNs = indexDetails.indexNamespace(); + + Status s = _renameSingleNamespace( oldIndexNs, newIndexNs, false ); + if ( !s.isOK() ) + return s; + + const BSONObj oldIndexSpecQueryObj = BSON("ns" << fromNS << + "name" << oldIndexSpec["name"]); + deleteObjects( _indexesName, oldIndexSpecQueryObj, true, false, true ); + } + + Top::global.collectionDropped( fromNS.toString() ); + + return Status::OK(); + } + + Status Database::_renameSingleNamespace( const StringData& fromNS, const StringData& toNS, + bool stayTemp ) { + + // TODO: make it so we dont't need to do this + string fromNSString = fromNS.toString(); + string toNSString = toNS.toString(); + + // some sanity checking + NamespaceDetails* fromDetails = _namespaceIndex.details( fromNS ); + if ( !fromDetails ) + return Status( ErrorCodes::BadValue, "from namespace doesn't exist" ); + + if ( _namespaceIndex.details( toNS ) ) + return Status( ErrorCodes::BadValue, "to namespace already exists" ); + + // remove anything cached + { + scoped_lock lk( _collectionLock ); + _clearCollectionCache_inlock( fromNSString ); + _clearCollectionCache_inlock( toNSString ); + } + + // at this point, we haven't done anything destructive yet + + // ---- + // actually start moving + // ---- + + // this could throw, but if it does we're ok + _namespaceIndex.add_ns( toNS, fromDetails ); + NamespaceDetails* toDetails = _namespaceIndex.details( toNS ); + + try { + toDetails->copyingFrom(toNSString.c_str(), fromDetails); // fixes extraOffset + } + catch( DBException& ) { + // could end up here if .ns is full - if so try to clean up / roll back a little + _namespaceIndex.kill_ns( toNSString ); + _clearCollectionCache(toNSString); + throw; + } + + // at this point, code .ns stuff moved + + _namespaceIndex.kill_ns( fromNSString ); + _clearCollectionCache(fromNSString); + fromDetails = NULL; + + // fix system.namespaces + BSONObj newSpec; + { + + BSONObj oldSpec; + if ( !Helpers::findOne( _namespacesName, BSON( "name" << fromNS ), oldSpec ) ) + return Status( ErrorCodes::InternalError, "can't find system.namespaces entry" ); + + BSONObjBuilder b; + BSONObjIterator i( oldSpec.getObjectField( "options" ) ); + while( i.more() ) { + BSONElement e = i.next(); + if ( strcmp( e.fieldName(), "create" ) != 0 ) { + if (stayTemp || (strcmp(e.fieldName(), "temp") != 0)) + b.append( e ); + } + else { + b << "create" << toNS; + } + } + newSpec = b.obj(); + } + + _addNamespaceToCatalog( toNSString, newSpec.isEmpty() ? 0 : &newSpec ); + + deleteObjects( _namespacesName, BSON( "name" << fromNS ), false, false, true ); + + return Status::OK(); + } + + Collection* Database::getOrCreateCollection( const StringData& ns ) { + Collection* c = getCollection( ns ); + if ( !c ) { + c = createCollection( ns ); + } + return c; + } + + namespace { + int _massageExtentSize( long long size ) { + if ( size < Extent::minSize() ) + return Extent::minSize(); + if ( size > Extent::maxSize() ) + return Extent::maxSize(); + return static_cast( size ); + } + } + + Collection* Database::createCollection( const StringData& ns, + const CollectionOptions& options, + bool allocateDefaultSpace, + bool createIdIndex ) { + massert( 17399, "collection already exists", _namespaceIndex.details( ns ) == NULL ); + massertNamespaceNotIndex( ns, "createCollection" ); + _namespaceIndex.init(); + + if ( serverGlobalParams.configsvr && + !( ns.startsWith( "config." ) || + ns.startsWith( "local." ) || + ns.startsWith( "admin." ) ) ) { + uasserted(14037, "can't create user databases on a --configsvr instance"); + } + + if (NamespaceString::normal(ns)) { + // This check only applies for actual collections, not indexes or other types of ns. + uassert(17381, str::stream() << "fully qualified namespace " << ns << " is too long " + << "(max is " << Namespace::MaxNsColletionLen << " bytes)", + ns.size() <= Namespace::MaxNsColletionLen); + } + + NamespaceString nss( ns ); + uassert( 17316, "cannot create a blank collection", nss.coll() > 0 ); + + audit::logCreateCollection( currentClient.get(), ns ); + + _namespaceIndex.add_ns( ns, DiskLoc(), options.capped ); + BSONObj optionsAsBSON = options.toBSON(); + _addNamespaceToCatalog( ns, &optionsAsBSON ); + + Collection* collection = getCollection( ns ); + massert( 17400, "_namespaceIndex.add_ns failed?", collection ); + + NamespaceDetails* nsd = collection->details(); + + // allocation strategy set explicitly in flags or by server-wide default + if ( !options.capped ) { + if ( options.flagsSet ) { + nsd->setUserFlag( options.flags ); + } + else if ( newCollectionsUsePowerOf2Sizes ) { + nsd->setUserFlag( NamespaceDetails::Flag_UsePowerOf2Sizes ); + } + } + + if ( options.cappedMaxDocs > 0 ) + nsd->setMaxCappedDocs( options.cappedMaxDocs ); + + if ( allocateDefaultSpace ) { + if ( options.initialNumExtents > 0 ) { + int size = _massageExtentSize( options.cappedSize ); + for ( int i = 0; i < options.initialNumExtents; i++ ) { + collection->increaseStorageSize( size, false ); + } + } + else if ( !options.initialExtentSizes.empty() ) { + for ( size_t i = 0; i < options.initialExtentSizes.size(); i++ ) { + int size = options.initialExtentSizes[i]; + size = _massageExtentSize( size ); + collection->increaseStorageSize( size, false ); + } + } + else if ( options.capped ) { + // normal + long long size = options.cappedSize; + while ( size > 0 ) { + int mySize = _massageExtentSize( size ); + mySize &= 0xffffff00; + Extent* e = collection->increaseStorageSize( mySize, true ); + size -= e->length; + } + } + else { + collection->increaseStorageSize( Extent::initialSize( 128 ), false ); + } + } + + if ( createIdIndex ) { + if ( collection->requiresIdIndex() ) { + if ( options.autoIndexId == CollectionOptions::YES || + options.autoIndexId == CollectionOptions::DEFAULT ) { + uassertStatusOK( collection->getIndexCatalog()->ensureHaveIdIndex() ); + } + } + + if ( nss.isSystem() ) { + authindex::createSystemIndexes( collection ); + } + + } + + return collection; + } + + void Database::cleanUpOrphanIndexesOnSystemCollection() { + Collection* namespacesCollection = getCollection( _namespacesName ); + if ( !namespacesCollection ) { + return; + } + Collection* systemCollection = getCollection( _name + ".system" ); // May be NULL. + Collection* indexesCollection = getCollection( _indexesName ); // May be NULL. + scoped_ptr it( + namespacesCollection->getIterator( DiskLoc(), false, CollectionScanParams::FORWARD ) ); + while ( !it->isEOF() ) { + NamespaceString ns( namespacesCollection->docFor( it->getNext() )["name"].String() ); + if ( ns.coll().startsWith( "system.$" ) ) { + // Found an index on collection named "system". + StringData indexName = ns.coll().substr( strlen( "system.$" ) ); + if ( systemCollection && + systemCollection->getIndexCatalog()->findIndexByName( indexName, true ) ) { + // Index is not an orphan, ignore it. + continue; + } + if ( indexesCollection && !Helpers::findOne( indexesCollection->ns().ns(), + BSON( "name" << indexName << + "ns" << ( _name + ".system" ) ), + false ).isNull() ) { + // Index is listed in system.indexes, but isn't in the catalog. Log a startup + // warning. + warning() << "found an index missing from catalog: " << ns.ns() + << startupWarningsLog; + continue; + } + // This index is an orphan. Either the "system" collection doesn't exist, or the + // "system" collection exists but the index isn't in the catalog. Clean it up. + log() << "dropping orphaned index: " << ns.ns(); + fassert( 17492, _dropNS( ns.ns() ) ); + } + } + } + + void Database::_addNamespaceToCatalog( const StringData& ns, const BSONObj* options ) { + LOG(1) << "Database::_addNamespaceToCatalog ns: " << ns << endl; + if ( nsToCollectionSubstring( ns ) == "system.namespaces" ) { + // system.namespaces holds all the others, so it is not explicitly listed in the catalog. + return; + } + + BSONObjBuilder b; + b.append("name", ns); + if ( options && !options->isEmpty() ) + b.append("options", *options); + BSONObj obj = b.done(); + + Collection* collection = getCollection( _namespacesName ); + if ( !collection ) + collection = createCollection( _namespacesName ); + StatusWith loc = collection->insertDocument( obj, false ); + uassertStatusOK( loc.getStatus() ); + } + + Status Database::_dropNS( const StringData& ns ) { + + NamespaceDetails* d = _namespaceIndex.details( ns ); + if ( !d ) + return Status( ErrorCodes::NamespaceNotFound, + str::stream() << "ns not found: " << ns ); + + BackgroundOperation::assertNoBgOpInProgForNs( ns ); + + { + // remove from the system catalog + BSONObj cond = BSON( "name" << ns ); // { name: "colltodropname" } + deleteObjects( _namespacesName, cond, false, false, true); + } + + // free extents + if( !d->firstExtent().isNull() ) { + _extentManager.freeExtents(d->firstExtent(), d->lastExtent()); + d->setFirstExtentInvalid(); + d->setLastExtentInvalid(); + } + + // remove from the catalog hashtable + _namespaceIndex.kill_ns( ns ); + + return Status::OK(); + } + + void Database::getFileFormat( int* major, int* minor ) { + if ( _extentManager.numFiles() == 0 ) { + *major = 0; + *minor = 0; + return; + } + const DataFile* df = _extentManager.getFile( 0 ); + *major = df->getHeader()->version; + *minor = df->getHeader()->versionMinor; + } + +} // namespace mongo diff --git a/src/mongo/db/catalog/database.h b/src/mongo/db/catalog/database.h new file mode 100644 index 00000000000..6dbacd0d21d --- /dev/null +++ b/src/mongo/db/catalog/database.h @@ -0,0 +1,258 @@ +// database.h + +/** +* Copyright (C) 2008 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/storage/extent_manager.h" +#include "mongo/db/storage/record.h" +#include "mongo/db/storage_options.h" +#include "mongo/util/string_map.h" + +namespace mongo { + + class Collection; + class Extent; + class DataFile; + class IndexCatalog; + class IndexDetails; + + struct CollectionOptions { + CollectionOptions() { + reset(); + } + + void reset() { + capped = false; + cappedSize = 0; + cappedMaxDocs = 0; + initialNumExtents = 0; + initialExtentSizes.clear(); + autoIndexId = DEFAULT; + flags = 0; + flagsSet = false; + temp = false; + } + + Status parse( const BSONObj& obj ); + BSONObj toBSON() const; + + // ---- + + bool capped; + long long cappedSize; + long long cappedMaxDocs; + + // following 2 are mutually exclusive, can only have one set + long long initialNumExtents; + vector initialExtentSizes; + + // behavior of _id index creation when collection created + void setNoIdIndex() { autoIndexId = NO; } + enum { + DEFAULT, // currently yes for most collections, NO for some system ones + YES, // create _id index + NO // do not create _id index + } autoIndexId; + + // user flags + int flags; + bool flagsSet; + + bool temp; + }; + + /** + * Database represents a database database + * Each database database has its own set of files -- dbname.ns, dbname.0, dbname.1, ... + * NOT memory mapped + */ + class Database { + public: + // you probably need to be in dbHolderMutex when constructing this + Database(const char *nm, /*out*/ bool& newDb, + const string& path = storageGlobalParams.dbpath); + + /* you must use this to close - there is essential code in this method that is not in the ~Database destructor. + thus the destructor is private. this could be cleaned up one day... + */ + static void closeDatabase( const string& db, const string& path ); + + const string& name() const { return _name; } + const string& path() const { return _path; } + + void clearTmpCollections(); + + /** + * tries to make sure that this hasn't been deleted + */ + bool isOk() const { return _magic == 781231; } + + bool isEmpty() { return ! _namespaceIndex.allocated(); } + + /** + * total file size of Database in bytes + */ + long long fileSize() const { return _extentManager.fileSize(); } + + int numFiles() const { return _extentManager.numFiles(); } + + void getFileFormat( int* major, int* minor ); + + /** + * makes sure we have an extra file at the end that is empty + * safe to call this multiple times - the implementation will only preallocate one file + */ + void preallocateAFile() { _extentManager.preallocateAFile(); } + + /** + * @return true if success. false if bad level or error creating profile ns + */ + bool setProfilingLevel( int newLevel , string& errmsg ); + + void flushFiles( bool sync ) { return _extentManager.flushFiles( sync ); } + + /** + * @return true if ns is part of the database + * ns=foo.bar, db=foo returns true + */ + bool ownsNS( const string& ns ) const { + if ( ! startsWith( ns , _name ) ) + return false; + return ns[_name.size()] == '.'; + } + + const RecordStats& recordStats() const { return _recordStats; } + RecordStats& recordStats() { return _recordStats; } + + int getProfilingLevel() const { return _profile; } + const char* getProfilingNS() const { return _profileName.c_str(); } + + const NamespaceIndex& namespaceIndex() const { return _namespaceIndex; } + NamespaceIndex& namespaceIndex() { return _namespaceIndex; } + + // TODO: do not think this method should exist, so should try and encapsulate better + ExtentManager& getExtentManager() { return _extentManager; } + const ExtentManager& getExtentManager() const { return _extentManager; } + + Status dropCollection( const StringData& fullns ); + + Collection* createCollection( const StringData& ns, + const CollectionOptions& options = CollectionOptions(), + bool allocateSpace = true, + bool createDefaultIndexes = true ); + + /** + * @param ns - this is fully qualified, which is maybe not ideal ??? + */ + Collection* getCollection( const StringData& ns ); + + Collection* getCollection( const NamespaceString& ns ) { return getCollection( ns.ns() ); } + + Collection* getOrCreateCollection( const StringData& ns ); + + Status renameCollection( const StringData& fromNS, const StringData& toNS, bool stayTemp ); + + /** + * @return name of an existing database with same text name but different + * casing, if one exists. Otherwise the empty string is returned. If + * 'duplicates' is specified, it is filled with all duplicate names. + */ + static string duplicateUncasedName( bool inholderlockalready, const string &name, const string &path, set< string > *duplicates = 0 ); + + static Status validateDBName( const StringData& dbname ); + + const string& getSystemIndexesName() const { return _indexesName; } + + /** + * Search system.namespaces for indexes on collection "system" that don't exist in the index + * catalog, and clean up any that are found. These would exist due to a bug in 2.4 + * (SERVER-13975). + */ + void cleanUpOrphanIndexesOnSystemCollection(); + private: + + void _clearCollectionCache( const StringData& fullns ); + + void _clearCollectionCache_inlock( const StringData& fullns ); + + ~Database(); // closes files and other cleanup see below. + + void _addNamespaceToCatalog( const StringData& ns, const BSONObj* options ); + + + /** + * removes from *.system.namespaces + * frees extents + * removes from NamespaceIndex + * NOT RIGHT NOW, removes cache entry in Database TODO? + */ + Status _dropNS( const StringData& ns ); + + /** + * @throws DatabaseDifferCaseCode if the name is a duplicate based on + * case insensitive matching. + */ + void checkDuplicateUncasedNames(bool inholderlockalready) const; + + void openAllFiles(); + + Status _renameSingleNamespace( const StringData& fromNS, const StringData& toNS, + bool stayTemp ); + + const string _name; // "alleyinsider" + const string _path; // "/data/db" + + NamespaceIndex _namespaceIndex; + ExtentManager _extentManager; + + const string _profileName; // "alleyinsider.system.profile" + const string _namespacesName; // "alleyinsider.system.namespaces" + const string _indexesName; // "alleyinsider.system.indexes" + + RecordStats _recordStats; + int _profile; // 0=off. + + int _magic; // used for making sure the object is still loaded in memory + + // TODO: make sure deletes go through + // this in some ways is a dupe of _namespaceIndex + // but it points to a much more useful data structure + typedef StringMap< Collection* > CollectionMap; + CollectionMap _collections; + mutex _collectionLock; + + friend class Collection; + friend class NamespaceDetails; + friend class IndexDetails; + friend class IndexCatalog; + }; + +} // namespace mongo diff --git a/src/mongo/db/catalog/database_holder.cpp b/src/mongo/db/catalog/database_holder.cpp new file mode 100644 index 00000000000..e3d7f200a63 --- /dev/null +++ b/src/mongo/db/catalog/database_holder.cpp @@ -0,0 +1,127 @@ +// database_holder.cpp + +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/pch.h" + +#include "mongo/db/auth/auth_index_d.h" +#include "mongo/db/background.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/catalog/database_holder.h" + +namespace mongo { + + Database* DatabaseHolder::getOrCreate( const string& ns, const string& path, bool& justCreated ) { + string dbname = _todb( ns ); + { + SimpleMutex::scoped_lock lk(_m); + Lock::assertAtLeastReadLocked(ns); + DBs& m = _paths[path]; + { + DBs::iterator i = m.find(dbname); + if( i != m.end() ) { + justCreated = false; + return i->second; + } + } + + // todo: protect against getting sprayed with requests for different db names that DNE - + // that would make the DBs map very large. not clear what to do to handle though, + // perhaps just log it, which is what we do here with the "> 40" : + bool cant = !Lock::isWriteLocked(ns); + if( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1)) || + m.size() > 40 || cant || DEBUG_BUILD ) { + log() << "opening db: " + << (path == storageGlobalParams.dbpath ? "" : path) << ' ' << dbname + << endl; + } + massert(15927, "can't open database in a read lock. if db was just closed, consider retrying the query. might otherwise indicate an internal error", !cant); + } + + // we mark our thread as having done writes now as we do not want any exceptions + // once we start creating a new database + cc().writeHappened(); + + // this locks _m for defensive checks, so we don't want to be locked right here : + Database *db = new Database( dbname.c_str() , justCreated , path ); + + { + SimpleMutex::scoped_lock lk(_m); + DBs& m = _paths[path]; + verify( m[dbname] == 0 ); + m[dbname] = db; + _size++; + } + + return db; + } + + bool DatabaseHolder::closeAll( const string& path , BSONObjBuilder& result , bool force ) { + log() << "DatabaseHolder::closeAll path:" << path << endl; + verify( Lock::isW() ); + getDur().commitNow(); // bad things happen if we close a DB with outstanding writes + + map& m = _paths[path]; + _size -= m.size(); + + set< string > dbs; + for ( map::iterator i = m.begin(); i != m.end(); i++ ) { + wassert( i->second->path() == path ); + dbs.insert( i->first ); + } + + currentClient.get()->getContext()->_clear(); + + BSONObjBuilder bb( result.subarrayStart( "dbs" ) ); + int n = 0; + int nNotClosed = 0; + for( set< string >::iterator i = dbs.begin(); i != dbs.end(); ++i ) { + string name = *i; + LOG(2) << "DatabaseHolder::closeAll path:" << path << " name:" << name << endl; + Client::Context ctx( name , path ); + if( !force && BackgroundOperation::inProgForDb(name) ) { + log() << "WARNING: can't close database " << name << " because a bg job is in progress - try killOp command" << endl; + nNotClosed++; + } + else { + Database::closeDatabase( name.c_str() , path ); + bb.append( bb.numStr( n++ ) , name ); + } + } + bb.done(); + if( nNotClosed ) { + result.append("nNotClosed", nNotClosed); + } + + return true; + } + + +} diff --git a/src/mongo/db/catalog/database_holder.h b/src/mongo/db/catalog/database_holder.h new file mode 100644 index 00000000000..a5971a0e75e --- /dev/null +++ b/src/mongo/db/catalog/database_holder.h @@ -0,0 +1,140 @@ +// @file databaseholder.h + +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/catalog/database.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { + + /** + * path + dbname -> Database + */ + class DatabaseHolder { + typedef map DBs; + typedef map Paths; + // todo: we want something faster than this if called a lot: + mutable SimpleMutex _m; + Paths _paths; + int _size; + public: + DatabaseHolder() : _m("dbholder"),_size(0) { } + + bool __isLoaded( const string& ns , const string& path ) const { + SimpleMutex::scoped_lock lk(_m); + Paths::const_iterator x = _paths.find( path ); + if ( x == _paths.end() ) + return false; + const DBs& m = x->second; + + string db = _todb( ns ); + + DBs::const_iterator it = m.find(db); + return it != m.end(); + } + // must be write locked as otherwise isLoaded could go false->true on you + // in the background and you might not expect that. + bool _isLoaded( const string& ns , const string& path ) const { + Lock::assertWriteLocked(ns); + return __isLoaded(ns,path); + } + + Database * get( const string& ns , const string& path ) const { + SimpleMutex::scoped_lock lk(_m); + Lock::assertAtLeastReadLocked(ns); + Paths::const_iterator x = _paths.find( path ); + if ( x == _paths.end() ) + return 0; + const DBs& m = x->second; + string db = _todb( ns ); + DBs::const_iterator it = m.find(db); + if ( it != m.end() ) + return it->second; + return 0; + } + + Database* getOrCreate( const string& ns , const string& path , bool& justCreated ); + + void erase( const string& ns , const string& path ) { + SimpleMutex::scoped_lock lk(_m); + verify( Lock::isW() ); + DBs& m = _paths[path]; + _size -= (int)m.erase( _todb( ns ) ); + } + + /** @param force - force close even if something underway - use at shutdown */ + bool closeAll( const string& path , BSONObjBuilder& result, bool force ); + + // "info" as this is informational only could change on you if you are not write locked + int sizeInfo() const { return _size; } + + /** + * gets all unique db names, ignoring paths + * need some lock + */ + void getAllShortNames( set& all ) const { + SimpleMutex::scoped_lock lk(_m); + for ( Paths::const_iterator i=_paths.begin(); i!=_paths.end(); i++ ) { + DBs m = i->second; + for( DBs::const_iterator j=m.begin(); j!=m.end(); j++ ) { + all.insert( j->first ); + } + } + } + + private: + static string _todb( const string& ns ) { + string d = __todb( ns ); + uassert( 13280 , (string)"invalid db name: " + ns , NamespaceString::validDBName( d ) ); + return d; + } + static string __todb( const string& ns ) { + size_t i = ns.find( '.' ); + if ( i == string::npos ) { + uassert( 13074 , "db name can't be empty" , ns.size() ); + return ns; + } + uassert( 13075 , "db name can't be empty" , i > 0 ); + return ns.substr( 0 , i ); + } + }; + + DatabaseHolder& dbHolderUnchecked(); + inline const DatabaseHolder& dbHolder() { + dassert( Lock::isLocked() ); + return dbHolderUnchecked(); + } + inline DatabaseHolder& dbHolderW() { + dassert( Lock::isW() ); + return dbHolderUnchecked(); + } + +} diff --git a/src/mongo/db/catalog/index_catalog.cpp b/src/mongo/db/catalog/index_catalog.cpp new file mode 100644 index 00000000000..9cee0d97585 --- /dev/null +++ b/src/mongo/db/catalog/index_catalog.cpp @@ -0,0 +1,1424 @@ +// index_catalog.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/index_catalog.h" + +#include + +#include "mongo/base/owned_pointer_map.h" +#include "mongo/db/audit.h" +#include "mongo/db/background.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/index_create.h" +#include "mongo/db/catalog/index_key_validate.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/curop.h" +#include "mongo/db/db.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/index/2d_access_method.h" +#include "mongo/db/index/btree_access_method.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/index/fts_access_method.h" +#include "mongo/db/index/hash_access_method.h" +#include "mongo/db/index/haystack_access_method.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index/s2_access_method.h" +#include "mongo/db/index_legacy.h" +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/jsobjmanipulator.h" +#include "mongo/db/keypattern.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/ops/delete.h" +#include "mongo/db/pagefault.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/repl/rs.h" // this is ugly +#include "mongo/db/storage/data_file.h" +#include "mongo/db/structure/catalog/namespace_details-inl.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/log.h" + +namespace mongo { + + static const int INDEX_CATALOG_INIT = 283711; + static const int INDEX_CATALOG_UNINIT = 654321; + + // What's the default version of our indices? + const int DefaultIndexVersionNumber = 1; + + const BSONObj IndexCatalog::_idObj = BSON( "_id" << 1 ); + + // ------------- + + IndexCatalog::IndexCatalog( Collection* collection, NamespaceDetails* details ) + : _magic(INDEX_CATALOG_UNINIT), _collection( collection ), _details( details ) { + } + + IndexCatalog::~IndexCatalog() { + if ( _magic != INDEX_CATALOG_UNINIT ) { + // only do this check if we haven't been initialized + _checkMagic(); + } + _magic = 123456; + } + + Status IndexCatalog::init() { + + NamespaceDetails::IndexIterator ii = _details->ii(true); + while ( ii.more() ) { + IndexDetails& id = ii.next(); + int idxNo = ii.pos() - 1; + + if ( idxNo >= _details->getCompletedIndexCount() ) { + _unfinishedIndexes.push_back( id.info.obj().getOwned() ); + continue; + } + + BSONObj ownedInfoObj = id.info.obj().getOwned(); + BSONObj keyPattern = ownedInfoObj.getObjectField("key"); + IndexDescriptor* descriptor = new IndexDescriptor( _collection, + _getAccessMethodName(keyPattern), + ownedInfoObj ); + IndexCatalogEntry* entry = _setupInMemoryStructures( descriptor ); + + fassert( 17340, entry->isReady() ); + } + + if ( _unfinishedIndexes.size() ) { + // if there are left over indexes, we don't let anyone add/drop indexes + // until someone goes and fixes them + log() << "found " << _unfinishedIndexes.size() + << " index(es) that wasn't finished before shutdown"; + } + + _magic = INDEX_CATALOG_INIT; + + GeneratorHolder::getInstance()->reset( _collection ); + + return Status::OK(); + } + + IndexCatalogEntry* IndexCatalog::_setupInMemoryStructures( IndexDescriptor* descriptor ) { + auto_ptr descriptorCleanup( descriptor ); + + NamespaceDetails* indexMetadata = + _collection->_database->namespaceIndex().details( descriptor->indexNamespace() ); + + massert( 17329, + str::stream() << "no NamespaceDetails for index: " << descriptor->toString(), + indexMetadata ); + + auto_ptr recordStore( new SimpleRecordStoreV1( descriptor->indexNamespace(), + indexMetadata, + _collection->getExtentManager(), + false ) ); + + auto_ptr entry( new IndexCatalogEntry( _collection, + descriptorCleanup.release(), + recordStore.release() ) ); + + entry->init( _createAccessMethod( entry->descriptor(), + entry.get() ) ); + + IndexCatalogEntry* save = entry.get(); + _entries.add( entry.release() ); + + invariant( save == _entries.find( descriptor ) ); + invariant( save == _entries.find( descriptor->indexName() ) ); + return save; + } + + bool IndexCatalog::ok() const { + return ( _magic == INDEX_CATALOG_INIT ); + } + + void IndexCatalog::_checkMagic() const { + if ( ok() ) { + return; + } + log() << "IndexCatalog::_magic wrong, is : " << _magic; + fassertFailed(17198); + } + + Status IndexCatalog::_checkUnfinished() const { + if ( _unfinishedIndexes.size() == 0 ) + return Status::OK(); + + return Status( ErrorCodes::InternalError, + str::stream() + << "IndexCatalog has left over indexes that must be cleared" + << " ns: " << _collection->ns().ns() ); + } + + bool IndexCatalog::_shouldOverridePlugin(const BSONObj& keyPattern) const { + string pluginName = IndexNames::findPluginName(keyPattern); + bool known = IndexNames::isKnownName(pluginName); + + if (_collection->_database->getExtentManager().getFile(0)->getHeader()->is24IndexClean()) { + // RulesFor24 + // This assert will be triggered when downgrading from a future version that + // supports an index plugin unsupported by this version. + uassert(17197, str::stream() << "Invalid index type '" << pluginName << "' " + << "in index " << keyPattern, + known); + return false; + } + + // RulesFor22 + if (!known) { + log() << "warning: can't find plugin [" << pluginName << "]" << endl; + return true; + } + + if (!IndexNames::existedBefore24(pluginName)) { + warning() << "Treating index " << keyPattern << " as ascending since " + << "it was created before 2.4 and '" << pluginName << "' " + << "was not a valid type at that time." + << endl; + return true; + } + + return false; + } + + string IndexCatalog::_getAccessMethodName(const BSONObj& keyPattern) const { + if ( _shouldOverridePlugin(keyPattern) ) { + return ""; + } + + return IndexNames::findPluginName(keyPattern); + } + + + // --------------------------- + + Status IndexCatalog::_upgradeDatabaseMinorVersionIfNeeded( const string& newPluginName ) { + + // first check if requested index requires pdfile minor version to be bumped + if ( IndexNames::existedBefore24(newPluginName) ) { + return Status::OK(); + } + + Database* db = _collection->_database; + + DataFileHeader* dfh = db->getExtentManager().getFile(0)->getHeader(); + if ( dfh->is24IndexClean() ) { + return Status::OK(); // these checks have already been done + } + + fassert(16737, dfh->isCurrentVersion()); + + auto_ptr runner( InternalPlanner::collectionScan( db->_indexesName ) ); + + BSONObj index; + Runner::RunnerState state; + while ( Runner::RUNNER_ADVANCED == (state = runner->getNext(&index, NULL)) ) { + const BSONObj key = index.getObjectField("key"); + const string plugin = IndexNames::findPluginName(key); + if ( IndexNames::existedBefore24(plugin) ) + continue; + + const string errmsg = str::stream() + << "Found pre-existing index " << index << " with invalid type '" << plugin << "'. " + << "Disallowing creation of new index type '" << newPluginName << "'. See " + << "http://dochub.mongodb.org/core/index-type-changes" + ; + + return Status( ErrorCodes::CannotCreateIndex, errmsg ); + } + + if ( Runner::RUNNER_EOF != state ) { + warning() << "Internal error while reading system.indexes collection"; + } + + dfh->setIs24IndexClean(); + + return Status::OK(); + } + + StatusWith IndexCatalog::prepareSpecForCreate( const BSONObj& original ) const { + Status status = _isSpecOk( original ); + if ( !status.isOK() ) + return StatusWith( status ); + + BSONObj fixed = _fixIndexSpec( original ); + + // we double check with new index spec + status = _isSpecOk( fixed ); + if ( !status.isOK() ) + return StatusWith( status ); + + status = _doesSpecConflictWithExisting( fixed ); + if ( !status.isOK() ) + return StatusWith( status ); + + return StatusWith( fixed ); + } + + Status IndexCatalog::createIndex( BSONObj spec, + bool mayInterrupt, + ShutdownBehavior shutdownBehavior ) { + Lock::assertWriteLocked( _collection->_database->name() ); + _checkMagic(); + Status status = _checkUnfinished(); + if ( !status.isOK() ) + return status; + + StatusWith statusWithSpec = prepareSpecForCreate( spec ); + status = statusWithSpec.getStatus(); + if ( !status.isOK() ) + return status; + spec = statusWithSpec.getValue(); + + string pluginName = IndexNames::findPluginName( spec["key"].Obj() ); + if ( pluginName.size() ) { + Status s = _upgradeDatabaseMinorVersionIfNeeded( pluginName ); + if ( !s.isOK() ) + return s; + } + + // now going to touch disk + IndexBuildBlock indexBuildBlock( _collection, spec ); + status = indexBuildBlock.init(); + if ( !status.isOK() ) + return status; + + // sanity checks, etc... + IndexCatalogEntry* entry = indexBuildBlock.getEntry(); + invariant( entry ); + IndexDescriptor* descriptor = entry->descriptor(); + invariant( descriptor ); + + string idxName = descriptor->indexName(); // out copy for yields, etc... + + invariant( entry == _entries.find( descriptor ) ); + invariant( _details->_catalogFindIndexByName( idxName, true ) >= 0 ); + + try { + Client& client = cc(); + + _inProgressIndexes[descriptor] = &client; + + // buildAnIndex can yield. During a yield, the Collection that owns this + // IndexCatalog can be dropped, which means both the Collection and IndexCatalog + // can be destructed out from under us. The runner used by the index build will + // throw a particular exception when it detects that this occurred. + buildAnIndex( _collection, entry, mayInterrupt ); + indexBuildBlock.success(); + + InProgressIndexesMap::iterator it = _inProgressIndexes.find(descriptor); + _inProgressIndexes.erase(it); + + // sanity check + int idxNo = _details->_catalogFindIndexByName( idxName, true ); + invariant( idxNo < numIndexesReady() ); + + return Status::OK(); + } + catch ( const AssertionException& exc ) { + // At this point, *this may have been destructed, if we dropped the collection + // while we were yielding. indexBuildBlock will not touch an invalid _collection + // pointer if you call abort() on it. + + log() << "index build failed." << " spec: " << spec << " error: " << exc; + + if ( shutdownBehavior == SHUTDOWN_LEAVE_DIRTY && + exc.getCode() == ErrorCodes::InterruptedAtShutdown ) { + indexBuildBlock.abort(); + } + else if ( exc.getCode() == ErrorCodes::CursorNotFound ) { + // The cursor was killed because the collection was dropped. No need to clean up. + indexBuildBlock.abort(); + } + else { + indexBuildBlock.fail(); + + InProgressIndexesMap::iterator it = _inProgressIndexes.find(descriptor); + _inProgressIndexes.erase(it); + } + + return exc.toStatus(); + } + } + + IndexCatalog::IndexBuildBlock::IndexBuildBlock( Collection* collection, + const BSONObj& spec ) + : _collection( collection ), + _catalog( collection->getIndexCatalog() ), + _ns( _catalog->_collection->ns().ns() ), + _spec( spec.getOwned() ), + _entry( NULL ), + _inProgress( false ) { + + invariant( collection ); + } + + Status IndexCatalog::IndexBuildBlock::init() { + // we do special cleanup until we're far enough in + invariant( _inProgress == false ); + + // need this first for names, etc... + BSONObj keyPattern = _spec.getObjectField("key"); + IndexDescriptor* descriptor = new IndexDescriptor( _collection, + IndexNames::findPluginName(keyPattern), + _spec ); + auto_ptr descriptorCleaner( descriptor ); + + _indexName = descriptor->indexName(); + _indexNamespace = descriptor->indexNamespace(); + + /// ---------- setup on disk structures ---------------- + + Database* db = _collection->_database; + + // 1) insert into system.indexes + + Collection* systemIndexes = db->getOrCreateCollection( db->_indexesName ); + invariant( systemIndexes ); + + StatusWith systemIndexesEntry = systemIndexes->insertDocument( _spec, false ); + if ( !systemIndexesEntry.isOK() ) + return systemIndexesEntry.getStatus(); + + // 2) collection's NamespaceDetails + IndexDetails& indexDetails = _collection->details()->getNextIndexDetails( _ns.c_str() ); + + try { + getDur().writingDiskLoc( indexDetails.info ) = systemIndexesEntry.getValue(); + getDur().writingDiskLoc( indexDetails.head ).Null(); + } + catch ( DBException& e ) { + log() << "got exception trying to assign loc to IndexDetails" << e; + _catalog->_removeFromSystemIndexes( descriptor->indexName() ); + return Status( ErrorCodes::InternalError, e.toString() ); + } + + int before = _collection->details()->_indexBuildsInProgress; + try { + getDur().writingInt( _collection->details()->_indexBuildsInProgress ) += 1; + } + catch ( DBException& e ) { + log() << "got exception trying to incrementStats _indexBuildsInProgress: " << e; + fassert( 17344, before == _collection->details()->_indexBuildsInProgress ); + _catalog->_removeFromSystemIndexes( descriptor->indexName() ); + return Status( ErrorCodes::InternalError, e.toString() ); + } + + // at this point we can do normal clean up procedure, so we mark ourselves + // as in progress. + _inProgress = true; + + // 3) indexes entry in .ns file + NamespaceIndex& nsi = db->namespaceIndex(); + invariant( nsi.details( descriptor->indexNamespace() ) == NULL ); + nsi.add_ns( descriptor->indexNamespace(), DiskLoc(), false ); + + // 4) system.namespaces entry index ns + db->_addNamespaceToCatalog( descriptor->indexNamespace(), NULL ); + + /// ---------- setup in memory structures ---------------- + + _entry = _catalog->_setupInMemoryStructures( descriptorCleaner.release() ); + + return Status::OK(); + } + + IndexCatalog::IndexBuildBlock::~IndexBuildBlock() { + if ( !_inProgress ) { + // taken care of already when success() is called + return; + } + + try { + fail(); + } + catch ( const AssertionException& exc ) { + log() << "exception in ~IndexBuildBlock trying to cleanup: " << exc; + log() << " going to fassert to preserve state"; + fassertFailed( 17345 ); + } + } + + void IndexCatalog::IndexBuildBlock::fail() { + if ( !_inProgress ) { + // taken care of already when success() is called + return; + } + + Client::Context context( _collection->ns().ns(), + _collection->_database ); + + // if we're here, the index build failed or was interrupted + + _inProgress = false; // defensive + fassert( 17204, _catalog->_collection->ok() ); // defensive + + int idxNo = _collection->details()->_catalogFindIndexByName( _indexName, true ); + fassert( 17205, idxNo >= 0 ); + + IndexCatalogEntry* entry = _catalog->_entries.find( _indexName ); + invariant( entry == _entry ); + + try { + if ( entry ) { + _catalog->_dropIndex( entry ); + } + else { + _catalog->_deleteIndexFromDisk( _indexName, + _indexNamespace, + idxNo ); + } + } + catch (const DBException& exc) { + error() << "exception while cleaning up in-progress index build: " << exc.what(); + fassertFailedWithStatus(17493, exc.toStatus()); + } + + } + + void IndexCatalog::IndexBuildBlock::abort() { + _inProgress = false; + } + + void IndexCatalog::IndexBuildBlock::success() { + + fassert( 17206, _inProgress ); + _inProgress = false; + + fassert( 17207, _catalog->_collection->ok() ); + + NamespaceDetails* nsd = _collection->details(); + + int idxNo = nsd->_catalogFindIndexByName( _indexName, true ); + fassert( 17202, idxNo >= 0 ); + + // Make sure the newly created index is relocated to nIndexes, if it isn't already there + if ( idxNo != nsd->getCompletedIndexCount() ) { + log() << "switching indexes at position " << idxNo << " and " + << nsd->getCompletedIndexCount() << endl; + + int toIdxNo = nsd->getCompletedIndexCount(); + + nsd->swapIndex( idxNo, toIdxNo ); + + idxNo = nsd->getCompletedIndexCount(); + } + + getDur().writingInt( nsd->_indexBuildsInProgress ) -= 1; + getDur().writingInt( nsd->_nIndexes ) += 1; + + _catalog->_collection->infoCache()->addedIndex(); + + IndexDescriptor* desc = _catalog->findIndexByName( _indexName, true ); + fassert( 17330, desc ); + IndexCatalogEntry* entry = _catalog->_entries.find( desc ); + fassert( 17331, entry && entry == _entry ); + + entry->setIsReady( true ); + + IndexLegacy::postBuildHook( _catalog->_collection, + _catalog->findIndexByName( _indexName )->keyPattern() ); + } + + + + Status IndexCatalog::_isSpecOk( const BSONObj& spec ) const { + + const NamespaceString& nss = _collection->ns(); + + + if ( nss.isSystemDotIndexes() ) + return Status( ErrorCodes::CannotCreateIndex, + "cannot create indexes on the system.indexes collection" ); + + if ( nss.isOplog() ) + return Status( ErrorCodes::CannotCreateIndex, + "cannot create indexes on the oplog" ); + + if ( nss.coll() == "$freelist" ) { + // this isn't really proper, but we never want it and its not an error per se + return Status( ErrorCodes::IndexAlreadyExists, "cannot index freelist" ); + } + + StringData specNamespace = spec.getStringField("ns"); + if ( specNamespace.size() == 0 ) + return Status( ErrorCodes::CannotCreateIndex, + "the index spec needs a 'ns' field'" ); + + if ( _collection->ns() != specNamespace ) + return Status( ErrorCodes::CannotCreateIndex, + "the index spec ns does not match" ); + + // logical name of the index + const char *name = spec.getStringField("name"); + if ( !name[0] ) + return Status( ErrorCodes::CannotCreateIndex, "no index name specified" ); + + string indexNamespace = IndexDetails::indexNamespaceFromObj(spec); + if ( indexNamespace.length() > Namespace::MaxNsLen ) + return Status( ErrorCodes::CannotCreateIndex, + str::stream() << "namespace name generated from index name \"" << + indexNamespace << "\" is too long (127 byte max)" ); + + const BSONObj key = spec.getObjectField("key"); + const Status keyStatus = validateKeyPattern(key); + if (!keyStatus.isOK()) { + return Status( ErrorCodes::CannotCreateIndex, + str::stream() << "bad index key pattern " << key << ": " + << keyStatus.reason() ); + } + + if ( _collection->isCapped() && spec["dropDups"].trueValue() ) { + return Status( ErrorCodes::CannotCreateIndex, + str::stream() << "Cannot create an index with dropDups=true on a " + << "capped collection, as capped collections do " + << "not allow document removal." ); + } + + if ( !IndexDetails::isIdIndexPattern( key ) ) { + // for non _id indexes, we check to see if replication has turned off all indexes + // we _always_ created _id index + if( theReplSet && !theReplSet->buildIndexes() ) { + // this is not exactly the right error code, but I think will make the most sense + return Status( ErrorCodes::IndexAlreadyExists, "no indexes per repl" ); + } + } + + return Status::OK(); + } + + Status IndexCatalog::_doesSpecConflictWithExisting( const BSONObj& spec ) const { + const char *name = spec.getStringField("name"); + invariant( name[0] ); + + const BSONObj key = spec.getObjectField("key"); + + { + // Check both existing and in-progress indexes (2nd param = true) + const IndexDescriptor* desc = findIndexByName( name, true ); + if ( desc ) { + // index already exists with same name + + if ( !desc->keyPattern().equal( key ) ) + return Status( ErrorCodes::IndexKeySpecsConflict, + str::stream() << "Trying to create an index " + << "with same name " << name + << " with different key spec " << key + << " vs existing spec " << desc->keyPattern() ); + + IndexDescriptor temp( _collection, + _getAccessMethodName( key ), + spec ); + if ( !desc->areIndexOptionsEquivalent( &temp ) ) + return Status( ErrorCodes::IndexOptionsConflict, + str::stream() << "Index with name: " << name + << " already exists with different options" ); + + // Index already exists with the same options, so no need to build a new + // one (not an error). Most likely requested by a client using ensureIndex. + return Status( ErrorCodes::IndexAlreadyExists, name ); + } + } + + { + // Check both existing and in-progress indexes (2nd param = true) + const IndexDescriptor* desc = findIndexByKeyPattern(key, true); + if (desc) { + LOG(2) << "index already exists with diff name " << name + << ' ' << key << endl; + + IndexDescriptor temp( _collection, + _getAccessMethodName( key ), + spec ); + if ( !desc->areIndexOptionsEquivalent( &temp ) ) + return Status( ErrorCodes::IndexOptionsConflict, + str::stream() << "Index with pattern: " << key + << " already exists with different options" ); + + return Status( ErrorCodes::IndexAlreadyExists, name ); + } + } + + if ( _details->getTotalIndexCount() >= NamespaceDetails::NIndexesMax ) { + string s = str::stream() << "add index fails, too many indexes for " + << _collection->ns().ns() << " key:" << key.toString(); + log() << s; + return Status( ErrorCodes::CannotCreateIndex, s ); + } + + // Refuse to build text index if another text index exists or is in progress. + // Collections should only have one text index. + string pluginName = IndexNames::findPluginName( key ); + if ( pluginName == IndexNames::TEXT ) { + vector textIndexes; + const bool includeUnfinishedIndexes = true; + findIndexByType( IndexNames::TEXT, textIndexes, includeUnfinishedIndexes ); + if ( textIndexes.size() > 0 ) { + return Status( ErrorCodes::CannotCreateIndex, + str::stream() << "only one text index per collection allowed, " + << "found existing text index \"" << textIndexes[0]->indexName() + << "\"" ); + } + } + return Status::OK(); + } + + Status IndexCatalog::ensureHaveIdIndex() { + if ( _details->isSystemFlagSet( NamespaceDetails::Flag_HaveIdIndex ) ) + return Status::OK(); + + dassert( _idObj["_id"].type() == NumberInt ); + + BSONObjBuilder b; + b.append( "name", "_id_" ); + b.append( "ns", _collection->ns().ns() ); + b.append( "key", _idObj ); + BSONObj o = b.done(); + + Status s = createIndex( o, false ); + if ( s.isOK() || s.code() == ErrorCodes::IndexAlreadyExists ) { + _details->setSystemFlag( NamespaceDetails::Flag_HaveIdIndex ); + return Status::OK(); + } + + return s; + } + + Status IndexCatalog::dropAllIndexes( bool includingIdIndex ) { + Lock::assertWriteLocked( _collection->_database->name() ); + + BackgroundOperation::assertNoBgOpInProgForNs( _collection->ns().ns() ); + + // there may be pointers pointing at keys in the btree(s). kill them. + // TODO: can this can only clear cursors on this index? + _collection->cursorCache()->invalidateAll( false ); + + // make sure nothing in progress + massert( 17348, + "cannot dropAllIndexes when index builds in progress", + numIndexesTotal() == numIndexesReady() ); + + bool haveIdIndex = false; + + vector indexNamesToDrop; + { + int seen = 0; + IndexIterator ii = getIndexIterator( true ); + while ( ii.more() ) { + seen++; + IndexDescriptor* desc = ii.next(); + if ( desc->isIdIndex() && includingIdIndex == false ) { + haveIdIndex = true; + continue; + } + indexNamesToDrop.push_back( desc->indexName() ); + } + invariant( seen == numIndexesTotal() ); + } + + for ( size_t i = 0; i < indexNamesToDrop.size(); i++ ) { + string indexName = indexNamesToDrop[i]; + IndexDescriptor* desc = findIndexByName( indexName, true ); + invariant( desc ); + LOG(1) << "\t dropAllIndexes dropping: " << desc->toString(); + IndexCatalogEntry* entry = _entries.find( desc ); + invariant( entry ); + _dropIndex( entry ); + } + + // verify state is sane post cleaning + + long long numSystemIndexesEntries = 0; + { + Collection* systemIndexes = + _collection->_database->getCollection( _collection->_database->_indexesName ); + if ( systemIndexes ) { + EqualityMatchExpression expr; + BSONObj nsBSON = BSON( "ns" << _collection->ns() ); + invariant( expr.init( "ns", nsBSON.firstElement() ).isOK() ); + numSystemIndexesEntries = systemIndexes->countTableScan( &expr ); + } + else { + // this is ok, 0 is the right number + } + } + + if ( haveIdIndex ) { + fassert( 17324, numIndexesTotal() == 1 ); + fassert( 17325, numIndexesReady() == 1 ); + fassert( 17326, numSystemIndexesEntries == 1 ); + fassert( 17336, _entries.size() == 1 ); + } + else { + if ( numIndexesTotal() || numSystemIndexesEntries || _entries.size() ) { + error() << "About to fassert - " + << " numIndexesTotal(): " << numIndexesTotal() + << " numSystemIndexesEntries: " << numSystemIndexesEntries + << " _entries.size(): " << _entries.size() + << " indexNamesToDrop: " << indexNamesToDrop.size() + << " haveIdIndex: " << haveIdIndex; + } + fassert( 17327, numIndexesTotal() == 0 ); + fassert( 17328, numSystemIndexesEntries == 0 ); + fassert( 17337, _entries.size() == 0 ); + } + + return Status::OK(); + } + + Status IndexCatalog::dropIndex( IndexDescriptor* desc ) { + Lock::assertWriteLocked( _collection->_database->name() ); + IndexCatalogEntry* entry = _entries.find( desc ); + if ( !entry ) + return Status( ErrorCodes::InternalError, "cannot find index to delete" ); + if ( !entry->isReady() ) + return Status( ErrorCodes::InternalError, "cannot delete not ready index" ); + BackgroundOperation::assertNoBgOpInProgForNs( _collection->ns().ns() ); + + return _dropIndex( entry ); + } + + Status IndexCatalog::_dropIndex( IndexCatalogEntry* entry ) { + /** + * IndexState in order + * .system.indexes + * NamespaceDetails + * .system.ns + */ + + // ----- SANITY CHECKS ------------- + if ( !entry ) + return Status( ErrorCodes::BadValue, "IndexCatalog::_dropIndex passed NULL" ); + + _checkMagic(); + Status status = _checkUnfinished(); + if ( !status.isOK() ) + return status; + + // there may be pointers pointing at keys in the btree(s). kill them. + // TODO: can this can only clear cursors on this index? + _collection->cursorCache()->invalidateAll( false ); + + // wipe out stats + _collection->infoCache()->reset(); + + string indexNamespace = entry->descriptor()->indexNamespace(); + string indexName = entry->descriptor()->indexName(); + + int idxNo = _details->_catalogFindIndexByName( indexName, true ); + invariant( idxNo >= 0 ); + + // --------- START REAL WORK ---------- + + audit::logDropIndex( currentClient.get(), indexName, _collection->ns().ns() ); + + _entries.remove( entry->descriptor() ); + entry = NULL; + + try { + _details->clearSystemFlag( NamespaceDetails::Flag_HaveIdIndex ); + + // **** this is the first disk change **** + _deleteIndexFromDisk( indexName, + indexNamespace, + idxNo ); + } + catch ( std::exception& ) { + // this is bad, and we don't really know state + // going to leak to make sure things are safe + + log() << "error dropping index: " << indexNamespace + << " going to leak some memory to be safe"; + + + _collection->_database->_clearCollectionCache( indexNamespace ); + + throw; + } + + _collection->_database->_clearCollectionCache( indexNamespace ); + + _checkMagic(); + + return Status::OK(); + } + + void IndexCatalog::_deleteIndexFromDisk( const string& indexName, + const string& indexNamespace, + int idxNo ) { + invariant( idxNo >= 0 ); + invariant( _details->_catalogFindIndexByName( indexName, true ) == idxNo ); + + // data + system.namespacesa + Status status = _collection->_database->_dropNS( indexNamespace ); + if ( status.code() == ErrorCodes::NamespaceNotFound ) { + // this is ok, as we may be partially through index creation + } + else if ( !status.isOK() ) { + warning() << "couldn't drop extents for " << indexNamespace << " " << status.toString(); + } + + // all info in the .ns file + _details->_removeIndexFromMe( idxNo ); + + // remove from system.indexes + // n is how many things were removed from this + // probably should clean this up + int n = _removeFromSystemIndexes( indexName ); + wassert( n == 1 ); + } + + int IndexCatalog::_removeFromSystemIndexes( const StringData& indexName ) { + BSONObjBuilder b; + b.append( "ns", _collection->ns() ); + b.append( "name", indexName ); + BSONObj cond = b.obj(); // e.g.: { name: "ts_1", ns: "foo.coll" } + return static_cast( deleteObjects( _collection->_database->_indexesName, + cond, + false, + false, + true ) ); + } + + vector IndexCatalog::getAndClearUnfinishedIndexes() { + vector toReturn = _unfinishedIndexes; + _unfinishedIndexes.clear(); + for ( size_t i = 0; i < toReturn.size(); i++ ) { + BSONObj spec = toReturn[i]; + + BSONObj keyPattern = spec.getObjectField("key"); + IndexDescriptor desc( _collection, _getAccessMethodName(keyPattern), spec ); + + int idxNo = _details->_catalogFindIndexByName( desc.indexName(), true ); + invariant( idxNo >= 0 ); + invariant( idxNo >= numIndexesReady() ); + + _deleteIndexFromDisk( desc.indexName(), + desc.indexNamespace(), + idxNo ); + } + return toReturn; + } + + const IndexDescriptor* IndexCatalog::updateTTLSetting( const IndexDescriptor* oldDesc, + long long newExpireSeconds ) { + IndexDetails* indexDetails = _getIndexDetails( oldDesc ); + + BSONElement oldExpireSecs = indexDetails->info.obj().getField("expireAfterSeconds"); + + // Important that we set the new value in-place. We are writing directly to the + // object here so must be careful not to overwrite with a longer numeric type. + + BSONElementManipulator manip( oldExpireSecs ); + switch( oldExpireSecs.type() ) { + case EOO: + massert( 16631, "index does not have an 'expireAfterSeconds' field", false ); + break; + case NumberInt: + manip.SetInt( static_cast( newExpireSeconds ) ); + break; + case NumberDouble: + manip.SetNumber( static_cast( newExpireSeconds ) ); + break; + case NumberLong: + manip.SetLong( newExpireSeconds ); + break; + default: + massert( 16632, "current 'expireAfterSeconds' is not a number", false ); + } + + // The value has now been updated on disk. We're now going to invalidate the index + // catalog's old IndexDescriptor, and register a new one with the updated spec object. + + BSONObj ownedInfoObj = indexDetails->info.obj().getOwned(); + BSONObj keyPattern = ownedInfoObj.getObjectField("key"); + + // Notify other users of the IndexCatalog that we're about to invalidate 'oldDesc'. + const bool collectionGoingAway = false; + _collection->cursorCache()->invalidateAll( collectionGoingAway ); + + // Delete the IndexCatalogEntry that owns this descriptor. After deletion, 'oldDesc' is + // invalid and should not be dereferenced. + const bool removed = _entries.remove( oldDesc ); + invariant( removed ); + + // Re-register this index in the index catalog with the new spec. + IndexDescriptor* newDesc = new IndexDescriptor( _collection, + _getAccessMethodName( keyPattern ), + ownedInfoObj ); + const IndexCatalogEntry* entry = _setupInMemoryStructures( newDesc ); + invariant( entry->isReady() ); + + // Return the new descriptor. + return entry->descriptor(); + } + + bool IndexCatalog::isMultikey( const IndexDescriptor* idx ) { + IndexCatalogEntry* entry = _entries.find( idx ); + invariant( entry ); + return entry->isMultikey(); + } + + + // --------------------------- + + int IndexCatalog::numIndexesTotal() const { + return _details->getTotalIndexCount(); + } + + int IndexCatalog::numIndexesReady() const { + return _details->getCompletedIndexCount(); + } + + bool IndexCatalog::haveIdIndex() const { + return _details->isSystemFlagSet( NamespaceDetails::Flag_HaveIdIndex ) + || findIdIndex() != NULL; + } + + IndexCatalog::IndexIterator::IndexIterator( const IndexCatalog* cat, + bool includeUnfinishedIndexes ) + : _includeUnfinishedIndexes( includeUnfinishedIndexes ), + _catalog( cat ), + _iterator( cat->_entries.begin() ), + _start( true ), + _prev( NULL ), + _next( NULL ) { + } + + bool IndexCatalog::IndexIterator::more() { + if ( _start ) { + _advance(); + _start = false; + } + return _next != NULL; + } + + IndexDescriptor* IndexCatalog::IndexIterator::next() { + if ( !more() ) + return NULL; + _prev = _next; + _advance(); + return _prev->descriptor(); + } + + IndexAccessMethod* IndexCatalog::IndexIterator::accessMethod( IndexDescriptor* desc ) { + invariant( desc == _prev->descriptor() ); + return _prev->accessMethod(); + } + + IndexCatalogEntry* IndexCatalog::IndexIterator::entry( IndexDescriptor* desc ) { + invariant( desc == _prev->descriptor() ); + return _prev; + } + + void IndexCatalog::IndexIterator::_advance() { + _next = NULL; + + while ( _iterator != _catalog->_entries.end() ) { + IndexCatalogEntry* entry = *_iterator; + ++_iterator; + + if ( _includeUnfinishedIndexes || + entry->isReady() ) { + _next = entry; + return; + } + } + + } + + + IndexDescriptor* IndexCatalog::findIdIndex() const { + IndexIterator ii = getIndexIterator( false ); + while ( ii.more() ) { + IndexDescriptor* desc = ii.next(); + if ( desc->isIdIndex() ) + return desc; + } + return NULL; + } + + IndexDescriptor* IndexCatalog::findIndexByName( const StringData& name, + bool includeUnfinishedIndexes ) const { + IndexIterator ii = getIndexIterator( includeUnfinishedIndexes ); + while ( ii.more() ) { + IndexDescriptor* desc = ii.next(); + if ( desc->indexName() == name ) + return desc; + } + return NULL; + } + + IndexDescriptor* IndexCatalog::findIndexByKeyPattern( const BSONObj& key, + bool includeUnfinishedIndexes ) const { + IndexIterator ii = getIndexIterator( includeUnfinishedIndexes ); + while ( ii.more() ) { + IndexDescriptor* desc = ii.next(); + if ( desc->keyPattern() == key ) + return desc; + } + return NULL; + } + + IndexDescriptor* IndexCatalog::findIndexByPrefix( const BSONObj &keyPattern, + bool requireSingleKey ) const { + IndexDescriptor* best = NULL; + + IndexIterator ii = getIndexIterator( false ); + while ( ii.more() ) { + IndexDescriptor* desc = ii.next(); + + if ( !keyPattern.isPrefixOf( desc->keyPattern() ) ) + continue; + + if( !desc->isMultikey() ) { + return desc; + } + else if ( desc->isIdIndex() ) { + warning() << "_id index is marked as multi-key" + << " ns: " << _collection->ns() + << " key: " << desc->keyPattern(); + } + + if ( !requireSingleKey ) + best = desc; + } + + return best; + } + + void IndexCatalog::findIndexByType( const string& type , vector& matches, + bool includeUnfinishedIndexes ) const { + IndexIterator ii = getIndexIterator( includeUnfinishedIndexes ); + while ( ii.more() ) { + IndexDescriptor* desc = ii.next(); + if ( IndexNames::findPluginName( desc->keyPattern() ) == type ) { + matches.push_back( desc ); + } + } + } + + IndexAccessMethod* IndexCatalog::getIndex( const IndexDescriptor* desc ) { + IndexCatalogEntry* entry = _entries.find( desc ); + massert( 17334, "cannot find index entry", entry ); + return entry->accessMethod(); + } + + const IndexAccessMethod* IndexCatalog::getIndex( const IndexDescriptor* desc ) const { + const IndexCatalogEntry* entry = _entries.find( desc ); + massert( 17357, "cannot find index entry", entry ); + return entry->accessMethod(); + } + + IndexAccessMethod* IndexCatalog::_createAccessMethod( const IndexDescriptor* desc, + IndexCatalogEntry* entry ) { + const string& type = desc->getAccessMethodName(); + + if (IndexNames::HASHED == type) + return new HashAccessMethod( entry ); + + if (IndexNames::GEO_2DSPHERE == type) + return new S2AccessMethod( entry ); + + if (IndexNames::TEXT == type) + return new FTSAccessMethod( entry ); + + if (IndexNames::GEO_HAYSTACK == type) + return new HaystackAccessMethod( entry ); + + if ("" == type) + return new BtreeAccessMethod( entry ); + + if (IndexNames::GEO_2D == type) + return new TwoDAccessMethod( entry ); + + log() << "Can't find index for keyPattern " << desc->keyPattern(); + invariant(0); + return NULL; + } + + IndexDetails* IndexCatalog::_getIndexDetails( const IndexDescriptor* descriptor ) const { + int idxNo = _details->_catalogFindIndexByName( descriptor->indexName(), true ); + invariant( idxNo >= 0 ); + return &_details->idx( idxNo ); + } + + // --------------------------- + + Status IndexCatalog::_indexRecord( IndexCatalogEntry* index, + const BSONObj& obj, + const DiskLoc &loc, + const PregeneratedKeysOnIndex* prep, + bool ignoreKeyTooLong ) { + InsertDeleteOptions options; + options.logIfError = false; + options.ignoreKeyTooLong = ignoreKeyTooLong; + + bool isUnique = + index->descriptor()->isIdIndex() || + index->descriptor()->unique(); + + options.dupsAllowed = ignoreUniqueIndex( index->descriptor() ) || !isUnique; + + int64_t inserted; + return index->accessMethod()->insert(obj, loc, options, &inserted, prep); + } + + Status IndexCatalog::_unindexRecord( IndexCatalogEntry* index, + const BSONObj& obj, + const DiskLoc &loc, + bool logIfError ) { + InsertDeleteOptions options; + options.logIfError = logIfError; + + int64_t removed; + Status status = index->accessMethod()->remove(obj, loc, options, &removed); + + if ( !status.isOK() ) { + problem() << "Couldn't unindex record " << obj.toString() + << " status: " << status.toString(); + } + + return Status::OK(); + } + + void IndexCatalog::touch( const PregeneratedKeys* preGen ) const { + if ( !cc().allowedToThrowPageFaultException() ) { + // no point touching if we can't throw + return; + } + + // touch what we can + for ( IndexCatalogEntryContainer::const_iterator i = _entries.begin(); + i != _entries.end(); + ++i ) { + + IndexCatalogEntry* entry = *i; + shared_ptr gen = entry->accessMethod()->getKeyGenerator(); + const PregeneratedKeysOnIndex* perIndex = preGen->get( entry ); + if ( perIndex && + perIndex->generator->getId() == gen->getId() ) { + entry->accessMethod()->touch( perIndex->keys ); + } + } + + } + + void IndexCatalog::indexRecord( const BSONObj& obj, + const DiskLoc &loc, + const PregeneratedKeys* preGen, + bool ignoreKeyTooLong ) { + + for ( IndexCatalogEntryContainer::const_iterator i = _entries.begin(); + i != _entries.end(); + ++i ) { + + IndexCatalogEntry* entry = *i; + + const PregeneratedKeysOnIndex* perIndex = NULL; + if ( preGen ) + perIndex = preGen->get( entry ); + + try { + Status s = _indexRecord( entry, obj, loc, perIndex, ignoreKeyTooLong ); + uassert(s.location(), s.reason(), s.isOK() ); + } + catch ( AssertionException& ae ) { + + LOG(2) << "IndexCatalog::indexRecord failed: " << ae; + + for ( IndexCatalogEntryContainer::const_iterator j = _entries.begin(); + j != _entries.end(); + ++j ) { + + IndexCatalogEntry* toDelete = *j; + + try { + _unindexRecord( toDelete, obj, loc, false ); + } + catch ( DBException& e ) { + LOG(1) << "IndexCatalog::indexRecord rollback failed: " << e; + } + + if ( toDelete == entry ) + break; + } + + throw; + } + } + + } + + void IndexCatalog::unindexRecord( const BSONObj& obj, const DiskLoc& loc, bool noWarn ) { + for ( IndexCatalogEntryContainer::const_iterator i = _entries.begin(); + i != _entries.end(); + ++i ) { + + IndexCatalogEntry* entry = *i; + + // If it's a background index, we DO NOT want to log anything. + bool logIfError = entry->isReady() ? !noWarn : false; + _unindexRecord( entry, obj, loc, logIfError ); + } + + } + + Status IndexCatalog::checkNoIndexConflicts( const BSONObj &obj, + const PregeneratedKeys* preGen ) { + // TODO: preGen + IndexIterator ii = getIndexIterator( true ); + while ( ii.more() ) { + IndexDescriptor* descriptor = ii.next(); + + if ( !descriptor->unique() ) + continue; + + if ( ignoreUniqueIndex(descriptor) ) + continue; + + IndexAccessMethod* iam = getIndex( descriptor ); + + InsertDeleteOptions options; + options.logIfError = false; + options.dupsAllowed = false; + + UpdateTicket ticket; + Status ret = iam->validateUpdate(BSONObj(), obj, DiskLoc(), options, &ticket); + if ( !ret.isOK() ) + return ret; + } + + return Status::OK(); + } + + BSONObj IndexCatalog::fixIndexKey( const BSONObj& key ) { + if ( IndexDetails::isIdIndexPattern( key ) ) { + return _idObj; + } + if ( key["_id"].type() == Bool && key.nFields() == 1 ) { + return _idObj; + } + return key; + } + + BSONObj IndexCatalog::_fixIndexSpec( const BSONObj& spec ) { + BSONObj o = IndexLegacy::adjustIndexSpecObject( spec ); + + BSONObjBuilder b; + + int v = DefaultIndexVersionNumber; + if( !o["v"].eoo() ) { + double vv = o["v"].Number(); + // note (one day) we may be able to fresh build less versions than we can use + // isASupportedIndexVersionNumber() is what we can use + uassert(14803, str::stream() << "this version of mongod cannot build new indexes of version number " << vv, + vv == 0 || vv == 1); + v = (int) vv; + } + // idea is to put things we use a lot earlier + b.append("v", v); + + if( o["unique"].trueValue() ) + b.appendBool("unique", true); // normalize to bool true in case was int 1 or something... + + BSONObj key = fixIndexKey( o["key"].Obj() ); + b.append( "key", key ); + + string name = o["name"].String(); + if ( IndexDetails::isIdIndexPattern( key ) ) { + name = "_id_"; + } + b.append( "name", name ); + + { + BSONObjIterator i(o); + while ( i.more() ) { + BSONElement e = i.next(); + string s = e.fieldName(); + + if ( s == "_id" ) { + // skip + } + else if ( s == "v" || s == "unique" || + s == "key" || s == "name" ) { + // covered above + } + else if ( s == "key" ) { + b.append( "key", fixIndexKey( e.Obj() ) ); + } + else { + b.append(e); + } + } + } + + return b.obj(); + } + + std::vector + IndexCatalog::killMatchingIndexBuilds(const IndexCatalog::IndexKillCriteria& criteria) { + verify(Lock::somethingWriteLocked()); + std::vector indexes; + for (InProgressIndexesMap::iterator it = _inProgressIndexes.begin(); + it != _inProgressIndexes.end(); + it++) { + // check criteria + IndexDescriptor* desc = it->first; + Client* client = it->second; + if (!criteria.ns.empty() && (desc->parentNS() != criteria.ns)) { + continue; + } + if (!criteria.name.empty() && (desc->indexName() != criteria.name)) { + continue; + } + if (!criteria.key.isEmpty() && (desc->keyPattern() != criteria.key)) { + continue; + } + indexes.push_back(desc->keyPattern()); + CurOp* op = client->curop(); + log() << "halting index build: " << desc->keyPattern(); + // Note that we can only be here if the background index build in question is + // yielding. The bg index code is set up specially to check for interrupt + // immediately after it recovers from yield, such that no further work is done + // on the index build. Thus this thread does not have to synchronize with the + // bg index operation; we can just assume that it is safe to proceed. + killCurrentOp.kill(op->opNum()); + } + + if (indexes.size() > 0) { + log() << "halted " << indexes.size() << " index build(s)" << endl; + } + + return indexes; + } +} diff --git a/src/mongo/db/catalog/index_catalog.h b/src/mongo/db/catalog/index_catalog.h new file mode 100644 index 00000000000..3ee9f467254 --- /dev/null +++ b/src/mongo/db/catalog/index_catalog.h @@ -0,0 +1,349 @@ +// index_catalog.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/db/catalog/index_catalog_entry.h" +#include "mongo/db/catalog/index_pregen.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/unordered_map.h" + +namespace mongo { + + class Client; + class Collection; + class NamespaceDetails; + + class BtreeInMemoryState; + class IndexDescriptor; + class IndexDetails; + class IndexAccessMethod; + class BtreeAccessMethod; + class BtreeBasedAccessMethod; + + /** + * how many: 1 per Collection + * lifecycle: attached to a Collection + */ + class IndexCatalog { + public: + IndexCatalog( Collection* collection, NamespaceDetails* details ); + ~IndexCatalog(); + + // must be called before used + Status init(); + + bool ok() const; + + // ---- accessors ----- + + int numIndexesTotal() const; + int numIndexesReady() const; + int numIndexesInProgress() const { return numIndexesTotal() - numIndexesReady(); } + + /** + * this is in "alive" until the Collection goes away + * in which case everything from this tree has to go away + */ + + bool haveIdIndex() const; + + IndexDescriptor* findIdIndex() const; + + /** + * @return null if cannot find + */ + IndexDescriptor* findIndexByName( const StringData& name, + bool includeUnfinishedIndexes = false ) const; + + /** + * @return null if cannot find + */ + IndexDescriptor* findIndexByKeyPattern( const BSONObj& key, + bool includeUnfinishedIndexes = false ) const; + + /* Returns the index entry for the first index whose prefix contains + * 'keyPattern'. If 'requireSingleKey' is true, skip indices that contain + * array attributes. Otherwise, returns NULL. + */ + IndexDescriptor* findIndexByPrefix( const BSONObj &keyPattern, + bool requireSingleKey ) const; + + void findIndexByType( const string& type, + vector& matches, + bool includeUnfinishedIndexes = false ) const; + + // never returns NULL + IndexAccessMethod* getIndex( const IndexDescriptor* desc ); + const IndexAccessMethod* getIndex( const IndexDescriptor* desc ) const; + + class IndexIterator { + public: + bool more(); + IndexDescriptor* next(); + + // returns the access method for the last return IndexDescriptor + IndexAccessMethod* accessMethod( IndexDescriptor* desc ); + + IndexCatalogEntry* entry( IndexDescriptor* desc ); + private: + IndexIterator( const IndexCatalog* cat, bool includeUnfinishedIndexes ); + + void _advance(); + + bool _includeUnfinishedIndexes; + const IndexCatalog* _catalog; + IndexCatalogEntryContainer::const_iterator _iterator; + + bool _start; // only true before we've called next() or more() + + IndexCatalogEntry* _prev; + IndexCatalogEntry* _next; + + friend class IndexCatalog; + }; + + IndexIterator getIndexIterator( bool includeUnfinishedIndexes ) const { + return IndexIterator( this, includeUnfinishedIndexes ); + }; + + // ---- index set modifiers ------ + + Status ensureHaveIdIndex(); + + enum ShutdownBehavior { + SHUTDOWN_CLEANUP, // fully clean up this build + SHUTDOWN_LEAVE_DIRTY // leave as if kill -9 happened, so have to deal with on restart + }; + + Status createIndex( BSONObj spec, + bool mayInterrupt, + ShutdownBehavior shutdownBehavior = SHUTDOWN_CLEANUP ); + + StatusWith prepareSpecForCreate( const BSONObj& original ) const; + + Status dropAllIndexes( bool includingIdIndex ); + + Status dropIndex( IndexDescriptor* desc ); + + /** + * will drop all incompleted indexes and return specs + * after this, the indexes can be rebuilt + */ + vector getAndClearUnfinishedIndexes(); + + + struct IndexKillCriteria { + std::string ns; + std::string name; + BSONObj key; + }; + + /** + * Given some criteria, will search through all in-progress index builds + * and will kill ones that match. (namespace, index name, and/or index key spec) + * Returns the list of index specs that were killed, for use in restarting them later. + */ + std::vector killMatchingIndexBuilds(const IndexKillCriteria& criteria); + + // ---- modify single index + + /** + * Updates the expireAfterSeconds field of the given index to the value in newExpireSecs. + * The specified index must already contain an expireAfterSeconds field, and the value in + * that field and newExpireSecs must both be numeric. Returns an unowned pointer to the + * descriptor for the new index definition. + * + * It is invalid to dereference 'oldDesc' after calling this method. This method broadcasts + * an invalidateAll() on the cursor cache to notify other users of the IndexCatalog that + * this descriptor is now invalid. + */ + const IndexDescriptor* updateTTLSetting( const IndexDescriptor* oldDesc, + long long newExpireSeconds ); + + bool isMultikey( const IndexDescriptor* idex ); + + // --- these probably become private? + + + /** + * disk creation order + * 1) system.indexes entry + * 2) collection's NamespaceDetails + * a) info + head + * b) _indexBuildsInProgress++ + * 3) indexes entry in .ns file + * 4) system.namespaces entry for index ns + */ + class IndexBuildBlock { + public: + IndexBuildBlock( Collection* collection, + const BSONObj& spec ); + ~IndexBuildBlock(); + + Status init(); + + void success(); + + /** + * index build failed, clean up meta data + */ + void fail(); + + /** + * we're stopping the build + * do NOT cleanup, leave meta data as is + */ + void abort(); + + IndexCatalogEntry* getEntry() { return _entry; } + + private: + + Collection* _collection; + IndexCatalog* _catalog; + string _ns; + + BSONObj _spec; + + string _indexName; + string _indexNamespace; + + IndexCatalogEntry* _entry; + bool _inProgress; + }; + + // ----- data modifiers ------ + + /** + * TODO: document + */ + void touch( const PregeneratedKeys* preGen ) const; + + // this throws for now + void indexRecord( const BSONObj& obj, const DiskLoc &loc, + const PregeneratedKeys* preGen, bool ignoreKeyTooLong ); + + void unindexRecord( const BSONObj& obj, const DiskLoc& loc, bool noWarn ); + + /** + * checks all unique indexes and checks for conflicts + * should not throw + */ + Status checkNoIndexConflicts( const BSONObj& obj, const PregeneratedKeys* preGen ); + + // ------- temp internal ------- + + string getAccessMethodName(const BSONObj& keyPattern) { + return _getAccessMethodName( keyPattern ); + } + + Status _upgradeDatabaseMinorVersionIfNeeded( const string& newPluginName ); + + // public static helpers + + static BSONObj fixIndexKey( const BSONObj& key ); + + private: + typedef unordered_map InProgressIndexesMap; + + // creates a new thing, no caching + IndexAccessMethod* _createAccessMethod( const IndexDescriptor* desc, + IndexCatalogEntry* entry ); + + int _removeFromSystemIndexes( const StringData& indexName ); + + bool _shouldOverridePlugin( const BSONObj& keyPattern ) const; + + /** + * This differs from IndexNames::findPluginName in that returns the plugin name we *should* + * use, not the plugin name inside of the provided key pattern. To understand when these + * differ, see shouldOverridePlugin. + */ + string _getAccessMethodName(const BSONObj& keyPattern) const; + + IndexDetails* _getIndexDetails( const IndexDescriptor* descriptor ) const; + + void _checkMagic() const; + + + // checks if there is anything in _leftOverIndexes + // meaning we shouldn't modify catalog + Status _checkUnfinished() const; + + Status _indexRecord( IndexCatalogEntry* index, + const BSONObj& obj, const DiskLoc &loc, + const PregeneratedKeysOnIndex* pregen, bool ignoreKeyTooLong ); + + Status _unindexRecord( IndexCatalogEntry* index, const BSONObj& obj, const DiskLoc &loc, + bool logIfError ); + + /** + * this does no sanity checks + */ + Status _dropIndex( IndexCatalogEntry* entry ); + + // just does disk hanges + // doesn't change memory state, etc... + void _deleteIndexFromDisk( const string& indexName, + const string& indexNamespace, + int idxNo ); + + // descriptor ownership passes to _setupInMemoryStructures + IndexCatalogEntry* _setupInMemoryStructures( IndexDescriptor* descriptor ); + + static BSONObj _fixIndexSpec( const BSONObj& spec ); + + Status _isSpecOk( const BSONObj& spec ) const; + + Status _doesSpecConflictWithExisting( const BSONObj& spec ) const; + + int _magic; + Collection* _collection; + NamespaceDetails* _details; + + IndexCatalogEntryContainer _entries; + + // These are the index specs of indexes that were "leftover" + // "Leftover" means they were unfinished when a mongod shut down + // Certain operations are prohibted until someone fixes + // get by calling getAndClearUnfinishedIndexes + std::vector _unfinishedIndexes; + + static const BSONObj _idObj; // { _id : 1 } + + // Track in-progress index builds, in order to find and stop them when necessary. + InProgressIndexesMap _inProgressIndexes; + }; + +} diff --git a/src/mongo/db/catalog/index_catalog_entry.cpp b/src/mongo/db/catalog/index_catalog_entry.cpp new file mode 100644 index 00000000000..9357ea469a9 --- /dev/null +++ b/src/mongo/db/catalog/index_catalog_entry.cpp @@ -0,0 +1,189 @@ +// index_catalog_entry.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/index_catalog_entry.h" + +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index/index_descriptor.h" + +namespace mongo { + + IndexCatalogEntry::IndexCatalogEntry( Collection* collection, + IndexDescriptor* descriptor, + RecordStore* recordstore ) + : _collection( collection ), + _descriptor( descriptor ), + _recordStore( recordstore ), + _accessMethod( NULL ), + _forcedBtreeIndex( NULL ), + _ordering( Ordering::make( descriptor->keyPattern() ) ), + _isReady( false ) { + _descriptor->_cachedEntry = this; + } + + IndexCatalogEntry::~IndexCatalogEntry() { + _descriptor->_cachedEntry = NULL; // defensive + + delete _forcedBtreeIndex; + delete _accessMethod; + + delete _recordStore; + + delete _descriptor; + } + + void IndexCatalogEntry::init( IndexAccessMethod* accessMethod ) { + verify( _accessMethod == NULL ); + _accessMethod = accessMethod; + + _isReady = _catalogIsReady(); + _head = _catalogHead(); + _isMultikey = _catalogIsMultikey(); + } + + const DiskLoc& IndexCatalogEntry::head() const { + DEV verify( _head == _catalogHead() ); + return _head; + } + + bool IndexCatalogEntry::isReady() const { + DEV verify( _isReady == _catalogIsReady() ); + return _isReady; + } + + bool IndexCatalogEntry::isMultikey() const { + DEV verify( _isMultikey == _catalogIsMultikey() ); + return _isMultikey; + } + + // --- + + void IndexCatalogEntry::setIsReady( bool newIsReady ) { + _isReady = newIsReady; + verify( isReady() == newIsReady ); + } + + void IndexCatalogEntry::setHead( DiskLoc newHead ) { + NamespaceDetails* nsd = _collection->details(); + int idxNo = _indexNo(); + IndexDetails& id = nsd->idx( idxNo ); + id.head.writing() = newHead; + _head = newHead; + } + + void IndexCatalogEntry::setMultikey() { + if ( isMultikey() ) + return; + NamespaceDetails* nsd = _collection->details(); + int idxNo = _indexNo(); + if ( nsd->setIndexIsMultikey( idxNo, true ) ) { + LOG(1) << _collection->ns().ns() << ": clearing plan cache - index " + << _descriptor->keyPattern() << " set to multi key."; + _collection->infoCache()->clearQueryCache(); + } + _isMultikey = true; + } + + // ---- + + bool IndexCatalogEntry::_catalogIsReady() const { + return _indexNo() < _collection->getIndexCatalog()->numIndexesReady(); + } + + DiskLoc IndexCatalogEntry::_catalogHead() const { + NamespaceDetails* nsd = _collection->details(); + int idxNo = _indexNo(); + return nsd->idx( idxNo ).head; + } + + bool IndexCatalogEntry::_catalogIsMultikey() const { + NamespaceDetails* nsd = _collection->details(); + int idxNo = _indexNo(); + return nsd->isMultikey( idxNo ); + } + + int IndexCatalogEntry::_indexNo() const { + int idxNo = _collection->details()->_catalogFindIndexByName( _descriptor->indexName(), + true ); + fassert( 17341, idxNo >= 0 ); + return idxNo; + } + + + // ------------------ + + const IndexCatalogEntry* IndexCatalogEntryContainer::find( const IndexDescriptor* desc ) const { + if ( desc->_cachedEntry ) + return desc->_cachedEntry; + + for ( const_iterator i = begin(); i != end(); ++i ) { + const IndexCatalogEntry* e = *i; + if ( e->descriptor() == desc ) + return e; + } + return NULL; + } + + IndexCatalogEntry* IndexCatalogEntryContainer::find( const IndexDescriptor* desc ) { + if ( desc->_cachedEntry ) + return desc->_cachedEntry; + + for ( iterator i = begin(); i != end(); ++i ) { + IndexCatalogEntry* e = *i; + if ( e->descriptor() == desc ) + return e; + } + return NULL; + } + + IndexCatalogEntry* IndexCatalogEntryContainer::find( const string& name ) { + for ( iterator i = begin(); i != end(); ++i ) { + IndexCatalogEntry* e = *i; + if ( e->descriptor()->indexName() == name ) + return e; + } + return NULL; + } + + bool IndexCatalogEntryContainer::remove( const IndexDescriptor* desc ) { + for ( std::vector::iterator i = _entries.mutableVector().begin(); + i != _entries.mutableVector().end(); + ++i ) { + IndexCatalogEntry* e = *i; + if ( e->descriptor() != desc ) + continue; + _entries.mutableVector().erase( i ); + delete e; + return true; + } + return false; + } + +} diff --git a/src/mongo/db/catalog/index_catalog_entry.h b/src/mongo/db/catalog/index_catalog_entry.h new file mode 100644 index 00000000000..17e5389a834 --- /dev/null +++ b/src/mongo/db/catalog/index_catalog_entry.h @@ -0,0 +1,154 @@ +// index_catalog_entry.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/bson/ordering.h" +#include "mongo/db/diskloc.h" + +namespace mongo { + + class Collection; + class IndexDescriptor; + class RecordStore; + class IndexAccessMethod; + + class IndexCatalogEntry { + MONGO_DISALLOW_COPYING( IndexCatalogEntry ); + public: + IndexCatalogEntry( Collection* collection, + IndexDescriptor* descriptor, // ownership passes to me + RecordStore* recordStore ); // ownership passes to me + + ~IndexCatalogEntry(); + + void init( IndexAccessMethod* accessMethod ); + + const Collection* collection() const { return _collection; } + + IndexDescriptor* descriptor() { return _descriptor; } + const IndexDescriptor* descriptor() const { return _descriptor; } + + IndexAccessMethod* accessMethod() { return _accessMethod; } + const IndexAccessMethod* accessMethod() const { return _accessMethod; } + + IndexAccessMethod* forcedBtreeIndex() { return _forcedBtreeIndex; } + // ownership passes + void setForcedBtreeIndex( IndexAccessMethod* iam ) { _forcedBtreeIndex = iam; } + + RecordStore* recordStore() { return _recordStore; } + const RecordStore* recordStore() const { return _recordStore; } + + const Ordering& ordering() const { return _ordering; } + + /// --------------------- + + const DiskLoc& head() const; + + void setHead( DiskLoc newHead ); + + void setIsReady( bool newIsReady ); + + // -- + + bool isMultikey() const; + + void setMultikey(); + + // if this ready is ready for queries + bool isReady() const; + + private: + + int _indexNo() const; + + bool _catalogIsReady() const; + DiskLoc _catalogHead() const; + bool _catalogIsMultikey() const; + + // ----- + + Collection* _collection; // not owned here + + IndexDescriptor* _descriptor; // owned here + + RecordStore* _recordStore; // owned here + + IndexAccessMethod* _accessMethod; // owned here + IndexAccessMethod* _forcedBtreeIndex; // owned here + + // cached stuff + + Ordering _ordering; // TODO: this might be b-tree specific + bool _isReady; // cache of NamespaceDetails info + DiskLoc _head; // cache of IndexDetails + bool _isMultikey; // cache of NamespaceDetails info + }; + + class IndexCatalogEntryContainer { + public: + + typedef std::vector::const_iterator const_iterator; + typedef std::vector::const_iterator iterator; + + const_iterator begin() const { return _entries.vector().begin(); } + const_iterator end() const { return _entries.vector().end(); } + + iterator begin() { return _entries.vector().begin(); } + iterator end() { return _entries.vector().end(); } + + // TODO: these have to be SUPER SUPER FAST + // maybe even some pointer trickery is in order + const IndexCatalogEntry* find( const IndexDescriptor* desc ) const; + IndexCatalogEntry* find( const IndexDescriptor* desc ); + + IndexCatalogEntry* find( const std::string& name ); + + + unsigned size() const { return _entries.size(); } + // ----------------- + + bool remove( const IndexDescriptor* desc ); + + // pass ownership to EntryContainer + void add( IndexCatalogEntry* entry ) { _entries.mutableVector().push_back( entry ); } + + // TODO: should the findIndexBy* methods be done here + // and proxied in IndexCatatalog + //IndexCatalogEntry* findIndexByName(); + + private: + OwnedPointerVector _entries; + }; + +} diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp new file mode 100644 index 00000000000..ffaacbb6d54 --- /dev/null +++ b/src/mongo/db/catalog/index_create.cpp @@ -0,0 +1,390 @@ +// index_create.cpp + +/** +* Copyright (C) 2008 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/index_create.h" + +#include "mongo/base/error_codes.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/audit.h" +#include "mongo/db/background.h" +#include "mongo/db/structure/btree/btreebuilder.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/curop.h" +#include "mongo/db/extsort.h" +#include "mongo/db/structure/catalog/index_details.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/pdfile_private.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/query/runner_yield_policy.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/progress_meter.h" + +namespace mongo { + + /** + * Add the provided (obj, dl) pair to the provided index. + */ + static void addKeysToIndex( Collection* collection, + const IndexDescriptor* descriptor, + IndexAccessMethod* accessMethod, + const BSONObj& obj, const DiskLoc &recordLoc ) { + + InsertDeleteOptions options; + options.logIfError = false; + options.dupsAllowed = true; + + if ( descriptor->isIdIndex() || descriptor->unique() ) { + if ( !ignoreUniqueIndex( descriptor ) ) { + options.dupsAllowed = false; + } + } + + int64_t inserted; + Status ret = accessMethod->insert(obj, recordLoc, options, &inserted); + uassertStatusOK( ret ); + } + + unsigned long long addExistingToIndex( Collection* collection, + const IndexDescriptor* descriptor, + IndexAccessMethod* accessMethod, + bool shouldYield ) { + + string ns = collection->ns().ns(); // our copy for sanity + + bool dupsAllowed = !descriptor->unique(); + bool dropDups = descriptor->dropDups(); + + + string curopMessage; + { + stringstream ss; + ss << "Index Build"; + if ( shouldYield ) + ss << "(background)"; + curopMessage = ss.str(); + } + + ProgressMeter& progress = + cc().curop()->setMessage( curopMessage.c_str(), + curopMessage, + collection->numRecords() ); + + unsigned long long n = 0; + unsigned long long numDropped = 0; + + auto_ptr runner(InternalPlanner::collectionScan(ns)); + + // We're not delegating yielding to the runner because we need to know when a yield + // happens. + RunnerYieldPolicy yieldPolicy; + + std::string idxName = descriptor->indexName(); + + // After this yields in the loop, idx may point at a different index (if indexes get + // flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should + // depend on idx. idxNo should be recalculated after each yield. + + BSONObj js; + DiskLoc loc; + while (Runner::RUNNER_ADVANCED == runner->getNext(&js, &loc)) { + try { + if ( !dupsAllowed && dropDups ) { + LastError::Disabled led( lastError.get() ); + addKeysToIndex(collection, descriptor, accessMethod, js, loc); + } + else { + addKeysToIndex(collection, descriptor, accessMethod, js, loc); + } + } + catch( AssertionException& e ) { + if (ErrorCodes::isInterruption(DBException::convertExceptionCode(e.getCode()))) { + killCurrentOp.checkForInterrupt(); + } + + // TODO: Does exception really imply dropDups exception? + if (dropDups) { + bool runnerEOF = runner->isEOF(); + runner->saveState(); + BSONObj toDelete; + collection->deleteDocument( loc, false, true, &toDelete ); + logOp( "d", ns.c_str(), toDelete ); + + if (!runner->restoreState()) { + // Runner got killed somehow. This probably shouldn't happen. + if (runnerEOF) { + // Quote: "We were already at the end. Normal. + // TODO: Why is this normal? + } + else { + uasserted(ErrorCodes::CursorNotFound, + "cursor gone during bg index; dropDups"); + } + break; + } + // We deleted a record, but we didn't actually yield the dblock. + // TODO: Why did the old code assume we yielded the lock? + numDropped++; + } + else { + log() << "background addExistingToIndex exception " << e.what() << endl; + throw; + } + } + + n++; + progress.hit(); + + getDur().commitIfNeeded(); + if (shouldYield && yieldPolicy.shouldYield()) { + // Note: yieldAndCheckIfOK checks for interrupt and thus can throw + if (!yieldPolicy.yieldAndCheckIfOK(runner.get())) { + uasserted(ErrorCodes::CursorNotFound, "cursor gone during bg index"); + break; + } + + // Checking for interrupt here is necessary because the bg index + // interruptors can only interrupt this index build while they hold + // a write lock, and yieldAndCheckIfOK only checks for + // interrupt prior to yielding our write lock. We need to check the kill flag + // here before another iteration of the loop. + killCurrentOp.checkForInterrupt(); + + progress.setTotalWhileRunning( collection->numRecords() ); + // Recalculate idxNo if we yielded + IndexDescriptor* idx = collection->getIndexCatalog()->findIndexByName( idxName, + true ); + verify( idx && idx == descriptor ); + } + } + + progress.finished(); + if ( dropDups && numDropped ) + log() << "\t index build dropped: " << numDropped << " dups"; + return n; + } + + // --------------------------- + + // throws DBException + void buildAnIndex( Collection* collection, + IndexCatalogEntry* btreeState, + bool mayInterrupt ) { + + string ns = collection->ns().ns(); // our copy + const IndexDescriptor* idx = btreeState->descriptor(); + const BSONObj& idxInfo = idx->infoObj(); + + MONGO_TLOG(0) << "build index on: " << ns + << " properties: " << idx->toString() << endl; + audit::logCreateIndex( currentClient.get(), &idxInfo, idx->indexName(), ns ); + + Timer t; + + verify( Lock::isWriteLocked( ns ) ); + // this is so that people know there are more keys to look at when doing + // things like in place updates, etc... + collection->infoCache()->addedIndex(); + + if ( collection->numRecords() == 0 ) { + Status status = btreeState->accessMethod()->initializeAsEmpty(); + massert( 17343, + str::stream() << "IndexAccessMethod::initializeAsEmpty failed" << status.toString(), + status.isOK() ); + MONGO_TLOG(0) << "\t added index to empty collection"; + return; + } + + scoped_ptr backgroundOperation; + bool doInBackground = false; + + if ( idxInfo["background"].trueValue() && !inDBRepair && mayInterrupt ) { + doInBackground = true; + backgroundOperation.reset( new BackgroundOperation(ns) ); + uassert( 13130, + "can't start bg index b/c in recursive lock (db.eval?)", + !Lock::nested() ); + log() << "\t building index in background"; + } + + Status status = btreeState->accessMethod()->initializeAsEmpty(); + massert( 17342, + str::stream() + << "IndexAccessMethod::initializeAsEmpty failed" + << status.toString(), + status.isOK() ); + + IndexAccessMethod* bulk = doInBackground ? NULL : btreeState->accessMethod()->initiateBulk(); + scoped_ptr bulkHolder(bulk); + IndexAccessMethod* iam = bulk ? bulk : btreeState->accessMethod(); + + if ( bulk ) + log() << "\t building index using bulk method"; + + unsigned long long n = addExistingToIndex( collection, + btreeState->descriptor(), + iam, + doInBackground ); + + if ( bulk ) { + LOG(1) << "\t bulk commit starting"; + std::set dupsToDrop; + + Status status = btreeState->accessMethod()->commitBulk( bulk, + mayInterrupt, + &dupsToDrop ); + massert( 17398, + str::stream() << "commitBulk failed: " << status.toString(), + status.isOK() ); + + if ( dupsToDrop.size() ) + log() << "\t bulk dropping " << dupsToDrop.size() << " dups"; + + for( set::const_iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); ++i ) { + BSONObj toDelete; + collection->deleteDocument( *i, + false /* cappedOk */, + true /* noWarn */, + &toDelete ); + if ( isMaster( ns.c_str() ) ) { + logOp( "d", ns.c_str(), toDelete ); + } + + getDur().commitIfNeeded(); + + RARELY if ( mayInterrupt ) { + killCurrentOp.checkForInterrupt(); + } + } + } + + verify( !btreeState->head().isNull() ); + MONGO_TLOG(0) << "build index done. scanned " << n << " total records. " + << t.millis() / 1000.0 << " secs" << endl; + + // this one is so people know that the index is finished + collection->infoCache()->addedIndex(); + } + + // ---------------------------- + + MultiIndexBlock::MultiIndexBlock( Collection* collection ) + : _collection( collection ) { + } + + MultiIndexBlock::~MultiIndexBlock() { + for ( size_t i = 0; i < _states.size(); i++ ) { + delete _states[i].bulk; + delete _states[i].block; + } + } + + Status MultiIndexBlock::init(std::vector& indexSpecs) { + + for ( size_t i = 0; i < indexSpecs.size(); i++ ) { + BSONObj info = indexSpecs[i]; + + string pluginName = IndexNames::findPluginName( info["key"].Obj() ); + if ( pluginName.size() ) { + Status s = + _collection->getIndexCatalog()->_upgradeDatabaseMinorVersionIfNeeded(pluginName); + if ( !s.isOK() ) + return s; + } + + } + + for ( size_t i = 0; i < indexSpecs.size(); i++ ) { + BSONObj info = indexSpecs[i]; + StatusWith statusWithInfo = + _collection->getIndexCatalog()->prepareSpecForCreate( info ); + Status status = statusWithInfo.getStatus(); + if ( !status.isOK() ) + return status; + info = statusWithInfo.getValue(); + + IndexState state; + state.block = new IndexCatalog::IndexBuildBlock( _collection, info ); + status = state.block->init(); + if ( !status.isOK() ) + return status; + + state.real = state.block->getEntry()->accessMethod(); + status = state.real->initializeAsEmpty(); + if ( !status.isOK() ) + return status; + + state.bulk = state.real->initiateBulk(); + + _states.push_back( state ); + } + + return Status::OK(); + } + + Status MultiIndexBlock::insert( const BSONObj& doc, + const DiskLoc& loc, + const InsertDeleteOptions& options ) { + + for ( size_t i = 0; i < _states.size(); i++ ) { + Status idxStatus = _states[i].forInsert()->insert( doc, + loc, + options, + NULL ); + if ( !idxStatus.isOK() ) + return idxStatus; + } + return Status::OK(); + } + + Status MultiIndexBlock::commit() { + for ( size_t i = 0; i < _states.size(); i++ ) { + if ( _states[i].bulk == NULL ) + continue; + Status status = _states[i].real->commitBulk( _states[i].bulk, + false, + NULL ); + if ( !status.isOK() ) + return status; + } + + for ( size_t i = 0; i < _states.size(); i++ ) { + _states[i].block->success(); + } + + return Status::OK(); + } + +} // namespace mongo + diff --git a/src/mongo/db/catalog/index_create.h b/src/mongo/db/catalog/index_create.h new file mode 100644 index 00000000000..c824f27f123 --- /dev/null +++ b/src/mongo/db/catalog/index_create.h @@ -0,0 +1,86 @@ +// index_create.h + +/** +* Copyright (C) 2008 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/index/index_access_method.h" + +namespace mongo { + + class BSONObj; + class Collection; + class IndexCatalogEntry; + + // Build an index in the foreground + // If background is false, uses fast index builder + // If background is true, uses background index builder; blocks until done. + void buildAnIndex( Collection* collection, + IndexCatalogEntry* btreeState, + bool mayInterrupt ); + + class MultiIndexBlock { + MONGO_DISALLOW_COPYING( MultiIndexBlock ); + public: + MultiIndexBlock( Collection* collection ); + ~MultiIndexBlock(); + + Status init( std::vector& specs ); + + Status insert( const BSONObj& doc, + const DiskLoc& loc, + const InsertDeleteOptions& options ); + + Status commit(); + + private: + Collection* _collection; + + struct IndexState { + IndexState() + : block( NULL ), real( NULL ), bulk( NULL ) { + } + + IndexAccessMethod* forInsert() { return bulk ? bulk : real; } + + IndexCatalog::IndexBuildBlock* block; + IndexAccessMethod* real; + IndexAccessMethod* bulk; + }; + + std::vector _states; + }; + +} // namespace mongo diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp new file mode 100644 index 00000000000..660f74f77bb --- /dev/null +++ b/src/mongo/db/catalog/index_key_validate.cpp @@ -0,0 +1,111 @@ +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/index_key_validate.h" + +#include "mongo/db/field_ref.h" +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + Status validateKeyPattern(const BSONObj& key) { + const ErrorCodes::Error code = ErrorCodes::CannotCreateIndex; + + if ( key.objsize() > 2048 ) + return Status(code, "Index key pattern too large."); + + if ( key.isEmpty() ) + return Status(code, "Index keys cannot be empty."); + + string pluginName = IndexNames::findPluginName( key ); + if ( pluginName.size() ) { + if ( !IndexNames::isKnownName( pluginName ) ) + return Status(code, + mongoutils::str::stream() << "Unknown index plugin '" + << pluginName << '\''); + } + + BSONObjIterator it( key ); + while ( it.more() ) { + BSONElement keyElement = it.next(); + + if( keyElement.type() == Object || keyElement.type() == Array ) + return Status(code, "Index keys cannot be Objects or Arrays."); + + if ( keyElement.type() == String && pluginName != keyElement.str() ) { + return Status(code, "Can't use more than one index plugin for a single index."); + } + + // Ensure that the fields on which we are building the index are valid: a field must not + // begin with a '$' unless it is part of a DBRef or text index, and a field path cannot + // contain an empty field. If a field cannot be created or updated, it should not be + // indexable. + + FieldRef keyField( keyElement.fieldName() ); + + const size_t numParts = keyField.numParts(); + if ( numParts == 0 ) { + return Status(code, "Index keys cannot be an empty field."); + } + + // "$**" is acceptable for a text index. + if ( mongoutils::str::equals( keyElement.fieldName(), "$**" ) && + keyElement.valuestrsafe() == IndexNames::TEXT ) + continue; + + + for ( size_t i = 0; i != numParts; ++i ) { + const StringData part = keyField.getPart(i); + + // Check if the index key path contains an empty field. + if ( part.empty() ) { + return Status(code, "Index keys cannot contain an empty field."); + } + + if ( part[0] != '$' ) + continue; + + // Check if the '$'-prefixed field is part of a DBRef: since we don't have the + // necessary context to validate whether this is a proper DBRef, we allow index + // creation on '$'-prefixed names that match those used in a DBRef. + const bool mightBePartOfDbRef = (i != 0) && + (part == "$db" || + part == "$id" || + part == "$ref"); + + if ( !mightBePartOfDbRef ) { + return Status(code, "Index key contains an illegal field name: " + "field name starts with '$'."); + } + } + } + + return Status::OK(); + } +} // namespace mongo diff --git a/src/mongo/db/catalog/index_key_validate.h b/src/mongo/db/catalog/index_key_validate.h new file mode 100644 index 00000000000..4392e7e01a8 --- /dev/null +++ b/src/mongo/db/catalog/index_key_validate.h @@ -0,0 +1,40 @@ +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" + +namespace mongo { + class BSONObj; + + /** + * Checks if the key is valid for building an index. + */ + Status validateKeyPattern(const BSONObj& key); +} // namespace mongo diff --git a/src/mongo/db/catalog/index_pregen.cpp b/src/mongo/db/catalog/index_pregen.cpp new file mode 100644 index 00000000000..37f26dc8233 --- /dev/null +++ b/src/mongo/db/catalog/index_pregen.cpp @@ -0,0 +1,137 @@ +// index_pregen.cpp + +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog/index_pregen.h" +#include "mongo/db/d_concurrency.h" +#include "mongo/db/index/index_access_method.h" + +namespace mongo { + + const PregeneratedKeysOnIndex* PregeneratedKeys::get( IndexCatalogEntry* entry ) const { + Map::const_iterator i = _indexes.find( entry ); + if ( i == _indexes.end() ) + return NULL; + return &i->second; + } + + void PregeneratedKeys::gen( const BSONObj& obj, + IndexCatalogEntry* entry, + const boost::shared_ptr& generator ) { + PregeneratedKeysOnIndex& onIndex = _indexes[entry]; + onIndex.generator = generator; + generator->getKeys( obj, &onIndex.keys ); + } + + GeneratorHolder::GeneratorHolder() + : _collectionsLock( "GeneratorHolder" ) { + } + + bool GeneratorHolder::prepare( const StringData& ns, + const BSONObj& obj, + PregeneratedKeys* out ) { + invariant( out ); + + if ( Lock::isLocked() ) { + // ewww, we do nothing + return true; + } + + shared_ptr collection; + { + string temp = ns.toString(); + SimpleMutex::scoped_lock lk( _collectionsLock ); + Collections::const_iterator i = _collections.find( temp ); + if ( i == _collections.end() ) + return false; + collection = i->second; + } + + try { + for ( size_t i = 0; i < collection->indexes.size(); i++ ) { + out->gen( obj, collection->indexes[i].entry, collection->indexes[i].generator ); + } + } + catch ( DBException& e ) { + log() << "GeneratorHolder::prepare failed: " << e; + out->clear(); + return false; + } + return true; + } + + void GeneratorHolder::reset( const Collection* aCollection ) { + shared_ptr myCollection( new MyCollection() ); + myCollection->ns = aCollection->ns().ns(); + + IndexCatalog::IndexIterator ii = aCollection->getIndexCatalog()->getIndexIterator( true ); + while ( ii.more() ) { + IndexDescriptor* desc = ii.next(); + IndexCatalogEntry* entry = ii.entry( desc ); + + MyIndex myIndex; + myIndex.entry = entry; + myIndex.generator = entry->accessMethod()->getKeyGenerator(); + + myCollection->indexes.push_back( myIndex ); + } + + SimpleMutex::scoped_lock lk( _collectionsLock ); + _collections[aCollection->ns().ns()] = myCollection; + } + + void GeneratorHolder::dropped( const std::string& ns ) { + SimpleMutex::scoped_lock lk( _collectionsLock ); + _collections.erase( ns ); + } + + void GeneratorHolder::droppedDatabase( const std::string& db ) { + SimpleMutex::scoped_lock lk( _collectionsLock ); + vector toDrop; + for ( Collections::const_iterator i = _collections.begin(); i != _collections.end(); ++i ) { + StringData temp = nsToDatabaseSubstring( i->first ); + if ( temp == db ) + toDrop.push_back( i->first ); + } + + for ( size_t i = 0; i < toDrop.size(); i++ ) { + _collections.erase( toDrop[i] ); + } + } + + namespace { + // this is ok because we don't access this via the construction of any globals + GeneratorHolder theHolder; + } + GeneratorHolder* GeneratorHolder::getInstance() { + return &theHolder; + } +} diff --git a/src/mongo/db/catalog/index_pregen.h b/src/mongo/db/catalog/index_pregen.h new file mode 100644 index 00000000000..1f133f28d66 --- /dev/null +++ b/src/mongo/db/catalog/index_pregen.h @@ -0,0 +1,126 @@ +// index_pregen.h + +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/index/key_generator.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/concurrency/mutex.h" + +/** + * This entire thing goes away with document level locking + */ +namespace mongo { + + class Collection; + class IndexCatalogEntry; + + /** + * One per index for pregenerated keys + */ + struct PregeneratedKeysOnIndex { + PregeneratedKeysOnIndex() {} + PregeneratedKeysOnIndex( const PregeneratedKeysOnIndex& other ) { + invariant( keys.empty() ); + generator = other.generator; + } + boost::shared_ptr generator; + BSONObjSet keys; + }; + + /** + * one per document + * not thread safe + */ + class PregeneratedKeys { + public: + PregeneratedKeys(){} + PregeneratedKeys( const PregeneratedKeys& other ){ + // we need the copy construct for insertion into map + // we don't want to actually copy data though + // so we fail in that case + invariant( _indexes.empty() ); + } + const PregeneratedKeysOnIndex* get( IndexCatalogEntry* entry ) const; + + void gen( const BSONObj& obj, + IndexCatalogEntry* entry, + const boost::shared_ptr& generator ); + + void clear() { _indexes.clear(); } + private: + typedef std::map Map; + Map _indexes; + }; + + /** + * this is a singleten + */ + class GeneratorHolder { + public: + GeneratorHolder(); + + /** + * @return if we have a cache entry for this + */ + bool prepare( const StringData& ns, + const BSONObj& obj, + PregeneratedKeys* out ); + + void reset( const Collection* aCollection ); + + void dropped( const std::string& ns ); + + void droppedDatabase( const std::string& ns ); + + static GeneratorHolder* getInstance(); + + private: + struct MyIndex { + IndexCatalogEntry* entry; // cannot use as a pointer, just a number + boost::shared_ptr generator; // safe to use + }; + + struct MyCollection { + std::string ns; + std::vector indexes; + }; + + typedef std::map< string,boost::shared_ptr > Collections; // map from namespace to the indexes + SimpleMutex _collectionsLock; // for modifying the map structure itself + Collections _collections; + }; +} diff --git a/src/mongo/db/cc_by_loc.h b/src/mongo/db/cc_by_loc.h deleted file mode 100644 index b9dfd7d2117..00000000000 --- a/src/mongo/db/cc_by_loc.h +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright (C) 20012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/** - * Defines CursorId and defines ByLocKey, which pairs up a DiskLoc - * and a CursorId and provides a comparison operation. - */ - -#pragma once - -#include -#include - -#include "mongo/db/diskloc.h" - -namespace mongo { - - typedef long long CursorId; /* passed to the client so it can send back on getMore */ - static const CursorId INVALID_CURSOR_ID = -1; // But see SERVER-5726. - - // TODO(acm|mathias): ByLocKey allows constructing a map so we can - // invalidate or advance all cursors pointing to a DiskLoc when we - // remove that record. During review of moving this class to this - // header, Mathias said: "On further thought, I think this need - // would be better served by an unordered_map>." At which point, this - // class should be removed. - - struct ByLocKey { - - ByLocKey(const DiskLoc& l, const CursorId& i) : loc(l), id(i) {} - - static ByLocKey min(const DiskLoc& l) { - return ByLocKey(l, std::numeric_limits::min()); - } - - static ByLocKey max(const DiskLoc& l) { - return ByLocKey(l, std::numeric_limits::max()); - } - - const DiskLoc loc; - const CursorId id; - }; - - inline bool operator<(const ByLocKey& lhs, const ByLocKey& rhs) { - int x = lhs.loc.compare(rhs.loc); - if (x) - return x < 0; - return lhs.id < rhs.id; - } - - class ClientCursor; - typedef std::map CCByLoc; - -} // namespace mongo diff --git a/src/mongo/db/client.cpp b/src/mongo/db/client.cpp index a3ec1c5c674..a523505433d 100644 --- a/src/mongo/db/client.cpp +++ b/src/mongo/db/client.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* Client represents a connection to the database (the server-side) and corresponds @@ -28,10 +40,12 @@ #include #include "mongo/base/status.h" +#include "mongo/bson/mutable/document.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/auth_external_state_d.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/authz_session_external_state_d.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/db.h" #include "mongo/db/commands.h" @@ -43,15 +57,25 @@ #include "mongo/db/jsobj.h" #include "mongo/db/pagefault.h" #include "mongo/db/repl/rs.h" +#include "mongo/db/storage_options.h" #include "mongo/s/chunk_version.h" #include "mongo/s/d_logic.h" #include "mongo/s/stale_exception.h" // for SendStaleConfigException #include "mongo/scripting/engine.h" +#include "mongo/util/concurrency/thread_name.h" #include "mongo/util/file_allocator.h" #include "mongo/util/mongoutils/checksum.h" -#include "mongo/util/mongoutils/html.h" #include "mongo/util/mongoutils/str.h" +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#define ASAN_ENABLED __has_feature(address_sanitizer) +#define MSAN_ENABLED __has_feature(memory_sanitizer) +#define TSAN_ENABLED __has_feature(thread_sanitizer) +#define XSAN_ENABLED (ASAN_ENABLED || MSAN_ENABLED || TSAN_ENABLED) + namespace mongo { mongo::mutex& Client::clientsMutex = *(new mutex("clientsMutex")); @@ -59,7 +83,7 @@ namespace mongo { TSP_DEFINE(Client, currentClient) -#if defined(_DEBUG) +#if defined(_DEBUG) && !defined(MONGO_OPTIMIZED_BUILD) && !XSAN_ENABLED struct StackChecker; ThreadLocalValue checker; @@ -112,7 +136,7 @@ namespace mongo { call this when your thread starts. */ Client& Client::initThread(const char *desc, AbstractMessagingPort *mp) { -#if defined(_DEBUG) +#if defined(_DEBUG) && !defined(MONGO_OPTIMIZED_BUILD) && !XSAN_ENABLED { if( sizeof(void*) == 8 ) { StackChecker sc; @@ -121,28 +145,35 @@ namespace mongo { } #endif verify( currentClient.get() == 0 ); - Client *c = new Client(desc, mp); + + string fullDesc = desc; + if ( str::equals( "conn" , desc ) && mp != NULL ) + fullDesc = str::stream() << desc << mp->connectionId(); + + setThreadName( fullDesc.c_str() ); + + // Create the client obj, attach to thread + Client *c = new Client( fullDesc, mp ); currentClient.reset(c); mongo::lastError.initThread(); - c->setAuthorizationManager(new AuthorizationManager(new AuthExternalStateMongod())); + c->setAuthorizationSession(new AuthorizationSession(new AuthzSessionExternalStateMongod( + getGlobalAuthorizationManager()))); return *c; } - Client::Client(const char *desc, AbstractMessagingPort *p) : + Client::Client(const string& desc, AbstractMessagingPort *p) : ClientBasic(p), _context(0), _shutdown(false), _desc(desc), _god(0), - _lastOp(0) + _lastOp(0), + _isWriteCmd(false) { - _hasWrittenThisPass = false; + _hasWrittenThisOperation = false; + _hasWrittenSinceCheckpoint = false; _pageFaultRetryableSection = 0; _connectionId = p ? p->connectionId() : 0; - - if ( str::equals( "conn" , desc ) && _connectionId > 0 ) - _desc = str::stream() << desc << _connectionId; - setThreadName(_desc.c_str()); _curOp = new CurOp( this ); #ifndef _WIN32 stringstream temp; @@ -156,12 +187,21 @@ namespace mongo { Client::~Client() { _god = 0; + // Because both Client object pointers and logging infrastructure are stored in Thread + // Specific Pointers and because we do not explicitly control the order in which TSPs are + // deleted, it is possible for the logging infrastructure to have been deleted before + // this code runs. This leads to segfaults (access violations) if this code attempts + // to log anything. Therefore, disable logging from this destructor until this is fixed. + // TODO(tad) Force the logging infrastructure to be the last TSP to be deleted for each + // thread and reenable this code once that is done. +#if 0 if ( _context ) error() << "Client::~Client _context should be null but is not; client:" << _desc << endl; if ( ! _shutdown ) { error() << "Client::shutdown not called: " << _desc << endl; } +#endif if ( ! inShutdown() ) { // we can't clean up safely once we're in shutdown @@ -181,7 +221,7 @@ namespace mongo { } bool Client::shutdown() { -#if defined(_DEBUG) +#if defined(_DEBUG) && !defined(MONGO_OPTIMIZED_BUILD) && !XSAN_ENABLED { if( sizeof(void*) == 8 ) { StackChecker::check( desc() ); @@ -199,11 +239,12 @@ namespace mongo { return false; } - BSONObj CachedBSONObj::_tooBig = fromjson("{\"$msg\":\"query not recording (too large)\"}"); + BSONObj CachedBSONObjBase::_tooBig = fromjson("{\"$msg\":\"query not recording (too large)\"}"); Client::Context::Context(const std::string& ns , Database * db) : _client( currentClient.get() ), _oldContext( _client->_context ), - _path( mongo::dbpath ), // is this right? could be a different db? may need a dassert for this + _path(storageGlobalParams.dbpath), // is this right? could be a different db? + // may need a dassert for this _justCreated(false), _doVersion( true ), _ns( ns ), @@ -272,22 +313,20 @@ namespace mongo { } - void Client::Context::checkNotStale() const { + void Client::Context::checkNotStale() const { + // Write commands do not rely on the writeback mechanism (handled after the request + // has been processed) to reroute writes, so the version check needs to be done here. + if (_client->isWriteCmd()) { + ensureShardVersionOKOrThrow(_ns); + } + switch ( _client->_curOp->getOp() ) { case dbGetMore: // getMore's are special and should be handled else where case dbUpdate: // update & delete check shard version in instance.cpp, so don't check here as well case dbDelete: break; - default: { - string errmsg; - ChunkVersion received; - ChunkVersion wanted; - if ( ! shardVersionOk( _ns , errmsg, received, wanted ) ) { - ostringstream os; - os << "[" << _ns << "] shard version not ok in Client::Context: " << errmsg; - throw SendStaleConfigException( _ns, os.str(), received, wanted ); - } - } + default: + ensureShardVersionOKOrThrow(_ns); } } @@ -370,7 +409,7 @@ namespace mongo { return c->toString(); } - void Client::gotHandshake( const BSONObj& o ) { + bool Client::gotHandshake( const BSONObj& o ) { BSONObjIterator i(o); { @@ -387,9 +426,11 @@ namespace mongo { _handshake = b.obj(); - if (theReplSet && o.hasField("member")) { - theReplSet->registerSlave(_remoteId, o["member"].Int()); + if (!theReplSet || !o.hasField("member") || !o["member"].isNumber()) { + return false; } + + return theReplSet->registerSlave(_remoteId, o["member"].Int()); } bool ClientBasic::hasCurrent() { @@ -411,8 +452,8 @@ namespace mongo { const BSONObj& cmdObj, std::vector* out) { ActionSet actions; - actions.addAction(ActionType::handshake); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + actions.addAction(ActionType::internal); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } virtual bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { Client& c = cc(); @@ -422,64 +463,6 @@ namespace mongo { } handshakeCmd; - class ClientListPlugin : public WebStatusPlugin { - public: - ClientListPlugin() : WebStatusPlugin( "clients" , 20 ) {} - virtual void init() {} - - virtual void run( stringstream& ss ) { - using namespace mongoutils::html; - - ss << "\n"; - ss << "" - << th( a("", "Connections to the database, both internal and external.", "Client") ) - << th( a("http://dochub.mongodb.org/core/viewingandterminatingcurrentoperation", "", "OpId") ) - << "" - << "" - << "" - << "" - << th( a("http://dochub.mongodb.org/core/whatisanamespace", "", "Namespace") ) - << "" - << "" - << "" - << "" - - << "\n"; - { - scoped_lock bl(Client::clientsMutex); - for( set::iterator i = Client::clients.begin(); i != Client::clients.end(); i++ ) { - Client *c = *i; - CurOp& co = *(c->curop()); - ss << ""; - - tablecell( ss , co.opNum() ); - tablecell( ss , co.active() ); - tablecell( ss , c->lockState().reportState() ); - if ( co.active() ) - tablecell( ss , co.elapsedSeconds() ); - else - tablecell( ss , "" ); - tablecell( ss , co.getOp() ); - tablecell( ss , html::escape( co.getNS() ) ); - if ( co.haveQuery() ) - tablecell( ss , html::escape( co.query().toString() ) ); - else - tablecell( ss , "" ); - tablecell( ss , co.getRemoteString() ); - - tablecell( ss , co.getMessage() ); - tablecell( ss , co.getProgressMeter().toString() ); - - - ss << "\n"; - } - } - ss << "
LockingWaitingSecsRunningOpQueryclientmsgprogress
" << c->desc() << "
\n"; - - } - - } clientListPlugin; - int Client::recommendedYieldMicros( int * writers , int * readers, bool needExact ) { int num = 0; int w = 0; @@ -539,9 +522,9 @@ namespace mongo { } bool Client::allowedToThrowPageFaultException() const { - if ( _hasWrittenThisPass ) + if ( _hasWrittenThisOperation ) return false; - + if ( ! _pageFaultRetryableSection ) return false; @@ -556,6 +539,14 @@ namespace mongo { return true; } + void Client::setIsWriteCmd(bool newSetting) { + _isWriteCmd = newSetting; + } + + bool Client::isWriteCmd() const { + return _isWriteCmd; + } + void OpDebug::reset() { extra.reset(); @@ -571,9 +562,11 @@ namespace mongo { exhaust = false; nscanned = -1; + nscannedObjects = -1; idhack = false; scanAndOrder = false; - nupdated = -1; + nMatched = -1; + nModified = -1; ninserted = -1; ndeleted = -1; nmoved = -1; @@ -581,6 +574,8 @@ namespace mongo { fastmodinsert = false; upsert = false; keyUpdates = 0; // unsigned, so -1 not possible + planSummary = ""; + execStats.reset(); exceptionInfo.reset(); @@ -601,11 +596,28 @@ namespace mongo { s << ns.toString(); if ( ! query.isEmpty() ) { - if ( iscommand ) + if ( iscommand ) { s << " command: "; - else + + Command* curCommand = curop.getCommand(); + if (curCommand) { + mutablebson::Document cmdToLog(query, mutablebson::Document::kInPlaceDisabled); + curCommand->redactForLogging(&cmdToLog); + s << curCommand->name << " "; + s << cmdToLog.toString(); + } + else { // Should not happen but we need to handle curCommand == NULL gracefully + s << query.toString(); + } + } + else { s << " query: "; - s << query.toString(); + s << query.toString(); + } + } + + if (!planSummary.empty()) { + s << " planSummary: " << planSummary.toString(); } if ( ! updateobj.isEmpty() ) { @@ -619,10 +631,12 @@ namespace mongo { OPDEBUG_TOSTRING_HELP_BOOL( exhaust ); OPDEBUG_TOSTRING_HELP( nscanned ); + OPDEBUG_TOSTRING_HELP( nscannedObjects ); OPDEBUG_TOSTRING_HELP_BOOL( idhack ); OPDEBUG_TOSTRING_HELP_BOOL( scanAndOrder ); OPDEBUG_TOSTRING_HELP( nmoved ); - OPDEBUG_TOSTRING_HELP( nupdated ); + OPDEBUG_TOSTRING_HELP( nMatched ); + OPDEBUG_TOSTRING_HELP( nModified ); OPDEBUG_TOSTRING_HELP( ninserted ); OPDEBUG_TOSTRING_HELP( ndeleted ); OPDEBUG_TOSTRING_HELP_BOOL( fastmod ); @@ -639,8 +653,7 @@ namespace mongo { s << " code:" << exceptionInfo.code; } - if ( curop.numYields() ) - s << " numYields: " << curop.numYields(); + s << " numYields:" << curop.numYields(); s << " "; curop.lockStat().report( s ); @@ -711,11 +724,13 @@ namespace mongo { OPDEBUG_APPEND_BOOL( exhaust ); OPDEBUG_APPEND_NUMBER( nscanned ); + OPDEBUG_APPEND_NUMBER( nscannedObjects ); OPDEBUG_APPEND_BOOL( idhack ); OPDEBUG_APPEND_BOOL( scanAndOrder ); OPDEBUG_APPEND_BOOL( moved ); OPDEBUG_APPEND_NUMBER( nmoved ); - OPDEBUG_APPEND_NUMBER( nupdated ); + OPDEBUG_APPEND_NUMBER( nMatched ); + OPDEBUG_APPEND_NUMBER( nModified ); OPDEBUG_APPEND_NUMBER( ninserted ); OPDEBUG_APPEND_NUMBER( ndeleted ); OPDEBUG_APPEND_BOOL( fastmod ); @@ -733,7 +748,13 @@ namespace mongo { OPDEBUG_APPEND_NUMBER( responseLength ); b.append( "millis" , executionTime ); + execStats.append(b, "execStats"); + return true; } + void saveGLEStats(const BSONObj& result, const std::string& conn) { + // This can be called in mongod, which is unfortunate. To fix this, + // we can redesign how connection pooling works on mongod for sharded operations. + } } diff --git a/src/mongo/db/client.h b/src/mongo/db/client.h index 67063ddb147..2017c51bf43 100644 --- a/src/mongo/db/client.h +++ b/src/mongo/db/client.h @@ -20,6 +20,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -30,8 +42,8 @@ #include "mongo/db/d_concurrency.h" #include "mongo/db/lasterror.h" #include "mongo/db/lockstate.h" -#include "mongo/db/namespace-inl.h" #include "mongo/db/stats/top.h" +#include "mongo/db/storage_options.h" #include "mongo/util/concurrency/rwlock.h" #include "mongo/util/concurrency/threadlocal.h" #include "mongo/util/paths.h" @@ -95,25 +107,40 @@ namespace mongo { void appendLastOp( BSONObjBuilder& b ) const; bool isGod() const { return _god; } /* this is for map/reduce writes */ + bool setGod(bool newVal) { const bool prev = _god; _god = newVal; return prev; } string toString() const; - void gotHandshake( const BSONObj& o ); + bool gotHandshake( const BSONObj& o ); BSONObj getRemoteID() const { return _remoteId; } BSONObj getHandshake() const { return _handshake; } ConnectionId getConnectionId() const { return _connectionId; } bool inPageFaultRetryableSection() const { return _pageFaultRetryableSection != 0; } PageFaultRetryableSection* getPageFaultRetryableSection() const { return _pageFaultRetryableSection; } - - bool hasWrittenThisPass() const { return _hasWrittenThisPass; } - void writeHappened() { _hasWrittenThisPass = true; } - void newTopLevelRequest() { _hasWrittenThisPass = false; } - + + void writeHappened() { _hasWrittenSinceCheckpoint = true; _hasWrittenThisOperation = true; } + bool hasWrittenSinceCheckpoint() const { return _hasWrittenSinceCheckpoint; } + void checkpointHappened() { _hasWrittenSinceCheckpoint = false; } + bool hasWrittenThisOperation() const { return _hasWrittenThisOperation; } + void newTopLevelRequest() { + _hasWrittenThisOperation = false; + _hasWrittenSinceCheckpoint = false; + } + + /** + * Call this to allow PageFaultExceptions even if writes happened before this was called. + * Writes after this is called still prevent PFEs from being thrown. + */ + void clearHasWrittenThisOperation() { _hasWrittenThisOperation = false; } + bool allowedToThrowPageFaultException() const; LockState& lockState() { return _ls; } + void setIsWriteCmd(bool newSetting); + bool isWriteCmd() const; + private: - Client(const char *desc, AbstractMessagingPort *p = 0); + Client(const std::string& desc, AbstractMessagingPort *p = 0); friend class CurOp; ConnectionId _connectionId; // > 0 for things "conn", 0 otherwise string _threadId; // "" on non support systems @@ -126,31 +153,25 @@ namespace mongo { BSONObj _handshake; BSONObj _remoteId; - bool _hasWrittenThisPass; + bool _hasWrittenThisOperation; + bool _hasWrittenSinceCheckpoint; PageFaultRetryableSection *_pageFaultRetryableSection; LockState _ls; friend class PageFaultRetryableSection; // TEMP friend class NoPageFaultsAllowed; // TEMP - public: - /* set _god=true temporarily, safely */ - class GodScope { - bool _prev; - public: - GodScope(); - ~GodScope(); - }; + bool _isWriteCmd; + + public: - //static void assureDatabaseIsOpen(const string& ns, string path=dbpath); - /** "read lock, and set my context, all in one operation" * This handles (if not recursively locked) opening an unopened database. */ class ReadContext : boost::noncopyable { public: - ReadContext(const std::string& ns, const std::string& path=dbpath); + ReadContext(const std::string& ns, const std::string& path=storageGlobalParams.dbpath); Context& ctx() { return *c.get(); } private: scoped_ptr lk; @@ -163,7 +184,8 @@ namespace mongo { class Context : boost::noncopyable { public: /** this is probably what you want */ - Context(const string& ns, const std::string& path=dbpath, bool doVersion=true); + Context(const string& ns, const std::string& path=storageGlobalParams.dbpath, + bool doVersion=true); /** note: this does not call finishInit -- i.e., does not call shardVersionOk() for example. @@ -178,13 +200,15 @@ namespace mongo { Client* getClient() const { return _client; } Database* db() const { return _db; } const char * ns() const { return _ns.c_str(); } - bool equals( const string& ns , const string& path=dbpath ) const { return _ns == ns && _path == path; } + bool equals(const string& ns, const string& path=storageGlobalParams.dbpath) const { + return _ns == ns && _path == path; + } /** @return if the db was created by this Context */ bool justCreated() const { return _justCreated; } /** @return true iff the current Context is using db/path */ - bool inDB( const string& db , const string& path=dbpath ) const; + bool inDB(const string& db, const string& path=storageGlobalParams.dbpath) const; void _clear() { // this is sort of an "early destruct" indication, _ns can never be uncleared const_cast(_ns).clear(); @@ -218,7 +242,7 @@ namespace mongo { class WriteContext : boost::noncopyable { public: - WriteContext(const string& ns, const std::string& path=dbpath); + WriteContext(const string& ns, const std::string& path=storageGlobalParams.dbpath); Context& ctx() { return _c; } private: Lock::DBWrite _lk; @@ -236,13 +260,6 @@ namespace mongo { return *c; } - inline Client::GodScope::GodScope() { - _prev = cc()._god; - cc()._god = true; - } - inline Client::GodScope::~GodScope() { cc()._god = _prev; } - - inline bool haveClient() { return currentClient.get() > 0; } }; diff --git a/src/mongo/db/client_basic.cpp b/src/mongo/db/client_basic.cpp index 33185b5aae1..d3b06557df8 100644 --- a/src/mongo/db/client_basic.cpp +++ b/src/mongo/db/client_basic.cpp @@ -12,12 +12,24 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/client_basic.h" #include "mongo/db/auth/authentication_session.h" -#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" namespace mongo { @@ -41,22 +53,22 @@ namespace mongo { _authenticationSession.swap(other); } - bool ClientBasic::hasAuthorizationManager() const { - return _authorizationManager.get(); + bool ClientBasic::hasAuthorizationSession() const { + return _authorizationSession.get(); } - AuthorizationManager* ClientBasic::getAuthorizationManager() const { + AuthorizationSession* ClientBasic::getAuthorizationSession() const { massert(16481, "No AuthorizationManager has been set up for this connection", - hasAuthorizationManager()); - return _authorizationManager.get(); + hasAuthorizationSession()); + return _authorizationSession.get(); } - void ClientBasic::setAuthorizationManager(AuthorizationManager* authorizationManager) { + void ClientBasic::setAuthorizationSession(AuthorizationSession* authorizationSession) { massert(16477, "An AuthorizationManager has already been set up for this connection", - !hasAuthorizationManager()); - _authorizationManager.reset(authorizationManager); + !hasAuthorizationSession()); + _authorizationSession.reset(authorizationSession); } } // namespace mongo diff --git a/src/mongo/db/client_basic.h b/src/mongo/db/client_basic.h index 703263e18f7..0a41400f3df 100644 --- a/src/mongo/db/client_basic.h +++ b/src/mongo/db/client_basic.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -25,7 +37,7 @@ namespace mongo { class AuthenticationInfo; class AuthenticationSession; - class AuthorizationManager; + class AuthorizationSession; /** * this is the base class for Client and ClientInfo @@ -41,9 +53,9 @@ namespace mongo { void resetAuthenticationSession(AuthenticationSession* newSession); void swapAuthenticationSession(boost::scoped_ptr& other); - bool hasAuthorizationManager() const; - AuthorizationManager* getAuthorizationManager() const; - void setAuthorizationManager(AuthorizationManager* authorizationManager); + bool hasAuthorizationSession() const; + AuthorizationSession* getAuthorizationSession() const; + void setAuthorizationSession(AuthorizationSession* authorizationSession); bool getIsLocalHostConnection() { if (!hasRemote()) { @@ -67,7 +79,7 @@ namespace mongo { private: boost::scoped_ptr _authenticationSession; - boost::scoped_ptr _authorizationManager; + boost::scoped_ptr _authorizationSession; AbstractMessagingPort* const _messagingPort; }; } diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp index fa39170d73b..63e44649475 100644 --- a/src/mongo/db/clientcursor.cpp +++ b/src/mongo/db/clientcursor.cpp @@ -1,26 +1,30 @@ /** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -/* clientcursor.cpp - - ClientCursor is a wrapper that represents a cursorid from our database - application's perspective. - - Cursor -- and its derived classes -- are our internal cursors. -*/ + * Copyright (C) 2008, 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #include "mongo/pch.h" @@ -30,10 +34,12 @@ #include #include +#include "mongo/base/counter.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/db/audit.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" #include "mongo/db/commands/server_status.h" @@ -43,310 +49,81 @@ #include "mongo/db/kill_current_op.h" #include "mongo/db/pagefault.h" #include "mongo/db/repl/rs.h" -#include "mongo/db/repl_block.h" -#include "mongo/db/scanandorder.h" +#include "mongo/db/repl/write_concern.h" +#include "mongo/db/server_parameters.h" #include "mongo/platform/random.h" #include "mongo/util/processinfo.h" #include "mongo/util/timer.h" namespace mongo { - CCById ClientCursor::clientCursorsById; - boost::recursive_mutex& ClientCursor::ccmutex( *(new boost::recursive_mutex()) ); - long long ClientCursor::numberTimedOut = 0; + static Counter64 cursorStatsOpen; // gauge + static Counter64 cursorStatsOpenPinned; // gauge + static Counter64 cursorStatsOpenNoTimeout; // gauge + static Counter64 cursorStatsTimedOut; - void aboutToDeleteForSharding( const Database* db, const NamespaceDetails* nsd, const DiskLoc& dl ); // from s/d_logic.h + static ServerStatusMetricField dCursorStatsOpen( "cursor.open.total", + &cursorStatsOpen ); + static ServerStatusMetricField dCursorStatsOpenPinned( "cursor.open.pinned", + &cursorStatsOpenPinned ); + static ServerStatusMetricField dCursorStatsOpenNoTimeout( "cursor.open.noTimeout", + &cursorStatsOpenNoTimeout ); + static ServerStatusMetricField dCursorStatusTimedout( "cursor.timedOut", + &cursorStatsTimedOut ); - /*static*/ void ClientCursor::assertNoCursors() { - recursive_scoped_lock lock(ccmutex); - if( clientCursorsById.size() ) { - log() << "ERROR clientcursors exist but should not at this point" << endl; - ClientCursor *cc = clientCursorsById.begin()->second; - log() << "first one: " << cc->_cursorid << ' ' << cc->_ns << endl; - clientCursorsById.clear(); - verify(false); - } - } - - - void ClientCursor::setLastLoc_inlock(DiskLoc L) { - verify( _pos != -2 ); // defensive - see ~ClientCursor - - if ( L == _lastLoc ) - return; - - CCByLoc& bl = byLoc(); - - if ( !_lastLoc.isNull() ) { - bl.erase( ByLocKey( _lastLoc, _cursorid ) ); - } - - if ( !L.isNull() ) - bl[ByLocKey(L,_cursorid)] = this; - _lastLoc = L; - } - - /* ------------------------------------------- */ - - /* must call this when a btree node is updated */ - //void removedKey(const DiskLoc& btreeLoc, int keyPos) { - //} - - // ns is either a full namespace or "dbname." when invalidating for a whole db - void ClientCursor::invalidate(const char *ns) { - Lock::assertWriteLocked(ns); - int len = strlen(ns); - const char* dot = strchr(ns, '.'); - verify( len > 0 && dot); - - bool isDB = (dot == &ns[len-1]); // first (and only) dot is the last char - - { - //cout << "\nTEMP invalidate " << ns << endl; - Database *db = cc().database(); - verify(db); - verify( str::startsWith(ns, db->name) ); - - for( LockedIterator i; i.ok(); ) { - ClientCursor *cc = i.current(); - - bool shouldDelete = false; - if ( cc->_db == db ) { - if (isDB) { - // already checked that db matched above - dassert( str::startsWith(cc->_ns.c_str(), ns) ); - shouldDelete = true; - } - else { - if ( str::equals(cc->_ns.c_str(), ns) ) - shouldDelete = true; - } - } - - if ( shouldDelete ) { - i.deleteAndAdvance(); - } - else { - i.advance(); - } - } + MONGO_EXPORT_SERVER_PARAMETER(cursorTimeoutMillis, int, 10 * 60 * 1000 /* 10 minutes */); - /* - note : we can't iterate byloc because clientcursors may exist with a loc of null in which case - they are not in the map. perhaps they should not exist though in the future? something to - change??? - - CCByLoc& bl = db->ccByLoc; - for ( CCByLoc::iterator i = bl.begin(); i != bl.end(); ++i ) { - ClientCursor *cc = i->second; - if ( strncmp(ns, cc->ns.c_str(), len) == 0 ) { - verify( cc->_db == db ); - toDelete.push_back(i->second); - } - }*/ - - /*cout << "TEMP after invalidate " << endl; - for( auto i = clientCursorsById.begin(); i != clientCursorsById.end(); ++i ) { - cout << " " << i->second->ns << endl; - } - cout << "TEMP after invalidate done" << endl;*/ - } - } - - /* note called outside of locks (other than ccmutex) so care must be exercised */ - bool ClientCursor::shouldTimeout( unsigned millis ) { - _idleAgeMillis += millis; - return _idleAgeMillis > 600000 && _pinValue == 0; + long long ClientCursor::totalOpen() { + return cursorStatsOpen.get(); } - /* called every 4 seconds. millis is amount of idle time passed since the last call -- could be zero */ - void ClientCursor::idleTimeReport(unsigned millis) { - bool foundSomeToTimeout = false; - - // two passes so that we don't need to readlock unless we really do some timeouts - // we assume here that incrementing _idleAgeMillis outside readlock is ok. - { - recursive_scoped_lock lock(ccmutex); - { - unsigned sz = clientCursorsById.size(); - static time_t last; - if( sz >= 100000 ) { - if( time(0) - last > 300 ) { - last = time(0); - log() << "warning number of open cursors is very large: " << sz << endl; - } - } - } - for ( CCById::iterator i = clientCursorsById.begin(); i != clientCursorsById.end(); ) { - CCById::iterator j = i; - i++; - if( j->second->shouldTimeout( millis ) ) { - foundSomeToTimeout = true; - } - } - } - - if( foundSomeToTimeout ) { - Lock::GlobalRead lk; - for( LockedIterator i; i.ok(); ) { - ClientCursor *cc = i.current(); - if( cc->shouldTimeout(0) ) { - numberTimedOut++; - LOG(1) << "killing old cursor " << cc->_cursorid << ' ' << cc->_ns - << " idle:" << cc->idleTime() << "ms\n"; - i.deleteAndAdvance(); - } - else { - i.advance(); - } - } + ClientCursor::ClientCursor(const Collection* collection, Runner* runner, + int qopts, const BSONObj query) + : _collection( collection ), + _countedYet( false ) { + _runner.reset(runner); + _ns = runner->ns(); + _query = query; + _queryOptions = qopts; + if ( runner->collection() ) { + invariant( collection == runner->collection() ); } + init(); } - /* must call when a btree bucket going away. - note this is potentially slow - */ - void ClientCursor::informAboutToDeleteBucket(const DiskLoc& b) { - recursive_scoped_lock lock(ccmutex); - Database *db = cc().database(); - CCByLoc& bl = db->ccByLoc; - RARELY if ( bl.size() > 70 ) { - log() << "perf warning: byLoc.size=" << bl.size() << " in aboutToDeleteBucket" << endl; - } - for ( CCByLoc::iterator i = bl.begin(); i != bl.end(); i++ ) - i->second->_c->aboutToDeleteBucket(b); - } - void aboutToDeleteBucket(const DiskLoc& b) { - ClientCursor::informAboutToDeleteBucket(b); + ClientCursor::ClientCursor(const Collection* collection) + : _ns(collection->ns().ns()), + _collection(collection), + _countedYet( false ), + _queryOptions(QueryOption_NoCursorTimeout) { + init(); } - /* must call this on a delete so we clean up the cursors. */ - void ClientCursor::aboutToDelete(const NamespaceDetails* nsd, const DiskLoc& dl) { - NoPageFaultsAllowed npfa; - - recursive_scoped_lock lock(ccmutex); - - Database *db = cc().database(); - verify(db); - - aboutToDeleteForSharding( db, nsd, dl ); - - CCByLoc& bl = db->ccByLoc; - CCByLoc::iterator j = bl.lower_bound(ByLocKey::min(dl)); - CCByLoc::iterator stop = bl.upper_bound(ByLocKey::max(dl)); - if ( j == stop ) - return; - - vector toAdvance; + void ClientCursor::init() { + invariant( _collection ); - while ( 1 ) { - toAdvance.push_back(j->second); - DEV verify( j->first.loc == dl ); - ++j; - if ( j == stop ) - break; - } - - if( toAdvance.size() >= 3000 ) { - log() << "perf warning MPW101: " << toAdvance.size() << " cursors for one diskloc " - << dl.toString() - << ' ' << toAdvance[1000]->_ns - << ' ' << toAdvance[2000]->_ns - << ' ' << toAdvance[1000]->_pinValue - << ' ' << toAdvance[2000]->_pinValue - << ' ' << toAdvance[1000]->_pos - << ' ' << toAdvance[2000]->_pos - << ' ' << toAdvance[1000]->_idleAgeMillis - << ' ' << toAdvance[2000]->_idleAgeMillis - << ' ' << toAdvance[1000]->_doingDeletes - << ' ' << toAdvance[2000]->_doingDeletes - << endl; - //wassert( toAdvance.size() < 5000 ); - } + isAggCursor = false; - for ( vector::iterator i = toAdvance.begin(); i != toAdvance.end(); ++i ) { - ClientCursor* cc = *i; - wassert(cc->_db == db); - - if ( cc->_doingDeletes ) continue; + _idleAgeMillis = 0; + _leftoverMaxTimeMicros = 0; + _pinValue = 0; + _pos = 0; - Cursor *c = cc->_c.get(); - if ( c->capped() ) { - /* note we cannot advance here. if this condition occurs, writes to the oplog - have "caught" the reader. skipping ahead, the reader would miss postentially - important data. - */ - delete cc; - continue; - } + Lock::assertAtLeastReadLocked(_ns); - c->recoverFromYield(); - DiskLoc tmp1 = c->refLoc(); - if ( tmp1 != dl ) { - // This might indicate a failure to call ClientCursor::prepareToYield() but it can - // also happen during correct operation, see SERVER-2009. - problem() << "warning: cursor loc " << tmp1 << " does not match byLoc position " << dl << " !" << endl; - } - else { - c->advance(); - } - while (!c->eof() && c->refLoc() == dl) { - /* We don't delete at EOF because we want to return "no more results" rather than "no such cursor". - * The loop is to handle MultiKey indexes where the deleted record is pointed to by multiple adjacent keys. - * In that case we need to advance until we get to the next distinct record or EOF. - * SERVER-4154 - * SERVER-5198 - * But see SERVER-5725. - */ - c->advance(); - } - cc->updateLocation(); + if (_queryOptions & QueryOption_NoCursorTimeout) { + // cursors normally timeout after an inactivity period to prevent excess memory use + // setting this prevents timeout of the cursor in question. + ++_pinValue; + cursorStatsOpenNoTimeout.increment(); } - } - void ClientCursor::LockedIterator::deleteAndAdvance() { - ClientCursor *cc = current(); - CursorId id = cc->cursorid(); - delete cc; - _i = clientCursorsById.upper_bound( id ); - } - - ClientCursor::ClientCursor(int queryOptions, const shared_ptr& c, const string& ns, BSONObj query ) : - _ns(ns), _db( cc().database() ), - _c(c), _pos(0), - _query(query), _queryOptions(queryOptions), - _idleAgeMillis(0), _pinValue(0), - _doingDeletes(false), _yieldSometimesTracker(128,10) { - - Lock::assertAtLeastReadLocked(ns); - - verify( _db ); - verify( str::startsWith(_ns, _db->name) ); - if( queryOptions & QueryOption_NoCursorTimeout ) - noTimeout(); - recursive_scoped_lock lock(ccmutex); - _cursorid = allocCursorId_inlock(); - clientCursorsById.insert( make_pair(_cursorid, this) ); - - if ( ! _c->modifiedKeys() ) { - // store index information so we can decide if we can - // get something out of the index key rather than full object - - int x = 0; - BSONObjIterator i( _c->indexKeyPattern() ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( e.isNumber() ) { - // only want basic index fields, not "2d" etc - _indexedFields[e.fieldName()] = x; - } - x++; - } - } + _cursorid = _collection->cursorCache()->registerCursor( this ); + cursorStatsOpen.increment(); + _countedYet = true; } - ClientCursor::~ClientCursor() { if( _pos == -2 ) { // defensive: destructor called twice @@ -354,204 +131,30 @@ namespace mongo { return; } - { - recursive_scoped_lock lock(ccmutex); - setLastLoc_inlock( DiskLoc() ); // removes us from bylocation multimap - clientCursorsById.erase(_cursorid); - - // defensive: - _cursorid = INVALID_CURSOR_ID; - _pos = -2; - _pinValue = 0; - } - } - - bool ClientCursor::getFieldsDotted( const string& name, BSONElementSet &ret, BSONObj& holder ) { - - map::const_iterator i = _indexedFields.find( name ); - if ( i == _indexedFields.end() ) { - current().getFieldsDotted( name , ret ); - return false; - } - - int x = i->second; - - holder = currKey(); - BSONObjIterator it( holder ); - while ( x && it.more() ) { - it.next(); - x--; - } - verify( x == 0 ); - ret.insert( it.next() ); - return true; - } - - BSONElement ClientCursor::getFieldDotted( const string& name , BSONObj& holder , bool * fromKey ) { - - map::const_iterator i = _indexedFields.find( name ); - if ( i == _indexedFields.end() ) { - if ( fromKey ) - *fromKey = false; - holder = current(); - return holder.getFieldDotted( name ); - } - - int x = i->second; - - holder = currKey(); - BSONObjIterator it( holder ); - while ( x && it.more() ) { - it.next(); - x--; - } - verify( x == 0 ); - - if ( fromKey ) - *fromKey = true; - return it.next(); - } - - BSONObj ClientCursor::extractFields(const BSONObj &pattern , bool fillWithNull ) { - BSONObjBuilder b( pattern.objsize() * 2 ); - - BSONObj holder; - - BSONObjIterator i( pattern ); - while ( i.more() ) { - BSONElement key = i.next(); - BSONElement value = getFieldDotted( key.fieldName() , holder ); - - if ( value.type() ) { - b.appendAs( value , key.fieldName() ); - continue; - } - - if ( fillWithNull ) - b.appendNull( key.fieldName() ); - - } - - return b.obj(); - } - - BSONObj ClientCursor::extractKey( const KeyPattern& usingKeyPattern ) const { - KeyPattern currentIndex( _c->indexKeyPattern() ); - if ( usingKeyPattern.isCoveredBy( currentIndex ) && ! currentIndex.isSpecial() ){ - BSONObj currKey = _c->currKey(); - BSONObj prettyKey = currKey.replaceFieldNames( currentIndex.toBSON() ); - return usingKeyPattern.extractSingleKey( prettyKey ); + if ( _countedYet ) { + _countedYet = false; + cursorStatsOpen.decrement(); + if ( _pinValue == 1 ) + cursorStatsOpenNoTimeout.decrement(); } - return usingKeyPattern.extractSingleKey( _c->current() ); - } - void ClientCursor::fillQueryResultFromObj( BufBuilder &b, const MatchDetails* details ) const { - const Projection::KeyOnly *keyFieldsOnly = c()->keyFieldsOnly(); - if ( keyFieldsOnly ) { - mongo::fillQueryResultFromObj( b, 0, keyFieldsOnly->hydrate( c()->currKey() ), details ); + if ( _collection ) { + // this could be null if kill() was killed + _collection->cursorCache()->deregisterCursor( this ); } - else { - DiskLoc loc = c()->currLoc(); - mongo::fillQueryResultFromObj( b, fields.get(), c()->current(), details, - ( ( pq && pq->showDiskLoc() ) ? &loc : 0 ) ); - } - } - /* call when cursor's location changes so that we can update the - cursorsbylocation map. if you are locked and internally iterating, only - need to call when you are ready to "unlock". - */ - void ClientCursor::updateLocation() { - verify( _cursorid ); - _idleAgeMillis = 0; - _c->prepareToYield(); - DiskLoc cl = _c->refLoc(); - if ( lastLoc() == cl ) { - //log() << "info: lastloc==curloc " << ns << endl; - } - else { - recursive_scoped_lock lock(ccmutex); - setLastLoc_inlock(cl); - } + // defensive: + _collection = NULL; + _cursorid = INVALID_CURSOR_ID; + _pos = -2; + _pinValue = 0; } - int ClientCursor::suggestYieldMicros() { - int writers = 0; - int readers = 0; - - int micros = Client::recommendedYieldMicros( &writers , &readers ); - - if ( micros > 0 && writers == 0 && Lock::isR() ) { - // we have a read lock, and only reads are coming on, so why bother unlocking - return 0; - } - - wassert( micros < 10000000 ); - dassert( micros < 1000001 ); - return micros; - } - - Record* ClientCursor::_recordForYield( ClientCursor::RecordNeeds need ) { - - if ( ! ok() ) - return 0; - - if ( need == DontNeed ) { - return 0; - } - else if ( need == MaybeCovered ) { - // TODO - return 0; - } - else if ( need == WillNeed ) { - // no-op - } - else { - warning() << "don't understand RecordNeeds: " << (int)need << endl; - return 0; - } - - DiskLoc l = currLoc(); - if ( l.isNull() ) - return 0; - - Record * rec = l.rec(); - if ( rec->likelyInPhysicalMemory() ) - return 0; - - return rec; - } - - bool ClientCursor::yieldSometimes( RecordNeeds need, bool *yielded ) { - if ( yielded ) { - *yielded = false; - } - if ( ! _yieldSometimesTracker.intervalHasElapsed() ) { - Record* rec = _recordForYield( need ); - if ( rec ) { - // yield for page fault - if ( yielded ) { - *yielded = true; - } - bool res = yield( suggestYieldMicros() , rec ); - if ( res ) - _yieldSometimesTracker.resetLastTime(); - return res; - } - return true; - } + void ClientCursor::kill() { + if ( _runner.get() ) + _runner->kill(); - int micros = suggestYieldMicros(); - if ( micros > 0 ) { - if ( yielded ) { - *yielded = true; - } - bool res = yield( micros , _recordForYield( need ) ); - if ( res ) - _yieldSometimesTracker.resetLastTime(); - return res; - } - return true; + _collection = NULL; } void yieldOrSleepFor1Microsecond() { @@ -564,10 +167,10 @@ namespace mongo { #endif } - void ClientCursor::staticYield( int micros , const StringData& ns , Record * rec ) { + void ClientCursor::staticYield(int micros, const StringData& ns, const Record* rec) { bool haveReadLock = Lock::isReadLocked(); - killCurrentOp.checkForInterrupt( false ); + killCurrentOp.checkForInterrupt(); { auto_ptr lk; if ( rec ) { @@ -594,7 +197,7 @@ namespace mongo { } else { if ( micros == -1 ) { - micros = Client::recommendedYieldMicros(); + sleepmicros(Client::recommendedYieldMicros()); } else if ( micros == 0 ) { yieldOrSleepFor1Microsecond(); @@ -612,10 +215,12 @@ namespace mongo { CurOp * c = cc().curop(); while ( c->parent() ) c = c->parent(); - warning() << "ClientCursor::yield can't unlock b/c of recursive lock" - << " ns: " << ns - << " top: " << c->info() - << endl; + RARELY { + warning() << "ClientCursor::staticYield can't unlock b/c of recursive lock" + << " ns: " << ns + << " top: " << c->info() + << endl; + } } if ( rec ) @@ -625,179 +230,89 @@ namespace mongo { } } - bool ClientCursor::prepareToYield( YieldData &data ) { - if ( ! _c->supportYields() ) - return false; - - // need to store in case 'this' gets deleted - data._id = _cursorid; - - data._doingDeletes = _doingDeletes; - _doingDeletes = false; - - updateLocation(); - - { - /* a quick test that our temprelease is safe. - todo: make a YieldingCursor class - and then make the following code part of a unit test. - */ - const int test = 0; - static bool inEmpty = false; - if( test && !inEmpty ) { - inEmpty = true; - log() << "TEST: manipulate collection during cc:yield" << endl; - if( test == 1 ) - Helpers::emptyCollection(_ns.c_str()); - else if( test == 2 ) { - BSONObjBuilder b; string m; - dropCollection(_ns.c_str(), m, b); - } - else { - dropDatabase(_ns.c_str()); - } - } - } - return true; - } - - bool ClientCursor::recoverFromYield( const YieldData &data ) { - ClientCursor *cc = ClientCursor::find( data._id , false ); - if ( cc == 0 ) { - // id was deleted - return false; - } + // + // Timing and timeouts + // - cc->_doingDeletes = data._doingDeletes; - cc->_c->recoverFromYield(); - return true; + bool ClientCursor::shouldTimeout(unsigned millis) { + _idleAgeMillis += millis; + return (static_cast(_idleAgeMillis) > cursorTimeoutMillis) && (_pinValue == 0); } - /** @return true if cursor is still ok */ - bool ClientCursor::yield( int micros , Record * recordToLoad ) { - - if ( ! _c->supportYields() ) // so me cursors (geo@oct2011) don't support yielding - return true; - - YieldData data; - prepareToYield( data ); - staticYield( micros , _ns , recordToLoad ); - return ClientCursor::recoverFromYield( data ); + void ClientCursor::setIdleTime( unsigned millis ) { + _idleAgeMillis = millis; } - namespace { - // so we don't have to do find() which is a little slow very often. - long long cursorGenTSLast = 0; - PseudoRandom* cursorGenRandom = NULL; + void ClientCursor::updateSlaveLocation( CurOp& curop ) { + if ( _slaveReadTill.isNull() ) + return; + mongo::updateSlaveLocation( curop , _ns.c_str() , _slaveReadTill ); } - long long ClientCursor::allocCursorId_inlock() { - // It is important that cursor IDs not be reused within a short period of time. - - if ( ! cursorGenRandom ) { - scoped_ptr sr( SecureRandom::create() ); - cursorGenRandom = new PseudoRandom( sr->nextInt64() ); - } - - const long long ts = Listener::getElapsedTimeMillis(); - - long long x; - - while ( 1 ) { - x = ts << 32; - x |= cursorGenRandom->nextInt32(); - - if ( x == 0 ) - continue; + int ClientCursor::suggestYieldMicros() { + int writers = 0; + int readers = 0; - if ( x < 0 ) - x *= -1; + int micros = Client::recommendedYieldMicros( &writers , &readers ); - if ( ts != cursorGenTSLast || ClientCursor::find_inlock(x, false) == 0 ) - break; + if ( micros > 0 && writers == 0 && Lock::isR() ) { + // we have a read lock, and only reads are coming on, so why bother unlocking + return 0; } - cursorGenTSLast = ts; - - return x; + wassert( micros < 10000000 ); + dassert( micros < 1000001 ); + return micros; } - void ClientCursor::storeOpForSlave( DiskLoc last ) { - if ( ! ( _queryOptions & QueryOption_OplogReplay )) - return; - - if ( last.isNull() ) - return; + // + // Pin methods + // TODO: Simplify when we kill Cursor. In particular, once we've pinned a CC, it won't be + // deleted from underneath us, so we can save the pointer and ignore the ID. + // - BSONElement e = last.obj()["ts"]; - if ( e.type() == Date || e.type() == Timestamp ) - _slaveReadTill = e._opTime(); + ClientCursorPin::ClientCursorPin( const Collection* collection, long long cursorid ) + : _cursor( NULL ) { + cursorStatsOpenPinned.increment(); + _cursor = collection->cursorCache()->find( cursorid, true ); } - void ClientCursor::updateSlaveLocation( CurOp& curop ) { - if ( _slaveReadTill.isNull() ) - return; - mongo::updateSlaveLocation( curop , _ns.c_str() , _slaveReadTill ); + ClientCursorPin::~ClientCursorPin() { + cursorStatsOpenPinned.decrement(); + DESTRUCTOR_GUARD( release(); ); } + void ClientCursorPin::release() { + if ( !_cursor ) + return; - void ClientCursor::appendStats( BSONObjBuilder& result ) { - recursive_scoped_lock lock(ccmutex); - result.appendNumber("totalOpen", clientCursorsById.size() ); - result.appendNumber("clientCursors_size", (int) numCursors()); - result.appendNumber("timedOut" , numberTimedOut); - unsigned pinned = 0; - unsigned notimeout = 0; - for ( CCById::iterator i = clientCursorsById.begin(); i != clientCursorsById.end(); i++ ) { - unsigned p = i->second->_pinValue; - if( p >= 100 ) - pinned++; - else if( p > 0 ) - notimeout++; - } - if( pinned ) - result.append("pinned", pinned); - if( notimeout ) - result.append("totalNoTimeout", notimeout); - } + invariant( _cursor->pinValue() >= 100 ); - // QUESTION: Restrict to the namespace from which this command was issued? - // Alternatively, make this command admin-only? - class CmdCursorInfo : public Command { - public: - CmdCursorInfo() : Command( "cursorInfo", true ) {} - virtual bool slaveOk() const { return true; } - virtual void help( stringstream& help ) const { - help << " example: { cursorInfo : 1 }"; + if ( _cursor->collection() == NULL ) { + // the ClientCursor was killed while we had it + // therefore its our responsibility to kill it + delete _cursor; + _cursor = NULL; // defensive } - virtual LockType locktype() const { return NONE; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::cursorInfo); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); - } - bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { - ClientCursor::appendStats( result ); - return true; + else { + _cursor->collection()->cursorCache()->unpin( _cursor ); } - } cmdCursorInfo; - - class CursorServerStats : public ServerStatusSection { - public: + } - CursorServerStats() : ServerStatusSection( "cursors" ){} - virtual bool includeByDefault() const { return true; } + void ClientCursorPin::deleteUnderlying() { + delete _cursor; + _cursor = NULL; + } - BSONObj generateSection(const BSONElement& configElement) const { - BSONObjBuilder b; - ClientCursor::appendStats( b ); - return b.obj(); - } + ClientCursor* ClientCursorPin::c() const { + return _cursor; + } - } cursorServerStats; + // + // ClientCursorMonitor + // + // Used by sayMemoryStatus below. struct Mem { Mem() { res = virt = mapped = 0; } long long res; @@ -810,13 +325,15 @@ namespace mongo { } }; - /** called once a minute from killcursors thread */ + /** + * called once a minute from killcursors thread + */ void sayMemoryStatus() { static time_t last; static Mem mlast; try { ProcessInfo p; - if ( !cmdLine.quiet && p.supported() ) { + if (!serverGlobalParams.quiet && p.supported()) { Mem m; m.res = p.getResidentSize(); m.virt = p.getVirtualMemorySize(); @@ -825,7 +342,7 @@ namespace mongo { if( now - last >= 300 || m.grew(mlast) ) { log() << "mem (MB) res:" << m.res << " virt:" << m.virt; long long totalMapped = m.mapped; - if (cmdLine.dur) { + if (storageGlobalParams.dur) { totalMapped *= 2; log() << " mapped (incl journal view):" << totalMapped; } @@ -848,7 +365,6 @@ namespace mongo { } } - /** thread for timing out old cursors */ void ClientCursorMonitor::run() { Client::initThread("clientcursormon"); Client& client = cc(); @@ -856,131 +372,71 @@ namespace mongo { const int Secs = 4; unsigned n = 0; while ( ! inShutdown() ) { - ClientCursor::idleTimeReport( t.millisReset() ); + cursorStatsTimedOut.increment( CollectionCursorCache::timeoutCursorsGlobal( t.millisReset() ) ); sleepsecs(Secs); - if( ++n % (60/4) == 0 /*once a minute*/ ) { + if( ++n % (60/Secs) == 0 /*once a minute*/ ) { sayMemoryStatus(); } } client.shutdown(); } - void ClientCursor::find( const string& ns , set& all ) { - recursive_scoped_lock lock(ccmutex); - - for ( CCById::iterator i=clientCursorsById.begin(); i!=clientCursorsById.end(); ++i ) { - if ( i->second->_ns == ns ) - all.insert( i->first ); - } - } + ClientCursorMonitor clientCursorMonitor; - bool ClientCursor::_erase_inlock(ClientCursor* cursor) { - // Must not have an active ClientCursor::Pin. - massert( 16089, - str::stream() << "Cannot kill active cursor " << cursor->cursorid(), - cursor->_pinValue < 100 ); + // + // cursorInfo command. + // - delete cursor; - return true; - } + void _appendCursorStats( BSONObjBuilder& b ) { + b.append( "note" , "deprecated, use server status metrics" ); - bool ClientCursor::erase(CursorId id) { - recursive_scoped_lock lock(ccmutex); - ClientCursor* cursor = find_inlock(id); - if (!cursor) { - return false; - } + b.appendNumber("clientCursors_size", cursorStatsOpen.get() ); + b.appendNumber("totalOpen", cursorStatsOpen.get() ); + b.appendNumber("pinned", cursorStatsOpenPinned.get() ); + b.appendNumber("totalNoTimeout", cursorStatsOpenNoTimeout.get() ); - return _erase_inlock(cursor); + b.appendNumber("timedOut" , cursorStatsTimedOut.get()); } - bool ClientCursor::eraseIfAuthorized(CursorId id) { - std::string ns; - { - recursive_scoped_lock lock(ccmutex); - ClientCursor* cursor = find_inlock(id); - if (!cursor) { - return false; - } - ns = cursor->ns(); - } - - // Can't be in a lock when checking authorization - if (!cc().getAuthorizationManager()->checkAuthorization(ns, ActionType::killCursors)) { - return false; - } - - // It is safe to lookup the cursor again after temporarily releasing the mutex because - // of 2 invariants: that the cursor ID won't be re-used in a short period of time, and that - // the namespace associated with a cursor cannot change. - recursive_scoped_lock lock(ccmutex); - ClientCursor* cursor = find_inlock(id); - if (!cursor) { - // Cursor was deleted in another thread since we found it earlier in this function. - return false; + // QUESTION: Restrict to the namespace from which this command was issued? + // Alternatively, make this command admin-only? + // TODO: remove this for 2.8 + class CmdCursorInfo : public Command { + public: + CmdCursorInfo() : Command( "cursorInfo", true ) {} + virtual bool slaveOk() const { return true; } + virtual void help( stringstream& help ) const { + help << " example: { cursorInfo : 1 }, deprecated"; } - if (cursor->ns() != ns) { - warning() << "Cursor namespace changed. Previous ns: " << ns << ", current ns: " - << cursor->ns() << endl; - return false; + virtual LockType locktype() const { return NONE; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::cursorInfo); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } - - return _erase_inlock(cursor); - } - - int ClientCursor::erase(int n, long long *ids) { - int found = 0; - for ( int i = 0; i < n; i++ ) { - if ( erase(ids[i])) - found++; - - if ( inShutdown() ) - break; + bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, + bool fromRepl ) { + _appendCursorStats( result ); + return true; } - return found; - } - - int ClientCursor::eraseIfAuthorized(int n, long long *ids) { - int found = 0; - for ( int i = 0; i < n; i++ ) { - if ( eraseIfAuthorized(ids[i])) - found++; + } cmdCursorInfo; - if ( inShutdown() ) - break; - } - return found; - } + // + // cursors stats. + // - ClientCursor::YieldLock::YieldLock( ptr cc ) - : _canYield(cc->_c->supportYields()) { - - if ( _canYield ) { - cc->prepareToYield( _data ); - _unlock.reset(new dbtempreleasecond()); - } + class CursorServerStats : public ServerStatusSection { + public: + CursorServerStats() : ServerStatusSection( "cursors" ){} + virtual bool includeByDefault() const { return true; } - } - - ClientCursor::YieldLock::~YieldLock() { - if ( _unlock ) { - warning() << "ClientCursor::YieldLock not closed properly" << endl; - relock(); + BSONObj generateSection(const BSONElement& configElement) const { + BSONObjBuilder b; + _appendCursorStats( b ); + return b.obj(); } - } - - bool ClientCursor::YieldLock::stillOk() { - if ( ! _canYield ) - return true; - relock(); - return ClientCursor::recoverFromYield( _data ); - } - - void ClientCursor::YieldLock::relock() { - _unlock.reset(); - } - - - ClientCursorMonitor clientCursorMonitor; + } cursorServerStats; } // namespace mongo diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index f2077282c66..26aed634556 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -1,436 +1,240 @@ -/* clientcursor.h */ - /** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -/* Cursor -- and its derived classes -- are our internal cursors. - - ClientCursor is a wrapper that represents a cursorid from our database - application's perspective. -*/ + * Copyright (C) 2008 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #pragma once -#include "mongo/pch.h" - #include -#include "cursor.h" -#include "jsobj.h" -#include "../util/net/message.h" -#include "../util/background.h" -#include "cc_by_loc.h" -#include "diskloc.h" -#include "dbhelpers.h" -#include "matcher.h" -#include "projection.h" -#include "s/d_chunk_manager.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" #include "mongo/db/keypattern.h" -#include "mongo/util/elapsed_tracker.h" +#include "mongo/db/query/runner.h" +#include "mongo/s/collection_metadata.h" +#include "mongo/util/background.h" +#include "mongo/util/net/message.h" namespace mongo { typedef boost::recursive_mutex::scoped_lock recursive_scoped_lock; - class Cursor; /* internal server cursor base class */ class ClientCursor; + class Collection; + class CurOp; + class Database; + class NamespaceDetails; class ParsedQuery; - /* todo: make this map be per connection. this will prevent cursor hijacking security attacks perhaps. - * ERH: 9/2010 this may not work since some drivers send getMore over a different connection - */ - typedef map CCById; - - extern BSONObj id_obj; + typedef long long CursorId; /* passed to the client so it can send back on getMore */ + static const CursorId INVALID_CURSOR_ID = -1; // But see SERVER-5726. + /** + * ClientCursor is a wrapper that represents a cursorid from our database application's + * perspective. + */ class ClientCursor : private boost::noncopyable { - friend class CmdCursorInfo; public: - static void assertNoCursors(); - - /* use this to assure we don't in the background time out cursor while it is under use. - if you are using noTimeout() already, there is no risk anyway. - Further, this mechanism guards against two getMore requests on the same cursor executing - at the same time - which might be bad. That should never happen, but if a client driver - had a bug, it could (or perhaps some sort of attack situation). - */ - class Pin : boost::noncopyable { - public: - Pin( long long cursorid ) : - _cursorid( INVALID_CURSOR_ID ) { - recursive_scoped_lock lock( ccmutex ); - ClientCursor *cursor = ClientCursor::find_inlock( cursorid, true ); - if ( cursor ) { - uassert( 12051, "clientcursor already in use? driver problem?", - cursor->_pinValue < 100 ); - cursor->_pinValue += 100; - _cursorid = cursorid; - } - } - void release() { - if ( _cursorid == INVALID_CURSOR_ID ) { - return; - } - ClientCursor *cursor = c(); - _cursorid = INVALID_CURSOR_ID; - if ( cursor ) { - verify( cursor->_pinValue >= 100 ); - cursor->_pinValue -= 100; - } - } - ~Pin() { DESTRUCTOR_GUARD( release(); ) } - ClientCursor *c() const { return ClientCursor::find( _cursorid ); } - private: - CursorId _cursorid; - }; - - /** Assures safe and reliable cleanup of a ClientCursor. */ - class Holder : boost::noncopyable { - public: - Holder( ClientCursor *c = 0 ) : - _c( 0 ), - _id( INVALID_CURSOR_ID ) { - reset( c ); - } - void reset( ClientCursor *c = 0 ) { - if ( c == _c ) - return; - if ( _c ) { - // be careful in case cursor was deleted by someone else - ClientCursor::erase( _id ); - } - if ( c ) { - _c = c; - _id = c->_cursorid; - } - else { - _c = 0; - _id = INVALID_CURSOR_ID; - } - } - ~Holder() { - DESTRUCTOR_GUARD ( reset(); ); - } - ClientCursor* get() { return _c; } - operator bool() { return _c; } - ClientCursor * operator-> () { return _c; } - const ClientCursor * operator-> () const { return _c; } - /** Release ownership of the ClientCursor. */ - void release() { - _c = 0; - _id = INVALID_CURSOR_ID; - } - private: - ClientCursor *_c; - CursorId _id; - }; + ClientCursor(const Collection* collection, Runner* runner, + int qopts = 0, const BSONObj query = BSONObj()); - /** - * Iterates through all ClientCursors, under its own ccmutex lock. - * Also supports deletion on the fly. - */ - class LockedIterator : boost::noncopyable { - public: - LockedIterator() : _lock( ccmutex ), _i( clientCursorsById.begin() ) {} - bool ok() const { return _i != clientCursorsById.end(); } - ClientCursor *current() const { return _i->second; } - void advance() { ++_i; } - /** - * Delete 'current' and advance. Properly handles cascading deletions that may occur - * when one ClientCursor is directly deleted. - */ - void deleteAndAdvance(); - private: - recursive_scoped_lock _lock; - CCById::const_iterator _i; - }; - - ClientCursor(int queryOptions, const shared_ptr& c, const string& ns, BSONObj query = BSONObj() ); + ClientCursor(const Collection* collection); ~ClientCursor(); - // *************** basic accessors ******************* + // + // Basic accessors + // CursorId cursorid() const { return _cursorid; } string ns() const { return _ns; } - Database * db() const { return _db; } - const BSONObj& query() const { return _query; } - int queryOptions() const { return _queryOptions; } - - DiskLoc lastLoc() const { return _lastLoc; } - - /* Get rid of cursors for namespaces 'ns'. When dropping a db, ns is "dbname." - Used by drop, dropIndexes, dropDatabase. - */ - static void invalidate(const char *ns); + const Collection* collection() const { return _collection; } /** - * @param microsToSleep -1 : ask client - * 0 : pthread_yield or equivilant - * >0 : sleep for that amount - * @param recordToLoad after yielding lock, load this record with only mmutex - * do a dbtemprelease - * note: caller should check matcher.docMatcher().atomic() first and not yield if atomic - - * we don't do herein as this->matcher (above) is only initialized for true queries/getmore. - * (ie not set for remote/update) - * @return if the cursor is still valid. - * if false is returned, then this ClientCursor should be considered deleted - - * in fact, the whole database could be gone. + * This is called when someone is dropping a collection or something else that + * goes through killing cursors. + * It removes the responsiilibty of de-registering from ClientCursor. + * Responsibility for deleting the ClientCursor doesn't change from this call + * see Runner::kill. */ - bool yield( int microsToSleep = -1, Record * recordToLoad = 0 ); + void kill(); - enum RecordNeeds { - DontNeed = -1 , MaybeCovered = 0 , WillNeed = 100 - }; - - /** - * @param needRecord whether or not the next record has to be read from disk for sure - * if this is true, will yield of next record isn't in memory - * @param yielded true if a yield occurred, and potentially if a yield did not occur - * @return same as yield() - */ - bool yieldSometimes( RecordNeeds need, bool *yielded = 0 ); + // + // Yielding. + // + static void staticYield(int micros, const StringData& ns, const Record* rec); static int suggestYieldMicros(); - static void staticYield( int micros , const StringData& ns , Record * rec ); - - struct YieldData { CursorId _id; bool _doingDeletes; }; - bool prepareToYield( YieldData &data ); - static bool recoverFromYield( const YieldData &data ); - - struct YieldLock : boost::noncopyable { - - explicit YieldLock( ptr cc ); - - ~YieldLock(); - - /** - * @return if the cursor is still ok - * if it is, we also relock - */ - bool stillOk(); - - void relock(); - - private: - const bool _canYield; - YieldData _data; - scoped_ptr _unlock; - }; - - // --- some pass through helpers for Cursor --- - - Cursor* c() const { return _c.get(); } - int pos() const { return _pos; } - void incPos( int n ) { _pos += n; } // TODO: this is bad - void setPos( int n ) { _pos = n; } // TODO : this is bad too - - BSONObj indexKeyPattern() { return _c->indexKeyPattern(); } - bool modifiedKeys() const { return _c->modifiedKeys(); } - bool isMultiKey() const { return _c->isMultiKey(); } - - bool ok() { return _c->ok(); } - bool advance() { return _c->advance(); } - BSONObj current() { return _c->current(); } - DiskLoc currLoc() { return _c->currLoc(); } - BSONObj currKey() const { return _c->currKey(); } + // + // Timing and timeouts + // /** - * same as BSONObj::getFieldsDotted - * if it can be retrieved from key, it is - * @param holder keeps the currKey in scope by keeping a reference to it here. generally you'll want - * holder and ret to destruct about the same time. - * @return if this was retrieved from key - */ - bool getFieldsDotted( const string& name, BSONElementSet &ret, BSONObj& holder ); - - /** - * same as BSONObj::getFieldDotted - * if it can be retrieved from key, it is - * @return if this was retrieved from key - */ - BSONElement getFieldDotted( const string& name , BSONObj& holder , bool * fromKey = 0 ) ; - - /** extract items from object which match a pattern object. - * e.g., if pattern is { x : 1, y : 1 }, builds an object with - * x and y elements of this object, if they are present. - * returns elements with original field names - * NOTE: copied from BSONObj::extractFields - */ - BSONObj extractFields(const BSONObj &pattern , bool fillWithNull = false) ; - - /** Extract elements from the object this cursor currently points to, using the expression - * specified in KeyPattern. Will use a covered index if the one in this cursor is usable. - * TODO: there are some cases where a covered index could be used but is not, for instance - * if both this index and the keyPattern are {a : "hashed"} - */ - BSONObj extractKey( const KeyPattern& usingKeyPattern ) const; - - void fillQueryResultFromObj( BufBuilder &b, const MatchDetails* details = NULL ) const; - - bool currentIsDup() { return _c->getsetdup( _c->currLoc() ); } - - bool currentMatches() { - if ( ! _c->matcher() ) - return true; - return _c->matcher()->matchesCurrent( _c.get() ); - } - - void setChunkManager( ShardChunkManagerPtr manager ){ _chunkManager = manager; } - ShardChunkManagerPtr getChunkManager(){ return _chunkManager; } - - private: - void setLastLoc_inlock(DiskLoc); - - static ClientCursor* find_inlock(CursorId id, bool warn = true) { - CCById::iterator it = clientCursorsById.find(id); - if ( it == clientCursorsById.end() ) { - if ( warn ) { - OCCASIONALLY out() << "ClientCursor::find(): cursor not found in map '" << id - << "' (ok after a drop)" << endl; - } - return 0; - } - return it->second; - } - - /* call when cursor's location changes so that we can update the - cursorsbylocation map. if you are locked and internally iterating, only - need to call when you are ready to "unlock". + * @param millis amount of idle passed time since last call + * note called outside of locks (other than ccmutex) so care must be exercised */ - void updateLocation(); + bool shouldTimeout( unsigned millis ); + void setIdleTime( unsigned millis ); + unsigned idleTime() const { return _idleAgeMillis; } - public: - static ClientCursor* find(CursorId id, bool warn = true) { - recursive_scoped_lock lock(ccmutex); - ClientCursor *c = find_inlock(id, warn); - // if this asserts, your code was not thread safe - you either need to set no timeout - // for the cursor or keep a ClientCursor::Pointer in scope for it. - massert( 12521, "internal error: use of an unlocked ClientCursor", c == 0 || c->_pinValue ); - return c; + uint64_t getLeftoverMaxTimeMicros() const { return _leftoverMaxTimeMicros; } + void setLeftoverMaxTimeMicros( uint64_t leftoverMaxTimeMicros ) { + _leftoverMaxTimeMicros = leftoverMaxTimeMicros; } - /** - * Deletes the cursor with the provided @param 'id' if one exists. - * @throw if the cursor with the provided id is pinned. - * This does not do any auth checking and should be used only when erasing cursors as part - * of cleaning up internal operations. - */ - static bool erase(CursorId id); - // Same as erase but checks to make sure this thread has read permission on the cursor's - // namespace. This should be called when receiving killCursors from a client. This should - // not be called when ccmutex is held. - static bool eraseIfAuthorized(CursorId id); + // + // Sharding-specific data. TODO: Document. + // - /** - * @return number of cursors found - */ - static int erase(int n, long long* ids); - static int eraseIfAuthorized(int n, long long* ids); - - void mayUpgradeStorage() { - /* if ( !ids_.get() ) - return; - stringstream ss; - ss << ns << "." << cursorid; - ids_->mayUpgradeStorage( ss.str() );*/ - } + void setCollMetadata( CollectionMetadataPtr metadata ){ _collMetadata = metadata; } + CollectionMetadataPtr getCollMetadata(){ return _collMetadata; } - /** - * @param millis amount of idle passed time since last call - */ - bool shouldTimeout( unsigned millis ); + // + // Replication-related stuff. TODO: Document and clean. + // - void storeOpForSlave( DiskLoc last ); void updateSlaveLocation( CurOp& curop ); - - unsigned idleTime() const { return _idleAgeMillis; } - - void setDoingDeletes( bool doingDeletes ) {_doingDeletes = doingDeletes; } - void slaveReadTill( const OpTime& t ) { _slaveReadTill = t; } - /** Just for testing. */ OpTime getSlaveReadTill() const { return _slaveReadTill; } - public: // static methods - - static void idleTimeReport(unsigned millis); + // + // Query-specific functionality that may be adapted for the Runner. + // - static void appendStats( BSONObjBuilder& result ); - static unsigned numCursors() { return clientCursorsById.size(); } - static void informAboutToDeleteBucket(const DiskLoc& b); - static void aboutToDelete(const NamespaceDetails* nsd, const DiskLoc& dl); - static void find( const string& ns , set& all ); + Runner* getRunner() const { return _runner.get(); } + int queryOptions() const { return _queryOptions; } + const BSONObj& getQuery() const { return _query; } + // Used by ops/query.cpp to stash how many results have been returned by a query. + int pos() const { return _pos; } + void incPos(int n) { _pos += n; } + void setPos(int n) { _pos = n; } - private: // methods + /** + * Is this ClientCursor backed by an aggregation pipeline. Defaults to false. + * + * Agg Runners differ from others in that they manage their own locking internally and + * should not be killed or destroyed when the underlying collection is deleted. + * + * Note: This should *not* be set for the internal cursor used as input to an aggregation. + */ + bool isAggCursor; - // cursors normally timeout after an inactivity period to prevent excess memory use - // setting this prevents timeout of the cursor in question. - void noTimeout() { _pinValue++; } + unsigned pinValue() const { return _pinValue; } - CCByLoc& byLoc() { return _db->ccByLoc; } - - Record* _recordForYield( RecordNeeds need ); - static bool _erase_inlock(ClientCursor* cursor); + static long long totalOpen(); private: + friend class ClientCursorMonitor; + friend class CmdCursorInfo; + friend class CollectionCursorCache; + + /** + * Initialization common between both constructors for the ClientCursor. + */ + void init(); + // + // ClientCursor-specific data, independent of the underlying execution type. + // + + // The ID of the ClientCursor. CursorId _cursorid; - const string _ns; - Database * _db; + // A variable indicating the state of the ClientCursor. Possible values: + // 0: Normal behavior. May time out. + // 1: No timing out of this ClientCursor. + // 100: Currently in use (via ClientCursorPin). + unsigned _pinValue; - const shared_ptr _c; - map _indexedFields; // map from indexed field to offset in key object - int _pos; // # objects into the cursor so far + // The namespace we're operating on. + string _ns; - const BSONObj _query; // used for logging diags only; optional in constructor - int _queryOptions; // see enum QueryOptions dbclient.h + const Collection* _collection; - OpTime _slaveReadTill; + // if we've added it to the total open counter yet + bool _countedYet; - DiskLoc _lastLoc; // use getter and setter not this (important) - unsigned _idleAgeMillis; // how long has the cursor been around, relative to server idle time + // How many objects have been returned by the find() so far? + int _pos; - /* 0 = normal - 1 = no timeout allowed - 100 = in use (pinned) -- see Pointer class - */ - unsigned _pinValue; + // If this cursor was created by a find operation, '_query' holds the query predicate for + // the find. If this cursor was created by a command (e.g. the aggregate command), then + // '_query' holds the command specification received from the client. + BSONObj _query; - bool _doingDeletes; // when true we are the delete and aboutToDelete shouldn't manipulate us - ElapsedTracker _yieldSometimesTracker; + // See the QueryOptions enum in dbclient.h + int _queryOptions; - ShardChunkManagerPtr _chunkManager; + // TODO: document better. + OpTime _slaveReadTill; - public: - shared_ptr pq; - shared_ptr fields; // which fields query wants returned + // How long has the cursor been idle? + unsigned _idleAgeMillis; - private: // static members + // TODO: Document. + uint64_t _leftoverMaxTimeMicros; - static CCById clientCursorsById; - static long long numberTimedOut; - static boost::recursive_mutex& ccmutex; // must use this for all statics above! - static CursorId allocCursorId_inlock(); + // For chunks that are being migrated, there is a period of time when that chunks data is in + // two shards, the donor and the receiver one. That data is picked up by a cursor on the + // receiver side, even before the migration was decided. The CollectionMetadata allow one + // to inquiry if any given document of the collection belongs indeed to this shard or if it + // is coming from (or a vestige of) an ongoing migration. + CollectionMetadataPtr _collMetadata; + + // + // The underlying execution machinery. + // + scoped_ptr _runner; + }; + /** + * use this to assure we don't in the background time out cursor while it is under use. if you + * are using noTimeout() already, there is no risk anyway. Further, this mechanism guards + * against two getMore requests on the same cursor executing at the same time - which might be + * bad. That should never happen, but if a client driver had a bug, it could (or perhaps some + * sort of attack situation). + * Must have a read lock on the collection already + */ + class ClientCursorPin : boost::noncopyable { + public: + ClientCursorPin( const Collection* collection, long long cursorid ); + ~ClientCursorPin(); + // This just releases the pin, does not delete the underlying + // unless ownership has passed to us after kill + void release(); + // Call this to delete the underlying ClientCursor. + void deleteUnderlying(); + ClientCursor *c() const; + private: + ClientCursor* _cursor; }; + /** thread for timing out old cursors */ class ClientCursorMonitor : public BackgroundJob { public: string name() const { return "ClientCursorMonitor"; } @@ -438,14 +242,3 @@ namespace mongo { }; } // namespace mongo - -// ClientCursor should only be used with auto_ptr because it needs to be -// release()ed after a yield if stillOk() returns false and these pointer types -// do not support releasing. This will prevent them from being used accidentally -// Instead of auto_ptr<>, which still requires some degree of manual management -// of this, consider using ClientCursor::Holder which handles ClientCursor's -// unusual self-deletion mechanics. -namespace boost{ - template<> class scoped_ptr {}; - template<> class shared_ptr {}; -} diff --git a/src/mongo/db/clientlistplugin.cpp b/src/mongo/db/clientlistplugin.cpp new file mode 100644 index 00000000000..9c5f6d3ce60 --- /dev/null +++ b/src/mongo/db/clientlistplugin.cpp @@ -0,0 +1,96 @@ +/** +* Copyright (C) 2009 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/platform/basic.h" + +#include "mongo/db/client.h" +#include "mongo/db/curop.h" +#include "mongo/db/dbwebserver.h" +#include "mongo/util/mongoutils/html.h" + +namespace mongo { +namespace { + class ClientListPlugin : public WebStatusPlugin { + public: + ClientListPlugin() : WebStatusPlugin( "clients" , 20 ) {} + virtual void init() {} + + virtual void run( std::stringstream& ss ) { + using namespace mongoutils::html; + + ss << "\n"; + ss << "" + << th( a("", "Connections to the database, both internal and external.", "Client") ) + << th( a("http://dochub.mongodb.org/core/viewingandterminatingcurrentoperation", "", "OpId") ) + << "" + << "" + << "" + << "" + << th( a("http://dochub.mongodb.org/core/whatisanamespace", "", "Namespace") ) + << "" + << "" + << "" + << "" + + << "\n"; + { + scoped_lock bl(Client::clientsMutex); + for( set::iterator i = Client::clients.begin(); i != Client::clients.end(); i++ ) { + Client *c = *i; + CurOp& co = *(c->curop()); + ss << ""; + + tablecell( ss , co.opNum() ); + tablecell( ss , co.active() ); + tablecell( ss , c->lockState().reportState() ); + if ( co.active() ) + tablecell( ss , co.elapsedSeconds() ); + else + tablecell( ss , "" ); + tablecell( ss , co.getOp() ); + tablecell( ss , html::escape( co.getNS() ) ); + if ( co.haveQuery() ) + tablecell( ss , html::escape( co.query().toString() ) ); + else + tablecell( ss , "" ); + tablecell( ss , co.getRemoteString() ); + + tablecell( ss , co.getMessage() ); + tablecell( ss , co.getProgressMeter().toString() ); + + + ss << "\n"; + } + } + ss << "
LockingWaitingSecsRunningOpQueryclientmsgprogress
" << c->desc() << "
\n"; + + } + + } clientListPlugin; +} // namespace +} // namespace mongo diff --git a/src/mongo/db/cloner.cpp b/src/mongo/db/cloner.cpp index d9600912715..3e2966f6e2e 100644 --- a/src/mongo/db/cloner.cpp +++ b/src/mongo/db/cloner.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -21,35 +33,53 @@ #include "mongo/base/init.h" #include "mongo/base/status.h" #include "mongo/bson/util/builder.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/cloner.h" #include "mongo/db/commands.h" +#include "mongo/db/commands/copydb.h" #include "mongo/db/commands/rename_collection.h" #include "mongo/db/db.h" +#include "mongo/db/dbhelpers.h" +#include "mongo/db/index_builder.h" #include "mongo/db/instance.h" #include "mongo/db/jsobj.h" #include "mongo/db/kill_current_op.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/ops/insert.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/oplogreader.h" #include "mongo/db/pdfile.h" -#include "mongo/db/repl.h" -#include "mongo/db/sort_phase_one.h" +#include "mongo/db/server_parameters.h" +#include "mongo/db/storage_options.h" +#include "mongo/db/catalog/collection.h" namespace mongo { - BSONElement getErrField(const BSONObj& o); + MONGO_EXPORT_SERVER_PARAMETER(skipCorruptDocumentsWhenCloning, bool, false); - bool replAuthenticate(DBClientBase *, bool); + BSONElement getErrField(const BSONObj& o); /** Selectively release the mutex based on a parameter. */ class dbtempreleaseif { + MONGO_DISALLOW_COPYING(dbtempreleaseif); public: dbtempreleaseif( bool release ) : _impl( release ? new dbtemprelease() : 0 ) {} + ~dbtempreleaseif() throw(DBException) { + if (_impl) + delete _impl; + } private: - shared_ptr< dbtemprelease > _impl; + // can't use a smart pointer because we need throw annotation on destructor + dbtemprelease* _impl; }; - + void mayInterrupt( bool mayBeInterrupted ) { - if ( mayBeInterrupted ) { - killCurrentOp.checkForInterrupt( false ); + if ( mayBeInterrupted ) { + killCurrentOp.checkForInterrupt( false ); } } @@ -75,7 +105,7 @@ namespace mongo { uassert( 10024 , "bad ns field for index during dbcopy", e.type() == String); const char *p = strchr(e.valuestr(), '.'); uassert( 10025 , "bad ns field for index during dbcopy [2]", p); - string newname = cc().database()->name + p; + string newname = cc().database()->name() + p; b.append("ns", newname); } else @@ -96,175 +126,187 @@ namespace mongo { Cloner::Cloner() { } struct Cloner::Fun { - Fun() : lastLog(0), _sortersForIndex(NULL) { } - time_t lastLog; + Fun( Client::Context& ctx ) : lastLog(0), context( ctx ) { } + void operator()( DBClientCursorBatchIterator &i ) { Lock::GlobalWrite lk; - if ( context ) { - context->relocked(); - } + context.relocked(); + + bool createdCollection = false; + Collection* collection = NULL; while( i.moreInCurrentBatch() ) { - if ( n % 128 == 127 /*yield some*/ ) { + if ( numSeen % 128 == 127 /*yield some*/ ) { + collection = NULL; time_t now = time(0); - if( now - lastLog >= 60 ) { + if( now - lastLog >= 60 ) { // report progress if( lastLog ) - log() << "clone " << to_collection << ' ' << n << endl; + log() << "clone " << to_collection << ' ' << numSeen << endl; lastLog = now; } mayInterrupt( _mayBeInterrupted ); dbtempreleaseif t( _mayYield ); } + uassert(ErrorCodes::NotMaster, + str::stream() << "Not primary while cloning collection " << from_collection + << " to " << to_collection, + !logForRepl || + isMasterNs(to_collection)); + + if ( isindex == false && collection == NULL ) { + collection = context.db()->getCollection( to_collection ); + if ( !collection ) { + massert( 17321, + str::stream() + << "collection dropped during clone [" + << to_collection << "]", + !createdCollection ); + createdCollection = true; + collection = context.db()->createCollection( to_collection ); + verify( collection ); + } + } + BSONObj tmp = i.nextSafe(); /* assure object is valid. note this will slow us down a little. */ - if ( !tmp.valid() ) { - stringstream ss; - ss << "Cloner: skipping corrupt object from " << from_collection; - BSONElement e = tmp.firstElement(); - try { - e.validate(); - ss << " firstElement: " << e; - } - catch( ... ) { - ss << " firstElement corrupt"; + const Status status = validateBSON(tmp.objdata(), tmp.objsize()); + if (!status.isOK()) { + str::stream ss; + ss << "Cloner: found corrupt document in " << from_collection + << ": " << status.reason(); + if (skipCorruptDocumentsWhenCloning) { + warning() << ss.ss.str() << "; skipping"; + continue; } - out() << ss.str() << endl; - continue; + msgasserted(28531, ss); } - ++n; + ++numSeen; BSONObj js = tmp; if ( isindex ) { - verify(NamespaceString(from_collection).coll == "system.indexes"); + verify(nsToCollectionSubstring(from_collection) == "system.indexes"); js = fixindex(tmp); - storedForLater->push_back( js.getOwned() ); + indexesToBuild->push_back( js.getOwned() ); continue; } - try { - // add keys for presorting - DiskLoc loc = theDataFileMgr.insertWithObjMod(to_collection, js); - loc.assertOk(); - if (_sortersForIndex != NULL) { - // add key to SortersForNS - for (SortersForIndex::iterator iSorter = _sortersForIndex->begin(); - iSorter != _sortersForIndex->end(); - ++iSorter) { - iSorter->second.preSortPhase.addKeys(iSorter->second.spec, js, - loc, false); - } - } - if ( logForRepl ) - logOp("i", to_collection, js); + verify(nsToCollectionSubstring(from_collection) != "system.indexes"); - getDur().commitIfNeeded(); - } - catch( UserException& e ) { - error() << "error: exception cloning object in " << from_collection << ' ' << e.what() << " obj:" << js.toString() << '\n'; - throw; + StatusWith loc = collection->insertDocument( js, true ); + if ( !loc.isOK() ) { + error() << "error: exception cloning object in " << from_collection + << ' ' << loc.toString() << " obj:" << js; } + uassertStatusOK( loc.getStatus() ); + if ( logForRepl ) + logOp("i", to_collection, js); + + getDur().commitIfNeeded(); RARELY if ( time( 0 ) - saveLast > 60 ) { - log() << n << " objects cloned so far from collection " << from_collection << endl; + log() << numSeen << " objects cloned so far from collection " << from_collection; saveLast = time( 0 ); } } } - int n; + + time_t lastLog; + Client::Context& context; + + int64_t numSeen; bool isindex; const char *from_collection; const char *to_collection; time_t saveLast; - list *storedForLater; // deferred query results (e.g. index insert/build) + list *indexesToBuild; // deferred query results (e.g. index insert/build) bool logForRepl; - Client::Context *context; bool _mayYield; bool _mayBeInterrupted; - SortersForIndex *_sortersForIndex; // sorters that build index keys during query }; /* copy the specified collection isindex - if true, this is system.indexes collection, in which we do some transformation when copying. */ - void Cloner::copy(const char *from_collection, const char *to_collection, bool isindex, + void Cloner::copy(Client::Context& ctx, + const char *from_collection, const char *to_collection, bool isindex, bool logForRepl, bool masterSameProcess, bool slaveOk, bool mayYield, bool mayBeInterrupted, Query query) { - list storedForLater; + list indexesToBuild; LOG(2) << "\t\tcloning collection " << from_collection << " to " << to_collection << " on " << _conn->getServerAddress() << " with filter " << query.toString() << endl; - Fun f; - f.n = 0; + Fun f( ctx ); + f.numSeen = 0; f.isindex = isindex; f.from_collection = from_collection; f.to_collection = to_collection; f.saveLast = time( 0 ); - f.storedForLater = &storedForLater; + f.indexesToBuild = &indexesToBuild; f.logForRepl = logForRepl; f._mayYield = mayYield; f._mayBeInterrupted = mayBeInterrupted; - if (!isindex) { - SortersForNS::iterator it = _sortersForNS.find(to_collection); - if (it != _sortersForNS.end()) - f._sortersForIndex = &it->second; - } - int options = QueryOption_NoCursorTimeout | ( slaveOk ? QueryOption_SlaveOk : 0 ); { - f.context = cc().getContext(); mayInterrupt( mayBeInterrupted ); dbtempreleaseif r( mayYield ); _conn->query(boost::function(f), from_collection, query, 0, options); } - if ( storedForLater.size() ) { - for (list::const_iterator i = storedForLater.begin(); - i != storedForLater.end(); - ++i) { - BSONObj js = *i; - scoped_lock precalcLock(theDataFileMgr._precalcedMutex); - try { - // set the 'precalculated' index data and add the index - SortersForNS::iterator sortIter = _sortersForNS.find(js["ns"].String()); - if (sortIter != _sortersForNS.end()) { - SortersForIndex::iterator it = sortIter->second.find(js["name"].String()); - if (it != sortIter->second.end()) { - theDataFileMgr.setPrecalced(&it->second.preSortPhase); - } - } - theDataFileMgr.insertWithObjMod(to_collection, js); - theDataFileMgr.setPrecalced(NULL); + uassert(ErrorCodes::NotMaster, + str::stream() << "Not primary while cloning collection " << from_collection + << " to " << to_collection << " with filter " + << query.toString(), + !logForRepl || + isMasterNs(from_collection)); - if ( logForRepl ) - logOp("i", to_collection, js); + if ( indexesToBuild.size() ) { + for (list::const_iterator i = indexesToBuild.begin(); + i != indexesToBuild.end(); + ++i) { - getDur().commitIfNeeded(); + BSONObj spec = *i; + string ns = spec["ns"].String(); // this was fixed when pulled off network + Collection* collection = f.context.db()->getCollection( ns ); + if ( !collection ) { + collection = f.context.db()->createCollection( ns ); + verify( collection ); } - catch( UserException& e ) { - theDataFileMgr.setPrecalced(NULL); - error() << "error: exception cloning object in " << from_collection << ' ' << e.what() << " obj:" << js.toString() << '\n'; - throw; + + Status status = collection->getIndexCatalog()->createIndex( spec, mayBeInterrupted ); + if ( status.code() == ErrorCodes::IndexAlreadyExists ) { + // no-op } - catch(const DBException&) { - theDataFileMgr.setPrecalced(NULL); - throw; + else if ( !status.isOK() ) { + error() << "error creating index when cloning spec: " << spec + << " error: " << status.toString(); + uassertStatusOK( status ); } + + if ( logForRepl ) + logOp("i", to_collection, spec); + + getDur().commitIfNeeded(); + } } } - bool Cloner::validateQueryResults(const auto_ptr& cur, int32_t* errCode) { + bool Cloner::validateQueryResults(const auto_ptr& cur, + int32_t* errCode, + string& errmsg) { if ( cur.get() == 0 ) return false; if ( cur->more() ) { BSONObj first = cur->next(); - if(!getErrField(first).eoo()) { + BSONElement errField = getErrField(first); + if(!errField.eoo()) { + errmsg = errField.str(); if (errCode) *errCode = first.getIntField("code"); return false; @@ -280,7 +322,7 @@ namespace mongo { DBClientConnection *tmpConn = new DBClientConnection(); // cloner owns _conn in auto_ptr cloner.setConnection(tmpConn); - uassert(15908, errmsg, tmpConn->connect(host, errmsg) && replAuthenticate(tmpConn, false)); + uassert(15908, errmsg, tmpConn->connect(host, errmsg) && replAuthenticate(tmpConn)); return cloner.copyCollection(ns, BSONObj(), errmsg, true, false, true, false); } @@ -289,17 +331,31 @@ namespace mongo { bool mayYield, bool mayBeInterrupted, bool copyIndexes, bool logForRepl) { + const NamespaceString nss(ns); + const string dbname = nss.db().toString(); + Client::WriteContext ctx(ns); + uassert(ErrorCodes::NotMaster, + str::stream() << "Not primary while copying collection " << ns << " (Cloner)", + !logForRepl || + isMasterNs(ns.c_str())); + // config - string temp = ctx.ctx().db()->name + ".system.namespaces"; - BSONObj config = _conn->findOne(temp , BSON("name" << ns)); - if (config["options"].isABSONObj()) - if (!userCreateNS(ns.c_str(), config["options"].Obj(), errmsg, logForRepl, 0)) - return false; + BSONObj filter = BSON("name" << nss.coll().toString()); + list collList = _conn->getCollectionInfos( dbname, filter); + if (!collList.empty()) { + invariant(collList.size() <= 1); + BSONObj col = collList.front(); + if (col["options"].isABSONObj()) { + if (!userCreateNS(ns.c_str(), col["options"].Obj(), errmsg, logForRepl, 0)) + return false; + } + } // main data - copy(ns.c_str(), ns.c_str(), false, logForRepl, false, true, mayYield, mayBeInterrupted, + copy(ctx.ctx(), + ns.c_str(), ns.c_str(), false, logForRepl, false, true, mayYield, mayBeInterrupted, Query(query).snapshot()); /* TODO : copyIndexes bool does not seem to be implemented! */ @@ -308,49 +364,31 @@ namespace mongo { } // indexes - temp = ctx.ctx().db()->name + ".system.indexes"; - copy(temp.c_str(), temp.c_str(), true, logForRepl, false, true, mayYield, mayBeInterrupted, - BSON( "ns" << ns )); + std::string temp = ctx.ctx().db()->name() + ".system.indexes"; + copy(ctx.ctx(), temp.c_str(), temp.c_str(), true, logForRepl, false, true, mayYield, + mayBeInterrupted, BSON( "ns" << ns )); getDur().commitIfNeeded(); return true; } extern bool inDBRepair; - extern const int DefaultIndexVersionNumber; // from indexkey.cpp - void ensureIdIndexForNewNs(const char *ns); - bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, bool slaveOk, bool useReplAuth, bool snapshot, bool mayYield, bool mayBeInterrupted, int *errCode) { - - CloneOptions opts; - - opts.fromDB = fromdb; - opts.logForRepl = logForRepl; - opts.slaveOk = slaveOk; - opts.useReplAuth = useReplAuth; - opts.snapshot = snapshot; - opts.mayYield = mayYield; - opts.mayBeInterrupted = mayBeInterrupted; - - set clonedColls; - return go( masterHost, opts, clonedColls, errmsg, errCode ); - - } - - bool Cloner::go(const char *masterHost, const CloneOptions& opts, set& clonedColls, + bool Cloner::go(Client::Context& context, + const string& masterHost, const CloneOptions& opts, set* clonedColls, string& errmsg, int* errCode) { if ( errCode ) { *errCode = 0; } massert( 10289 , "useReplAuth is not written to replication log", !opts.useReplAuth || !opts.logForRepl ); - string todb = cc().database()->name; + string todb = cc().database()->name(); stringstream a,b; - a << "localhost:" << cmdLine.port; - b << "127.0.0.1:" << cmdLine.port; + a << "localhost:" << serverGlobalParams.port; + b << "127.0.0.1:" << serverGlobalParams.port; bool masterSameProcess = ( a.str() == masterHost || b.str() == masterHost ); if ( masterSameProcess ) { - if ( opts.fromDB == todb && cc().database()->path == dbpath ) { + if (opts.fromDB == todb && cc().database()->path() == storageGlobalParams.dbpath) { // guard against an "infinite" loop /* if you are replicating, the local.sources config may be wrong if you get this */ errmsg = "can't clone from self (localhost)."; @@ -368,7 +406,7 @@ namespace mongo { auto_ptr con( cs.connect( errmsg )); if ( !con.get() ) return false; - if( !replAuthenticate(con.get(), false) ) + if( !replAuthenticate(con.get())) return false; _conn = con; @@ -378,10 +416,8 @@ namespace mongo { } } - string ns = opts.fromDB + ".system.namespaces"; - string idxns = opts.fromDB + ".system.indexes"; list toClone; - clonedColls.clear(); + if ( clonedColls ) clonedColls->clear(); if ( opts.syncData ) { /* todo: we can put these releases inside dbclient or a dbclient specialization. or just wait until we get rid of global lock anyway. @@ -389,54 +425,23 @@ namespace mongo { mayInterrupt( opts.mayBeInterrupted ); dbtempreleaseif r( opts.mayYield ); -#if 0 - // fetch index info - auto_ptr cur = _conn->query(idxns.c_str(), BSONObj(), 0, 0, 0, - opts.slaveOk ? QueryOption_SlaveOk : 0 ); - if (!validateQueryResults(cur, errCode)) { - errmsg = "index query failed " + ns; - return false; - } - while(cur->more()) { - BSONObj idxEntry = cur->next(); - massert(16536, "sync source has invalid index data", - idxEntry.hasField("key") && - idxEntry.hasField("ns") && - idxEntry.hasField("name")); - - // validate index version (similar to fixIndexVersion()) - SortPhaseOne initialSort; - IndexInterface* interface = &IndexInterface::defaultVersion(); - - // initialize sorter for this index - PreSortDetails details; - details.preSortPhase.sorter.reset( - new BSONObjExternalSorter(*interface,idxEntry["key"].Obj().copy())); - details.spec = IndexSpec(idxEntry["key"].Obj().copy(), idxEntry.copy()); - _sortersForNS[idxEntry["ns"].String()].insert(make_pair(idxEntry["name"].String(), - details)); - } -#endif - // just using exhaust for collection copying right now - - // todo: if snapshot (bool param to this func) is true, we need to snapshot this query? - // only would be relevant if a thousands of collections -- maybe even then it is hard - // to exceed a single cursor batch. - // for repl it is probably ok as we apply oplog section after the clone (i.e. repl - // doesnt not use snapshot=true). - auto_ptr cursor = _conn->query(ns.c_str(), BSONObj(), 0, 0, 0, - opts.slaveOk ? QueryOption_SlaveOk : 0); - - if (!validateQueryResults(cursor, errCode)) { - errmsg = "namespace query failed " + ns; - return false; - } + list raw = _conn->getCollectionInfos( opts.fromDB ); + for ( list::iterator it = raw.begin(); it != raw.end(); ++it ) { + BSONObj collection = *it; - while ( cursor->more() ) { - BSONObj collection = cursor->next(); LOG(2) << "\t cloner got " << collection << endl; + BSONElement collectionOptions = collection["options"]; + if ( collectionOptions.isABSONObj() ) { + Status parseOptionsStatus = CollectionOptions().parse(collectionOptions.Obj()); + if ( !parseOptionsStatus.isOK() ) { + errmsg = str::stream() << "invalid collection options: " << collection + << ", reason: " << parseOptionsStatus.reason(); + return false; + } + } + BSONElement e = collection.getField("name"); if ( e.eoo() ) { string s = "bad system.namespaces object " + collection.toString(); @@ -444,30 +449,34 @@ namespace mongo { } verify( !e.eoo() ); verify( e.type() == String ); - const char *from_name = e.valuestr(); - if( strstr(from_name, ".system.") ) { + const NamespaceString ns(opts.fromDB, e.valuestr()); + + if( ns.isSystem() ) { /* system.users and s.js is cloned -- but nothing else from system. * system.indexes is handled specially at the end*/ - if( legalClientSystemNS( from_name , true ) == 0 ) { + if( legalClientSystemNS( ns.ns() , true ) == 0 ) { LOG(2) << "\t\t not cloning because system collection" << endl; continue; } } - if( ! NamespaceString::normal( from_name ) ) { + if( !ns.isNormal() ) { LOG(2) << "\t\t not cloning because has $ " << endl; continue; } - if( opts.collsToIgnore.find( string( from_name ) ) != opts.collsToIgnore.end() ){ - LOG(2) << "\t\t ignoring collection " << from_name << endl; + if( opts.collsToIgnore.find( ns.ns() ) != opts.collsToIgnore.end() ){ + LOG(2) << "\t\t ignoring collection " << ns << endl; continue; } else { - LOG(2) << "\t\t not ignoring collection " << from_name << endl; + LOG(2) << "\t\t not ignoring collection " << ns << endl; + } + + if (clonedColls) { + clonedColls->insert(ns.ns()); } - clonedColls.insert( from_name ); toClone.push_back( collection.getOwned() ); } } @@ -477,37 +486,48 @@ namespace mongo { mayInterrupt( opts.mayBeInterrupted ); dbtempreleaseif r( opts.mayYield ); } + + uassert(ErrorCodes::NotMaster, + str::stream() << "Not primary while cloning database " << opts.fromDB + << " (after getting list of collections to clone)", + !opts.logForRepl || + isMaster(todb.c_str())); + BSONObj collection = *i; LOG(2) << " really will clone: " << collection << endl; - const char * from_name = collection["name"].valuestr(); BSONObj options = collection.getObjectField("options"); - /* change name ".collection" -> .collection */ - const char *p = strchr(from_name, '.'); - verify(p); - string to_name = todb + p; - - bool wantIdIndex = false; + string to_name = todb + "." + collection["name"].valuestr(); + string from_name = opts.fromDB + "." + collection["name"].valuestr(); { string err; const char *toname = to_name.c_str(); /* we defer building id index for performance - building it in batch is much faster */ - userCreateNS(toname, options, err, opts.logForRepl, &wantIdIndex); + bool createStatus = userCreateNS(toname, options, err, opts.logForRepl, false); + if ( !createStatus ) { + errmsg = str::stream() << "failed to create collection \"" << to_name << "\": " + << err; + return false; + } } + LOG(1) << "\t\t cloning " << from_name << " -> " << to_name << endl; Query q; if( opts.snapshot ) q.snapshot(); - copy(from_name, to_name.c_str(), false, opts.logForRepl, masterSameProcess, opts.slaveOk, opts.mayYield, opts.mayBeInterrupted, q); + copy(context, from_name.c_str(), to_name.c_str(), false, opts.logForRepl, + masterSameProcess, opts.slaveOk, opts.mayYield, opts.mayBeInterrupted, q); - if( wantIdIndex ) { + { /* we need dropDups to be true as we didn't do a true snapshot and this is before applying oplog operations that occur during the initial sync. inDBRepair makes dropDups be true. */ bool old = inDBRepair; try { inDBRepair = true; - ensureIdIndexForNewNs(to_name.c_str()); + Collection* c = cc().database()->getCollection( to_name ); + if ( c ) + c->getIndexCatalog()->ensureHaveIdIndex(); inDBRepair = old; } catch(...) { @@ -537,36 +557,16 @@ namespace mongo { BSONObj query = BSON( "name" << NE << "_id_" << "ns" << NIN << arr ); // won't need a snapshot of the query of system.indexes as there can never be very many. - copy(system_indexes_from.c_str(), system_indexes_to.c_str(), true, opts.logForRepl, masterSameProcess, opts.slaveOk, opts.mayYield, opts.mayBeInterrupted, query ); + copy(context,system_indexes_from.c_str(), system_indexes_to.c_str(), true, + opts.logForRepl, masterSameProcess, opts.slaveOk, opts.mayYield, opts.mayBeInterrupted, query ); } return true; } - // same as above, but ignores the collection names - bool Cloner::go(const char *masterHost, const CloneOptions& opts, string& errmsg, - int *errCode) { - set unusedCollections; - return go(masterHost, opts, unusedCollections, errmsg, errCode); - } - - bool Cloner::cloneFrom(const char *masterHost, string& errmsg, const string& fromdb, - bool logForReplication, bool slaveOk, bool useReplAuth, bool snapshot, - bool mayYield, bool mayBeInterrupted, int *errCode) { - Cloner cloner; - return cloner.go(masterHost, errmsg, fromdb, logForReplication, slaveOk, useReplAuth, snapshot, - mayYield, mayBeInterrupted, errCode); - } - - bool Cloner::cloneFrom(const string& masterHost, const CloneOptions& options, + bool Cloner::cloneFrom(Client::Context& context, const string& masterHost, const CloneOptions& options, string& errmsg, int* errCode, set* clonedCollections) { - scoped_ptr< set > myset; - if (!clonedCollections) { - myset.reset(new set()); - clonedCollections = myset.get(); - } - Cloner cloner; - return cloner.go(masterHost.c_str(), options, *clonedCollections, errmsg, errCode); + return cloner.go(context, masterHost.c_str(), options, clonedCollections, errmsg, errCode); } /* Usage: @@ -584,14 +584,17 @@ namespace mongo { help << "clone this database from an instance of the db on another host\n"; help << "{ clone : \"host13\" }"; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - // Note: privileges required are currently only granted to old-style users for backwards - // compatibility, and to internal connections (used in movePrimary). + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { ActionSet actions; - actions.addAction(ActionType::clone); - out->push_back(Privilege(dbname, actions)); + actions.addAction(ActionType::insert); + actions.addAction(ActionType::createIndex); + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(dbname), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + return Status::OK(); } CmdClone() : Command("clone") { } virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { @@ -615,9 +618,11 @@ namespace mongo { } } - Cloner cloner; set clonedColls; - bool rval = cloner.go(from.c_str(), opts, clonedColls, errmsg); + Client::Context context( dbname ); + + Cloner cloner; + bool rval = cloner.go(context, from, opts, &clonedColls, errmsg); BSONArrayBuilder barr; barr.append( clonedColls ); @@ -636,16 +641,24 @@ namespace mongo { } virtual LockType locktype() const { return NONE; } CmdCloneCollection() : Command("cloneCollection") { } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - // Will fail if source instance has auth on. - string collection = cmdObj.getStringField("cloneCollection"); - uassert(16709, "bad 'cloneCollection' value", !collection.empty()); + + virtual std::string parseNs(const std::string& dbname, const BSONObj& cmdObj) const { + return parseNsFullyQualified(dbname, cmdObj); + } + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + std::string ns = parseNs(dbname, cmdObj); ActionSet actions; - actions.addAction(ActionType::cloneCollectionTarget); - out->push_back(Privilege(collection, actions)); + actions.addAction(ActionType::insert); + actions.addAction(ActionType::createIndex); // SERVER-11418 + + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString(ns)), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + return Status::OK(); } virtual void help( stringstream &help ) const { help << "{ cloneCollection: , from: [,query: ] [,copyIndexes:] }" @@ -666,10 +679,10 @@ namespace mongo { return false; } } - string collection = cmdObj.getStringField("cloneCollection"); - if ( collection.empty() ) { - errmsg = "bad 'cloneCollection' value"; - return false; + string collection = parseNs(dbname, cmdObj); + Status allowedWriteStatus = userAllowedWriteNS(dbname, collection); + if (!allowedWriteStatus.isOK()) { + return appendCommandStatus(result, allowedWriteStatus); } BSONObj query = cmdObj.getObjectField("query"); if ( query.isEmpty() ) @@ -695,9 +708,13 @@ namespace mongo { // SERVER-4328 todo review for concurrency - thread_specific_ptr< DBClientConnection > authConn_; + thread_specific_ptr< DBClientBase > authConn_; /* Usage: - admindb.$cmd.findOne( { copydbgetnonce: 1, fromhost: } ); + * admindb.$cmd.findOne( { copydbgetnonce: 1, fromhost: } ); + * + * Run against the mongod that is the intended target for the "copydb" command. Used to get a + * nonce from the source of a "copydb" operation for authentication purposes. See the + * description of the "copydb" command below. */ class CmdCopyDbGetNonce : public Command { public: @@ -708,7 +725,7 @@ namespace mongo { virtual bool slaveOk() const { return false; } - virtual LockType locktype() const { return WRITE; } + virtual LockType locktype() const { return NONE; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) {} // No auth required @@ -721,19 +738,21 @@ namespace mongo { if ( fromhost.empty() ) { /* copy from self */ stringstream ss; - ss << "localhost:" << cmdLine.port; + ss << "localhost:" << serverGlobalParams.port; fromhost = ss.str(); } - authConn_.reset( new DBClientConnection() ); BSONObj ret; - { - dbtemprelease t; - if ( !authConn_->connect( fromhost, errmsg ) ) - return false; - if( !authConn_->runCommand( "admin", BSON( "getnonce" << 1 ), ret ) ) { - errmsg = "couldn't get nonce " + ret.toString(); - return false; - } + ConnectionString cs = ConnectionString::parse(fromhost, errmsg); + if (!cs.isValid()) { + return false; + } + authConn_.reset(cs.connect(errmsg)); + if (!authConn_.get()) { + return false; + } + if( !authConn_->runCommand( "admin", BSON( "getnonce" << 1 ), ret ) ) { + errmsg = "couldn't get nonce " + ret.toString(); + return false; } result.appendElements( ret ); return true; @@ -741,9 +760,43 @@ namespace mongo { } cmdCopyDBGetNonce; /* Usage: - admindb.$cmd.findOne( { copydb: 1, fromhost: , fromdb: , todb: [, username: , nonce: , key: ] } ); - Note: doesn't work with authentication enabled, except as old-style users. - */ + * admindb.$cmd.findOne( { copydb: 1, fromhost: , fromdb: , + * todb: [, username: , nonce: , key: ] } ); + * + * The "copydb" command is used to copy a database. Note that this is a very broad definition. + * This means that the "copydb" command can be used in the following ways: + * + * 1. To copy a database within a single node + * 2. To copy a database within a sharded cluster, possibly to another shard + * 3. To copy a database from one cluster to another + * + * Note that in all cases both the target and source database must be unsharded. + * + * The "copydb" command gets sent by the client or the mongos to the destination of the copy + * operation. The node, cluster, or shard that recieves the "copydb" command must then query + * the source of the database to be copied for all the contents and metadata of the database. + * + * + * + * When used with auth, there are two different considerations. + * + * The first is authentication with the target. The only entity that needs to authenticate with + * the target node is the client, so authentication works there the same as it would with any + * other command. + * + * The second is the authentication of the target with the source, which is needed because the + * target must query the source directly for the contents of the database. To do this, the + * client must use the "copydbgetnonce" command, in which the target will get a nonce from the + * source and send it back to the client. The client can then hash its password with the nonce, + * send it to the target when it runs the "copydb" command, which can then use that information + * to authenticate with the source. + * + * NOTE: mongos doesn't know how to call or handle the "copydbgetnonce" command. See + * SERVER-6427. + * + * NOTE: Since internal cluster auth works differently, "copydb" currently doesn't work between + * shards in a cluster when auth is enabled. See SERVER-13080. + */ class CmdCopyDb : public Command { public: CmdCopyDb() : Command("copydb") { } @@ -754,40 +807,45 @@ namespace mongo { return false; } virtual LockType locktype() const { return NONE; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - // Note: privileges required are currently only granted to old-style users for backwards - // compatibility, since we can't properly handle auth checking for the read from the - // source DB. - ActionSet actions; - actions.addAction(ActionType::copyDBTarget); - out->push_back(Privilege(dbname, actions)); // NOTE: dbname is always admin + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + return copydb::checkAuthForCopydbCommand(client, dbname, cmdObj); } virtual void help( stringstream &help ) const { help << "copy a database from another host to this host\n"; - help << "usage: {copydb: 1, fromhost: , fromdb: , todb: [, slaveOk: , username: , nonce: , key: ]}"; + help << "usage: {copydb: 1, fromhost: , fromdb: , todb: " + << "[, slaveOk: , username: , nonce: , key: ]}"; } virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - bool slaveOk = cmdObj["slaveOk"].trueValue(); string fromhost = cmdObj.getStringField("fromhost"); bool fromSelf = fromhost.empty(); if ( fromSelf ) { /* copy from self */ stringstream ss; - ss << "localhost:" << cmdLine.port; + ss << "localhost:" << serverGlobalParams.port; fromhost = ss.str(); } - string fromdb = cmdObj.getStringField("fromdb"); + + CloneOptions cloneOptions; + cloneOptions.fromDB = cmdObj.getStringField("fromdb"); + cloneOptions.logForRepl = !fromRepl; + cloneOptions.slaveOk = cmdObj["slaveOk"].trueValue(); + cloneOptions.useReplAuth = false; + cloneOptions.snapshot = true; + cloneOptions.mayYield = true; + cloneOptions.mayBeInterrupted = false; + string todb = cmdObj.getStringField("todb"); - if ( fromhost.empty() || todb.empty() || fromdb.empty() ) { - errmsg = "parms missing - {copydb: 1, fromhost: , fromdb: , todb: }"; + if ( fromhost.empty() || todb.empty() || cloneOptions.fromDB.empty() ) { + errmsg = "parms missing - {copydb: 1, fromhost: , " + "fromdb: , todb: }"; return false; } // SERVER-4328 todo lock just the two db's not everything for the fromself case - scoped_ptr lk( fromSelf ? - static_cast( new Lock::GlobalWrite() ) : + scoped_ptr lk( fromSelf ? + static_cast( new Lock::GlobalWrite() ) : static_cast( new Lock::DBWrite( todb ) ) ); Cloner cloner; @@ -799,170 +857,31 @@ namespace mongo { BSONObj ret; { dbtemprelease t; - if ( !authConn_->runCommand( fromdb, BSON( "authenticate" << 1 << "user" << username << "nonce" << nonce << "key" << key ), ret ) ) { + if ( !authConn_->runCommand( cloneOptions.fromDB, + BSON( "authenticate" << 1 << "user" << username + << "nonce" << nonce << "key" << key ), ret ) ) { errmsg = "unable to login " + ret.toString(); return false; } } cloner.setConnection( authConn_.release() ); } - Client::Context ctx(todb); - bool res = cloner.go(fromhost.c_str(), errmsg, fromdb, /*logForReplication=*/!fromRepl, slaveOk, /*replauth*/false, /*snapshot*/true, /*mayYield*/true, /*mayBeInterrupted*/ false); - return res; - } - } cmdCopyDB; + else if (!fromSelf) { + // If fromSelf leave the cloner's conn empty, it will use a DBDirectClient instead. - class CmdRenameCollection : public Command { - public: - // Absolute maximum Namespace is 128 incl NUL - // Namespace is 128 minus .$ and $extra so 120 before additions - static const int maxNamespaceLen = 120; - CmdRenameCollection() : Command( "renameCollection" ) {} - virtual bool adminOnly() const { - return true; - } - virtual bool requiresAuth() { return true; } - virtual bool slaveOk() const { - return false; - } - virtual LockType locktype() const { return WRITE; } - virtual bool lockGlobally() const { return true; } - virtual bool logTheOp() { - return true; // can't log steps when doing fast rename within a db, so always log the op rather than individual steps comprising it. - } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - rename_collection::addPrivilegesRequiredForRenameCollection(cmdObj, out); - } - virtual void help( stringstream &help ) const { - help << " example: { renameCollection: foo.a, to: bar.b }"; - } - virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - string source = cmdObj.getStringField( name.c_str() ); - string target = cmdObj.getStringField( "to" ); - uassert(15967,"invalid collection name: " + target, NamespaceString::validCollectionName(target.c_str())); - if ( source.empty() || target.empty() ) { - errmsg = "invalid command syntax"; - return false; - } - - string sourceDB = nsToDatabase(source); - string targetDB = nsToDatabase(target); - string databaseName = sourceDB; - databaseName += ".system.indexes"; - - int longestIndexNameLength = 0; - vector oldIndSpec = Helpers::findAll(databaseName, BSON("ns" << source)); - for (size_t i = 0; i < oldIndSpec.size(); ++i) { - int thisLength = oldIndSpec[i].getField("name").valuesize(); - if (thisLength > longestIndexNameLength) { - longestIndexNameLength = thisLength; - } - } - unsigned int longestAllowed = maxNamespaceLen - longestIndexNameLength - 1; - if (target.size() > longestAllowed) { - StringBuilder sb; - sb << "collection name length of " << target.size() - << " exceeds maximum length of " << longestAllowed - << ", allowing for index names"; - uasserted(16451, sb.str()); - } - - bool capped = false; - long long size = 0; - { - Client::Context ctx( source ); - NamespaceDetails *nsd = nsdetails( source ); - uassert( 10026 , "source namespace does not exist", nsd ); - capped = nsd->isCapped(); - if ( capped ) - for( DiskLoc i = nsd->firstExtent; !i.isNull(); i = i.ext()->xnext ) - size += i.ext()->length; - } - - Client::Context ctx( target ); - - if ( nsdetails( target ) ) { - uassert( 10027 , "target namespace exists", cmdObj["dropTarget"].trueValue() ); - BSONObjBuilder bb( result.subobjStart( "dropTarget" ) ); - dropCollection( target , errmsg , bb ); - bb.done(); - if ( errmsg.size() > 0 ) + ConnectionString cs = ConnectionString::parse(fromhost, errmsg); + if (!cs.isValid()) { return false; - } - - - // if we are renaming in the same database, just - // rename the namespace and we're done. - { - if ( sourceDB == targetDB ) { - renameNamespace( source.c_str(), target.c_str(), cmdObj["stayTemp"].trueValue() ); - // make sure we drop counters etc - Top::global.collectionDropped( source ); - return true; - } - } - - // renaming across databases, so we must copy all - // the data and then remove the source collection. - BSONObjBuilder spec; - if ( capped ) { - spec.appendBool( "capped", true ); - spec.append( "size", double( size ) ); - } - if ( !userCreateNS( target.c_str(), spec.done(), errmsg, false ) ) - return false; - - auto_ptr< DBClientCursor > c; - DBDirectClient bridge; - - { - c = bridge.query( source, BSONObj(), 0, 0, 0, fromRepl ? QueryOption_SlaveOk : 0 ); - } - while( 1 ) { - { - if ( !c->more() ) - break; - } - BSONObj o = c->next(); - theDataFileMgr.insertWithObjMod( target.c_str(), o ); - } - - string sourceIndexes = nsToDatabase( source ) + ".system.indexes"; - string targetIndexes = nsToDatabase( target ) + ".system.indexes"; - { - c = bridge.query( sourceIndexes, QUERY( "ns" << source ), 0, 0, 0, fromRepl ? QueryOption_SlaveOk : 0 ); - } - while( 1 ) { - { - if ( !c->more() ) - break; } - BSONObj o = c->next(); - BSONObjBuilder b; - BSONObjIterator i( o ); - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - if ( strcmp( e.fieldName(), "ns" ) == 0 ) { - b.append( "ns", target ); - } - else { - b.append( e ); - } + DBClientBase* conn = cs.connect(errmsg); + if (!conn) { + return false; } - BSONObj n = b.done(); - theDataFileMgr.insertWithObjMod( targetIndexes.c_str(), n ); - } - - { - Client::Context ctx( source ); - dropCollection( source, errmsg, result ); + cloner.setConnection(conn); } - return true; + Client::Context ctx(todb); + return cloner.go(ctx, fromhost, cloneOptions, NULL, errmsg ); } - } cmdrenamecollection; + } cmdCopyDB; } // namespace mongo diff --git a/src/mongo/db/cloner.h b/src/mongo/db/cloner.h index 6f4e104ec60..255c1442f0b 100644 --- a/src/mongo/db/cloner.h +++ b/src/mongo/db/cloner.h @@ -14,17 +14,29 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/client.h" #include "mongo/db/jsobj.h" -#include "mongo/db/sort_phase_one.h" namespace mongo { struct CloneOptions; - class IndexSpec; class DBClientBase; class DBClientCursor; class Query; @@ -42,15 +54,11 @@ namespace mongo { void setConnection( DBClientBase *c ) { _conn.reset( c ); } /** copy the entire database */ - bool go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, - bool slaveOk, bool useReplAuth, bool snapshot, bool mayYield, - bool mayBeInterrupted, int *errCode = 0); - - bool go(const char *masterHost, const CloneOptions& opts, set& clonedColls, + bool go(Client::Context& ctx, + const string& masterHost, const CloneOptions& opts, + set* clonedColls, string& errmsg, int *errCode = 0); - bool go(const char *masterHost, const CloneOptions& opts, string& errmsg, int *errCode = 0); - bool copyCollection(const string& ns, const BSONObj& query, string& errmsg, bool mayYield, bool mayBeInterrupted, bool copyIndexes = true, bool logForRepl = true ); @@ -58,27 +66,19 @@ namespace mongo { * validate the cloner query was successful * @param cur Cursor the query was executed on * @param errCode out Error code encountered during the query + * @param errmsg out Error message encountered during the query */ - static bool validateQueryResults(const auto_ptr& cur, int32_t* errCode); + static bool validateQueryResults(const auto_ptr& cur, int32_t* errCode, + string& errmsg); /** * @param errmsg out - Error message (if encountered). - * @param slaveOk - if true it is ok if the source of the data is !ismaster. - * @param useReplAuth - use the credentials we normally use as a replication slave for the - * cloning. - * @param snapshot - use $snapshot mode for copying collections. note this should not be - * used when it isn't required, as it will be slower. for example - * repairDatabase need not use it. * @param errCode out - If provided, this will be set on error to the server's error code. * Currently this will only be set if there is an error in the initial * system.namespaces query. */ - static bool cloneFrom(const char *masterHost, string& errmsg, const string& fromdb, - bool logForReplication, bool slaveOk, bool useReplAuth, - bool snapshot, bool mayYield, bool mayBeInterrupted, - int *errCode = 0); - - static bool cloneFrom(const string& masterHost, const CloneOptions& options, + static bool cloneFrom(Client::Context& context, + const string& masterHost, const CloneOptions& options, string& errmsg, int* errCode = 0, set* clonedCollections = 0); @@ -88,22 +88,13 @@ namespace mongo { static bool copyCollectionFromRemote(const string& host, const string& ns, string& errmsg); private: - void copy(const char *from_ns, const char *to_ns, bool isindex, bool logForRepl, + void copy(Client::Context& ctx, + const char *from_ns, const char *to_ns, bool isindex, bool logForRepl, bool masterSameProcess, bool slaveOk, bool mayYield, bool mayBeInterrupted, Query q); - // index presort info - typedef struct { - IndexSpec spec; - SortPhaseOne preSortPhase; - } PreSortDetails; - - typedef map SortersForIndex; // map from index name to presorter - typedef map SortersForNS; // map from ns to indices/sorters - struct Fun; auto_ptr _conn; - SortersForNS _sortersForNS; }; struct CloneOptions { @@ -119,7 +110,7 @@ namespace mongo { syncData = true; syncIndexes = true; } - + string fromDB; set collsToIgnore; diff --git a/src/mongo/db/cmdline.cpp b/src/mongo/db/cmdline.cpp deleted file mode 100644 index 0dea50faeef..00000000000 --- a/src/mongo/db/cmdline.cpp +++ /dev/null @@ -1,533 +0,0 @@ -// cmdline.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/cmdline.h" - -#include "mongo/base/status.h" -#include "mongo/bson/util/builder.h" -#include "mongo/db/server_parameters.h" -#include "mongo/util/map_util.h" -#include "mongo/util/mongoutils/str.h" -#include "mongo/util/net/listen.h" -#include "mongo/util/password.h" - -#ifdef _WIN32 -#include -#endif - -#define MAX_LINE_LENGTH 256 - -#include - -namespace po = boost::program_options; - -namespace mongo { - - static bool _isPasswordArgument(char const* argumentName); - static bool _isPasswordSwitch(char const* switchName); - -namespace { - BSONArray argvArray; - BSONObj parsedOpts; -} // namespace - - BSONArray CmdLine::getArgvArray() { - return argvArray; - } - - BSONObj CmdLine::getParsedOpts() { - return parsedOpts; - } - - void CmdLine::addGlobalOptions( boost::program_options::options_description& general , - boost::program_options::options_description& hidden , - boost::program_options::options_description& ssl_options ) { - /* support for -vv -vvvv etc. */ - for (string s = "vv"; s.length() <= 12; s.append("v")) { - hidden.add_options()(s.c_str(), "verbose"); - } - - StringBuilder portInfoBuilder; - StringBuilder maxConnInfoBuilder; - - portInfoBuilder << "specify port number - " << DefaultDBPort << " by default"; - maxConnInfoBuilder << "max number of simultaneous connections - " << DEFAULT_MAX_CONN << " by default"; - - general.add_options() - ("help,h", "show this usage information") - ("version", "show version information") - ("config,f", po::value(), "configuration file specifying additional options") - ("verbose,v", "be more verbose (include multiple times for more verbosity e.g. -vvvvv)") - ("quiet", "quieter output") - ("port", po::value(&cmdLine.port), portInfoBuilder.str().c_str()) - ("bind_ip", po::value(&cmdLine.bind_ip), "comma separated list of ip addresses to listen on - all local ips by default") - ("maxConns",po::value(), maxConnInfoBuilder.str().c_str()) - ("logpath", po::value() , "log file to send write to instead of stdout - has to be a file, not directory" ) - ("logappend" , "append to logpath instead of over-writing" ) - ("pidfilepath", po::value(), "full path to pidfile (if not set, no pidfile is created)") - ("keyFile", po::value(), "private key for cluster authentication") - ("setParameter", po::value< std::vector >()->composing(), - "Set a configurable parameter") -#ifndef _WIN32 - ("nounixsocket", "disable listening on unix sockets") - ("unixSocketPrefix", po::value(), "alternative directory for UNIX domain sockets (defaults to /tmp)") - ("fork" , "fork server process" ) - ("syslog" , "log to system's syslog facility instead of file or stdout" ) -#endif - ; - - -#ifdef MONGO_SSL - ssl_options.add_options() - ("sslOnNormalPorts" , "use ssl on configured ports" ) - ("sslPEMKeyFile" , po::value(&cmdLine.sslPEMKeyFile), "PEM file for ssl" ) - ("sslPEMKeyPassword" , new PasswordValue(&cmdLine.sslPEMKeyPassword) , "PEM file password" ) - ("sslCAFile", po::value(&cmdLine.sslCAFile), - "Certificate Authority file for SSL") - ("sslCRLFile", po::value(&cmdLine.sslCRLFile), - "Certificate Revocation List file for SSL") - ("sslWeakCertificateValidation", "allow client to connect without presenting a certificate") - ("sslFIPSMode", "activate FIPS 140-2 mode at startup") -#endif - ; - - // Extra hidden options - hidden.add_options() - ("objcheck", "inspect client data for validity on receipt (DEFAULT)") - ("noobjcheck", "do NOT inspect client data for validity on receipt") - ("traceExceptions", "log stack traces for every exception") - ("enableExperimentalIndexStatsCmd", po::bool_switch(&cmdLine.experimental.indexStatsCmdEnabled), - "EXPERIMENTAL (UNSUPPORTED). Enable command computing aggregate statistics on indexes.") - ("enableExperimentalStorageDetailsCmd", po::bool_switch(&cmdLine.experimental.storageDetailsCmdEnabled), - "EXPERIMENTAL (UNSUPPORTED). Enable command computing aggregate statistics on storage.") - ; - } - -#if defined(_WIN32) - void CmdLine::addWindowsOptions( boost::program_options::options_description& windows , - boost::program_options::options_description& hidden ) { - windows.add_options() - ("install", "install Windows service") - ("remove", "remove Windows service") - ("reinstall", "reinstall Windows service (equivalent to --remove followed by --install)") - ("serviceName", po::value(), "Windows service name") - ("serviceDisplayName", po::value(), "Windows service display name") - ("serviceDescription", po::value(), "Windows service description") - ("serviceUser", po::value(), "account for service execution") - ("servicePassword", po::value(), "password used to authenticate serviceUser") - ; - hidden.add_options()("service", "start mongodb service"); - } -#endif - - void CmdLine::parseConfigFile( istream &f, stringstream &ss ) { - string s; - char line[MAX_LINE_LENGTH]; - - while ( f ) { - f.getline(line, MAX_LINE_LENGTH); - s = line; - std::remove(s.begin(), s.end(), ' '); - std::remove(s.begin(), s.end(), '\t'); - boost::to_upper(s); - - if ( s.find( "FASTSYNC" ) != string::npos ) - cout << "warning \"fastsync\" should not be put in your configuration file" << endl; - - if ( s.c_str()[0] == '#' ) { - // skipping commented line - } else if ( s.find( "=FALSE" ) == string::npos ) { - ss << line << endl; - } else { - cout << "warning: remove or comment out this line by starting it with \'#\', skipping now : " << line << endl; - } - } - return; - } - - bool CmdLine::store( const std::vector& argv, - boost::program_options::options_description& visible, - boost::program_options::options_description& hidden, - boost::program_options::positional_options_description& positional, - boost::program_options::variables_map ¶ms ) { - - - if (argv.empty()) - return false; - - { - // setup binary name - cmdLine.binaryName = argv[0]; - size_t i = cmdLine.binaryName.rfind( '/' ); - if ( i != string::npos ) - cmdLine.binaryName = cmdLine.binaryName.substr( i + 1 ); - - // setup cwd - char buffer[1024]; -#ifdef _WIN32 - verify( _getcwd( buffer , 1000 ) ); -#else - verify( getcwd( buffer , 1000 ) ); -#endif - cmdLine.cwd = buffer; - } - - - /* don't allow guessing - creates ambiguities when some options are - * prefixes of others. allow long disguises and don't allow guessing - * to get away with our vvvvvvv trick. */ - int style = (((po::command_line_style::unix_style ^ - po::command_line_style::allow_guessing) | - po::command_line_style::allow_long_disguise) ^ - po::command_line_style::allow_sticky); - - - try { - - po::options_description all; - all.add( visible ); - all.add( hidden ); - - po::store( po::command_line_parser(std::vector(argv.begin() + 1, - argv.end())) - .options( all ) - .positional( positional ) - .style( style ) - .run(), - params ); - - if ( params.count("config") ) { - ifstream f( params["config"].as().c_str() ); - if ( ! f.is_open() ) { - cout << "ERROR: could not read from config file" << endl << endl; - cout << visible << endl; - return false; - } - - stringstream ss; - CmdLine::parseConfigFile( f, ss ); - po::store( po::parse_config_file( ss , all ) , params ); - f.close(); - } - - po::notify(params); - } - catch (po::error &e) { - cout << "error command line: " << e.what() << endl; - cout << "use --help for help" << endl; - //cout << visible << endl; - return false; - } - - { - BSONArrayBuilder b; - std::vector censoredArgv = argv; - censor(&censoredArgv); - for (size_t i=0; i < censoredArgv.size(); i++) { - b << censoredArgv[i]; - } - argvArray = b.arr(); - } - - { - BSONObjBuilder b; - for (po::variables_map::const_iterator it(params.begin()), end(params.end()); it != end; it++){ - if (!it->second.defaulted()){ - const string& key = it->first; - const po::variable_value& value = it->second; - const type_info& type = value.value().type(); - - if (type == typeid(string)){ - if (value.as().empty()) - b.appendBool(key, true); // boost po uses empty string for flags like --quiet - else { - if ( _isPasswordArgument(key.c_str()) ) { - b.append( key, "" ); - } - else { - b.append( key, value.as() ); - } - } - } - else if (type == typeid(int)) - b.append(key, value.as()); - else if (type == typeid(double)) - b.append(key, value.as()); - else if (type == typeid(bool)) - b.appendBool(key, value.as()); - else if (type == typeid(long)) - b.appendNumber(key, (long long)value.as()); - else if (type == typeid(unsigned)) - b.appendNumber(key, (long long)value.as()); - else if (type == typeid(unsigned long long)) - b.appendNumber(key, (long long)value.as()); - else if (type == typeid(vector)) - b.append(key, value.as >()); - else - b.append(key, "UNKNOWN TYPE: " + demangleName(type)); - } - } - parsedOpts = b.obj(); - } - - if (params.count("verbose")) { - logLevel = 1; - } - - for (string s = "vv"; s.length() <= 12; s.append("v")) { - if (params.count(s)) { - logLevel = s.length(); - } - } - - if (params.count("quiet")) { - cmdLine.quiet = true; - } - - if (params.count("traceExceptions")) { - DBException::traceExceptions = true; - } - - if (params.count("maxConns")) { - cmdLine.maxConns = params["maxConns"].as(); - - if ( cmdLine.maxConns < 5 ) { - out() << "maxConns has to be at least 5" << endl; - return false; - } - else if ( cmdLine.maxConns > MAX_MAX_CONN ) { - out() << "maxConns can't be greater than " << MAX_MAX_CONN << endl; - return false; - } - } - - if (params.count("objcheck")) { - cmdLine.objcheck = true; - } - if (params.count("noobjcheck")) { - if (params.count("objcheck")) { - out() << "can't have both --objcheck and --noobjcheck" << endl; - return false; - } - cmdLine.objcheck = false; - } - - if (params.count("bind_ip")) { - // passing in wildcard is the same as default behavior; remove and warn - if ( cmdLine.bind_ip == "0.0.0.0" ) { - cout << "warning: bind_ip of 0.0.0.0 is unnecessary; listens on all ips by default" << endl; - cmdLine.bind_ip = ""; - } - } - -#ifndef _WIN32 - if (params.count("unixSocketPrefix")) { - cmdLine.socket = params["unixSocketPrefix"].as(); - } - - if (params.count("nounixsocket")) { - cmdLine.noUnixSocket = true; - } - - if (params.count("fork") && !params.count("shutdown")) { - cmdLine.doFork = true; - } -#endif // _WIN32 - - if (params.count("logpath")) { - cmdLine.logpath = params["logpath"].as(); - if (cmdLine.logpath.empty()) { - cout << "logpath cannot be empty if supplied" << endl; - return false; - } - } - - cmdLine.logWithSyslog = params.count("syslog"); - cmdLine.logAppend = params.count("logappend"); - if (!cmdLine.logpath.empty() && cmdLine.logWithSyslog) { - cout << "Cant use both a logpath and syslog " << endl; - return false; - } - - if (cmdLine.doFork && cmdLine.logpath.empty() && !cmdLine.logWithSyslog) { - cout << "--fork has to be used with --logpath or --syslog" << endl; - return false; - } - - if (params.count("keyFile")) { - cmdLine.keyFile = params["keyFile"].as(); - } - - if ( params.count("pidfilepath")) { - cmdLine.pidFile = params["pidfilepath"].as(); - } - - if (params.count("setParameter")) { - std::vector parameters = - params["setParameter"].as >(); - for (size_t i = 0, length = parameters.size(); i < length; ++i) { - std::string name; - std::string value; - if (!mongoutils::str::splitOn(parameters[i], '=', name, value)) { - cout << "Illegal option assignment: \"" << parameters[i] << "\"" << endl; - return false; - } - ServerParameter* parameter = mapFindWithDefault( - ServerParameterSet::getGlobal()->getMap(), - name, - static_cast(NULL)); - if (NULL == parameter) { - cout << "Illegal --setParameter parameter: \"" << name << "\"" << endl; - return false; - } - if (!parameter->allowedToChangeAtStartup()) { - cout << "Cannot use --setParameter to set \"" << name << "\" at startup" << - endl; - return false; - } - Status status = parameter->setFromString(value); - if (!status.isOK()) { - cout << "Bad value for parameter \"" << name << "\": " << status.reason() - << endl; - return false; - } - } - } - -#ifdef MONGO_SSL - if (params.count("sslWeakCertificateValidation")) { - cmdLine.sslWeakCertificateValidation = true; - } - if (params.count("sslOnNormalPorts")) { - cmdLine.sslOnNormalPorts = true; - if ( cmdLine.sslPEMKeyFile.size() == 0 ) { - log() << "need sslPEMKeyFile with sslOnNormalPorts" << endl; - return false; - } - if (cmdLine.sslWeakCertificateValidation && - cmdLine.sslCAFile.empty()) { - log() << "need sslCAFile with sslWeakCertificateValidation" << endl; - return false; - } - if (!cmdLine.sslCRLFile.empty() && - cmdLine.sslCAFile.empty()) { - log() << "need sslCAFile with sslCRLFile" << endl; - return false; - } - if (params.count("sslFIPSMode")) { - cmdLine.sslFIPSMode = true; - } - } - else if (cmdLine.sslPEMKeyFile.size() || - cmdLine.sslPEMKeyPassword.size() || - cmdLine.sslCAFile.size() || - cmdLine.sslCRLFile.size() || - cmdLine.sslWeakCertificateValidation || - cmdLine.sslFIPSMode) { - log() << "need to enable sslOnNormalPorts" << endl; - return false; - } -#endif - - return true; - } - - static bool _isPasswordArgument(const char* argumentName) { - static const char* const passwordArguments[] = { - "sslPEMKeyPassword", - "servicePassword", - NULL // Last entry sentinel. - }; - for (const char* const* current = passwordArguments; *current; ++current) { - if (mongoutils::str::equals(argumentName, *current)) - return true; - } - return false; - } - - static bool _isPasswordSwitch(const char* switchName) { - if (switchName[0] != '-') - return false; - size_t i = 1; - if (switchName[1] == '-') - i = 2; - switchName += i; - - return _isPasswordArgument(switchName); - } - - static void _redact(char* arg) { - for (; *arg; ++arg) - *arg = 'x'; - } - - void CmdLine::censor(std::vector* args) { - for (size_t i = 0; i < args->size(); ++i) { - std::string& arg = args->at(i); - const std::string::iterator endSwitch = std::find(arg.begin(), arg.end(), '='); - std::string switchName(arg.begin(), endSwitch); - if (_isPasswordSwitch(switchName.c_str())) { - if (endSwitch == arg.end()) { - if (i + 1 < args->size()) { - args->at(i + 1) = ""; - } - } - else { - arg = switchName + "="; - } - } - } - } - - void CmdLine::censor(int argc, char** argv) { - // Algorithm: For each arg in argv: - // Look for an equal sign in arg; if there is one, temporarily nul it out. - // check to see if arg is a password switch. If so, overwrite the value - // component with xs. - // restore the nul'd out equal sign, if any. - for (int i = 0; i < argc; ++i) { - - char* const arg = argv[i]; - char* const firstEqSign = strchr(arg, '='); - if (NULL != firstEqSign) { - *firstEqSign = '\0'; - } - - if (_isPasswordSwitch(arg)) { - if (NULL == firstEqSign) { - if (i + 1 < argc) { - _redact(argv[i + 1]); - } - } - else { - _redact(firstEqSign + 1); - } - } - - if (NULL != firstEqSign) { - *firstEqSign = '='; - } - } - } - - void printCommandLineOpts() { - log() << "options: " << parsedOpts << endl; - } -} diff --git a/src/mongo/db/cmdline.h b/src/mongo/db/cmdline.h deleted file mode 100644 index 22bd63f9eb4..00000000000 --- a/src/mongo/db/cmdline.h +++ /dev/null @@ -1,220 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include -#include - -#include "mongo/db/jsobj.h" -#include "mongo/util/net/listen.h" - -namespace boost { - namespace program_options { - class options_description; - class positional_options_description; - class variables_map; - } -} - -namespace mongo { - - /* command line options - */ - /* concurrency: OK/READ */ - struct CmdLine { - - CmdLine(); - - std::string binaryName; // mongod or mongos - std::string cwd; // cwd of when process started - - // this is suboptimal as someone could rename a binary. todo... - bool isMongos() const { return binaryName == "mongos"; } - - int port; // --port - enum { - DefaultDBPort = 27017, - ConfigServerPort = 27019, - ShardServerPort = 27018 - }; - bool isDefaultPort() const { return port == DefaultDBPort; } - - std::string bind_ip; // --bind_ip - bool rest; // --rest - bool jsonp; // --jsonp - - std::string _replSet; // --replSet[/] - std::string ourSetName() const { - std::string setname; - size_t sl = _replSet.find('/'); - if( sl == std::string::npos ) - return _replSet; - return _replSet.substr(0, sl); - } - bool usingReplSets() const { return !_replSet.empty(); } - - std::string rsIndexPrefetch;// --indexPrefetch - bool indexBuildRetry; // --noIndexBuildRetry - - // for master/slave replication - std::string source; // --source - std::string only; // --only - - bool quiet; // --quiet - bool noTableScan; // --notablescan no table scans allowed - bool prealloc; // --noprealloc no preallocation of data files - bool preallocj; // --nopreallocj no preallocation of journal files - bool smallfiles; // --smallfiles allocate smaller data files - - bool configsvr; // --configsvr - - bool quota; // --quota - int quotaFiles; // --quotaFiles - bool cpu; // --cpu show cpu time periodically - - bool dur; // --dur durability (now --journal) - unsigned journalCommitInterval; // group/batch commit interval ms - - /** --durOptions 7 dump journal and terminate without doing anything further - --durOptions 4 recover and terminate without listening - */ - enum { // bits to be ORed - DurDumpJournal = 1, // dump diagnostics on the journal during recovery - DurScanOnly = 2, // don't do any real work, just scan and dump if dump specified - DurRecoverOnly = 4, // terminate after recovery step - DurParanoid = 8, // paranoid mode enables extra checks - DurAlwaysCommit = 16, // do a group commit every time the writelock is released - DurAlwaysRemap = 32, // remap the private view after every group commit (may lag to the next write lock acquisition, but will do all files then) - DurNoCheckSpace = 64 // don't check that there is enough room for journal files before startup (for diskfull tests) - }; - int durOptions; // --durOptions for debugging - - bool objcheck; // --objcheck - - long long oplogSize; // --oplogSize - int defaultProfile; // --profile - int slowMS; // --time in ms that is "slow" - int defaultLocalThresholdMillis; // --localThreshold in ms to consider a node local - int pretouch; // --pretouch for replication application (experimental) - bool moveParanoia; // for move chunk paranoia - double syncdelay; // seconds between fsyncs - - bool noUnixSocket; // --nounixsocket - bool doFork; // --fork - std::string socket; // UNIX domain socket directory - - int maxConns; // Maximum number of simultaneous open connections. - - std::string keyFile; // Path to keyfile, or empty if none. - std::string pidFile; // Path to pid file, or empty if none. - - std::string logpath; // Path to log file, if logging to a file; otherwise, empty. - bool logAppend; // True if logging to a file in append mode. - bool logWithSyslog; // True if logging to syslog; must not be set if logpath is set. - -#ifndef _WIN32 - pid_t parentProc; // --fork pid of initial process - pid_t leaderProc; // --fork pid of leader process -#endif - -#ifdef MONGO_SSL - bool sslOnNormalPorts; // --sslOnNormalPorts - std::string sslPEMKeyFile; // --sslPEMKeyFile - std::string sslPEMKeyPassword; // --sslPEMKeyPassword - std::string sslCAFile; // --sslCAFile - std::string sslCRLFile; // --sslCRLFile - bool sslWeakCertificateValidation; - bool sslFIPSMode; -#endif - - /** - * Switches to enable experimental (unsupported) features. - */ - struct ExperimentalFeatures { - ExperimentalFeatures() - : indexStatsCmdEnabled(false) - , storageDetailsCmdEnabled(false) - {} - bool indexStatsCmdEnabled; // -- enableExperimentalIndexStatsCmd - bool storageDetailsCmdEnabled; // -- enableExperimentalStorageDetailsCmd - } experimental; - - static void launchOk(); - - static void addGlobalOptions( boost::program_options::options_description& general , - boost::program_options::options_description& hidden , - boost::program_options::options_description& ssl_options ); - - static void addWindowsOptions( boost::program_options::options_description& windows , - boost::program_options::options_description& hidden ); - - - static void parseConfigFile( istream &f, std::stringstream &ss); - /** - * @return true if should run program, false if should exit - */ - static bool store( const std::vector& argv, - boost::program_options::options_description& visible, - boost::program_options::options_description& hidden, - boost::program_options::positional_options_description& positional, - boost::program_options::variables_map &output ); - - /** - * Blot out sensitive fields in the argv array. - */ - static void censor(int argc, char** argv); - static void censor(std::vector* args); - - static BSONArray getArgvArray(); - static BSONObj getParsedOpts(); - - time_t started; - }; - - // todo move to cmdline.cpp? - inline CmdLine::CmdLine() : - port(DefaultDBPort), rest(false), jsonp(false), indexBuildRetry(true), quiet(false), - noTableScan(false), prealloc(true), preallocj(true), smallfiles(sizeof(int*) == 4), - configsvr(false), quota(false), quotaFiles(8), cpu(false), - durOptions(0), objcheck(true), oplogSize(0), defaultProfile(0), - slowMS(100), defaultLocalThresholdMillis(15), pretouch(0), moveParanoia( false ), - syncdelay(60), noUnixSocket(false), doFork(0), socket("/tmp"), maxConns(DEFAULT_MAX_CONN), - logAppend(false), logWithSyslog(false) - { - started = time(0); - - journalCommitInterval = 0; // 0 means use default - dur = false; -#if defined(_DURABLEDEFAULTON) - dur = true; -#endif - if( sizeof(void*) == 8 ) - dur = true; -#if defined(_DURABLEDEFAULTOFF) - dur = false; -#endif - -#ifdef MONGO_SSL - sslOnNormalPorts = false; -#endif - } - - extern CmdLine cmdLine; - - void printCommandLineOpts(); -} - diff --git a/src/mongo/db/cmdline_test.cpp b/src/mongo/db/cmdline_test.cpp deleted file mode 100644 index 9af8423f929..00000000000 --- a/src/mongo/db/cmdline_test.cpp +++ /dev/null @@ -1,224 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include - -#include "mongo/db/cmdline.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { - - CmdLine cmdLine; - -namespace { - - void testCensoringArgv(const char* const * expected, - const char* const * toCensor, - int elementCount) { - - std::vector toCensorStringVec(toCensor, toCensor + elementCount); - std::vector arrayStandin; - for (size_t i = 0; i < toCensorStringVec.size(); ++i) - arrayStandin.push_back(&*toCensorStringVec[i].begin()); - - char** argv = &*arrayStandin.begin(); - - CmdLine::censor(elementCount, argv); - - for (int i = 0; i < elementCount; ++i) { - ASSERT_EQUALS(std::string(expected[i]), std::string(argv[i])); - } - } - - void testCensoringVector(const char* const * expected, - const char* const * toCensor, - int elementCount) { - - std::vector actual(toCensor, toCensor + elementCount); - - CmdLine::censor(&actual); - - for (int i = 0; i < elementCount; ++i) { - ASSERT_EQUALS(std::string(expected[i]), actual[i]); - } - } - - TEST(ArgvCensorTests, NothingCensored) { - const char* const argv[] = { - "first", - "second", - "sslPEMKeyPassword=KEEP", - "---sslPEMKeyPassword=KEEP", - "sslPEMKeyPassword", - "KEEP", - "servicePassword=KEEP", - "--servicePassword-", - "KEEP", - "--servicePasswordFake=KEEP" - }; - const int argc = boost::size(argv); - testCensoringArgv(argv, argv, argc); - } - - TEST(ArgvCensorTests, SomeStuffCensoredDoubleHyphen) { - const char* const argv[] = { - "first", - "second", - "--sslPEMKeyPassword=LOSEME", - "--sslPEMKeyPassword", - "Really, loose me!", - "--servicePassword=bad news", - "--servicePassword-", - "KEEP", - "--servicePassword", - "get out of dodge" - }; - const int argc = boost::size(argv); - - const char* const expected[] = { - "first", - "second", - "--sslPEMKeyPassword=xxxxxx", - "--sslPEMKeyPassword", - "xxxxxxxxxxxxxxxxx", - "--servicePassword=xxxxxxxx", - "--servicePassword-", - "KEEP", - "--servicePassword", - "xxxxxxxxxxxxxxxx" - }; - ASSERT_EQUALS(boost::size(expected), argc); - - testCensoringArgv(expected, argv, argc); - } - - TEST(ArgvCensorTests, SomeStuffCensoredSingleHyphen) { - const char* const argv[] = { - "first", - "second", - "-sslPEMKeyPassword=LOSEME", - "-sslPEMKeyPassword", - "Really, loose me!", - "-servicePassword=bad news", - "-servicePassword-", - "KEEP", - "-servicePassword", - "get out of dodge" - }; - const int argc = boost::size(argv); - - const char* const expected[] = { - "first", - "second", - "-sslPEMKeyPassword=xxxxxx", - "-sslPEMKeyPassword", - "xxxxxxxxxxxxxxxxx", - "-servicePassword=xxxxxxxx", - "-servicePassword-", - "KEEP", - "-servicePassword", - "xxxxxxxxxxxxxxxx" - }; - ASSERT_EQUALS(boost::size(expected), argc); - - testCensoringArgv(expected, argv, argc); - } - - TEST(VectorCensorTests, NothingCensored) { - const char* const argv[] = { - "first", - "second", - "sslPEMKeyPassword=KEEP", - "---sslPEMKeyPassword=KEEP", - "sslPEMKeyPassword", - "KEEP", - "servicePassword=KEEP", - "--servicePassword-", - "KEEP", - "--servicePasswordFake=KEEP" - }; - const int argc = boost::size(argv); - testCensoringVector(argv, argv, argc); - } - - TEST(VectorCensorTests, SomeStuffCensoredDoubleHyphen) { - const char* const argv[] = { - "first", - "second", - "--sslPEMKeyPassword=LOSEME", - "--sslPEMKeyPassword", - "Really, loose me!", - "--servicePassword=bad news", - "--servicePassword-", - "KEEP", - "--servicePassword", - "get out of dodge" - }; - const int argc = boost::size(argv); - - const char* const expected[] = { - "first", - "second", - "--sslPEMKeyPassword=", - "--sslPEMKeyPassword", - "", - "--servicePassword=", - "--servicePassword-", - "KEEP", - "--servicePassword", - "" - }; - ASSERT_EQUALS(boost::size(expected), argc); - - testCensoringVector(expected, argv, argc); - } - - TEST(VectorCensorTests, SomeStuffCensoredSingleHyphen) { - const char* const argv[] = { - "first", - "second", - "-sslPEMKeyPassword=LOSEME", - "-sslPEMKeyPassword", - "Really, loose me!", - "-servicePassword=bad news", - "-servicePassword-", - "KEEP", - "-servicePassword", - "get out of dodge" - }; - const int argc = boost::size(argv); - - const char* const expected[] = { - "first", - "second", - "-sslPEMKeyPassword=", - "-sslPEMKeyPassword", - "", - "-servicePassword=", - "-servicePassword-", - "KEEP", - "-servicePassword", - "" - }; - ASSERT_EQUALS(boost::size(expected), argc); - - testCensoringVector(expected, argv, argc); - } - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/collection.h b/src/mongo/db/collection.h deleted file mode 100644 index 0c8c728e991..00000000000 --- a/src/mongo/db/collection.h +++ /dev/null @@ -1,31 +0,0 @@ -// @file collection.h - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "namespace.h" - -namespace mongo { - - class Collection { - public: - NamespaceDetails * const d; - NamespaceDetailsTransient * const nsd; - }; - -} diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp index 87e9129e612..ff0a04fcfa8 100644 --- a/src/mongo/db/commands.cpp +++ b/src/mongo/db/commands.cpp @@ -1,257 +1,382 @@ -/* commands.cpp - db "commands" (sent via db.$cmd.findOne(...)) - */ - -/* Copyright 2009 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "pch.h" - -#include "mongo/db/commands.h" - -#include -#include - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/client.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/replutil.h" -#include "mongo/db/server_parameters.h" - -namespace mongo { - - map * Command::_commandsByBestName; - map * Command::_webCommands; - map * Command::_commands; - - int Command::testCommandsEnabled = 0; - - namespace { - ExportedServerParameter testCommandsParameter(ServerParameterSet::getGlobal(), - "enableTestCommands", - &Command::testCommandsEnabled, - true, - false); - } - - string Command::parseNsFullyQualified(const string& dbname, const BSONObj& cmdObj) const { - string s = cmdObj.firstElement().valuestr(); - NamespaceString nss(s); - // these are for security, do not remove: - massert(15962, "need to specify namespace" , !nss.db.empty() ); - massert(15966, str::stream() << "dbname not ok in Command::parseNsFullyQualified: " << dbname , dbname == nss.db || dbname == "admin" ); - return s; - } - - /*virtual*/ string Command::parseNs(const string& dbname, const BSONObj& cmdObj) const { - string coll = cmdObj.firstElement().valuestr(); -#if defined(CLC) - DEV if( mongoutils::str::startsWith(coll, dbname+'.') ) { - log() << "DEBUG parseNs Command's collection name looks like it includes the db name\n" - << dbname << '\n' - << coll << '\n' - << cmdObj.toString() << endl; - dassert(false); - } -#endif - return dbname + '.' + coll; - } - - void Command::htmlHelp(stringstream& ss) const { - string helpStr; - { - stringstream h; - help(h); - helpStr = h.str(); - } - ss << "\n"; - bool web = _webCommands->count(name) != 0; - if( web ) ss << ""; - ss << name; - if( web ) ss << ""; - ss << "\n"; - ss << ""; - int l = locktype(); - //if( l == NONE ) ss << "N "; - if( l == READ ) ss << "R "; - else if( l == WRITE ) ss << "W "; - if( slaveOk() ) - ss << "S "; - if( adminOnly() ) - ss << "A"; - if( lockGlobally() ) - ss << " lockGlobally "; - ss << ""; - ss << ""; - if( helpStr != "no help defined" ) { - const char *p = helpStr.c_str(); - while( *p ) { - if( *p == '<' ) { - ss << "<"; - p++; continue; - } - else if( *p == '{' ) - ss << ""; - else if( *p == '}' ) { - ss << "}"; - p++; - continue; - } - if( strncmp(p, "http:", 5) == 0 ) { - ss << ""; - q = p; - if( startsWith(q, "http://www.mongodb.org/display/") ) - q += 31; - while( *q && *q != ' ' && *q != '\n' ) { - ss << (*q == '+' ? ' ' : *q); - q++; - if( *q == '#' ) - while( *q && *q != ' ' && *q != '\n' ) q++; - } - ss << ""; - p = q; - continue; - } - if( *p == '\n' ) ss << "
"; - else ss << *p; - p++; - } - } - ss << ""; - ss << "\n"; - } - - Command::Command(const char *_name, bool web, const char *oldName) : name(_name) { - // register ourself. - if ( _commands == 0 ) - _commands = new map; - if( _commandsByBestName == 0 ) - _commandsByBestName = new map; - Command*& c = (*_commands)[name]; - if ( c ) - log() << "warning: 2 commands with name: " << _name << endl; - c = this; - (*_commandsByBestName)[name] = this; - - if( web ) { - if( _webCommands == 0 ) - _webCommands = new map; - (*_webCommands)[name] = this; - } - - if( oldName ) - (*_commands)[oldName] = this; - } - - void Command::help( stringstream& help ) const { - help << "no help defined"; - } - - Command* Command::findCommand( const string& name ) { - map::iterator i = _commands->find( name ); - if ( i == _commands->end() ) - return 0; - return i->second; - } - - Command::LockType Command::locktype( const string& name ) { - Command * c = findCommand( name ); - if ( ! c ) - return WRITE; - return c->locktype(); - } - - void Command::appendCommandStatus(BSONObjBuilder& result, bool ok, const std::string& errmsg) { - BSONObj tmp = result.asTempObj(); - bool have_ok = tmp.hasField("ok"); - bool have_errmsg = tmp.hasField("errmsg"); - - if (!have_ok) - result.append( "ok" , ok ? 1.0 : 0.0 ); - - if (!ok && !have_errmsg) { - result.append("errmsg", errmsg); - } - } - - void Command::logIfSlow( const Timer& timer, const string& msg ) { - int ms = timer.millis(); - if ( ms > cmdLine.slowMS ) { - out() << msg << " took " << ms << " ms." << endl; - } - } - -} - -#include "../client/connpool.h" - -namespace mongo { - - extern DBConnectionPool pool; - - class PoolFlushCmd : public Command { - public: - PoolFlushCmd() : Command( "connPoolSync" , false , "connpoolsync" ) {} - virtual void help( stringstream &help ) const { help<<"internal"; } - virtual LockType locktype() const { return NONE; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::connPoolSync); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); - } - virtual bool run(const string&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder& result, bool) { - pool.flush(); - return true; - } - virtual bool slaveOk() const { - return true; - } - - } poolFlushCmd; - - class PoolStats : public Command { - public: - PoolStats() : Command( "connPoolStats" ) {} - virtual void help( stringstream &help ) const { help<<"stats about connection pool"; } - virtual LockType locktype() const { return NONE; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::connPoolStats); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); - } - virtual bool run(const string&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder& result, bool) { - pool.appendInfo( result ); - result.append( "numDBClientConnection" , DBClientConnection::getNumConnections() ); - result.append( "numAScopedConnection" , AScopedConnection::getNumConnections() ); - return true; - } - virtual bool slaveOk() const { - return true; - } - - } poolStatsCmd; - -} // namespace mongo +/* commands.cpp + db "commands" (sent via db.$cmd.findOne(...)) + */ + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pch.h" + +#include "mongo/db/commands.h" + +#include +#include + +#include "mongo/bson/mutable/document.h" +#include "mongo/db/audit.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/client.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/server_parameters.h" + +namespace mongo { + + map * Command::_commandsByBestName; + map * Command::_webCommands; + map * Command::_commands; + + int Command::testCommandsEnabled = 0; + + namespace { + ExportedServerParameter testCommandsParameter(ServerParameterSet::getGlobal(), + "enableTestCommands", + &Command::testCommandsEnabled, + true, + false); + } + + string Command::parseNsFullyQualified(const string& dbname, const BSONObj& cmdObj) const { + BSONElement first = cmdObj.firstElement(); + uassert(17005, + mongoutils::str::stream() << "Main argument to " << first.fieldNameStringData() << + " must be a fully qualified namespace string. Found: " << + first.toString(false), + first.type() == mongo::String && + NamespaceString::validCollectionComponent(first.valuestr())); + return first.String(); + } + + /*virtual*/ string Command::parseNs(const string& dbname, const BSONObj& cmdObj) const { + BSONElement first = cmdObj.firstElement(); + if (first.type() != mongo::String) + return dbname; + + string coll = cmdObj.firstElement().valuestr(); +#if defined(CLC) + DEV if( mongoutils::str::startsWith(coll, dbname+'.') ) { + log() << "DEBUG parseNs Command's collection name looks like it includes the db name\n" + << dbname << '\n' + << coll << '\n' + << cmdObj.toString() << endl; + dassert(false); + } +#endif + return dbname + '.' + coll; + } + + ResourcePattern Command::parseResourcePattern(const std::string& dbname, + const BSONObj& cmdObj) const { + std::string ns = parseNs(dbname, cmdObj); + if (ns.find('.') == std::string::npos) { + return ResourcePattern::forDatabaseName(ns); + } + return ResourcePattern::forExactNamespace(NamespaceString(ns)); + } + + + void Command::htmlHelp(stringstream& ss) const { + string helpStr; + { + stringstream h; + help(h); + helpStr = h.str(); + } + ss << "\n"; + bool web = _webCommands->count(name) != 0; + if( web ) ss << ""; + ss << name; + if( web ) ss << ""; + ss << "\n"; + ss << ""; + int l = locktype(); + //if( l == NONE ) ss << "N "; + if( l == READ ) ss << "R "; + else if( l == WRITE ) ss << "W "; + if( slaveOk() ) + ss << "S "; + if( adminOnly() ) + ss << "A"; + if( lockGlobally() ) + ss << " lockGlobally "; + ss << ""; + ss << ""; + if( helpStr != "no help defined" ) { + const char *p = helpStr.c_str(); + while( *p ) { + if( *p == '<' ) { + ss << "<"; + p++; continue; + } + else if( *p == '{' ) + ss << ""; + else if( *p == '}' ) { + ss << "}"; + p++; + continue; + } + if( strncmp(p, "http:", 5) == 0 ) { + ss << ""; + q = p; + if( startsWith(q, "http://www.mongodb.org/display/") ) + q += 31; + while( *q && *q != ' ' && *q != '\n' ) { + ss << (*q == '+' ? ' ' : *q); + q++; + if( *q == '#' ) + while( *q && *q != ' ' && *q != '\n' ) q++; + } + ss << ""; + p = q; + continue; + } + if( *p == '\n' ) ss << "
"; + else ss << *p; + p++; + } + } + ss << ""; + ss << "\n"; + } + + Command::Command(StringData _name, bool web, StringData oldName) : name(_name.toString()) { + // register ourself. + if ( _commands == 0 ) + _commands = new map; + if( _commandsByBestName == 0 ) + _commandsByBestName = new map; + Command*& c = (*_commands)[name]; + if ( c ) + log() << "warning: 2 commands with name: " << _name << endl; + c = this; + (*_commandsByBestName)[name] = this; + + if( web ) { + if( _webCommands == 0 ) + _webCommands = new map; + (*_webCommands)[name] = this; + } + + if( !oldName.empty() ) + (*_commands)[oldName.toString()] = this; + } + + void Command::help( stringstream& help ) const { + help << "no help defined"; + } + + std::vector Command::stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + return std::vector(); + } + + Command* Command::findCommand( const string& name ) { + map::iterator i = _commands->find( name ); + if ( i == _commands->end() ) + return 0; + return i->second; + } + + Command::LockType Command::locktype( const string& name ) { + Command * c = findCommand( name ); + if ( ! c ) + return WRITE; + return c->locktype(); + } + + bool Command::appendCommandStatus(BSONObjBuilder& result, const Status& status) { + appendCommandStatus(result, status.isOK(), status.reason()); + BSONObj tmp = result.asTempObj(); + if (!status.isOK() && !tmp.hasField("code")) { + result.append("code", status.code()); + } + return status.isOK(); + } + + void Command::appendCommandStatus(BSONObjBuilder& result, bool ok, const std::string& errmsg) { + BSONObj tmp = result.asTempObj(); + bool have_ok = tmp.hasField("ok"); + bool have_errmsg = tmp.hasField("errmsg"); + + if (!have_ok) + result.append( "ok" , ok ? 1.0 : 0.0 ); + + if (!ok && !have_errmsg) { + result.append("errmsg", errmsg); + } + } + + Status Command::getStatusFromCommandResult(const BSONObj& result) { + BSONElement okElement = result["ok"]; + BSONElement codeElement = result["code"]; + BSONElement errmsgElement = result["errmsg"]; + if (okElement.eoo()) { + return Status(ErrorCodes::CommandResultSchemaViolation, + mongoutils::str::stream() << "No \"ok\" field in command result " << + result); + } + if (okElement.trueValue()) { + return Status::OK(); + } + int code = codeElement.numberInt(); + if (0 == code) + code = ErrorCodes::UnknownError; + std::string errmsg; + if (errmsgElement.type() == String) { + errmsg = errmsgElement.String(); + } + else if (!errmsgElement.eoo()) { + errmsg = errmsgElement.toString(); + } + return Status(ErrorCodes::Error(code), errmsg); + } + + Status Command::checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + std::vector privileges; + this->addRequiredPrivileges(dbname, cmdObj, &privileges); + if (client->getAuthorizationSession()->isAuthorizedForPrivileges(privileges)) + return Status::OK(); + return Status(ErrorCodes::Unauthorized, "unauthorized"); + } + + void Command::redactForLogging(mutablebson::Document* cmdObj) {} + + void Command::logIfSlow( const Timer& timer, const string& msg ) { + int ms = timer.millis(); + if (ms > serverGlobalParams.slowMS) { + out() << msg << " took " << ms << " ms." << endl; + } + } + + static Status _checkAuthorizationImpl(Command* c, + ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj, + bool fromRepl) { + namespace mmb = mutablebson; + if ( c->adminOnly() && ! fromRepl && dbname != "admin" ) { + return Status(ErrorCodes::Unauthorized, str::stream() << c->name << + " may only be run against the admin database."); + } + if (client->getAuthorizationSession()->getAuthorizationManager().isAuthEnabled()) { + Status status = c->checkAuthForCommand(client, dbname, cmdObj); + if (status == ErrorCodes::Unauthorized) { + mmb::Document cmdToLog(cmdObj, mmb::Document::kInPlaceDisabled); + c->redactForLogging(&cmdToLog); + return Status(ErrorCodes::Unauthorized, + str::stream() << "not authorized on " << dbname << + " to execute command " << cmdToLog.toString()); + } + if (!status.isOK()) { + return status; + } + } + else if (c->adminOnly() && + c->localHostOnlyIfNoAuth(cmdObj) && + !client->getIsLocalHostConnection()) { + + return Status(ErrorCodes::Unauthorized, str::stream() << c->name << + " must run from localhost when running db without auth"); + } + return Status::OK(); + } + + Status Command::_checkAuthorization(Command* c, + ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj, + bool fromRepl) { + namespace mmb = mutablebson; + Status status = _checkAuthorizationImpl(c, client, dbname, cmdObj, fromRepl); + if (!status.isOK()) { + log() << status << std::endl; + } + mmb::Document cmdToLog(cmdObj, mmb::Document::kInPlaceDisabled); + c->redactForLogging(&cmdToLog); + audit::logCommandAuthzCheck(client, + NamespaceString(c->parseNs(dbname, cmdObj)), + cmdToLog, + status.code()); + return status; + } +} + +#include "mongo/client/connpool.h" + +namespace mongo { + + extern DBConnectionPool pool; + // This is mainly used by the internal writes using write commands. + extern DBConnectionPool shardConnectionPool; + + class PoolFlushCmd : public Command { + public: + PoolFlushCmd() : Command( "connPoolSync" , false , "connpoolsync" ) {} + virtual void help( stringstream &help ) const { help<<"internal"; } + virtual LockType locktype() const { return NONE; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::connPoolSync); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); + } + + virtual bool run(const string&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder& result, bool) { + shardConnectionPool.flush(); + pool.flush(); + return true; + } + virtual bool slaveOk() const { + return true; + } + + } poolFlushCmd; + + class PoolStats : public Command { + public: + PoolStats() : Command( "connPoolStats" ) {} + virtual void help( stringstream &help ) const { help<<"stats about connection pool"; } + virtual LockType locktype() const { return NONE; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::connPoolStats); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); + } + virtual bool run(const string&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder& result, bool) { + pool.appendInfo( result ); + result.append( "numDBClientConnection" , DBClientConnection::getNumConnections() ); + result.append( "numAScopedConnection" , AScopedConnection::getNumConnections() ); + return true; + } + virtual bool slaveOk() const { + return true; + } + + } poolStatsCmd; + +} // namespace mongo diff --git a/src/mongo/db/commands.h b/src/mongo/db/commands.h index b041a726412..fcd1ae12ed0 100644 --- a/src/mongo/db/commands.h +++ b/src/mongo/db/commands.h @@ -17,33 +17,50 @@ #pragma once +#include #include -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" +#include "mongo/base/status.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/resource_pattern.h" #include "mongo/db/client_basic.h" #include "mongo/db/jsobj.h" -#include "mongo/util/mongoutils/str.h" namespace mongo { class BSONObj; class BSONObjBuilder; class Client; + class Database; class Timer; +namespace mutablebson { + class Document; +} // namespace mutablebson + /** mongodb "commands" (sent via db.$cmd.findOne(...)) subclass to make a command. define a singleton object for it. */ class Command { protected: + // The type of the first field in 'cmdObj' must be mongo::String. The first field is + // interpreted as a collection name. string parseNsFullyQualified(const string& dbname, const BSONObj& cmdObj) const; public: - // only makes sense for commands where 1st parm is the collection. + + // Return the namespace for the command. If the first field in 'cmdObj' is of type + // mongo::String, then that field is interpreted as the collection name, and is + // appended to 'dbname' after a '.' character. If the first field is not of type + // mongo::String, then 'dbname' is returned unmodified. virtual string parseNs(const string& dbname, const BSONObj& cmdObj) const; + // Utility that returns a ResourcePattern for the namespace returned from + // parseNs(dbname, cmdObj). This will be either an exact namespace resource pattern + // or a database resource pattern, depending on whether parseNs returns a fully qualifed + // collection name or just a database name. + ResourcePattern parseResourcePattern(const std::string& dbname, + const BSONObj& cmdObj) const; + // warning: isAuthorized uses the lockType() return values, and values are being passed // around as ints so be careful as it isn't really typesafe and will need cleanup later enum LockType { READ = -1 , NONE = 0 , WRITE = 1 }; @@ -106,20 +123,28 @@ namespace mongo { */ virtual bool logTheOp() { return false; } + /** + * Override and return fales if the command opcounters should not be incremented on + * behalf of this command. + */ + virtual bool shouldAffectCommandCounter() const { return true; } + virtual void help( stringstream& help ) const; - /* Return true if authentication and security applies to the commands. Some commands - (e.g., getnonce, authenticate) can be done by anyone even unauthorized. - */ - virtual bool requiresAuth() { return true; } + /** + * Checks if the given client is authorized to run this command on database "dbname" + * with the invocation described by "cmdObj". + */ + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj); /** - * Appends to "*out" the privileges required to run this command on database "dbname" with - * the invocation described by "cmdObj". + * Redacts "cmdObj" in-place to a form suitable for writing to logs. + * + * The default implementation does nothing. */ - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) = 0; + virtual void redactForLogging(mutablebson::Document* cmdObj); /* Return true if a replica set secondary should go into "recovering" (unreadable) state while running this command. @@ -134,11 +159,24 @@ namespace mongo { /** @param webUI expose the command in the web ui as localhost:28017/ @param oldName an optional old, deprecated name for the command */ - Command(const char *_name, bool webUI = false, const char *oldName = 0); + Command(StringData _name, bool webUI = false, StringData oldName = StringData()); virtual ~Command() {} protected: + + /** + * Appends to "*out" the privileges required to run this command on database "dbname" with + * the invocation described by "cmdObj". New commands shouldn't implement this, they should + * implement checkAuthForCommand instead. + */ + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + // The default implementation of addRequiredPrivileges should never be hit. + fassertFailed(16940); + } + BSONObj getQuery( const BSONObj& cmdObj ) { if ( cmdObj["query"].type() == Object ) return cmdObj["query"].embeddedObject(); @@ -154,6 +192,10 @@ namespace mongo { static map * _webCommands; public: + // Stops all index builds required to run this command and returns index builds killed. + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj); + static const map* commandsByBestName() { return _commandsByBestName; } static const map* webCommands() { return _webCommands; } /** @return if command was found */ @@ -183,34 +225,35 @@ namespace mongo { // Helper for setting errmsg and ok field in command result object. static void appendCommandStatus(BSONObjBuilder& result, bool ok, const std::string& errmsg); + // @return s.isOK() + static bool appendCommandStatus(BSONObjBuilder& result, const Status& status); + + // Converts "result" into a Status object. The input is expected to be the object returned + // by running a command. Returns ErrorCodes::CommandResultSchemaViolation if "result" does + // not look like the result of a command. + static Status getStatusFromCommandResult(const BSONObj& result); + // Set by command line. Controls whether or not testing-only commands should be available. static int testCommandsEnabled; - }; - class CmdShutdown : public Command { - public: - virtual bool requiresAuth() { return true; } - virtual bool adminOnly() const { return true; } - virtual bool localHostOnlyIfNoAuth(const BSONObj& cmdObj) { return true; } - virtual bool logTheOp() { - return false; - } - virtual bool slaveOk() const { - return true; - } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::shutdown); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); - } - virtual LockType locktype() const { return NONE; } - virtual void help( stringstream& help ) const; - CmdShutdown() : Command("shutdown") {} - bool run(const string& dbname, BSONObj& cmdObj, int options, string& errmsg, BSONObjBuilder& result, bool fromRepl); private: - bool shutdownHelper(); + /** + * Checks to see if the client is authorized to run the given command with the given + * parameters on the given named database. + * + * fromRepl is true if this command is running as part of oplog application, which for + * historic reasons has slightly different authorization semantics. TODO(schwerin): Check + * to see if this oddity can now be eliminated. + * + * Returns Status::OK() if the command is authorized. Most likely returns + * ErrorCodes::Unauthorized otherwise, but any return other than Status::OK implies not + * authorized. + */ + static Status _checkAuthorization(Command* c, + ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj, + bool fromRepl); }; bool _runCommands(const char *ns, BSONObj& jsobj, BufBuilder &b, BSONObjBuilder& anObjBuilder, bool fromRepl, int queryOptions); diff --git a/src/mongo/db/commands/apply_ops.cpp b/src/mongo/db/commands/apply_ops.cpp new file mode 100644 index 00000000000..782ffe0a8fa --- /dev/null +++ b/src/mongo/db/commands/apply_ops.cpp @@ -0,0 +1,167 @@ +/** +* Copyright (C) 2008 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include +#include +#include + +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/commands.h" +#include "mongo/db/commands/dbhash.h" +#include "mongo/db/instance.h" +#include "mongo/db/matcher.h" +#include "mongo/db/repl/oplog.h" + +namespace mongo { + class ApplyOpsCmd : public Command { + public: + virtual bool slaveOk() const { return false; } + virtual LockType locktype() const { return WRITE; } + virtual bool lockGlobally() const { return true; } // SERVER-4328 todo : is global ok or does this take a long time? i believe multiple ns used so locking individually requires more analysis + ApplyOpsCmd() : Command( "applyOps" ) {} + virtual void help( stringstream &help ) const { + help << "internal (sharding)\n{ applyOps : [ ] , preCondition : [ { ns : ... , q : ... , res : ... } ] }"; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + // applyOps can do pretty much anything, so require all privileges. + RoleGraph::generateUniversalPrivileges(out); + } + virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + + if ( cmdObj.firstElement().type() != Array ) { + errmsg = "ops has to be an array"; + return false; + } + + BSONObj ops = cmdObj.firstElement().Obj(); + + { + // check input + BSONObjIterator i( ops ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( e.type() == Object ) + continue; + errmsg = "op not an object: "; + errmsg += e.fieldName(); + return false; + } + } + + if ( cmdObj["preCondition"].type() == Array ) { + BSONObjIterator i( cmdObj["preCondition"].Obj() ); + while ( i.more() ) { + BSONObj f = i.next().Obj(); + + BSONObj realres = db.findOne( f["ns"].String() , f["q"].Obj() ); + + Matcher m( f["res"].Obj() ); + if ( ! m.matches( realres ) ) { + result.append( "got" , realres ); + result.append( "whatFailed" , f ); + errmsg = "pre-condition failed"; + return false; + } + } + } + + // apply + int num = 0; + int errors = 0; + + BSONObjIterator i( ops ); + BSONArrayBuilder ab; + const bool alwaysUpsert = cmdObj.hasField("alwaysUpsert") ? + cmdObj["alwaysUpsert"].trueValue() : true; + + while ( i.more() ) { + BSONElement e = i.next(); + const BSONObj& temp = e.Obj(); + + string ns = temp["ns"].String(); + + // Run operations under a nested lock as a hack to prevent them from yielding. + // + // The list of operations is supposed to be applied atomically; yielding would break + // atomicity by allowing an interruption or a shutdown to occur after only some + // operations are applied. We are already locked globally at this point, so taking + // a DBWrite on the namespace creates a nested lock, and yields are disallowed for + // operations that hold a nested lock. + Lock::DBWrite lk(ns); + invariant(Lock::nested()); + + Client::Context ctx(ns); + bool failed = applyOperation_inlock(temp, false, alwaysUpsert); + ab.append(!failed); + if ( failed ) + errors++; + + num++; + + logOpForDbHash( "u", ns.c_str(), BSONObj(), NULL, NULL, false ); + } + + result.append( "applied" , num ); + result.append( "results" , ab.arr() ); + + if ( ! fromRepl ) { + // We want this applied atomically on slaves + // so we re-wrap without the pre-condition for speed + + string tempNS = str::stream() << dbname << ".$cmd"; + + // TODO: possibly use mutable BSON to remove preCondition field + // once it is available + BSONObjIterator iter(cmdObj); + BSONObjBuilder cmdBuilder; + + while (iter.more()) { + BSONElement elem(iter.next()); + if (strcmp(elem.fieldName(), "preCondition") != 0) { + cmdBuilder.append(elem); + } + } + + logOp("c", tempNS.c_str(), cmdBuilder.done()); + } + + return errors == 0; + } + + DBDirectClient db; + + } applyOpsCmd; + +} diff --git a/src/mongo/db/commands/auth_schema_upgrade_d.cpp b/src/mongo/db/commands/auth_schema_upgrade_d.cpp new file mode 100644 index 00000000000..d18fb04356c --- /dev/null +++ b/src/mongo/db/commands/auth_schema_upgrade_d.cpp @@ -0,0 +1,158 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authz_documents_update_guard.h" +#include "mongo/db/auth/user_management_commands_parser.h" +#include "mongo/db/commands/user_management_commands.h" +#include "mongo/db/repl/multicmd.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/repl/rs_config.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/version.h" + +namespace mongo { +namespace { + + Status checkReplicaMemberVersions() { + if (!theReplSet) + return Status::OK(); + + + std::list rsMembers; + try { + const unsigned rsSelfId = theReplSet->selfId(); + const std::vector& rsMemberConfigs = + theReplSet->config().members; + for (size_t i = 0; i < rsMemberConfigs.size(); ++i) { + const unsigned otherId = rsMemberConfigs[i]._id; + if (rsSelfId == otherId) + continue; + const Member* other = theReplSet->findById(otherId); + if (!other) { + log() << "During authSchemaUpgrade, no information about replica set member " + "with id " << otherId << "; ignoring."; + continue; + } + if (!other->hbinfo().maybeUp()) { + log() << "During authSchemaUpgrade, replica set member " << other->h() << + " is down; ignoring."; + continue; + } + rsMembers.push_back(Target(other->fullName())); + } + + multiCommand(BSON("buildInfo" << 1), rsMembers); + } + catch (const DBException& ex) { + return ex.toStatus(); + } + + for (std::list::const_iterator iter = rsMembers.begin(), end = rsMembers.end(); + iter != end; + ++iter) { + + if (!iter->ok) { + logger::LogstreamBuilder wlog = warning(); + wlog << "During authSchemaUpgrade, could not run buildInfo command on " << + iter->toHost; + if (!iter->result.isEmpty()) + wlog << "; response was " << iter->result.toString(); + wlog << "; ignoring."; + continue; + } + + const char* version = iter->result["version"].valuestrsafe(); + if (!*version) { + return Status(ErrorCodes::RemoteValidationError, mongoutils::str::stream() << + "Missing or non-string \"version\" field in result of buildInfo " + "command sent to " << iter->toHost << "; found " << + iter->result["version"]); + } + + if (!isSameMajorVersion(version)) { + BSONArray foundVersionArray = toVersionArray(version); + return Status(ErrorCodes::RemoteValidationError, mongoutils::str::stream() << + "To upgrade auth schema in a replica set, all members must be " + "running the same release series of mongod; found " << + foundVersionArray["0"] << '.' << foundVersionArray["1"] << + " on host " << iter->toHost << " but expected " << + versionArray["0"] << '.' << versionArray["1"]); + } + } + return Status::OK(); + } + + class CmdAuthSchemaUpgradeD : public CmdAuthSchemaUpgrade { + virtual bool run( + const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + + int maxSteps; + bool upgradeShardServers; + BSONObj writeConcern; + Status status = auth::parseAuthSchemaUpgradeStepCommand( + cmdObj, + dbname, + &maxSteps, + &upgradeShardServers, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("auth schema upgrade")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = checkReplicaMemberVersions(); + if (!status.isOK()) + return appendCommandStatus(result, status); + + status = authzManager->upgradeSchema(maxSteps, writeConcern); + if (status.isOK()) + result.append("done", true); + return appendCommandStatus(result, status); + } + + } cmdAuthSchemaUpgradeStep; + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/commands/authentication_commands.cpp b/src/mongo/db/commands/authentication_commands.cpp index bcc5a2f16a0..0b462cab95d 100644 --- a/src/mongo/db/commands/authentication_commands.cpp +++ b/src/mongo/db/commands/authentication_commands.cpp @@ -12,35 +12,66 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/commands/authentication_commands.h" +#include #include #include #include #include "mongo/base/status.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" #include "mongo/client/sasl_client_authenticate.h" +#include "mongo/db/audit.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/mongo_authentication_session.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/security_key.h" #include "mongo/db/client_basic.h" #include "mongo/db/commands.h" #include "mongo/db/jsobj.h" #include "mongo/platform/random.h" #include "mongo/util/concurrency/mutex.h" #include "mongo/util/md5.hpp" +#include "mongo/util/net/ssl_manager.h" +#include "mongo/util/text.h" namespace mongo { - static bool _areNonceAuthenticateCommandsEnabled = true; - static const char _nonceAuthenticateCommandsDisabledMessage[] = + static bool _isCRAuthDisabled; + static bool _isX509AuthDisabled; + static const char _nonceAuthenticationDisabledMessage[] = "Challenge-response authentication using getnonce and authenticate commands is disabled."; - - void CmdAuthenticate::disableCommand() { _areNonceAuthenticateCommandsEnabled = false; } + static const char _x509AuthenticationDisabledMessage[] = + "x.509 authentication is disabled."; + + void CmdAuthenticate::disableAuthMechanism(std::string authMechanism) { + if (authMechanism == "MONGODB-CR") { + _isCRAuthDisabled = true; + } + if (authMechanism == "MONGODB-X509") { + _isX509AuthDisabled = true; + } + } /* authentication @@ -62,7 +93,6 @@ namespace mongo { _random(SecureRandom::create()) { } - virtual bool requiresAuth() { return false; } virtual bool logTheOp() { return false; } virtual bool slaveOk() const { return true; @@ -92,80 +122,144 @@ namespace mongo { boost::scoped_ptr _random; } cmdGetNonce; - bool CmdAuthenticate::run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + void CmdAuthenticate::redactForLogging(mutablebson::Document* cmdObj) { + namespace mmb = mutablebson; + static const int numRedactedFields = 2; + static const char* redactedFields[numRedactedFields] = { "key", "nonce" }; + for (int i = 0; i < numRedactedFields; ++i) { + for (mmb::Element element = mmb::findFirstChildNamed(cmdObj->root(), redactedFields[i]); + element.ok(); + element = mmb::findElementNamed(element.rightSibling(), redactedFields[i])) { - log() << " authenticate db: " << dbname << " " << cmdObj << endl; + element.setValueString("xxx"); + } + } + } - string user = cmdObj.getStringField("user"); + bool CmdAuthenticate::run(const string& dbname, + BSONObj& cmdObj, + int, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + + mutablebson::Document cmdToLog(cmdObj, mutablebson::Document::kInPlaceDisabled); + redactForLogging(&cmdToLog); + log() << " authenticate db: " << dbname << " " << cmdToLog << endl; + + UserName user(cmdObj.getStringField("user"), dbname); + if (Command::testCommandsEnabled && + user.getDB() == "admin" && + user.getUser() == internalSecurity.user->getName().getUser()) { + // Allows authenticating as the internal user against the admin database. This is to + // support the auth passthrough test framework on mongos (since you can't use the local + // database on a mongos, so you can't auth as the internal user without this). + user = internalSecurity.user->getName(); + } - if (!_areNonceAuthenticateCommandsEnabled) { + std::string mechanism = cmdObj.getStringField("mechanism"); + if (mechanism.empty()) { + mechanism = "MONGODB-CR"; + } + Status status = _authenticate(mechanism, user, cmdObj); + audit::logAuthentication(ClientBasic::getCurrent(), + mechanism, + user, + status.code()); + if (!status.isOK()) { + log() << "Failed to authenticate " << user << " with mechanism " << mechanism << ": " << + status; + if (status.code() == ErrorCodes::AuthenticationFailed) { + // Statuses with code AuthenticationFailed may contain messages we do not wish to + // reveal to the user, so we return a status with the message "auth failed". + appendCommandStatus(result, + Status(ErrorCodes::AuthenticationFailed, "auth failed")); + } + else { + appendCommandStatus(result, status); + } + return false; + } + result.append("dbname", user.getDB()); + result.append("user", user.getUser()); + return true; + } + + Status CmdAuthenticate::_authenticate(const std::string& mechanism, + const UserName& user, + const BSONObj& cmdObj) { + + if (mechanism == "MONGODB-CR") { + return _authenticateCR(user, cmdObj); + } +#ifdef MONGO_SSL + if (mechanism == "MONGODB-X509") { + return _authenticateX509(user, cmdObj); + } +#endif + return Status(ErrorCodes::BadValue, "Unsupported mechanism: " + mechanism); + } + + Status CmdAuthenticate::_authenticateCR(const UserName& user, const BSONObj& cmdObj) { + + if (user == internalSecurity.user->getName() && + serverGlobalParams.clusterAuthMode.load() == + ServerGlobalParams::ClusterAuthMode_x509) { + return Status(ErrorCodes::AuthenticationFailed, + "Mechanism x509 is required for internal cluster authentication"); + } + + if (_isCRAuthDisabled) { // SERVER-8461, MONGODB-CR must be enabled for authenticating the internal user, so that // cluster members may communicate with each other. - if (dbname != StringData("local", StringData::LiteralTag()) || - user != internalSecurity.user) { - errmsg = _nonceAuthenticateCommandsDisabledMessage; - result.append(saslCommandCodeFieldName, ErrorCodes::AuthenticationFailed); - return false; + if (user != internalSecurity.user->getName()) { + return Status(ErrorCodes::BadValue, _nonceAuthenticationDisabledMessage); } } string key = cmdObj.getStringField("key"); string received_nonce = cmdObj.getStringField("nonce"); - if( user.empty() || key.empty() || received_nonce.empty() ) { - log() << "field missing/wrong type in received authenticate command " - << dbname - << endl; - errmsg = "auth fails"; + if( user.getUser().empty() || key.empty() || received_nonce.empty() ) { sleepmillis(10); - result.append(saslCommandCodeFieldName, ErrorCodes::AuthenticationFailed); - return false; + return Status(ErrorCodes::ProtocolError, + "field missing/wrong type in received authenticate command"); } stringstream digestBuilder; { - bool reject = false; ClientBasic *client = ClientBasic::getCurrent(); - AuthenticationSession *session = client->getAuthenticationSession(); + boost::scoped_ptr session; + client->swapAuthenticationSession(session); if (!session || session->getType() != AuthenticationSession::SESSION_TYPE_MONGO) { - reject = true; - LOG(1) << "auth: No pending nonce" << endl; + sleepmillis(30); + return Status(ErrorCodes::ProtocolError, "No pending nonce"); } else { - nonce64 nonce = static_cast(session)->getNonce(); + nonce64 nonce = static_cast(session.get())->getNonce(); digestBuilder << hex << nonce; - reject = digestBuilder.str() != received_nonce; - if ( reject ) { - LOG(1) << "auth: Authentication failed for " << dbname << '$' << user << endl; + if (digestBuilder.str() != received_nonce) { + sleepmillis(30); + return Status(ErrorCodes::AuthenticationFailed, "Received wrong nonce."); } } - client->resetAuthenticationSession(NULL); - - if ( reject ) { - log() << "auth: bad nonce received or getnonce not called. could be a driver bug or a security attack. db:" << dbname << endl; - errmsg = "auth fails"; - sleepmillis(30); - result.append(saslCommandCodeFieldName, ErrorCodes::AuthenticationFailed); - return false; - } } - BSONObj userObj; - string pwd; - Status status = ClientBasic::getCurrent()->getAuthorizationManager()->getPrivilegeDocument( - dbname, PrincipalName(user, dbname), &userObj); + User* userObj; + Status status = getGlobalAuthorizationManager()->acquireUser(user, &userObj); if (!status.isOK()) { - log() << status.reason() << std::endl; - errmsg = "auth fails"; - result.append(saslCommandCodeFieldName, ErrorCodes::AuthenticationFailed); - return false; + // Failure to find the privilege document indicates no-such-user, a fact that we do not + // wish to reveal to the client. So, we return AuthenticationFailed rather than passing + // through the returned status. + return Status(ErrorCodes::AuthenticationFailed, status.toString()); } - pwd = userObj["pwd"].String(); + string pwd = userObj->getCredentials().password; + getGlobalAuthorizationManager()->releaseUser(userObj); md5digest d; { - digestBuilder << user << pwd; + digestBuilder << user.getUser() << pwd; string done = digestBuilder.str(); md5_state_t st; @@ -177,22 +271,101 @@ namespace mongo { string computed = digestToString( d ); if ( key != computed ) { - log() << "auth: key mismatch " << user << ", ns:" << dbname << endl; - errmsg = "auth fails"; - result.append(saslCommandCodeFieldName, ErrorCodes::AuthenticationFailed); - return false; + return Status(ErrorCodes::AuthenticationFailed, "key mismatch"); + } + + AuthorizationSession* authorizationSession = + ClientBasic::getCurrent()->getAuthorizationSession(); + status = authorizationSession->addAndAuthorizeUser(user); + if (!status.isOK()) { + return status; + } + + return Status::OK(); + } + +#ifdef MONGO_SSL + void canonicalizeClusterDN(std::vector* dn) { + // remove all RDNs we don't care about + for (std::vector::iterator it=dn->begin(); it != dn->end(); it++) { + boost::algorithm::trim(*it); + if (!mongoutils::str::startsWith(it->c_str(), "DC=") && + !mongoutils::str::startsWith(it->c_str(), "O=") && + !mongoutils::str::startsWith(it->c_str(), "OU=")) { + dn->erase(it--); + } } + std::stable_sort(dn->begin(), dn->end()); + } - AuthorizationManager* authorizationManager = - ClientBasic::getCurrent()->getAuthorizationManager(); - Principal* principal = new Principal(PrincipalName(user, dbname)); - principal->setImplicitPrivilegeAcquisition(true); - authorizationManager->addAuthorizedPrincipal(principal); + bool CmdAuthenticate::_clusterIdMatch(const std::string& subjectName, + const std::string& srvSubjectName) { + std::vector clientRDN = StringSplitter::split(subjectName, ","); + std::vector serverRDN = StringSplitter::split(srvSubjectName, ","); + + canonicalizeClusterDN(&clientRDN); + canonicalizeClusterDN(&serverRDN); + + if (clientRDN.size() == 0 || clientRDN.size() != serverRDN.size()) { + return false; + } - result.append( "dbname" , dbname ); - result.append( "user" , user ); + for (size_t i=0; i < serverRDN.size(); i++) { + if(clientRDN[i] != serverRDN[i]) { + return false; + } + } return true; } + + Status CmdAuthenticate::_authenticateX509(const UserName& user, const BSONObj& cmdObj) { + if (!getSSLManager()) { + return Status(ErrorCodes::ProtocolError, + "SSL support is required for the MONGODB-X509 mechanism."); + } + if(user.getDB() != "$external") { + return Status(ErrorCodes::ProtocolError, + "X.509 authentication must always use the $external database."); + } + + ClientBasic *client = ClientBasic::getCurrent(); + AuthorizationSession* authorizationSession = client->getAuthorizationSession(); + std::string subjectName = client->port()->getX509SubjectName(); + + if (user.getUser() != subjectName) { + return Status(ErrorCodes::AuthenticationFailed, + "There is no x.509 client certificate matching the user."); + } + else { + std::string srvSubjectName = getSSLManager()->getServerSubjectName(); + + // Handle internal cluster member auth, only applies to server-server connections + if (_clusterIdMatch(subjectName, srvSubjectName)) { + int clusterAuthMode = serverGlobalParams.clusterAuthMode.load(); + if (clusterAuthMode == ServerGlobalParams::ClusterAuthMode_undefined || + clusterAuthMode == ServerGlobalParams::ClusterAuthMode_keyFile) { + return Status(ErrorCodes::AuthenticationFailed, "The provided certificate " + "can only be used for cluster authentication, not client " + "authentication. The current configuration does not allow " + "x.509 cluster authentication, check the --clusterAuthMode flag"); + } + authorizationSession->grantInternalAuthorization(); + } + // Handle normal client authentication, only applies to client-server connections + else { + if (_isX509AuthDisabled) { + return Status(ErrorCodes::BadValue, + _x509AuthenticationDisabledMessage); + } + Status status = authorizationSession->addAndAuthorizeUser(user); + if (!status.isOK()) { + return status; + } + } + return Status::OK(); + } + } +#endif CmdAuthenticate cmdAuthenticate; class CmdLogout : public Command { @@ -215,8 +388,9 @@ namespace mongo { string& errmsg, BSONObjBuilder& result, bool fromRepl) { - AuthorizationManager* authManager = ClientBasic::getCurrent()->getAuthorizationManager(); - authManager->logoutDatabase(dbname); + AuthorizationSession* authSession = + ClientBasic::getCurrent()->getAuthorizationSession(); + authSession->logoutDatabase(dbname); return true; } } cmdLogout; diff --git a/src/mongo/db/commands/authentication_commands.h b/src/mongo/db/commands/authentication_commands.h index 0e785d416a9..e7891329946 100644 --- a/src/mongo/db/commands/authentication_commands.h +++ b/src/mongo/db/commands/authentication_commands.h @@ -12,19 +12,34 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once +#include + +#include "mongo/base/status.h" +#include "mongo/db/auth/user_name.h" #include "mongo/db/commands.h" namespace mongo { class CmdAuthenticate : public Command { public: - static void disableCommand(); + static void disableAuthMechanism(std::string authMechanism); - virtual bool requiresAuth() { return false; } virtual bool logTheOp() { return false; } @@ -36,8 +51,35 @@ namespace mongo { virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) {} // No auth required + virtual void redactForLogging(mutablebson::Document* cmdObj); + CmdAuthenticate() : Command("authenticate") {} - bool run(const string& dbname , BSONObj& cmdObj, int options, string& errmsg, BSONObjBuilder& result, bool fromRepl); + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl); + + private: + /** + * Completes the authentication of "user" using "mechanism" and parameters from "cmdObj". + * + * Returns Status::OK() on success. All other statuses indicate failed authentication. The + * entire status returned here may always be used for logging. However, if the code is + * AuthenticationFailed, the "reason" field of the return status may contain information + * that should not be revealed to the connected client. + * + * Other than AuthenticationFailed, common returns are BadValue, indicating unsupported + * mechanism, and ProtocolError, indicating an error in the use of the authentication + * protocol. + */ + Status _authenticate(const std::string& mechanism, + const UserName& user, + const BSONObj& cmdObj); + Status _authenticateCR(const UserName& user, const BSONObj& cmdObj); + Status _authenticateX509(const UserName& user, const BSONObj& cmdObj); + bool _clusterIdMatch(const std::string& subjectName, const std::string& srvSubjectName); }; extern CmdAuthenticate cmdAuthenticate; diff --git a/src/mongo/db/commands/cleanup_orphaned_cmd.cpp b/src/mongo/db/commands/cleanup_orphaned_cmd.cpp new file mode 100644 index 00000000000..f58cb9a5ac6 --- /dev/null +++ b/src/mongo/db/commands/cleanup_orphaned_cmd.cpp @@ -0,0 +1,267 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include +#include + +#include "mongo/base/init.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/field_parser.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/range_deleter_service.h" +#include "mongo/s/collection_metadata.h" +#include "mongo/s/d_logic.h" +#include "mongo/s/range_arithmetic.h" + +namespace mongo { + + using mongoutils::str::stream; + + enum CleanupResult { + CleanupResult_Done, CleanupResult_Continue, CleanupResult_Error + }; + + /** + * Cleans up one range of orphaned data starting from a range that overlaps or starts at + * 'startingFromKey'. If empty, startingFromKey is the minimum key of the sharded range. + * + * @return CleanupResult_Continue and 'stoppedAtKey' if orphaned range was found and cleaned + * @return CleanupResult_Done if no orphaned ranges remain + * @return CleanupResult_Error and 'errMsg' if an error occurred + * + * If the collection is not sharded, returns CleanupResult_Done. + */ + CleanupResult cleanupOrphanedData( const NamespaceString& ns, + const BSONObj& startingFromKeyConst, + bool secondaryThrottle, + BSONObj* stoppedAtKey, + string* errMsg ) { + + BSONObj startingFromKey = startingFromKeyConst; + + CollectionMetadataPtr metadata = shardingState.getCollectionMetadata( ns.toString() ); + if ( !metadata || metadata->getKeyPattern().isEmpty() ) { + + warning() << "skipping orphaned data cleanup for " << ns.toString() + << ", collection is not sharded" << endl; + + return CleanupResult_Done; + } + + BSONObj keyPattern = metadata->getKeyPattern(); + if ( !startingFromKey.isEmpty() ) { + if ( !metadata->isValidKey( startingFromKey ) ) { + + *errMsg = stream() << "could not cleanup orphaned data, start key " + << startingFromKey + << " does not match shard key pattern " << keyPattern; + + warning() << *errMsg << endl; + return CleanupResult_Error; + } + } + else { + startingFromKey = metadata->getMinKey(); + } + + KeyRange orphanRange; + if ( !metadata->getNextOrphanRange( startingFromKey, &orphanRange ) ) { + + LOG( 1 ) << "orphaned data cleanup requested for " << ns.toString() + << " starting from " << startingFromKey + << ", no orphan ranges remain" << endl; + + return CleanupResult_Done; + } + *stoppedAtKey = orphanRange.maxKey; + + // We're done with this metadata now, no matter what happens + metadata.reset(); + + LOG( 1 ) << "orphaned data cleanup requested for " << ns.toString() + << " starting from " << startingFromKey + << ", removing next orphan range" + << " [" << orphanRange.minKey << "," << orphanRange.maxKey << ")" + << endl; + + // Metadata snapshot may be stale now, but deleter checks metadata again in write lock + // before delete. + if ( !getDeleter()->deleteNow( ns.toString(), + orphanRange.minKey, + orphanRange.maxKey, + keyPattern, + secondaryThrottle, + errMsg ) ) { + + warning() << *errMsg << endl; + return CleanupResult_Error; + } + + return CleanupResult_Continue; + } + + /** + * Cleanup orphaned data command. Called on a particular namespace, and if the collection + * is sharded will clean up a single orphaned data range which overlaps or starts after a + * passed-in 'startingFromKey'. Returns true and a 'stoppedAtKey' (which will start a + * search for the next orphaned range if the command is called again) or no key if there + * are no more orphaned ranges in the collection. + * + * If the collection is not sharded, returns true but no 'stoppedAtKey'. + * On failure, returns false and an error message. + * + * Calling this command repeatedly until no 'stoppedAtKey' is returned ensures that the + * full collection range is searched for orphaned documents, but since sharding state may + * change between calls there is no guarantee that all orphaned documents were found unless + * the balancer is off. + * + * Safe to call with the balancer on. + */ + class CleanupOrphanedCommand : public Command { + public: + CleanupOrphanedCommand() : + Command( "cleanupOrphaned" ) {} + + virtual bool slaveOk() const { return false; } + virtual bool adminOnly() const { return true; } + virtual bool localHostOnlyIfNoAuth( const BSONObj& cmdObj ) { return false; } + + virtual Status checkAuthForCommand( ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj ) { + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::cleanupOrphaned)) { + return Status(ErrorCodes::Unauthorized, + "Not authorized for cleanupOrphaned command."); + } + return Status::OK(); + } + + virtual LockType locktype() const { return NONE; } + + // Input + static BSONField nsField; + static BSONField startingFromKeyField; + static BSONField secondaryThrottleField; + + // Output + static BSONField stoppedAtKeyField; + + bool run( string const &db, + BSONObj &cmdObj, + int, + string &errmsg, + BSONObjBuilder &result, + bool ) { + + string ns; + if ( !FieldParser::extract( cmdObj, nsField, &ns, &errmsg ) ) { + return false; + } + + if ( ns == "" ) { + errmsg = "no collection name specified"; + return false; + } + + BSONObj startingFromKey; + if ( !FieldParser::extract( cmdObj, + startingFromKeyField, + &startingFromKey, + &errmsg ) ) { + return false; + } + + bool secondaryThrottle = true; + if ( !FieldParser::extract( cmdObj, + secondaryThrottleField, + &secondaryThrottle, + &errmsg ) ) { + return false; + } + + if (!shardingState.enabled()) { + errmsg = str::stream() << "server is not part of a sharded cluster or " + << "the sharding metadata is not yet initialized."; + return false; + } + + ChunkVersion shardVersion; + Status status = shardingState.refreshMetadataNow( ns, &shardVersion ); + if ( !status.isOK() ) { + if ( status.code() == ErrorCodes::RemoteChangeDetected ) { + warning() << "Shard version in transition detected while refreshing " + << "metadata for " << ns << " at version " << shardVersion << endl; + } + else { + errmsg = str::stream() << "failed to refresh shard metadata: " + << status.reason(); + return false; + } + } + + BSONObj stoppedAtKey; + CleanupResult cleanupResult = cleanupOrphanedData( NamespaceString( ns ), + startingFromKey, + secondaryThrottle, + &stoppedAtKey, + &errmsg ); + + if ( cleanupResult == CleanupResult_Error ) { + return false; + } + + if ( cleanupResult == CleanupResult_Continue ) { + result.append( stoppedAtKeyField(), stoppedAtKey ); + } + else { + dassert( cleanupResult == CleanupResult_Done ); + } + + return true; + } + }; + + BSONField CleanupOrphanedCommand::nsField( "cleanupOrphaned" ); + BSONField CleanupOrphanedCommand::startingFromKeyField( "startingFromKey" ); + BSONField CleanupOrphanedCommand::secondaryThrottleField( "secondaryThrottle" ); + BSONField CleanupOrphanedCommand::stoppedAtKeyField( "stoppedAtKey" ); + + MONGO_INITIALIZER(RegisterCleanupOrphanedCommand)(InitializerContext* context) { + // Leaked intentionally: a Command registers itself when constructed. + new CleanupOrphanedCommand(); + return Status::OK(); + } + +} // namespace mongo + diff --git a/src/mongo/db/commands/collection_to_capped.cpp b/src/mongo/db/commands/collection_to_capped.cpp new file mode 100644 index 00000000000..8a21db08ada --- /dev/null +++ b/src/mongo/db/commands/collection_to_capped.cpp @@ -0,0 +1,256 @@ +// collection_to_capped.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/background.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" // XXX-remove +#include "mongo/db/commands.h" +#include "mongo/db/index_builder.h" +#include "mongo/db/instance.h" // XXX-remove +#include "mongo/db/pdfile.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/query/new_find.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/storage/extent.h" + +namespace mongo { + + Status cloneCollectionAsCapped( Database* db, + const string& shortFrom, + const string& shortTo, + double size, + bool temp, + bool logForReplication ) { + + string fromNs = db->name() + "." + shortFrom; + string toNs = db->name() + "." + shortTo; + + Collection* fromCollection = db->getCollection( fromNs ); + if ( !fromCollection ) + return Status( ErrorCodes::NamespaceNotFound, + str::stream() << "source collection " << fromNs << " does not exist" ); + + if ( db->getCollection( toNs ) ) + return Status( ErrorCodes::NamespaceExists, "to collection already exists" ); + + // create new collection + { + Client::Context ctx( toNs ); + BSONObjBuilder spec; + spec.appendBool( "capped", true ); + spec.append( "size", size ); + if ( temp ) + spec.appendBool( "temp", true ); + + string errmsg; + if ( !userCreateNS( toNs.c_str(), spec.done(), errmsg, logForReplication ) ) + return Status( ErrorCodes::InternalError, errmsg ); + } + + auto_ptr runner; + + { + const NamespaceDetails* details = fromCollection->details(); + DiskLoc extent = details->firstExtent(); + + // datasize and extentSize can't be compared exactly, so add some padding to 'size' + long long excessSize = + static_cast( fromCollection->dataSize() - size * 2 ); + + // skip ahead some extents since not all the data fits, + // so we have to chop a bunch off + for( ; + excessSize > extent.ext()->length && extent != details->lastExtent(); + extent = extent.ext()->xnext ) { + + excessSize -= extent.ext()->length; + LOG( 2 ) << "cloneCollectionAsCapped skipping extent of size " + << extent.ext()->length << endl; + LOG( 6 ) << "excessSize: " << excessSize << endl; + } + DiskLoc startLoc = extent.ext()->firstRecord; + + runner.reset( InternalPlanner::collectionScan(fromNs, + InternalPlanner::FORWARD, + startLoc) ); + } + + Collection* toCollection = db->getCollection( toNs ); + verify( toCollection ); + + while ( true ) { + BSONObj obj; + Runner::RunnerState state = runner->getNext(&obj, NULL); + + switch( state ) { + case Runner::RUNNER_EOF: + return Status::OK(); + case Runner::RUNNER_DEAD: + db->dropCollection( toNs ); + return Status( ErrorCodes::InternalError, "runner turned dead while iterating" ); + case Runner::RUNNER_ERROR: + return Status( ErrorCodes::InternalError, "runner error while iterating" ); + case Runner::RUNNER_ADVANCED: + toCollection->insertDocument( obj, true ); + if ( logForReplication ) + logOp( "i", toNs.c_str(), obj ); + getDur().commitIfNeeded(); + } + } + + verify( false ); // unreachable + } + + /* convertToCapped seems to use this */ + class CmdCloneCollectionAsCapped : public Command { + public: + CmdCloneCollectionAsCapped() : Command( "cloneCollectionAsCapped" ) {} + virtual bool slaveOk() const { return false; } + virtual LockType locktype() const { return WRITE; } + virtual void help( stringstream &help ) const { + help << "{ cloneCollectionAsCapped:, toCollection:, size: }"; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet sourceActions; + sourceActions.addAction(ActionType::find); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), sourceActions)); + + ActionSet targetActions; + targetActions.addAction(ActionType::insert); + targetActions.addAction(ActionType::createIndex); + targetActions.addAction(ActionType::convertToCapped); + std::string collection = cmdObj.getStringField("toCollection"); + uassert(16708, "bad 'toCollection' value", !collection.empty()); + + out->push_back(Privilege(ResourcePattern::forExactNamespace( + NamespaceString(dbname, collection)), + targetActions)); + } + bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { + string from = jsobj.getStringField( "cloneCollectionAsCapped" ); + string to = jsobj.getStringField( "toCollection" ); + double size = jsobj.getField( "size" ).number(); + bool temp = jsobj.getField( "temp" ).trueValue(); + + if ( from.empty() || to.empty() || size == 0 ) { + errmsg = "invalid command spec"; + return false; + } + + Status status = cloneCollectionAsCapped( cc().database(), from, to, size, temp, true ); + return appendCommandStatus( result, status ); + } + } cmdCloneCollectionAsCapped; + + /* jan2010: + Converts the given collection to a capped collection w/ the specified size. + This command is not highly used, and is not currently supported with sharded + environments. + */ + class CmdConvertToCapped : public Command { + public: + CmdConvertToCapped() : Command( "convertToCapped" ) {} + virtual bool slaveOk() const { return false; } + virtual LockType locktype() const { return WRITE; } + // calls renamecollection which does a global lock, so we must too: + virtual bool lockGlobally() const { return true; } + virtual bool logTheOp() { + // see CmdRenameCollection::logTheOp as to why this is best + return true; + } + virtual void help( stringstream &help ) const { + help << "{ convertToCapped:, size: }"; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::convertToCapped); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); + } + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + std::string collName = cmdObj.firstElement().valuestr(); + std::string ns = db->name() + "." + collName; + + IndexCatalog::IndexKillCriteria criteria; + criteria.ns = ns; + Collection* coll = db->getCollection(ns); + if (coll) { + return IndexBuilder::killMatchingIndexBuilds(coll, criteria); + } + return std::vector(); + } + + bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { + Database* db = cc().database(); + + stopIndexBuilds(db, jsobj); + BackgroundOperation::assertNoBgOpInProgForDb(dbname.c_str()); + + string shortSource = jsobj.getStringField( "convertToCapped" ); + string longSource = dbname + "." + shortSource; + double size = jsobj.getField( "size" ).number(); + + if ( shortSource.empty() || size == 0 ) { + errmsg = "invalid command spec"; + return false; + } + + string shortTmpName = str::stream() << "tmp.convertToCapped." << shortSource; + string longTmpName = str::stream() << dbname << "." << shortTmpName; + + if ( db->getCollection( longTmpName ) ) { + Status status = db->dropCollection( longTmpName ); + if ( !status.isOK() ) + return appendCommandStatus( result, status ); + } + + Status status = cloneCollectionAsCapped( db, shortSource, shortTmpName, size, true, false ); + + if ( !status.isOK() ) + return appendCommandStatus( result, status ); + + verify( db->getCollection( longTmpName ) ); + + status = db->dropCollection( longSource ); + if ( !status.isOK() ) + return appendCommandStatus( result, status ); + + status = db->renameCollection( longTmpName, longSource, false ); + return appendCommandStatus( result, status ); + } + } cmdConvertToCapped; + +} diff --git a/src/mongo/db/commands/compact.cpp b/src/mongo/db/commands/compact.cpp new file mode 100644 index 00000000000..b020bf1e4e8 --- /dev/null +++ b/src/mongo/db/commands/compact.cpp @@ -0,0 +1,180 @@ +/// compact.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful,b +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include +#include + +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/background.h" +#include "mongo/db/commands.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/d_concurrency.h" +#include "mongo/db/curop-inl.h" +#include "mongo/db/index_builder.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + // from repl/rs.cpp + bool isCurrentlyAReplSetPrimary(); + + class CompactCmd : public Command { + public: + virtual LockType locktype() const { return NONE; } + virtual bool adminOnly() const { return false; } + virtual bool slaveOk() const { return true; } + virtual bool maintenanceMode() const { return true; } + virtual bool logTheOp() { return false; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::compact); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); + } + virtual void help( stringstream& help ) const { + help << "compact collection\n" + "warning: this operation locks the database and is slow. you can cancel with killOp()\n" + "{ compact : , [force:], [validate:],\n" + " [paddingFactor:], [paddingBytes:] }\n" + " force - allows to run on a replica set primary\n" + " validate - check records are noncorrupt before adding to newly compacting extents. slower but safer (defaults to true in this version)\n"; + } + CompactCmd() : Command("compact") { } + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + std::string coll = cmdObj.firstElement().valuestr(); + std::string ns = db->name() + "." + coll; + + IndexCatalog::IndexKillCriteria criteria; + criteria.ns = ns; + return IndexBuilder::killMatchingIndexBuilds(db->getCollection(ns), criteria); + } + + virtual bool run(const string& db, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + string coll = cmdObj.firstElement().valuestr(); + if( coll.empty() || db.empty() ) { + errmsg = "no collection name specified"; + return false; + } + + if( isCurrentlyAReplSetPrimary() && !cmdObj["force"].trueValue() ) { + errmsg = "will not run compact on an active replica set primary as this is a slow blocking operation. use force:true to force"; + return false; + } + + NamespaceString ns(db,coll); + if ( !ns.isNormal() ) { + errmsg = "bad namespace name"; + return false; + } + + if ( ns.isSystem() ) { + // items in system.* cannot be moved as there might be pointers to them + // i.e. system.indexes entries are pointed to from NamespaceDetails + errmsg = "can't compact a system namespace"; + return false; + } + + CompactOptions compactOptions; + + if ( cmdObj["preservePadding"].trueValue() ) { + compactOptions.paddingMode = CompactOptions::PRESERVE; + if ( cmdObj.hasElement( "paddingFactor" ) || + cmdObj.hasElement( "paddingBytes" ) ) { + errmsg = "cannot mix preservePadding and paddingFactor|paddingBytes"; + return false; + } + } + else if ( cmdObj.hasElement( "paddingFactor" ) || cmdObj.hasElement( "paddingBytes" ) ) { + compactOptions.paddingMode = CompactOptions::MANUAL; + if ( cmdObj.hasElement("paddingFactor") ) { + compactOptions.paddingFactor = cmdObj["paddingFactor"].Number(); + if ( compactOptions.paddingFactor < 1 || + compactOptions.paddingFactor > 4 ){ + errmsg = "invalid padding factor"; + return false; + } + } + if ( cmdObj.hasElement("paddingBytes") ) { + compactOptions.paddingBytes = cmdObj["paddingBytes"].numberInt(); + if ( compactOptions.paddingBytes < 0 || + compactOptions.paddingBytes > ( 1024 * 1024 ) ) { + errmsg = "invalid padding bytes"; + return false; + } + } + } + + if ( cmdObj.hasElement("validate") ) + compactOptions.validateDocuments = cmdObj["validate"].trueValue(); + + + Lock::DBWrite lk(ns.ns()); + BackgroundOperation::assertNoBgOpInProgForNs(ns.ns()); + Client::Context ctx(ns); + + Collection* collection = ctx.db()->getCollection(ns.ns()); + if( ! collection ) { + errmsg = "namespace does not exist"; + return false; + } + + if ( collection->isCapped() ) { + errmsg = "cannot compact a capped collection"; + return false; + } + + log() << "compact " << ns << " begin, options: " << compactOptions.toString(); + + std::vector indexesInProg = stopIndexBuilds(ctx.db(), cmdObj); + + StatusWith status = collection->compact( &compactOptions ); + if ( !status.isOK() ) + return appendCommandStatus( result, status.getStatus() ); + + if ( status.getValue().corruptDocuments > 0 ) + result.append("invalidObjects", status.getValue().corruptDocuments ); + + log() << "compact " << ns << " end"; + + IndexBuilder::restoreIndexes(indexesInProg); + + return true; + } + }; + static CompactCmd compactCmd; + +} diff --git a/src/mongo/db/commands/connection_status.cpp b/src/mongo/db/commands/connection_status.cpp index 10403d8bc69..a4d173c0143 100644 --- a/src/mongo/db/commands/connection_status.cpp +++ b/src/mongo/db/commands/connection_status.cpp @@ -12,18 +12,30 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands.h" namespace mongo { class CmdConnectionStatus : public Command { public: CmdConnectionStatus() : Command("connectionStatus") {} - virtual bool requiresAuth() { return false; } virtual bool logTheOp() { return false; } virtual bool slaveOk() const { return true; } virtual LockType locktype() const { return NONE; } @@ -37,18 +49,21 @@ namespace mongo { bool run(const string&, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - AuthorizationManager* authMgr = ClientBasic::getCurrent()->getAuthorizationManager(); + AuthorizationSession* authSession = + ClientBasic::getCurrent()->getAuthorizationSession(); BSONObjBuilder authInfo(result.subobjStart("authInfo")); { BSONArrayBuilder authenticatedUsers(authInfo.subarrayStart("authenticatedUsers")); - PrincipalSet::NameIterator nameIter = authMgr->getAuthenticatedPrincipalNames(); + UserNameIterator nameIter = authSession->getAuthenticatedUserNames(); for ( ; nameIter.more(); nameIter.next()) { - BSONObjBuilder principal(authenticatedUsers.subobjStart()); - principal.append("user", nameIter->getUser()); - principal.append("userSource", nameIter->getDB()); - principal.doneFast(); + BSONObjBuilder userInfoBuilder(authenticatedUsers.subobjStart()); + userInfoBuilder.append(AuthorizationManager::USER_NAME_FIELD_NAME, + nameIter->getUser()); + userInfoBuilder.append(AuthorizationManager::USER_DB_FIELD_NAME, + nameIter->getDB()); + userInfoBuilder.doneFast(); } authenticatedUsers.doneFast(); } diff --git a/src/mongo/db/commands/copydb.h b/src/mongo/db/commands/copydb.h new file mode 100644 index 00000000000..f7b2adfbe6d --- /dev/null +++ b/src/mongo/db/commands/copydb.h @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class ClientBasic; + +namespace copydb { + + Status checkAuthForCopydbCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj); + +} // namespace copydb +} // namespace mongo + + diff --git a/src/mongo/db/commands/copydb_common.cpp b/src/mongo/db/commands/copydb_common.cpp new file mode 100644 index 00000000000..5cebb8db021 --- /dev/null +++ b/src/mongo/db/commands/copydb_common.cpp @@ -0,0 +1,100 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/commands/copydb.h" + +#include +#include + +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/client_basic.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { +namespace copydb { + + Status checkAuthForCopydbCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + bool fromSelf = StringData(cmdObj.getStringField("fromhost")).empty(); + StringData fromdb = cmdObj.getStringField("fromdb"); + StringData todb = cmdObj.getStringField("todb"); + + // get system collections + std::vector legalClientSystemCollections; + legalClientSystemCollections.push_back("system.js"); + if (fromdb == "admin") { + legalClientSystemCollections.push_back("system.users"); + legalClientSystemCollections.push_back("system.roles"); + legalClientSystemCollections.push_back("system.version"); + } else if (fromdb == "local") { // TODO(spencer): shouldn't be possible. See SERVER-11383 + legalClientSystemCollections.push_back("system.replset"); + } + + // Check authorization on destination db + ActionSet actions; + actions.addAction(ActionType::insert); + actions.addAction(ActionType::createIndex); + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(todb), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + actions.removeAllActions(); + actions.addAction(ActionType::insert); + for (size_t i = 0; i < legalClientSystemCollections.size(); ++i) { + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnNamespace( + NamespaceString(todb, legalClientSystemCollections[i]), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + } + + if (fromSelf) { + // If copying from self, also require privileges on source db + actions.removeAllActions(); + actions.addAction(ActionType::find); + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(fromdb), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + for (size_t i = 0; i < legalClientSystemCollections.size(); ++i) { + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnNamespace( + NamespaceString(fromdb, legalClientSystemCollections[i]), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + } + } + return Status::OK(); + } + +} // namespace copydb +} // namespace mongo diff --git a/src/mongo/db/commands/cpuprofile.cpp b/src/mongo/db/commands/cpuprofile.cpp index 51883a21c27..4a1605d3929 100644 --- a/src/mongo/db/commands/cpuprofile.cpp +++ b/src/mongo/db/commands/cpuprofile.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /** @@ -67,7 +79,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::cpuProfiler); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } // This is an abuse of the global dbmutex. We only really need to diff --git a/src/mongo/db/commands/create_indexes.cpp b/src/mongo/db/commands/create_indexes.cpp new file mode 100644 index 00000000000..89b679673e1 --- /dev/null +++ b/src/mongo/db/commands/create_indexes.cpp @@ -0,0 +1,234 @@ +// create_indexes.cpp + +/** +* Copyright (C) 2013 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/ops/insert.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/s/d_logic.h" +#include "mongo/s/shard_key_pattern.h" + +namespace mongo { + + /** + * { createIndexes : "bar", indexes : [ { ns : "test.bar", key : { x : 1 }, name: "x_1" } ] } + */ + class CmdCreateIndex : public Command { + public: + CmdCreateIndex() : Command( "createIndexes" ){} + + virtual LockType locktype() const { return NONE; } + virtual bool logTheOp() { return false; } + virtual bool slaveOk() const { return false; } // TODO: this could be made true... + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + ActionSet actions; + actions.addAction(ActionType::createIndex); + Privilege p(parseResourcePattern(dbname, cmdObj), actions); + if ( client->getAuthorizationSession()->isAuthorizedForPrivilege(p) ) + return Status::OK(); + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + + BSONObj _addNsToSpec( const NamespaceString& ns, const BSONObj& obj ) { + BSONObjBuilder b; + b.append( "ns", ns ); + b.appendElements( obj ); + return b.obj(); + } + + virtual bool run( const string& dbname, BSONObj& cmdObj, int options, + string& errmsg, BSONObjBuilder& result, + bool fromRepl = false ) { + + // --- parse + + NamespaceString ns( dbname, cmdObj[name].String() ); + Status status = userAllowedWriteNS( ns ); + if ( !status.isOK() ) + return appendCommandStatus( result, status ); + + if ( cmdObj["indexes"].type() != Array ) { + errmsg = "indexes has to be an array"; + result.append( "cmdObj", cmdObj ); + return false; + } + + std::vector specs; + { + BSONObjIterator i( cmdObj["indexes"].Obj() ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( e.type() != Object ) { + errmsg = "everything in indexes has to be an Object"; + result.append( "cmdObj", cmdObj ); + return false; + } + specs.push_back( e.Obj() ); + } + } + + if ( specs.size() == 0 ) { + errmsg = "no indexes to add"; + return false; + } + + // check specs + for ( size_t i = 0; i < specs.size(); i++ ) { + BSONObj spec = specs[i]; + if ( spec["ns"].eoo() ) { + spec = _addNsToSpec( ns, spec ); + specs[i] = spec; + } + + if ( spec["ns"].type() != String ) { + errmsg = "spec has no ns"; + result.append( "spec", spec ); + return false; + } + if ( ns != spec["ns"].String() ) { + errmsg = "namespace mismatch"; + result.append( "spec", spec ); + return false; + } + } + + + { + // We first take a read lock to see if we need to do anything + // as many calls are ensureIndex (and hence no-ops), this is good so its a shared + // lock for common calls. We only take write lock if needed. + Client::ReadContext readContext( ns ); + const Collection* collection = readContext.ctx().db()->getCollection( ns.ns() ); + if ( collection ) { + for ( size_t i = 0; i < specs.size(); i++ ) { + BSONObj spec = specs[i]; + StatusWith statusWithSpec = + collection->getIndexCatalog()->prepareSpecForCreate( spec ); + status = statusWithSpec.getStatus(); + if ( status.code() == ErrorCodes::IndexAlreadyExists ) { + specs.erase( specs.begin() + i ); + i--; + continue; + } + if ( !status.isOK() ) + return appendCommandStatus( result, status ); + } + + if ( specs.size() == 0 ) { + result.append( "numIndexesBefore", + collection->getIndexCatalog()->numIndexesTotal() ); + result.append( "note", "all indexes already exist" ); + return true; + } + + // need to create index + } + } + + // now we know we have to create index(es) + Client::WriteContext writeContext( ns.ns() ); + Database* db = writeContext.ctx().db(); + + Collection* collection = db->getCollection( ns.ns() ); + result.appendBool( "createdCollectionAutomatically", collection == NULL ); + if ( !collection ) { + collection = db->createCollection( ns.ns() ); + invariant( collection ); + } + + result.append( "numIndexesBefore", collection->getIndexCatalog()->numIndexesTotal() ); + + for ( size_t i = 0; i < specs.size(); i++ ) { + BSONObj spec = specs[i]; + + if ( spec["unique"].trueValue() ) { + status = checkUniqueIndexConstraints( ns.ns(), spec["key"].Obj() ); + + if ( !status.isOK() ) { + appendCommandStatus( result, status ); + return false; + } + } + + status = collection->getIndexCatalog()->createIndex( spec, true ); + if ( status.code() == ErrorCodes::IndexAlreadyExists ) { + if ( !result.hasField( "note" ) ) + result.append( "note", "index already exists" ); + continue; + } + + if ( !status.isOK() ) { + appendCommandStatus( result, status ); + return false; + } + + if ( !fromRepl ) { + std::string systemIndexes = ns.getSystemIndexesCollection(); + logOp( "i", systemIndexes.c_str(), spec ); + } + } + + result.append( "numIndexesAfter", collection->getIndexCatalog()->numIndexesTotal() ); + + return true; + } + + private: + static Status checkUniqueIndexConstraints(const StringData& ns, + const BSONObj& newIdxKey) { + Lock::assertWriteLocked( ns ); + + if ( shardingState.enabled() ) { + CollectionMetadataPtr metadata( + shardingState.getCollectionMetadata( ns.toString() )); + + if ( metadata ) { + BSONObj shardKey(metadata->getKeyPattern()); + if ( !isUniqueIndexCompatible( shardKey, newIdxKey )) { + return Status(ErrorCodes::CannotCreateIndex, + str::stream() << "cannot create unique index over " << newIdxKey + << " with shard key pattern " << shardKey); + } + } + } + + return Status::OK(); + } + + } cmdCreateIndex; + +} diff --git a/src/mongo/db/commands/dbhash.cpp b/src/mongo/db/commands/dbhash.cpp new file mode 100644 index 00000000000..20a815535ae --- /dev/null +++ b/src/mongo/db/commands/dbhash.cpp @@ -0,0 +1,212 @@ +// dbhash.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/commands/dbhash.h" + +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/util/md5.hpp" +#include "mongo/util/timer.h" + +namespace mongo { + + DBHashCmd dbhashCmd; + + + void logOpForDbHash( const char* opstr, + const char* ns, + const BSONObj& obj, + BSONObj* patt, + const BSONObj* fullObj, + bool forMigrateCleanup ) { + dbhashCmd.wipeCacheForCollection( ns ); + } + + // ---- + + DBHashCmd::DBHashCmd() + : Command( "dbHash", false, "dbhash" ), + _cachedHashedMutex( "_cachedHashedMutex" ){ + } + + void DBHashCmd::addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::dbHash); + out->push_back(Privilege(ResourcePattern::forDatabaseName(dbname), actions)); + } + + string DBHashCmd::hashCollection( const string& fullCollectionName, bool* fromCache ) { + + scoped_ptr cachedHashedLock; + + if ( isCachable( fullCollectionName ) ) { + cachedHashedLock.reset( new scoped_lock( _cachedHashedMutex ) ); + string hash = _cachedHashed[fullCollectionName]; + if ( hash.size() > 0 ) { + *fromCache = true; + return hash; + } + } + + *fromCache = false; + Collection* collection = cc().database()->getCollection( fullCollectionName ); + if ( !collection ) + return ""; + + IndexDescriptor* desc = collection->getIndexCatalog()->findIdIndex(); + + auto_ptr runner; + if ( desc ) { + runner.reset(InternalPlanner::indexScan(collection, + desc, + BSONObj(), + BSONObj(), + false, + InternalPlanner::FORWARD, + InternalPlanner::IXSCAN_FETCH)); + } + else if ( collection->details()->isCapped() ) { + runner.reset(InternalPlanner::collectionScan(fullCollectionName)); + } + else { + log() << "can't find _id index for: " << fullCollectionName << endl; + return "no _id _index"; + } + + md5_state_t st; + md5_init(&st); + + long long n = 0; + Runner::RunnerState state; + BSONObj c; + verify(NULL != runner.get()); + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&c, NULL))) { + md5_append( &st , (const md5_byte_t*)c.objdata() , c.objsize() ); + n++; + } + if (Runner::RUNNER_EOF != state) { + warning() << "error while hashing, db dropped? ns=" << fullCollectionName << endl; + } + md5digest d; + md5_finish(&st, d); + string hash = digestToString( d ); + + if ( cachedHashedLock.get() ) { + _cachedHashed[fullCollectionName] = hash; + } + + return hash; + } + + bool DBHashCmd::run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { + Timer timer; + + set desiredCollections; + if ( cmdObj["collections"].type() == Array ) { + BSONObjIterator i( cmdObj["collections"].Obj() ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( e.type() != String ) { + errmsg = "collections entries have to be strings"; + return false; + } + desiredCollections.insert( e.String() ); + } + } + + list colls; + Database* db = cc().database(); + if ( db ) + db->namespaceIndex().getNamespaces( colls ); + colls.sort(); + + result.appendNumber( "numCollections" , (long long)colls.size() ); + result.append( "host" , prettyHostName() ); + + md5_state_t globalState; + md5_init(&globalState); + + vector cached; + + BSONObjBuilder bb( result.subobjStart( "collections" ) ); + for ( list::iterator i=colls.begin(); i != colls.end(); i++ ) { + string fullCollectionName = *i; + if ( fullCollectionName.size() -1 <= dbname.size() ) { + errmsg = str::stream() << "weird fullCollectionName [" << fullCollectionName << "]"; + return false; + } + string shortCollectionName = fullCollectionName.substr( dbname.size() + 1 ); + + if ( shortCollectionName.find( "system." ) == 0 ) + continue; + + if ( desiredCollections.size() > 0 && + desiredCollections.count( shortCollectionName ) == 0 ) + continue; + + bool fromCache = false; + string hash = hashCollection( fullCollectionName, &fromCache ); + + bb.append( shortCollectionName, hash ); + + md5_append( &globalState , (const md5_byte_t*)hash.c_str() , hash.size() ); + if ( fromCache ) + cached.push_back( fullCollectionName ); + } + bb.done(); + + md5digest d; + md5_finish(&globalState, d); + string hash = digestToString( d ); + + result.append( "md5" , hash ); + result.appendNumber( "timeMillis", timer.millis() ); + + result.append( "fromCache", cached ); + + return 1; + } + + void DBHashCmd::wipeCacheForCollection( const StringData& ns ) { + if ( !isCachable( ns ) ) + return; + scoped_lock lk( _cachedHashedMutex ); + _cachedHashed.erase( ns.toString() ); + } + + bool DBHashCmd::isCachable( const StringData& ns ) const { + return ns.startsWith( "config." ); + } + +} diff --git a/src/mongo/db/commands/dbhash.h b/src/mongo/db/commands/dbhash.h new file mode 100644 index 00000000000..0887026a763 --- /dev/null +++ b/src/mongo/db/commands/dbhash.h @@ -0,0 +1,69 @@ +// dbhash.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/commands.h" + +namespace mongo { + + void logOpForDbHash( const char* opstr, + const char* ns, + const BSONObj& obj, + BSONObj* patt, + const BSONObj* fullObj, + bool forMigrateCleanup ); + + class DBHashCmd : public Command { + public: + DBHashCmd(); + + virtual bool slaveOk() const { return true; } + virtual LockType locktype() const { return READ; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out); + + virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool); + + void wipeCacheForCollection( const StringData& ns ); + + private: + + bool isCachable( const StringData& ns ) const; + + string hashCollection( const string& fullCollectionName, bool* fromCache ); + + map _cachedHashed; + mutex _cachedHashedMutex; + + }; + +} diff --git a/src/mongo/db/commands/distinct.cpp b/src/mongo/db/commands/distinct.cpp index 17656a1226f..154015d75bc 100644 --- a/src/mongo/db/commands/distinct.cpp +++ b/src/mongo/db/commands/distinct.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include @@ -21,13 +33,16 @@ #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" #include "mongo/db/instance.h" #include "mongo/db/jsobj.h" #include "mongo/db/kill_current_op.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/type_explain.h" #include "mongo/util/timer.h" namespace mongo { @@ -35,20 +50,26 @@ namespace mongo { class DistinctCommand : public Command { public: DistinctCommand() : Command("distinct") {} - virtual bool slaveOk() const { return true; } + + virtual bool slaveOk() const { return false; } + virtual bool slaveOverrideOk() const { return true; } virtual LockType locktype() const { return READ; } + virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { ActionSet actions; actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } + virtual void help( stringstream &help ) const { help << "{ distinct : 'collection name' , key : 'a.b' , query : {} }"; } - bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { + bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, + bool fromRepl ) { + Timer t; string ns = dbname + '.' + cmdObj.firstElement().valuestr(); @@ -67,89 +88,64 @@ namespace mongo { long long nscanned = 0; // locations looked at long long nscannedObjects = 0; // full objects looked at long long n = 0; // matches - MatchDetails md; - NamespaceDetails * d = nsdetails( ns ); + Collection* collection = cc().database()->getCollection( ns ); - if ( ! d ) { + if (!collection) { result.appendArray( "values" , BSONObj() ); - result.append( "stats" , BSON( "n" << 0 << "nscanned" << 0 << "nscannedObjects" << 0 ) ); + result.append("stats", BSON("n" << 0 << + "nscanned" << 0 << + "nscannedObjects" << 0)); return true; } - shared_ptr cursor; - if ( ! query.isEmpty() ) { - cursor = NamespaceDetailsTransient::getCursor(ns.c_str() , query , BSONObj() ); + Runner* rawRunner; + Status status = getRunnerDistinct(collection, query, key, &rawRunner); + if (!status.isOK()) { + uasserted(17216, mongoutils::str::stream() << "Can't get runner for query " + << query << ": " << status.toString()); + return 0; } - else { - - // query is empty, so lets see if we can find an index - // with the key so we don't have to hit the raw data - NamespaceDetails::IndexIterator ii = d->ii(); - while ( ii.more() ) { - IndexDetails& idx = ii.next(); - - if ( d->isMultikey( ii.pos() - 1 ) ) - continue; - - if ( idx.inKeyPattern( key ) ) { - cursor = NamespaceDetailsTransient::bestGuessCursor( ns.c_str() , - BSONObj() , - idx.keyPattern() ); - if( cursor.get() ) break; - } + auto_ptr runner(rawRunner); + const ScopedRunnerRegistration safety(runner.get()); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + string cursorName; + BSONObj obj; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { + // Distinct expands arrays. + // + // If our query is covered, each value of the key should be in the index key and + // available to us without this. If a collection scan is providing the data, we may + // have to expand an array. + BSONElementSet elts; + obj.getFieldsDotted(key, elts); + + for (BSONElementSet::iterator it = elts.begin(); it != elts.end(); ++it) { + BSONElement elt = *it; + if (values.count(elt)) { continue; } + int currentBufPos = bb.len(); + + uassert(17217, "distinct too big, 16mb cap", + (currentBufPos + elt.size() + 1024) < bufSize); + + arr.append(elt); + BSONElement x(start + currentBufPos); + values.insert(x); } - - if ( ! cursor.get() ) - cursor = NamespaceDetailsTransient::getCursor(ns.c_str() , query , BSONObj() ); - } - - - verify( cursor ); - string cursorName = cursor->toString(); - - auto_ptr cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns)); - - while ( cursor->ok() ) { - nscanned++; - bool loadedRecord = false; - - if ( cursor->currentMatches( &md ) && !cursor->getsetdup( cursor->currLoc() ) ) { - n++; - - BSONObj holder; - BSONElementSet temp; - loadedRecord = ! cc->getFieldsDotted( key , temp, holder ); - - for ( BSONElementSet::iterator i=temp.begin(); i!=temp.end(); ++i ) { - BSONElement e = *i; - if ( values.count( e ) ) - continue; - - int now = bb.len(); - - uassert(10044, "distinct too big, 16mb cap", ( now + e.size() + 1024 ) < bufSize ); - - arr.append( e ); - BSONElement x( start + now ); - - values.insert( x ); - } + TypeExplain* bareExplain; + Status res = runner->getInfo(&bareExplain, NULL); + if (res.isOK()) { + auto_ptr explain(bareExplain); + if (explain->isCursorSet()) { + cursorName = explain->getCursor(); } - - if ( loadedRecord || md.hasLoadedRecord() ) - nscannedObjects++; - - cursor->advance(); - - if (!cc->yieldSometimes( ClientCursor::MaybeCovered )) { - cc.release(); - break; - } - - RARELY killCurrentOp.checkForInterrupt(); + n = explain->getN(); + nscanned = explain->getNScanned(); + nscannedObjects = explain->getNScannedObjects(); } verify( start == bb.buf() ); @@ -168,7 +164,6 @@ namespace mongo { return true; } - } distinctCmd; -} +} // namespace mongo diff --git a/src/mongo/db/commands/drop_indexes.cpp b/src/mongo/db/commands/drop_indexes.cpp new file mode 100644 index 00000000000..0f2fdce6dc5 --- /dev/null +++ b/src/mongo/db/commands/drop_indexes.cpp @@ -0,0 +1,263 @@ +// drop_indexes.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/background.h" +#include "mongo/db/commands.h" +#include "mongo/db/index_builder.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/instance.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog/index_key_validate.h" +#include "mongo/db/pdfile.h" + +namespace mongo { + + /* "dropIndexes" is now the preferred form - "deleteIndexes" deprecated */ + class CmdDropIndexes : public Command { + public: + virtual bool logTheOp() { + return true; + } + virtual bool slaveOk() const { + return false; + } + virtual LockType locktype() const { return WRITE; } + virtual void help( stringstream& help ) const { + help << "drop indexes for a collection"; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::dropIndex); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); + } + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + std::string toDeleteNs = db->name() + "." + cmdObj.firstElement().valuestr(); + Collection* collection = db->getCollection(toDeleteNs); + IndexCatalog::IndexKillCriteria criteria; + + // Get index name to drop + BSONElement toDrop = cmdObj.getField("index"); + + if (toDrop.type() == String) { + // Kill all in-progress indexes + if (strcmp("*", toDrop.valuestr()) == 0) { + criteria.ns = toDeleteNs; + return IndexBuilder::killMatchingIndexBuilds(collection, criteria); + } + // Kill an in-progress index by name + else { + criteria.name = toDrop.valuestr(); + return IndexBuilder::killMatchingIndexBuilds(collection, criteria); + } + } + // Kill an in-progress index build by index key + else if (toDrop.type() == Object) { + criteria.key = toDrop.Obj(); + return IndexBuilder::killMatchingIndexBuilds(collection, criteria); + } + + return std::vector(); + } + + CmdDropIndexes() : Command("dropIndexes", false, "deleteIndexes") { } + bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& anObjBuilder, bool /*fromRepl*/) { + BSONElement e = jsobj.firstElement(); + string toDeleteNs = dbname + '.' + e.valuestr(); + if (!serverGlobalParams.quiet) { + MONGO_TLOG(0) << "CMD: dropIndexes " << toDeleteNs << endl; + } + + Collection* collection = cc().database()->getCollection( toDeleteNs ); + if ( ! collection ) { + errmsg = "ns not found"; + return false; + } + + stopIndexBuilds(cc().database(), jsobj); + + IndexCatalog* indexCatalog = collection->getIndexCatalog(); + anObjBuilder.appendNumber("nIndexesWas", indexCatalog->numIndexesTotal() ); + + + BSONElement f = jsobj.getField("index"); + if ( f.type() == String ) { + + string indexToDelete = f.valuestr(); + + if ( indexToDelete == "*" ) { + Status s = indexCatalog->dropAllIndexes( false ); + if ( !s.isOK() ) { + appendCommandStatus( anObjBuilder, s ); + return false; + } + anObjBuilder.append("msg", "non-_id indexes dropped for collection"); + return true; + } + + IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName( indexToDelete ); + if ( desc == NULL ) { + errmsg = str::stream() << "index not found with name [" << indexToDelete << "]"; + return false; + } + + if ( desc->isIdIndex() ) { + errmsg = "cannot drop _id index"; + return false; + } + + Status s = indexCatalog->dropIndex( desc ); + if ( !s.isOK() ) { + appendCommandStatus( anObjBuilder, s ); + return false; + } + + return true; + } + + if ( f.type() == Object ) { + IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern( f.embeddedObject() ); + if ( desc == NULL ) { + errmsg = "can't find index with key:"; + errmsg += f.embeddedObject().toString(); + return false; + } + + if ( desc->isIdIndex() ) { + errmsg = "cannot drop _id index"; + return false; + } + + Status s = indexCatalog->dropIndex( desc ); + if ( !s.isOK() ) { + appendCommandStatus( anObjBuilder, s ); + return false; + } + + return true; + } + + errmsg = "invalid index name spec"; + return false; + } + + } cmdDropIndexes; + + class CmdReIndex : public Command { + public: + virtual bool logTheOp() { return false; } // only reindexes on the one node + virtual bool slaveOk() const { return true; } // can reindex on a secondary + virtual LockType locktype() const { return WRITE; } + virtual void help( stringstream& help ) const { + help << "re-index a collection"; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::reIndex); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); + } + CmdReIndex() : Command("reIndex") { } + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + std::string ns = db->name() + '.' + cmdObj["reIndex"].valuestrsafe(); + IndexCatalog::IndexKillCriteria criteria; + criteria.ns = ns; + return IndexBuilder::killMatchingIndexBuilds(db->getCollection(ns), criteria); + } + + bool run(const string& dbname , BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) { + static DBDirectClient db; + + BSONElement e = jsobj.firstElement(); + string toDeleteNs = dbname + '.' + e.valuestr(); + + MONGO_TLOG(0) << "CMD: reIndex " << toDeleteNs << endl; + + Collection* collection = cc().database()->getCollection( toDeleteNs ); + + if ( !collection ) { + errmsg = "ns not found"; + return false; + } + + BackgroundOperation::assertNoBgOpInProgForNs( toDeleteNs ); + + std::vector indexesInProg = stopIndexBuilds(cc().database(), jsobj); + + list all; + auto_ptr i = db.query( dbname + ".system.indexes" , BSON( "ns" << toDeleteNs ) , 0 , 0 , 0 , QueryOption_SlaveOk ); + BSONObjBuilder b; + while ( i->more() ) { + const BSONObj spec = i->next().removeField("v").getOwned(); + const BSONObj key = spec.getObjectField("key"); + const Status keyStatus = validateKeyPattern(key); + if (!keyStatus.isOK()) { + errmsg = str::stream() + << "Cannot rebuild index " << spec << ": " << keyStatus.reason() + << " For more info see http://dochub.mongodb.org/core/index-validation"; + return false; + } + + b.append( BSONObjBuilder::numStr( all.size() ) , spec ); + all.push_back( spec ); + } + result.appendNumber( "nIndexesWas", collection->getIndexCatalog()->numIndexesTotal() ); + + Status s = collection->getIndexCatalog()->dropAllIndexes( true ); + if ( !s.isOK() ) { + errmsg = "dropIndexes failed"; + return appendCommandStatus( result, s ); + } + + for ( list::iterator i=all.begin(); i!=all.end(); i++ ) { + BSONObj o = *i; + LOG(1) << "reIndex ns: " << toDeleteNs << " index: " << o << endl; + Status s = collection->getIndexCatalog()->createIndex( o, false ); + if ( !s.isOK() ) + return appendCommandStatus( result, s ); + } + + result.append( "nIndexes" , (int)all.size() ); + result.appendArray( "indexes" , b.obj() ); + + IndexBuilder::restoreIndexes(indexesInProg); + return true; + } + } cmdReIndex; + + +} diff --git a/src/mongo/db/commands/fail_point_cmd.cpp b/src/mongo/db/commands/fail_point_cmd.cpp index f55d28485ab..8924e4450e3 100644 --- a/src/mongo/db/commands/fail_point_cmd.cpp +++ b/src/mongo/db/commands/fail_point_cmd.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include @@ -63,10 +75,6 @@ namespace mongo { } // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { - return false; - } - virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) {} diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index 325e879ac0e..a287ce1a301 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -14,20 +14,36 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/db/commands/find_and_modify.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" +#include "mongo/db/dbhelpers.h" #include "mongo/db/instance.h" -#include "mongo/db/clientcursor.h" #include "mongo/db/pagefault.h" -#include "mongo/db/dbhelpers.h" +#include "mongo/db/projection.h" #include "mongo/db/ops/delete.h" +#include "mongo/db/ops/insert.h" #include "mongo/db/ops/update.h" +#include "mongo/db/ops/update_lifecycle_impl.h" #include "mongo/db/queryutil.h" +#include "mongo/db/query/get_runner.h" namespace mongo { @@ -49,13 +65,17 @@ namespace mongo { virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { - find_and_modify::addPrivilegesRequiredForFindAndModify(dbname, cmdObj, out); + find_and_modify::addPrivilegesRequiredForFindAndModify(this, dbname, cmdObj, out); } /* this will eventually replace run, once sort is handled */ bool runNoDirectClient( const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { verify( cmdObj["sort"].eoo() ); string ns = dbname + '.' + cmdObj.firstElement().valuestr(); + Status allowedWriteStatus = userAllowedWriteNS(ns); + if (!allowedWriteStatus.isOK()) { + return appendCommandStatus(result, allowedWriteStatus); + } BSONObj query = cmdObj.getObjectField("query"); BSONObj fields = cmdObj.getObjectField("fields"); @@ -80,20 +100,20 @@ namespace mongo { return false; } - PageFaultRetryableSection s; - while ( 1 ) { - try { - return runNoDirectClient( ns , - query , fields , update , - upsert , returnNew , remove , - result , errmsg ); - } - catch ( PageFaultException& e ) { - e.touch(); + { + PageFaultRetryableSection s; + while ( 1 ) { + try { + return runNoDirectClient( ns , + query , fields , update , + upsert , returnNew , remove , + result , errmsg ); + } + catch ( PageFaultException& e ) { + e.touch(); + } } - } - - + } // end PageFaultRetryableSection } void _appendHelper( BSONObjBuilder& result , const BSONObj& doc , bool found , const BSONObj& fields ) { @@ -123,8 +143,27 @@ namespace mongo { Client::Context cx( ns ); BSONObj doc; - - bool found = Helpers::findOne( ns.c_str() , queryOriginal , doc ); + bool found = false; + { + CanonicalQuery* cq; + massert(17383, "Could not canonicalize " + queryOriginal.toString(), + CanonicalQuery::canonicalize(ns, queryOriginal, &cq).isOK()); + + Runner* rawRunner; + massert(17384, "Could not get runner for query " + queryOriginal.toString(), + getRunner(cq, &rawRunner, QueryPlannerParams::DEFAULT).isOK()); + + auto_ptr runner(rawRunner); + + // Set up automatic yielding + const ScopedRunnerRegistration safety(runner.get()); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + Runner::RunnerState state; + if (Runner::RUNNER_ADVANCED == (state = runner->getNext(&doc, NULL))) { + found = true; + } + } BSONObj queryModified = queryOriginal; if ( found && doc["_id"].type() && ! isSimpleIdQuery( queryOriginal ) ) { @@ -187,7 +226,7 @@ namespace mongo { if ( remove ) { _appendHelper( result , doc , found , fields ); if ( found ) { - deleteObjects( ns.c_str() , queryModified , true , true ); + deleteObjects( ns , queryModified , true , true ); BSONObjBuilder le( result.subobjStart( "lastErrorObject" ) ); le.appendNumber( "n" , 1 ); le.done(); @@ -206,16 +245,33 @@ namespace mongo { _appendHelper( result , doc , found , fields ); } - UpdateResult res = updateObjects( ns.c_str() , update , queryModified , upsert , false , true , cc().curop()->debug() ); - + const NamespaceString requestNs(ns); + UpdateRequest request(requestNs); + + request.setQuery(queryModified); + request.setUpdates(update); + request.setUpsert(upsert); + request.setUpdateOpLog(); + // TODO(greg) We need to send if we are ignoring + // the shard version below, but for now no + UpdateLifecycleImpl updateLifecycle(false, requestNs); + request.setLifecycle(&updateLifecycle); + UpdateResult res = mongo::update(request, &cc().curop()->debug()); + + LOG(3) << "update result: " << res ; if ( returnNew ) { - if ( res.upserted.isSet() ) { - queryModified = BSON( "_id" << res.upserted ); + if ( !res.upserted.isEmpty() ) { + BSONElement upsertedElem = res.upserted[kUpsertedFieldName]; + LOG(3) << "using new _id to get new doc: " + << upsertedElem; + queryModified = upsertedElem.wrap("_id"); } else if ( queryModified["_id"].type() ) { // we do this so that if the update changes the fields, it still matches queryModified = queryModified["_id"].wrap(); } + + LOG(3) << "using modified query to return the new doc: " << queryModified; if ( ! Helpers::findOne( ns.c_str() , queryModified , doc ) ) { errmsg = str::stream() << "can't find object after modification " << " ns: " << ns @@ -229,9 +285,10 @@ namespace mongo { BSONObjBuilder le( result.subobjStart( "lastErrorObject" ) ); le.appendBool( "updatedExisting" , res.existing ); - le.appendNumber( "n" , res.num ); - if ( res.upserted.isSet() ) - le.append( "upserted" , res.upserted ); + le.appendNumber( "n" , res.numMatched ); + if ( !res.upserted.isEmpty() ) { + le.append( res.upserted[kUpsertedFieldName] ); + } le.done(); } @@ -247,6 +304,10 @@ namespace mongo { return runNoDirectClient( dbname , cmdObj , x, errmsg , result, y ); string ns = dbname + '.' + cmdObj.firstElement().valuestr(); + Status allowedWriteStatus = userAllowedWriteNS(ns); + if (!allowedWriteStatus.isOK()) { + return appendCommandStatus(result, allowedWriteStatus); + } BSONObj origQuery = cmdObj.getObjectField("query"); // defaults to {} Query q (origQuery); @@ -286,11 +347,14 @@ namespace mongo { } if (cmdObj["new"].trueValue()) { - BSONElement _id = gle["upserted"]; - if (_id.eoo()) - _id = origQuery["_id"]; - - out = db.findOne(ns, QUERY("_id" << _id), fields); + BSONObjBuilder bob; + BSONElement _id = gle[kUpsertedFieldName]; + if (!_id.eoo()) + bob.appendAs(_id, "_id"); + else + bob.appendAs(origQuery["_id"], "_id"); + + out = db.findOne(ns, bob.done(), fields); } } diff --git a/src/mongo/db/commands/find_and_modify.h b/src/mongo/db/commands/find_and_modify.h index 7d9500d35db..ab1a6eff18c 100644 --- a/src/mongo/db/commands/find_and_modify.h +++ b/src/mongo/db/commands/find_and_modify.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -23,11 +35,15 @@ #include "mongo/db/jsobj.h" namespace mongo { + + class Command; + namespace find_and_modify { - void addPrivilegesRequiredForFindAndModify(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out); + void addPrivilegesRequiredForFindAndModify(Command* commandTemplate, + const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out); } // namespace find_and_modify } // namespace mongo diff --git a/src/mongo/db/commands/find_and_modify_common.cpp b/src/mongo/db/commands/find_and_modify_common.cpp index 46eca0d79b6..796b6db415a 100644 --- a/src/mongo/db/commands/find_and_modify_common.cpp +++ b/src/mongo/db/commands/find_and_modify_common.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/commands/find_and_modify.h" @@ -24,12 +36,15 @@ #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/commands.h" #include "mongo/db/jsobj.h" namespace mongo { namespace find_and_modify { - void addPrivilegesRequiredForFindAndModify(const std::string& dbname, + void addPrivilegesRequiredForFindAndModify(Command* commandTemplate, + const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { bool update = cmdObj["update"].trueValue(); @@ -47,8 +62,10 @@ namespace find_and_modify { if (remove) { actions.addAction(ActionType::remove); } - std::string ns = dbname + '.' + cmdObj.firstElement().valuestr(); - out->push_back(Privilege(ns, actions)); + ResourcePattern resource(commandTemplate->parseResourcePattern(dbname, cmdObj)); + uassert(17137, "Invalid target namespace " + resource.toString(), + resource.isExactNamespacePattern()); + out->push_back(Privilege(resource, actions)); } } // namespace find_and_modify diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp new file mode 100644 index 00000000000..f79a00fac8a --- /dev/null +++ b/src/mongo/db/commands/find_cmd.cpp @@ -0,0 +1,146 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/commands/find_cmd.h" + +#include + +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/query/explain.h" +#include "mongo/db/query/get_executor.h" +#include "mongo/db/query/find.h" +#include "mongo/s/d_state.h" + +namespace mongo { + + using boost::scoped_ptr; + using std::auto_ptr; + using std::string; + + static FindCmd findCmd; + + Status FindCmd::checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + ResourcePattern pattern = parseResourcePattern(dbname, cmdObj); + + if (authzSession->isAuthorizedForActionsOnResource(pattern, ActionType::find)) { + return Status::OK(); + } + + return Status(ErrorCodes::Unauthorized, "unauthorized"); + } + + Status FindCmd::explain(OperationContext* txn, + const std::string& dbname, + const BSONObj& cmdObj, + ExplainCommon::Verbosity verbosity, + BSONObjBuilder* out) const { + const string fullns = parseNs(dbname, cmdObj); + + // Parse the command BSON to a LiteParsedQuery. + LiteParsedQuery* rawLpq; + bool isExplain = true; + Status lpqStatus = LiteParsedQuery::make(fullns, cmdObj, isExplain, &rawLpq); + if (!lpqStatus.isOK()) { + return lpqStatus; + } + auto_ptr lpq(rawLpq); + + const NamespaceString nss(fullns); + + // Finish the parsing step by using the LiteParsedQuery to create a CanonicalQuery. + // This requires a lock on the collection in case we're parsing $where: where-specific + // parsing code assumes we have a lock and creates execution machinery that requires it. + CanonicalQuery* rawCq; + WhereCallbackReal whereCallback(txn, nss.db()); + Status canonStatus = CanonicalQuery::canonicalize(lpq.release(), &rawCq, whereCallback); + if (!canonStatus.isOK()) { + return canonStatus; + } + auto_ptr cq(rawCq); + + AutoGetCollectionForRead ctx(txn, nss); + // The collection may be NULL. If so, getExecutor() should handle it by returning + // an execution tree with an EOFStage. + Collection* collection = ctx.getCollection(); + + // We have a parsed query. Time to get the execution plan for it. + PlanExecutor* rawExec; + Status execStatus = Status::OK(); + if (cq->getParsed().getOptions().oplogReplay) { + execStatus = getOplogStartHack(txn, collection, cq.release(), &rawExec); + } + else { + size_t options = QueryPlannerParams::DEFAULT; + // TODO: The version attached to the TLS cannot be relied upon, the shard + // version should be passed as part of the command parameter. + if (shardingState.needCollectionMetadata(cq->getParsed().ns())) { + options |= QueryPlannerParams::INCLUDE_SHARD_FILTER; + } + + execStatus = getExecutor(txn, + collection, + cq.release(), + PlanExecutor::YIELD_AUTO, + &rawExec, + options); + } + + if (!execStatus.isOK()) { + return execStatus; + } + + scoped_ptr exec(rawExec); + + // Got the execution tree. Explain it. + Explain::explainStages(exec.get(), verbosity, out); + return Status::OK(); + } + + bool FindCmd::run(OperationContext* txn, + const string& dbname, + BSONObj& cmdObj, int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + // Currently only explains of finds run through the find command. Queries that are not + // explained use the legacy OP_QUERY path. + // TODO: check the comment above regarding shard versioning. + errmsg = "find command not yet implemented"; + return false; + } + +} // namespace mongo diff --git a/src/mongo/db/commands/fsync.cpp b/src/mongo/db/commands/fsync.cpp index e822374ed48..4da7b007d88 100644 --- a/src/mongo/db/commands/fsync.cpp +++ b/src/mongo/db/commands/fsync.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -78,7 +90,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::fsync); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { diff --git a/src/mongo/db/commands/fsync.h b/src/mongo/db/commands/fsync.h index 05722c82a1f..f3fa2adb993 100644 --- a/src/mongo/db/commands/fsync.h +++ b/src/mongo/db/commands/fsync.h @@ -12,10 +12,24 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once +#include "mongo/util/concurrency/mutex.h" + namespace mongo { // Use this for blocking during an fsync-and-lock extern SimpleMutex filesLockedFsync; diff --git a/src/mongo/db/commands/geonear.cpp b/src/mongo/db/commands/geonear.cpp new file mode 100644 index 00000000000..a1166ddd1dd --- /dev/null +++ b/src/mongo/db/commands/geonear.cpp @@ -0,0 +1,306 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include + +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/curop.h" +#include "mongo/db/geo/geoconstants.h" +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/geo/s2common.h" +#include "mongo/db/index_names.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/type_explain.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/platform/unordered_map.h" + +namespace mongo { + + class Geo2dFindNearCmd : public Command { + public: + Geo2dFindNearCmd() : Command("geoNear") {} + + virtual LockType locktype() const { return READ; } + bool slaveOk() const { return true; } + bool slaveOverrideOk() const { return true; } + + void help(stringstream& h) const { + h << "http://dochub.mongodb.org/core/geo#GeospatialIndexing-geoNearCommand"; + } + + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::find); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); + } + + bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + string ns = dbname + "." + cmdObj.firstElement().valuestr(); + + if (!cmdObj["start"].eoo()) { + errmsg = "using deprecated 'start' argument to geoNear"; + return false; + } + + Database* db = cc().database(); + if ( !db ) { + errmsg = "can't find ns"; + return false; + } + + Collection* collection = db->getCollection( ns ); + if ( !collection ) { + errmsg = "can't find ns"; + return false; + } + + IndexCatalog* indexCatalog = collection->getIndexCatalog(); + + // cout << "raw cmd " << cmdObj.toString() << endl; + + // We seek to populate this. + string nearFieldName; + bool using2DIndex = false; + if (!getFieldName(collection, indexCatalog, &nearFieldName, &errmsg, &using2DIndex)) { + return false; + } + + uassert(17304, "'near' field must be point", + !cmdObj["near"].eoo() && cmdObj["near"].isABSONObj() + && GeoParser::isPoint(cmdObj["near"].Obj())); + + bool isSpherical = cmdObj["spherical"].trueValue(); + if (!using2DIndex) { + uassert(17301, "2dsphere index must have spherical: true", isSpherical); + } + + // Build the $near expression for the query. + BSONObjBuilder nearBob; + if (isSpherical) { + nearBob.append("$nearSphere", cmdObj["near"].Obj()); + } + else { + nearBob.append("$near", cmdObj["near"].Obj()); + } + + if (!cmdObj["maxDistance"].eoo()) { + uassert(17299, "maxDistance must be a number",cmdObj["maxDistance"].isNumber()); + nearBob.append("$maxDistance", cmdObj["maxDistance"].number()); + } + + if (!cmdObj["minDistance"].eoo()) { + uassert(17298, "minDistance doesn't work on 2d index", !using2DIndex); + uassert(17300, "minDistance must be a number",cmdObj["minDistance"].isNumber()); + nearBob.append("$minDistance", cmdObj["minDistance"].number()); + } + + if (!cmdObj["uniqueDocs"].eoo()) { + warning() << ns << ": ignoring deprecated uniqueDocs option in geoNear command"; + } + + // And, build the full query expression. + BSONObjBuilder queryBob; + queryBob.append(nearFieldName, nearBob.obj()); + if (!cmdObj["query"].eoo() && cmdObj["query"].isABSONObj()) { + queryBob.appendElements(cmdObj["query"].Obj()); + } + BSONObj rewritten = queryBob.obj(); + + // cout << "rewritten query: " << rewritten.toString() << endl; + + int numWanted = 100; + const char* limitName = !cmdObj["num"].eoo() ? "num" : "limit"; + BSONElement eNumWanted = cmdObj[limitName]; + if (!eNumWanted.eoo()) { + uassert(17303, "limit must be number", eNumWanted.isNumber()); + numWanted = eNumWanted.numberInt(); + uassert(17302, "limit must be >=0", numWanted >= 0); + } + + bool includeLocs = false; + if (!cmdObj["includeLocs"].eoo()) { + includeLocs = cmdObj["includeLocs"].trueValue(); + } + + double distanceMultiplier = 1.0; + BSONElement eDistanceMultiplier = cmdObj["distanceMultiplier"]; + if (!eDistanceMultiplier.eoo()) { + uassert(17296, "distanceMultiplier must be a number", eDistanceMultiplier.isNumber()); + distanceMultiplier = eDistanceMultiplier.number(); + uassert(17297, "distanceMultiplier must be non-negative", distanceMultiplier >= 0); + } + + BSONObj projObj = BSON("$pt" << BSON("$meta" << LiteParsedQuery::metaGeoNearPoint) << + "$dis" << BSON("$meta" << LiteParsedQuery::metaGeoNearDistance)); + + CanonicalQuery* cq; + if (!CanonicalQuery::canonicalize(ns, rewritten, BSONObj(), projObj, 0, numWanted, BSONObj(), &cq).isOK()) { + errmsg = "Can't parse filter / create query"; + return false; + } + + Runner* rawRunner; + if (!getRunner(cq, &rawRunner, 0).isOK()) { + errmsg = "can't get query runner"; + return false; + } + + auto_ptr runner(rawRunner); + const ScopedRunnerRegistration safety(runner.get()); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + double totalDistance = 0; + BSONObjBuilder resultBuilder(result.subarrayStart("results")); + double farthestDist = 0; + + BSONObj currObj; + int results = 0; + while ((results < numWanted) && Runner::RUNNER_ADVANCED == runner->getNext(&currObj, NULL)) { + + // Come up with the correct distance. + double dist = currObj["$dis"].number() * distanceMultiplier; + totalDistance += dist; + if (dist > farthestDist) { farthestDist = dist; } + + // Strip out '$dis' and '$pt' from the result obj. The rest gets added as 'obj' + // in the command result. + BSONObjIterator resIt(currObj); + BSONObjBuilder resBob; + while (resIt.more()) { + BSONElement elt = resIt.next(); + if (!mongoutils::str::equals("$pt", elt.fieldName()) + && !mongoutils::str::equals("$dis", elt.fieldName())) { + resBob.append(elt); + } + } + BSONObj resObj = resBob.obj(); + + // Don't make a too-big result object. + if (resultBuilder.len() + resObj.objsize()> BSONObjMaxUserSize) { + warning() << "Too many geoNear results for query " << rewritten.toString() + << ", truncating output."; + break; + } + + // Add the next result to the result builder. + BSONObjBuilder oneResultBuilder( + resultBuilder.subobjStart(BSONObjBuilder::numStr(results))); + oneResultBuilder.append("dis", dist); + if (includeLocs) { + oneResultBuilder.appendAs(currObj["$pt"], "loc"); + } + oneResultBuilder.append("obj", resObj); + oneResultBuilder.done(); + ++results; + } + + resultBuilder.done(); + + // Fill out the stats subobj. + BSONObjBuilder stats(result.subobjStart("stats")); + + // Fill in nscanned from the explain. + TypeExplain* bareExplain; + Status res = runner->getInfo(&bareExplain, NULL); + if (res.isOK()) { + auto_ptr explain(bareExplain); + stats.append("nscanned", explain->getNScanned()); + stats.append("objectsLoaded", explain->getNScannedObjects()); + } + + stats.append("avgDistance", totalDistance / results); + stats.append("maxDistance", farthestDist); + stats.append("time", cc().curop()->elapsedMillis()); + stats.done(); + + return true; + } + + private: + bool getFieldName(Collection* collection, IndexCatalog* indexCatalog, string* fieldOut, + string* errOut, bool *isFrom2D) { + vector idxs; + + // First, try 2d. + collection->getIndexCatalog()->findIndexByType(IndexNames::GEO_2D, idxs); + if (idxs.size() > 1) { + *errOut = "more than one 2d index, not sure which to run geoNear on"; + return false; + } + + if (1 == idxs.size()) { + BSONObj indexKp = idxs[0]->keyPattern(); + BSONObjIterator kpIt(indexKp); + while (kpIt.more()) { + BSONElement elt = kpIt.next(); + if (String == elt.type() && IndexNames::GEO_2D == elt.valuestr()) { + *fieldOut = elt.fieldName(); + *isFrom2D = true; + return true; + } + } + } + + // Next, 2dsphere. + idxs.clear(); + collection->getIndexCatalog()->findIndexByType(IndexNames::GEO_2DSPHERE, idxs); + if (0 == idxs.size()) { + *errOut = "no geo indices for geoNear"; + return false; + } + + if (idxs.size() > 1) { + *errOut = "more than one 2dsphere index, not sure which to run geoNear on"; + return false; + } + + // 1 == idx.size() + BSONObj indexKp = idxs[0]->keyPattern(); + BSONObjIterator kpIt(indexKp); + while (kpIt.more()) { + BSONElement elt = kpIt.next(); + if (String == elt.type() && IndexNames::GEO_2DSPHERE == elt.valuestr()) { + *fieldOut = elt.fieldName(); + *isFrom2D = false; + return true; + } + } + + return false; + } + } geo2dFindNearCmd; +} // namespace mongo diff --git a/src/mongo/db/commands/get_last_error.cpp b/src/mongo/db/commands/get_last_error.cpp new file mode 100644 index 00000000000..460029b6ab4 --- /dev/null +++ b/src/mongo/db/commands/get_last_error.cpp @@ -0,0 +1,284 @@ +// get_last_error.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/client.h" +#include "mongo/db/curop.h" +#include "mongo/db/commands.h" +#include "mongo/db/field_parser.h" +#include "mongo/db/lasterror.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/write_concern.h" + +namespace mongo { + + /* reset any errors so that getlasterror comes back clean. + + useful before performing a long series of operations where we want to + see if any of the operations triggered an error, but don't want to check + after each op as that woudl be a client/server turnaround. + */ + class CmdResetError : public Command { + public: + virtual LockType locktype() const { return NONE; } + virtual bool logTheOp() { + return false; + } + virtual bool slaveOk() const { + return true; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) {} // No auth required + virtual void help( stringstream& help ) const { + help << "reset error state (used with getpreverror)"; + } + CmdResetError() : Command("resetError", false, "reseterror") {} + bool run(const string& db, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + LastError *le = lastError.get(); + verify( le ); + le->reset(); + return true; + } + } cmdResetError; + + /* set by replica sets if specified in the configuration. + a pointer is used to avoid any possible locking issues with lockless reading (see below locktype() is NONE + and would like to keep that) + (for now, it simply orphans any old copy as config changes should be extremely rare). + note: once non-null, never goes to null again. + */ + BSONObj *getLastErrorDefault = 0; + + class CmdGetLastError : public Command { + public: + CmdGetLastError() : Command("getLastError", false, "getlasterror") { } + virtual LockType locktype() const { return NONE; } + virtual bool logTheOp() { return false; } + virtual bool slaveOk() const { return true; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) {} // No auth required + virtual void help( stringstream& help ) const { + lastError.disableForCommand(); // SERVER-11492 + help << "return error status of the last operation on this connection\n" + << "options:\n" + << " { fsync:true } - fsync before returning, or wait for journal commit if running with --journal\n" + << " { j:true } - wait for journal commit if running with --journal\n" + << " { w:n } - await replication to n servers (including self) before returning\n" + << " { w:'majority' } - await replication to majority of set\n" + << " { wtimeout:m} - timeout for w in m milliseconds"; + } + + bool run( const string& dbname, + BSONObj& cmdObj, + int, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl ) { + + // + // Correct behavior here is very finicky. + // + // 1. The first step is to append the error that occurred on the previous operation. + // This adds an "err" field to the command, which is *not* the command failing. + // + // 2. Next we parse and validate write concern options. If these options are invalid + // the command fails no matter what, even if we actually had an error earlier. The + // reason for checking here is to match legacy behavior on these kind of failures - + // we'll still get an "err" field for the write error. + // + // 3. If we had an error on the previous operation, we then return immediately. + // + // 4. Finally, we actually enforce the write concern. All errors *except* timeout are + // reported with ok : 0.0, to match legacy behavior. + // + // There is a special case when "wOpTime" and "wElectionId" are explicitly provided by + // the client (mongos) - in this case we *only* enforce the write concern if it is + // valid. + // + // We always need to either report "err" (if ok : 1) or "errmsg" (if ok : 0), even if + // err is null. + // + + LastError *le = lastError.disableForCommand(); + + // Always append lastOp and connectionId + Client& c = cc(); + c.appendLastOp( result ); + + // for sharding; also useful in general for debugging + result.appendNumber( "connectionId" , c.getConnectionId() ); + + OpTime lastOpTime; + BSONField wOpTimeField("wOpTime"); + FieldParser::FieldState extracted = FieldParser::extract(cmdObj, wOpTimeField, + &lastOpTime, &errmsg); + if (!extracted) { + result.append("badGLE", cmdObj); + appendCommandStatus(result, false, errmsg); + return false; + } + bool lastOpTimePresent = extracted != FieldParser::FIELD_NONE; + if (!lastOpTimePresent) { + // Use the client opTime if no wOpTime is specified + lastOpTime = cc().getLastOp(); + } + + OID electionId; + BSONField wElectionIdField("wElectionId"); + extracted = FieldParser::extract(cmdObj, wElectionIdField, + &electionId, &errmsg); + if (!extracted) { + result.append("badGLE", cmdObj); + appendCommandStatus(result, false, errmsg); + return false; + } + + bool electionIdPresent = extracted != FieldParser::FIELD_NONE; + bool errorOccurred = false; + + // Errors aren't reported when wOpTime is used + if ( !lastOpTimePresent ) { + if ( le->nPrev != 1 ) { + errorOccurred = LastError::noError.appendSelf( result, false ); + le->appendSelfStatus( result ); + } + else { + errorOccurred = le->appendSelf( result, false ); + } + } + + BSONObj writeConcernDoc = cmdObj; + // Use the default options if we have no gle options aside from wOpTime/wElectionId + const int nFields = cmdObj.nFields(); + bool useDefaultGLEOptions = (nFields == 1) || + (nFields == 2 && lastOpTimePresent) || + (nFields == 3 && lastOpTimePresent && electionIdPresent); + + if ( useDefaultGLEOptions && getLastErrorDefault ) { + writeConcernDoc = *getLastErrorDefault; + } + + // + // Validate write concern no matter what, this matches 2.4 behavior + // + + WriteConcernOptions writeConcern; + Status status = writeConcern.parse( writeConcernDoc ); + + if ( status.isOK() ) { + // Ensure options are valid for this host + status = validateWriteConcern( writeConcern ); + } + + if ( !status.isOK() ) { + result.append( "badGLE", writeConcernDoc ); + return appendCommandStatus( result, status ); + } + + // Don't wait for replication if there was an error reported - this matches 2.4 behavior + if ( errorOccurred ) { + dassert( !lastOpTimePresent ); + return true; + } + + // No error occurred, so we won't duplicate these fields with write concern errors + dassert( result.asTempObj()["err"].eoo() ); + dassert( result.asTempObj()["code"].eoo() ); + + // If we got an electionId, make sure it matches + if (electionIdPresent) { + if (!theReplSet) { + // Ignore electionIds of 0 from mongos. + if (electionId != OID()) { + errmsg = "wElectionId passed but no replication active"; + result.append("code", ErrorCodes::BadValue); + return false; + } + } + else { + if (electionId != theReplSet->getElectionId()) { + LOG(3) << "oid passed in is " << electionId + << ", but our id is " << theReplSet->getElectionId(); + errmsg = "election occurred after write"; + result.append("code", ErrorCodes::WriteConcernFailed); + return false; + } + } + } + + cc().curop()->setMessage( "waiting for write concern" ); + + WriteConcernResult wcResult; + status = waitForWriteConcern( writeConcern, lastOpTime, &wcResult ); + wcResult.appendTo( writeConcern, &result ); + + // For backward compatibility with 2.4, wtimeout returns ok : 1.0 + if ( wcResult.wTimedOut ) { + dassert( !wcResult.err.empty() ); // so we always report err + dassert( !status.isOK() ); + result.append( "errmsg", "timed out waiting for slaves" ); + result.append( "code", status.code() ); + return true; + } + + return appendCommandStatus( result, status ); + } + + } cmdGetLastError; + + class CmdGetPrevError : public Command { + public: + virtual LockType locktype() const { return NONE; } + virtual bool logTheOp() { + return false; + } + virtual void help( stringstream& help ) const { + help << "check for errors since last reseterror commandcal"; + } + virtual bool slaveOk() const { + return true; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) {} // No auth required + CmdGetPrevError() : Command("getPrevError", false, "getpreverror") {} + bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + LastError *le = lastError.disableForCommand(); + le->appendSelf( result ); + if ( le->valid ) + result.append( "nPrev", le->nPrev ); + else + result.append( "nPrev", -1 ); + return true; + } + } cmdGetPrevError; + +} diff --git a/src/mongo/db/commands/group.cpp b/src/mongo/db/commands/group.cpp index 441a1192905..e7fae6e4ba4 100644 --- a/src/mongo/db/commands/group.cpp +++ b/src/mongo/db/commands/group.cpp @@ -14,19 +14,36 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/client_basic.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" +#include "mongo/db/catalog/database.h" #include "mongo/db/instance.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/catalog/collection.h" #include "mongo/scripting/engine.h" -#include "mongo/db/clientcursor.h" namespace mongo { @@ -39,13 +56,23 @@ namespace mongo { virtual void help( stringstream &help ) const { help << "http://dochub.mongodb.org/core/aggregation"; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + std::string ns = parseNs(dbname, cmdObj); + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnNamespace( + NamespaceString(ns), ActionType::find)) { + return Status(ErrorCodes::Unauthorized, "unauthorized"); + } + return Status::OK(); } + + string parseNs(const string& dbname, const BSONObj& cmdObj) const { + const BSONObj& p = cmdObj.firstElement().embeddedObjectUserCheck(); + uassert(17211, "ns has to be set", p["ns"].type() == String); + return dbname + "." + p["ns"].String(); + } + BSONObj getKey( const BSONObj& obj , const BSONObj& keyPattern , ScriptingFunction func , double avgSize , Scope * s ) { if ( func ) { BSONObjBuilder b( obj.objsize() + 32 ); @@ -72,7 +99,9 @@ namespace mongo { string& errmsg, BSONObjBuilder& result ) { - auto_ptr s = globalScriptEngine->getPooledScope( realdbname, "group"); + const string userToken = ClientBasic::getCurrent()->getAuthorizationSession() + ->getAuthenticatedUserNamesToken(); + auto_ptr s = globalScriptEngine->getPooledScope(realdbname, "group" + userToken); if ( reduceScope ) s->init( reduceScope ); @@ -102,55 +131,56 @@ namespace mongo { double keysize = keyPattern.objsize() * 3; double keynum = 1; + Collection* collection = cc().database()->getCollection( ns ); + map map; list blah; - shared_ptr cursor = NamespaceDetailsTransient::getCursor(ns.c_str() , query); - ClientCursor::Holder ccPointer( new ClientCursor( QueryOption_NoCursorTimeout, cursor, - ns ) ); - - while ( cursor->ok() ) { - - if ( !ccPointer->yieldSometimes( ClientCursor::MaybeCovered ) || - !cursor->ok() ) { - break; + if (collection) { + CanonicalQuery* cq; + if (!CanonicalQuery::canonicalize(ns, query, &cq).isOK()) { + uasserted(17212, "Can't canonicalize query " + query.toString()); + return 0; } - - if ( !cursor->currentMatches() || cursor->getsetdup( cursor->currLoc() ) ) { - cursor->advance(); - continue; - } - - if ( !ccPointer->yieldSometimes( ClientCursor::WillNeed ) || - !cursor->ok() ) { - break; - } - - BSONObj obj = cursor->current(); - cursor->advance(); - - BSONObj key = getKey( obj , keyPattern , keyFunction , keysize / keynum , s.get() ); - keysize += key.objsize(); - keynum++; - int& n = map[key]; - if ( n == 0 ) { - n = map.size(); - s->setObject( "$key" , key , true ); - - uassert( 10043 , "group() can't handle more than 20000 unique keys" , n <= 20000 ); + Runner* rawRunner; + if (!getRunner(cq, &rawRunner).isOK()) { + uasserted(17213, "Can't get runner for query " + query.toString()); + return 0; } - s->setObject( "obj" , obj , true ); - s->setNumber( "n" , n - 1 ); - if ( s->invoke( f , 0, 0 , 0 , true ) ) { - throw UserException( 9010 , (string)"reduce invoke failed: " + s->getError() ); + auto_ptr runner(rawRunner); + const ScopedRunnerRegistration safety(runner.get()); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + BSONObj obj; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { + BSONObj key = getKey(obj , keyPattern , keyFunction , keysize / keynum, + s.get() ); + keysize += key.objsize(); + keynum++; + + int& n = map[key]; + if ( n == 0 ) { + n = map.size(); + s->setObject( "$key" , key , true ); + uassert(17203, "group() can't handle more than 20000 unique keys", + n <= 20000 ); + } + + s->setObject( "obj" , obj , true ); + s->setNumber( "n" , n - 1 ); + if ( s->invoke( f , 0, 0 , 0 , true ) ) { + throw UserException(17214, + (string)"reduce invoke failed: " + s->getError()); + } } } - ccPointer.reset(); if (!finalize.empty()) { - s->exec( "$finalize = " + finalize , "$group finalize define" , false , true , true , 100 ); + s->exec( "$finalize = " + finalize , "$group finalize define" , + false , true , true , 100 ); ScriptingFunction g = s->createFunction( "function(){ " " for(var i=0; i < $arr.length; i++){ " @@ -189,12 +219,7 @@ namespace mongo { else q = getQuery( p ); - if ( p["ns"].type() != String ) { - errmsg = "ns has to be set"; - return false; - } - - string ns = dbname + "." + p["ns"].String(); + string ns = parseNs(dbname, jsobj); BSONObj key; string keyf; diff --git a/src/mongo/db/commands/hashcmd.cpp b/src/mongo/db/commands/hashcmd.cpp index acf4000505e..2e8c09bc062 100644 --- a/src/mongo/db/commands/hashcmd.cpp +++ b/src/mongo/db/commands/hashcmd.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* @@ -27,7 +39,6 @@ #include "mongo/base/status.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" #include "mongo/db/hasher.h" @@ -42,7 +53,6 @@ namespace mongo { virtual LockType locktype() const { return NONE; } virtual bool slaveOk() const { return true; } // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) {} diff --git a/src/mongo/db/commands/index_filter_commands.cpp b/src/mongo/db/commands/index_filter_commands.cpp new file mode 100644 index 00000000000..20fb72fcdfd --- /dev/null +++ b/src/mongo/db/commands/index_filter_commands.cpp @@ -0,0 +1,369 @@ +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include +#include + +#include "mongo/base/init.h" +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/base/status.h" +#include "mongo/db/client.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/commands/index_filter_commands.h" +#include "mongo/db/commands/plan_cache_commands.h" +#include "mongo/db/catalog/collection.h" + +namespace { + + using std::string; + using std::vector; + using namespace mongo; + + /** + * Utility function to extract error code and message from status + * and append to BSON results. + */ + void addStatus(const Status& status, BSONObjBuilder& builder) { + builder.append("ok", status.isOK() ? 1.0 : 0.0); + if (!status.isOK()) { + builder.append("code", status.code()); + } + if (!status.reason().empty()) { + builder.append("errmsg", status.reason()); + } + } + + /** + * Retrieves a collection's query settings and plan cache from the database. + */ + Status getQuerySettingsAndPlanCache(Database* db, const string& ns, + QuerySettings** querySettingsOut, + PlanCache** planCacheOut) { + invariant(db); + + Collection* collection = db->getCollection(ns); + if (NULL == collection) { + return Status(ErrorCodes::BadValue, "no such collection"); + } + + CollectionInfoCache* infoCache = collection->infoCache(); + invariant(infoCache); + + QuerySettings* querySettings = infoCache->getQuerySettings(); + invariant(querySettings); + + *querySettingsOut = querySettings; + + PlanCache* planCache = infoCache->getPlanCache(); + invariant(planCache); + + *planCacheOut = planCache; + + return Status::OK(); + } + + // + // Command instances. + // Registers commands with the command system and make commands + // available to the client. + // + + MONGO_INITIALIZER_WITH_PREREQUISITES(SetupIndexFilterCommands, MONGO_NO_PREREQUISITES)( + InitializerContext* context) { + + new ListFilters(); + new ClearFilters(); + new SetFilter(); + + return Status::OK(); + } + +} // namespace + +namespace mongo { + + using std::string; + using std::stringstream; + using std::vector; + using boost::scoped_ptr; + + IndexFilterCommand::IndexFilterCommand(const string& name, const string& helpText) + : Command(name), + helpText(helpText) { } + + bool IndexFilterCommand::run(const string& dbname, BSONObj& cmdObj, int options, + string& errmsg, BSONObjBuilder& result, bool fromRepl) { + string ns = parseNs(dbname, cmdObj); + + Status status = runIndexFilterCommand(ns, cmdObj, &result); + + if (!status.isOK()) { + addStatus(status, result); + return false; + } + + return true; + } + + Command::LockType IndexFilterCommand::locktype() const { + return NONE; + } + + bool IndexFilterCommand::slaveOk() const { + return false; + } + + bool IndexFilterCommand::slaveOverrideOk() const { + return true; + } + + void IndexFilterCommand::help(stringstream& ss) const { + ss << helpText; + } + + Status IndexFilterCommand::checkAuthForCommand(ClientBasic* client, const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + ResourcePattern pattern = parseResourcePattern(dbname, cmdObj); + + if (authzSession->isAuthorizedForActionsOnResource(pattern, ActionType::planCacheIndexFilter)) { + return Status::OK(); + } + + return Status(ErrorCodes::Unauthorized, "unauthorized"); + } + + ListFilters::ListFilters() : IndexFilterCommand("planCacheListFilters", + "Displays index filters for all query shapes in a collection.") { } + + Status ListFilters::runIndexFilterCommand(const string& ns, BSONObj& cmdObj, BSONObjBuilder* bob) { + // This is a read lock. The query settings is owned by the collection. + Client::ReadContext readCtx(ns); + Client::Context& ctx = readCtx.ctx(); + QuerySettings* querySettings; + PlanCache* unused; + Status status = getQuerySettingsAndPlanCache(ctx.db(), ns, &querySettings, &unused); + if (!status.isOK()) { + // No collection - return empty array of filters. + BSONArrayBuilder hintsBuilder(bob->subarrayStart("filters")); + hintsBuilder.doneFast(); + return Status::OK(); + } + return list(*querySettings, bob); + } + + // static + Status ListFilters::list(const QuerySettings& querySettings, BSONObjBuilder* bob) { + invariant(bob); + + // Format of BSON result: + // + // { + // hints: [ + // { + // query: , + // sort: , + // projection: , + // indexes: [, , , ...] + // } + // } + BSONArrayBuilder hintsBuilder(bob->subarrayStart("filters")); + OwnedPointerVector entries; + entries.mutableVector() = querySettings.getAllAllowedIndices(); + for (vector::const_iterator i = entries.begin(); + i != entries.end(); ++i) { + AllowedIndexEntry* entry = *i; + invariant(entry); + + BSONObjBuilder hintBob(hintsBuilder.subobjStart()); + hintBob.append("query", entry->query); + hintBob.append("sort", entry->sort); + hintBob.append("projection", entry->projection); + BSONArrayBuilder indexesBuilder(hintBob.subarrayStart("indexes")); + for (vector::const_iterator j = entry->indexKeyPatterns.begin(); + j != entry->indexKeyPatterns.end(); ++j) { + const BSONObj& index = *j; + indexesBuilder.append(index); + } + indexesBuilder.doneFast(); + } + hintsBuilder.doneFast(); + return Status::OK(); + } + + ClearFilters::ClearFilters() : IndexFilterCommand("planCacheClearFilters", + "Clears index filter for a single query shape or, " + "if the query shape is omitted, all filters for the collection.") { } + + Status ClearFilters::runIndexFilterCommand(const string& ns, BSONObj& cmdObj, BSONObjBuilder* bob) { + // This is a read lock. The query settings is owned by the collection. + Client::ReadContext readCtx(ns); + Client::Context& ctx = readCtx.ctx(); + QuerySettings* querySettings; + PlanCache* planCache; + Status status = getQuerySettingsAndPlanCache(ctx.db(), ns, &querySettings, &planCache); + if (!status.isOK()) { + // No collection - do nothing. + return Status::OK(); + } + return clear(querySettings, planCache, ns, cmdObj); + } + + // static + Status ClearFilters::clear(QuerySettings* querySettings, PlanCache* planCache, + const std::string& ns, const BSONObj& cmdObj) { + invariant(querySettings); + + // According to the specification, the planCacheClearFilters command runs in two modes: + // - clear all hints; or + // - clear hints for single query shape when a query shape is described in the + // command arguments. + if (cmdObj.hasField("query")) { + CanonicalQuery* cqRaw; + Status status = PlanCacheCommand::canonicalize(ns, cmdObj, &cqRaw); + if (!status.isOK()) { + return status; + } + + scoped_ptr cq(cqRaw); + querySettings->removeAllowedIndices(*cq); + + // Remove entry from plan cache + planCache->remove(*cq); + return Status::OK(); + } + + // If query is not provided, make sure sort and projection are not in arguments. + // We do not want to clear the entire cache inadvertently when the user + // forgot to provide a value for "query". + if (cmdObj.hasField("sort") || cmdObj.hasField("projection")) { + return Status(ErrorCodes::BadValue, "sort or projection provided without query"); + } + + // Get entries from query settings. We need to remove corresponding entries from the plan + // cache shortly. + OwnedPointerVector entries; + entries.mutableVector() = querySettings->getAllAllowedIndices(); + + // OK to proceed with clearing entire cache. + querySettings->clearAllowedIndices(); + + // Remove corresponding entries from plan cache. + // Admin hints affect the planning process directly. If there were + // plans generated as a result of applying index filter, these need to be + // invalidated. This allows the planner to re-populate the plan cache with + // non-filtered indexed solutions next time the query is run. + // Resolve plan cache key from (query, sort, projection) in query settings entry. + // Concurrency note: There's no harm in removing plan cache entries one at at time. + // Only way that PlanCache::remove() can fail is when the query shape has been removed from + // the cache by some other means (re-index, collection info reset, ...). This is OK since + // that's the intended effect of calling the remove() function with the key from the hint entry. + for (vector::const_iterator i = entries.begin(); + i != entries.end(); ++i) { + AllowedIndexEntry* entry = *i; + invariant(entry); + + // Create canonical query. + CanonicalQuery* cqRaw; + Status result = CanonicalQuery::canonicalize(ns, entry->query, entry->sort, + entry->projection, &cqRaw); + invariant(result.isOK()); + scoped_ptr cq(cqRaw); + + // Remove plan cache entry. + planCache->remove(*cq); + } + + return Status::OK(); + } + + SetFilter::SetFilter() : IndexFilterCommand("planCacheSetFilter", + "Sets index filter for a query shape. Overrides existing filter.") { } + + Status SetFilter::runIndexFilterCommand(const string& ns, BSONObj& cmdObj, BSONObjBuilder* bob) { + // This is a read lock. The query settings is owned by the collection. + Client::ReadContext readCtx(ns); + Client::Context& ctx = readCtx.ctx(); + QuerySettings* querySettings; + PlanCache* planCache; + Status status = getQuerySettingsAndPlanCache(ctx.db(), ns, &querySettings, &planCache); + if (!status.isOK()) { + return status; + } + return set(querySettings, planCache, ns, cmdObj); + } + + // static + Status SetFilter::set(QuerySettings* querySettings, PlanCache* planCache, + const string& ns, const BSONObj& cmdObj) { + // indexes - required + BSONElement indexesElt = cmdObj.getField("indexes"); + if (indexesElt.eoo()) { + return Status(ErrorCodes::BadValue, "required field indexes missing"); + } + if (indexesElt.type() != mongo::Array) { + return Status(ErrorCodes::BadValue, "required field indexes must be an array"); + } + vector indexesEltArray = indexesElt.Array(); + if (indexesEltArray.empty()) { + return Status(ErrorCodes::BadValue, + "required field indexes must contain at least one index"); + } + vector indexes; + for (vector::const_iterator i = indexesEltArray.begin(); + i != indexesEltArray.end(); ++i) { + const BSONElement& elt = *i; + if (!elt.isABSONObj()) { + return Status(ErrorCodes::BadValue, "each item in indexes must be an object"); + } + BSONObj obj = elt.Obj(); + if (obj.isEmpty()) { + return Status(ErrorCodes::BadValue, "index specification cannot be empty"); + } + indexes.push_back(obj.getOwned()); + } + + CanonicalQuery* cqRaw; + Status status = PlanCacheCommand::canonicalize(ns, cmdObj, &cqRaw); + if (!status.isOK()) { + return status; + } + scoped_ptr cq(cqRaw); + + // Add allowed indices to query settings, overriding any previous entries. + querySettings->setAllowedIndices(*cq, indexes); + + // Remove entry from plan cache. + planCache->remove(*cq); + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/commands/index_filter_commands.h b/src/mongo/db/commands/index_filter_commands.h new file mode 100644 index 00000000000..a2ccd9a39f9 --- /dev/null +++ b/src/mongo/db/commands/index_filter_commands.h @@ -0,0 +1,169 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/commands.h" +#include "mongo/db/query/query_settings.h" +#include "mongo/db/query/plan_cache.h" + +namespace mongo { + + /** + * DB commands for index filters. + * Index filter commands work on a different data structure in the collection + * info cache from the plan cache. + * The user still thinks of index filter commands as part of the plan cache functionality + * so the command name prefix is still "planCache". + * + * These are in a header to facilitate unit testing. See index_filter_commands_test.cpp. + */ + + /** + * IndexFilterCommand + * Defines common attributes for all index filter related commands + * such as slaveOk and locktype. + */ + class IndexFilterCommand : public Command { + public: + IndexFilterCommand(const std::string& name, const std::string& helpText); + + /** + * Entry point from command subsystem. + * Implementation provides standardization of error handling + * such as adding error code and message to BSON result. + * + * Do not override in derived classes. + * Override runPlanCacheCommands instead to + * implement plan cache command functionality. + */ + + bool run(const std::string& dbname, BSONObj& cmdObj, int options, + std::string& errmsg, BSONObjBuilder& result, bool fromRepl); + + /** + * It's fine to return NONE here because plan cache commands + * create explicit read context to access collection info cache. + * Refer to dbcommands.cpp on how locktype() is handled. + */ + virtual LockType locktype() const; + + virtual bool slaveOk() const; + + virtual bool slaveOverrideOk() const; + + virtual void help(std::stringstream& ss) const; + + /** + * One action type defined for index filter commands: + * - planCacheIndexFilter + */ + virtual Status checkAuthForCommand(ClientBasic* client, const std::string& dbname, + const BSONObj& cmdObj); + + /** + * Subset of command arguments used by index filter commands + * Override to provide command functionality. + * Should contain just enough logic to invoke run*Command() function + * in query_settings.h + */ + virtual Status runIndexFilterCommand(const std::string& ns, BSONObj& cmdObj, + BSONObjBuilder* bob) = 0; + + private: + std::string helpText; + }; + + /** + * ListFilters + * + * { planCacheListFilters: } + * + */ + class ListFilters : public IndexFilterCommand { + public: + ListFilters(); + + virtual Status runIndexFilterCommand(const std::string& ns, BSONObj& cmdObj, BSONObjBuilder* bob); + + /** + * Looks up index filters from collection's query settings. + * Inserts index filters into BSON builder. + */ + static Status list(const QuerySettings& querySettings, BSONObjBuilder* bob); + }; + + /** + * ClearFilters + * + * { planCacheClearFilters: , query: , sort: , projection: } + * + */ + class ClearFilters : public IndexFilterCommand { + public: + ClearFilters(); + + virtual Status runIndexFilterCommand(const std::string& ns, BSONObj& cmdObj, BSONObjBuilder* bob); + + /** + * If query shape is provided, clears index filter for a query. + * Otherwise, clears collection's filters. + * Namespace argument ns is ignored if we are clearing the entire cache. + * Removes corresponding entries from plan cache. + */ + static Status clear(QuerySettings* querySettings, PlanCache* planCache, const std::string& ns, + const BSONObj& cmdObj); + }; + + /** + * SetFilter + * + * { + * planCacheSetFilter: , + * query: , + * sort: , + * projection: , + * indexes: [ , , , ... ] + * } + * + */ + class SetFilter : public IndexFilterCommand { + public: + SetFilter(); + + virtual Status runIndexFilterCommand(const std::string& ns, BSONObj& cmdObj, BSONObjBuilder* bob); + + /** + * Sets index filter for a query shape. + * Removes entry for query shape from plan cache. + */ + static Status set(QuerySettings* querySettings, PlanCache* planCache, const std::string& ns, + const BSONObj& cmdObj); + }; + +} // namespace mongo diff --git a/src/mongo/db/commands/index_filter_commands_test.cpp b/src/mongo/db/commands/index_filter_commands_test.cpp new file mode 100644 index 00000000000..d80d51d7d82 --- /dev/null +++ b/src/mongo/db/commands/index_filter_commands_test.cpp @@ -0,0 +1,319 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/commands/index_filter_commands.h + */ + +#include "mongo/db/commands/index_filter_commands.h" + +#include "mongo/db/json.h" +#include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + using std::string; + using std::vector; + + static const char* ns = "somebogusns"; + + /** + * Utility function to get list of index filters from the query settings. + */ + vector getFilters(const QuerySettings& querySettings) { + BSONObjBuilder bob; + ASSERT_OK(ListFilters::list(querySettings, &bob)); + BSONObj resultObj = bob.obj(); + BSONElement filtersElt = resultObj.getField("filters"); + ASSERT_EQUALS(filtersElt.type(), mongo::Array); + vector filtersEltArray = filtersElt.Array(); + vector filters; + for (vector::const_iterator i = filtersEltArray.begin(); + i != filtersEltArray.end(); ++i) { + const BSONElement& elt = *i; + + ASSERT_TRUE(elt.isABSONObj()); + BSONObj obj = elt.Obj(); + + // Check required fields. + // query + BSONElement queryElt = obj.getField("query"); + ASSERT_TRUE(queryElt.isABSONObj()); + + // sort + BSONElement sortElt = obj.getField("sort"); + ASSERT_TRUE(sortElt.isABSONObj()); + + // projection + BSONElement projectionElt = obj.getField("projection"); + ASSERT_TRUE(projectionElt.isABSONObj()); + + // indexes + BSONElement indexesElt = obj.getField("indexes"); + ASSERT_EQUALS(indexesElt.type(), mongo::Array); + + // All fields OK. Append to vector. + filters.push_back(obj.getOwned()); + } + + return filters; + } + + /** + * Utility function to create a PlanRankingDecision + */ + PlanRankingDecision* createDecision(size_t numPlans) { + auto_ptr why(new PlanRankingDecision()); + for (size_t i = 0; i < numPlans; ++i) { + auto_ptr stats(new PlanStageStats(CommonStats(), STAGE_COLLSCAN)); + stats->specific.reset(new CollectionScanStats()); + why->stats.mutableVector().push_back(stats.release()); + why->scores.push_back(0U); + why->candidateOrder.push_back(i); + } + return why.release(); + } + + /** + * Injects an entry into plan cache for query shape. + */ + void addQueryShapeToPlanCache(PlanCache* planCache, const char* queryStr, const char* sortStr, + const char* projectionStr) { + BSONObj queryObj = fromjson(queryStr); + BSONObj sortObj = fromjson(sortStr); + BSONObj projectionObj = fromjson(projectionStr); + + // Create canonical query. + CanonicalQuery* cqRaw; + ASSERT_OK(CanonicalQuery::canonicalize(ns, queryObj, sortObj, projectionObj, &cqRaw)); + scoped_ptr cq(cqRaw); + + QuerySolution qs; + qs.cacheData.reset(new SolutionCacheData()); + qs.cacheData->tree.reset(new PlanCacheIndexTree()); + std::vector solns; + solns.push_back(&qs); + ASSERT_OK(planCache->add(*cq, solns, createDecision(1U))); + } + + /** + * Checks if plan cache contains query shape. + */ + bool planCacheContains(const PlanCache& planCache, const char* queryStr, const char* sortStr, + const char* projectionStr) { + BSONObj queryObj = fromjson(queryStr); + BSONObj sortObj = fromjson(sortStr); + BSONObj projectionObj = fromjson(projectionStr); + + // Create canonical query. + CanonicalQuery* cqRaw; + ASSERT_OK(CanonicalQuery::canonicalize(ns, queryObj, sortObj, projectionObj, &cqRaw)); + scoped_ptr cq(cqRaw); + + // Retrieve cache entries from plan cache. + vector entries = planCache.getAllEntries(); + + // Search keys. + bool found = false; + for (vector::const_iterator i = entries.begin(); i != entries.end(); i++) { + PlanCacheEntry* entry = *i; + + // Canonicalizing query shape in cache entry to get cache key. + // Alternatively, we could add key to PlanCacheEntry but that would be used in one place only. + ASSERT_OK(CanonicalQuery::canonicalize(ns, entry->query, entry->sort, + entry->projection, &cqRaw)); + scoped_ptr currentQuery(cqRaw); + + const PlanCacheKey& currentKey = currentQuery->getPlanCacheKey(); + if (currentKey == cq->getPlanCacheKey()) { + found = true; + } + // Release resources for cache entry after extracting key. + delete entry; + } + return found; + } + + /** + * Tests for ListFilters + */ + + TEST(IndexFilterCommandsTest, ListFiltersEmpty) { + QuerySettings empty; + vector filters = getFilters(empty); + ASSERT_TRUE(filters.empty()); + } + + /** + * Tests for ClearFilters + */ + + TEST(IndexFilterCommandsTest, ClearFiltersInvalidParameter) { + QuerySettings empty; + PlanCache planCache; + // If present, query has to be an object. + ASSERT_NOT_OK(ClearFilters::clear(&empty, &planCache, ns, fromjson("{query: 1234}"))); + // If present, sort must be an object. + ASSERT_NOT_OK(ClearFilters::clear(&empty, &planCache, ns, + fromjson("{query: {a: 1}, sort: 1234}"))); + // If present, projection must be an object. + ASSERT_NOT_OK(ClearFilters::clear(&empty, &planCache, ns, + fromjson("{query: {a: 1}, projection: 1234}"))); + // Query must pass canonicalization. + ASSERT_NOT_OK(ClearFilters::clear(&empty, &planCache, ns, + fromjson("{query: {a: {$no_such_op: 1}}}"))); + // Sort present without query is an error. + ASSERT_NOT_OK(ClearFilters::clear(&empty, &planCache, ns, fromjson("{sort: {a: 1}}"))); + // Projection present without query is an error. + ASSERT_NOT_OK(ClearFilters::clear(&empty, &planCache, ns, + fromjson("{projection: {_id: 0, a: 1}}"))); + } + + TEST(IndexFilterCommandsTest, ClearNonexistentHint) { + QuerySettings querySettings; + PlanCache planCache; + ASSERT_OK(SetFilter::set(&querySettings, &planCache, ns, + fromjson("{query: {a: 1}, indexes: [{a: 1}]}"))); + vector filters = getFilters(querySettings); + ASSERT_EQUALS(filters.size(), 1U); + + // Clear nonexistent hint. + // Command should succeed and cache should remain unchanged. + ASSERT_OK(ClearFilters::clear(&querySettings, &planCache, ns, fromjson("{query: {b: 1}}"))); + filters = getFilters(querySettings); + ASSERT_EQUALS(filters.size(), 1U); + } + + /** + * Tests for SetFilter + */ + + TEST(IndexFilterCommandsTest, SetFilterInvalidParameter) { + QuerySettings empty; + PlanCache planCache; + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, fromjson("{}"))); + // Missing required query field. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, fromjson("{indexes: [{a: 1}]}"))); + // Missing required indexes field. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, fromjson("{query: {a: 1}}"))); + // Query has to be an object. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: 1234, indexes: [{a: 1}, {b: 1}]}"))); + // Indexes field has to be an array. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: {a: 1}, indexes: 1234}"))); + // Array indexes field cannot empty. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: {a: 1}, indexes: []}"))); + // Elements in indexes have to be objects. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: {a: 1}, indexes: [{a: 1}, 99]}"))); + // Objects in indexes cannot be empty. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: {a: 1}, indexes: [{a: 1}, {}]}"))); + // If present, sort must be an object. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: {a: 1}, sort: 1234, indexes: [{a: 1}, {b: 1}]}"))); + // If present, projection must be an object. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: {a: 1}, projection: 1234, indexes: [{a: 1}, {b: 1}]}"))); + // Query must pass canonicalization. + ASSERT_NOT_OK(SetFilter::set(&empty, &planCache, ns, + fromjson("{query: {a: {$no_such_op: 1}}, indexes: [{a: 1}, {b: 1}]}"))); + } + + TEST(IndexFilterCommandsTest, SetAndClearFilters) { + QuerySettings querySettings; + PlanCache planCache; + + // Inject query shape into plan cache. + addQueryShapeToPlanCache(&planCache, "{a: 1, b: 1}", "{a: -1}", "{_id: 0, a: 1}"); + ASSERT_TRUE(planCacheContains(planCache, "{a: 1, b: 1}", "{a: -1}", "{_id: 0, a: 1}")); + + ASSERT_OK(SetFilter::set(&querySettings, &planCache, ns, + fromjson("{query: {a: 1, b: 1}, sort: {a: -1}, projection: {_id: 0, a: 1}, " + "indexes: [{a: 1}]}"))); + vector filters = getFilters(querySettings); + ASSERT_EQUALS(filters.size(), 1U); + + // Query shape should not exist in plan cache after hint is updated. + ASSERT_FALSE(planCacheContains(planCache, "{a: 1, b: 1}", "{a: -1}", "{_id: 0, a: 1}")); + + // Fields in filter should match criteria in most recent query settings update. + ASSERT_EQUALS(filters[0].getObjectField("query"), fromjson("{a: 1, b: 1}")); + ASSERT_EQUALS(filters[0].getObjectField("sort"), fromjson("{a: -1}")); + ASSERT_EQUALS(filters[0].getObjectField("projection"), fromjson("{_id: 0, a: 1}")); + + // Replacing the hint for the same query shape ({a: 1, b: 1} and {b: 2, a: 3} + // share same shape) should not change the query settings size. + ASSERT_OK(SetFilter::set(&querySettings, &planCache, ns, + fromjson("{query: {b: 2, a: 3}, sort: {a: -1}, projection: {_id: 0, a: 1}, " + "indexes: [{a: 1, b: 1}]}"))); + filters = getFilters(querySettings); + ASSERT_EQUALS(filters.size(), 1U); + + // Add hint for different query shape. + ASSERT_OK(SetFilter::set(&querySettings, &planCache, ns, + fromjson("{query: {b: 1}, indexes: [{b: 1}]}"))); + filters = getFilters(querySettings); + ASSERT_EQUALS(filters.size(), 2U); + + // Add hint for 3rd query shape. This is to prepare for ClearHint tests. + ASSERT_OK(SetFilter::set(&querySettings, &planCache, ns, + fromjson("{query: {a: 1}, indexes: [{a: 1}]}"))); + filters = getFilters(querySettings); + ASSERT_EQUALS(filters.size(), 3U); + + // Add 2 entries to plan cache and check plan cache after clearing one/all filters. + addQueryShapeToPlanCache(&planCache, "{a: 1}", "{}", "{}"); + addQueryShapeToPlanCache(&planCache, "{b: 1}", "{}", "{}"); + + // Clear single hint. + ASSERT_OK(ClearFilters::clear(&querySettings, &planCache, ns, + fromjson("{query: {a: 1}}"))); + filters = getFilters(querySettings); + ASSERT_EQUALS(filters.size(), 2U); + + // Query shape should not exist in plan cache after cleaing 1 hint. + ASSERT_FALSE(planCacheContains(planCache, "{a: 1}", "{}", "{}")); + ASSERT_TRUE(planCacheContains(planCache, "{b: 1}", "{}", "{}")); + + // Clear all filters + ASSERT_OK(ClearFilters::clear(&querySettings, &planCache, ns, fromjson("{}"))); + filters = getFilters(querySettings); + ASSERT_TRUE(filters.empty()); + + // {b: 1} should be gone from plan cache after flushing query settings. + ASSERT_FALSE(planCacheContains(planCache, "{b: 1}", "{}", "{}")); + } + +} // namespace diff --git a/src/mongo/db/commands/index_stats.cpp b/src/mongo/db/commands/index_stats.cpp index 1309e078ee9..de1bd4bf834 100644 --- a/src/mongo/db/commands/index_stats.cpp +++ b/src/mongo/db/commands/index_stats.cpp @@ -30,13 +30,13 @@ #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" -#include "mongo/db/btree.h" +#include "mongo/db/structure/btree/btree.h" #include "mongo/db/commands.h" #include "mongo/db/db.h" -#include "mongo/db/index.h" +#include "mongo/db/structure/catalog/index_details.h" #include "mongo/db/jsobj.h" #include "mongo/db/kill_current_op.h" -#include "mongo/db/namespace_details.h" +#include "mongo/db/structure/catalog/namespace_details.h" #include "mongo/util/descriptive_stats.h" namespace mongo { @@ -492,7 +492,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::indexStats); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, @@ -500,8 +500,8 @@ namespace mongo { string ns = dbname + "." + cmdObj.firstElement().valuestrsafe(); const NamespaceDetails* nsd = nsdetails(ns); - if (!cmdLine.quiet) { - tlog() << "CMD: indexStats " << ns << endl; + if (!serverGlobalParams.quiet) { + MONGO_TLOG(0) << "CMD: indexStats " << ns << endl; } if (!nsd) { errmsg = "ns not found"; @@ -544,7 +544,7 @@ namespace mongo { }; MONGO_INITIALIZER(IndexStatsCmd)(InitializerContext* context) { - if (cmdLine.experimental.indexStatsCmdEnabled) { + if (serverGlobalParams.experimental.indexStatsCmdEnabled) { // Leaked intentionally: a Command registers itself when constructed. new IndexStatsCmd(); } diff --git a/src/mongo/db/commands/isself.cpp b/src/mongo/db/commands/isself.cpp index 698f865b73c..57159d39099 100644 --- a/src/mongo/db/commands/isself.cpp +++ b/src/mongo/db/commands/isself.cpp @@ -14,25 +14,39 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" +#include #include #include #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/security_key.h" +#include "mongo/db/commands.h" #include "mongo/db/jsobj.h" -#include "../../util/net/listen.h" -#include "../commands.h" +#include "mongo/db/server_options.h" +#include "mongo/util/net/listen.h" #include "mongo/util/net/hostandport.h" #include "mongo/client/dbclientinterface.h" -#include - #ifndef _WIN32 # ifndef __sunos__ # include @@ -64,8 +78,8 @@ namespace mongo { vector out; ifaddrs * addrs; - if ( ! cmdLine.bind_ip.empty() ) { - boost::split( out, cmdLine.bind_ip, boost::is_any_of( ", " ) ); + if (!serverGlobalParams.bind_ip.empty()) { + boost::split(out, serverGlobalParams.bind_ip, boost::is_any_of(", ")); return out; } @@ -96,12 +110,15 @@ namespace mongo { freeifaddrs( addrs ); addrs = NULL; - if (logLevel >= 1) { - LOG(1) << "getMyAddrs():"; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + LogstreamBuilder builder(logger::globalLogDomain(), + getThreadName(), + logger::LogSeverity::Debug(1)); + builder << "getMyAddrs():"; for (vector::const_iterator it=out.begin(), end=out.end(); it!=end; ++it) { - LOG(1) << " [" << *it << ']'; + builder << " [" << *it << ']'; } - LOG(1) << endl; + builder << endl; } return out; @@ -114,7 +131,7 @@ namespace mongo { hints.ai_socktype = SOCK_STREAM; hints.ai_family = (IPv6Enabled() ? AF_UNSPEC : AF_INET); - static string portNum = BSONObjBuilder::numStr(cmdLine.port); + static string portNum = BSONObjBuilder::numStr(serverGlobalParams.port); vector out; @@ -140,12 +157,15 @@ namespace mongo { freeaddrinfo(addrs); - if (logLevel >= 1) { - LOG(1) << "getallIPs(\"" << iporhost << "\"):"; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + LogstreamBuilder builder(logger::globalLogDomain(), + getThreadName(), + logger::LogSeverity::Debug(1)); + builder << "getallIPs(\"" << iporhost << "\"):"; for (vector::const_iterator it=out.begin(), end=out.end(); it!=end; ++it) { - LOG(1) << " [" << *it << ']'; + builder << " [" << *it << ']'; } - LOG(1) << endl; + builder << endl; } return out; @@ -185,12 +205,7 @@ namespace mongo { bool HostAndPort::isSelf() const { int _p = port(); - int p = _p == -1 ? CmdLine::DefaultDBPort : _p; - - if( p != cmdLine.port ) { - // shortcut - ports have to match at the very least - return false; - } + int p = _p == -1 ? ServerGlobalParams::DefaultDBPort : _p; string host = str::stream() << this->host() << ":" << p; @@ -206,22 +221,27 @@ namespace mongo { #if !defined(_WIN32) && !defined(__sunos__) // on linux and os x we can do a quick check for an ip match - const vector myaddrs = getMyAddrs(); - const vector addrs = getAllIPs(_host); - - for (vector::const_iterator i=myaddrs.begin(), iend=myaddrs.end(); i!=iend; ++i) { - for (vector::const_iterator j=addrs.begin(), jend=addrs.end(); j!=jend; ++j) { - string a = *i; - string b = *j; - - if ( a == b || - ( str::startsWith( a , "127." ) && str::startsWith( b , "127." ) ) // 127. is all loopback - ) { - - // add to cache - scoped_lock lk( isSelfCommand._cacheLock ); - isSelfCommand._cache[host] = true; - return true; + // no need for ip match if the ports do not match + if (p == serverGlobalParams.port) { + const vector myaddrs = getMyAddrs(); + const vector addrs = getAllIPs(_host); + + for (vector::const_iterator i=myaddrs.begin(), iend=myaddrs.end(); + i!=iend; ++i) { + for (vector::const_iterator j=addrs.begin(), jend=addrs.end(); + j!=jend; ++j) { + string a = *i; + string b = *j; + + if ( a == b || ( str::startsWith( a , "127." ) && + str::startsWith( b , "127." ) ) // 127. is all loopback + ) { + + // add to cache + scoped_lock lk( isSelfCommand._cacheLock ); + isSelfCommand._cache[host] = true; + return true; + } } } } @@ -243,8 +263,8 @@ namespace mongo { return false; } - if (!noauth && !cmdLine.keyFile.empty() ) { - if (!conn.auth("local", internalSecurity.user, internalSecurity.pwd, errmsg, false)) { + if (getGlobalAuthorizationManager()->isAuthEnabled() && isInternalAuthSet()) { + if (!authenticateInternalUser(&conn)) { return false; } } diff --git a/src/mongo/db/commands/merge_chunks_cmd.cpp b/src/mongo/db/commands/merge_chunks_cmd.cpp new file mode 100644 index 00000000000..4f6b5048f3e --- /dev/null +++ b/src/mongo/db/commands/merge_chunks_cmd.cpp @@ -0,0 +1,179 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/base/init.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/field_parser.h" +#include "mongo/db/namespace_string.h" +#include "mongo/s/d_logic.h" +#include "mongo/s/d_merge.h" + +namespace mongo { + + /** + * Mongod-side command for merging chunks. + */ + class MergeChunksCommand : public Command { + public: + MergeChunksCommand() : Command("mergeChunks") {} + + virtual void help(stringstream& h) const { + h << "Merge Chunks command\n" + << "usage: { mergeChunks : , bounds : [ , ]," + << " (opt) epoch : , (opt) config : ," + << " (opt) shardName : }"; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString(parseNs(dbname, cmdObj))), + ActionType::splitChunk)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + return Status::OK(); + } + + virtual std::string parseNs(const std::string& dbname, const BSONObj& cmdObj) const { + return parseNsFullyQualified(dbname, cmdObj); + } + + virtual bool adminOnly() const { return true; } + virtual bool slaveOk() const { return false; } + virtual LockType locktype() const { return NONE; } + + // Required + static BSONField nsField; + static BSONField > boundsField; + // Optional, if the merge is only valid for a particular epoch + static BSONField epochField; + // Optional, if our sharding state has not previously been initializeed + static BSONField shardNameField; + static BSONField configField; + + bool run( const string& dbname, + BSONObj& cmdObj, + int, + string& errmsg, + BSONObjBuilder& result, + bool ) { + + string ns = parseNs(dbname, cmdObj); + + if ( ns.size() == 0 ) { + errmsg = "no namespace specified"; + return false; + } + + vector bounds; + if ( !FieldParser::extract( cmdObj, boundsField, &bounds, &errmsg ) ) { + return false; + } + + if ( bounds.size() == 0 ) { + errmsg = "no bounds were specified"; + return false; + } + + if ( bounds.size() != 2 ) { + errmsg = "only a min and max bound may be specified"; + return false; + } + + BSONObj minKey = bounds[0]; + BSONObj maxKey = bounds[1]; + + if ( minKey.isEmpty() ) { + errmsg = "no min key specified"; + return false; + } + + if ( maxKey.isEmpty() ) { + errmsg = "no max key specified"; + return false; + } + + // + // This might be the first call from mongos, so we may need to pass the config and shard + // information to initialize the shardingState. + // + + string config; + FieldParser::FieldState extracted = FieldParser::extract( cmdObj, + configField, + &config, + &errmsg ); + if ( !extracted ) return false; + if ( extracted != FieldParser::FIELD_NONE ) { + ShardingState::initialize( config ); + } + else if ( !shardingState.enabled() ) { + errmsg = + "sharding state must be enabled or config server specified to merge chunks"; + return false; + } + + // ShardName is optional, but might not be set yet + string shardName; + extracted = FieldParser::extract( cmdObj, shardNameField, &shardName, &errmsg ); + + if ( !extracted ) return false; + if ( extracted != FieldParser::FIELD_NONE ) { + shardingState.gotShardName( shardName ); + } + + // + // Epoch is optional, and if not set indicates we should use the latest epoch + // + + OID epoch; + if ( !FieldParser::extract( cmdObj, epochField, &epoch, &errmsg ) ) { + return false; + } + + return mergeChunks( NamespaceString( ns ), minKey, maxKey, epoch, true, &errmsg ); + } + }; + + BSONField MergeChunksCommand::nsField( "mergeChunks" ); + BSONField > MergeChunksCommand::boundsField( "bounds" ); + + BSONField MergeChunksCommand::configField( "config" ); + BSONField MergeChunksCommand::shardNameField( "shardName" ); + BSONField MergeChunksCommand::epochField( "epoch" ); + + MONGO_INITIALIZER(InitMergeChunksCommand)(InitializerContext* context) { + // Leaked intentionally: a Command registers itself when constructed. + new MergeChunksCommand(); + return Status::OK(); + } +} diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index 9528e495ded..deeac2b1318 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -14,23 +14,43 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" -#include "mr.h" +#include "mongo/db/commands/mr.h" #include "mongo/client/connpool.h" #include "mongo/client/parallel.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/catalog/index_catalog.h" #include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" #include "mongo/db/db.h" +#include "mongo/db/dbhelpers.h" #include "mongo/db/instance.h" #include "mongo/db/kill_current_op.h" #include "mongo/db/matcher.h" -#include "mongo/db/replutil.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/range_preserver.h" +#include "mongo/db/storage_options.h" #include "mongo/scripting/engine.h" -#include "mongo/s/d_chunk_manager.h" +#include "mongo/s/collection_metadata.h" #include "mongo/s/d_logic.h" #include "mongo/s/grid.h" #include "mongo/s/stale_exception.h" @@ -163,7 +183,6 @@ namespace mongo { // need to build the reduce args: ( key, [values] ) BSONObjBuilder reduceArgs( sizeEstimate ); boost::scoped_ptr valueBuilder; - int sizeSoFar = 0; unsigned n = 0; for ( ; n BSONObjMaxUserSize ) { + // If adding this element to the array would cause it to be too large, break. The + // remainder of the tuples will be processed recursively at the end of this + // function. + if ( valueBuilder->len() + ee.size() > BSONObjMaxUserSize ) { verify( n > 1 ); // if not, inf. loop break; } valueBuilder->append( ee ); - sizeSoFar += ee.size(); } verify(valueBuilder); valueBuilder->done(); @@ -299,9 +319,19 @@ namespace mongo { * Clean up the temporary and incremental collections */ void State::dropTempCollections() { + // Dropping the tempNamespace must be logged as that collection is replicated. _db.dropCollection(_config.tempNamespace); - if (_useIncremental) - _db.dropCollection(_config.incLong); + // Always forget about temporary namespaces, so we don't cache lots of them + ShardConnection::forgetNS( _config.tempNamespace ); + if (_useIncremental) { + // We don't want to log the deletion of incLong as it isn't replicated. While + // harmless, this would lead to a scary looking warning on the secondaries. + Client::WriteContext ctx(_config.incLong); + ctx.ctx().db()->dropCollection(_config.incLong); + + ShardConnection::forgetNS( _config.incLong ); + } + } /** @@ -313,52 +343,81 @@ namespace mongo { dropTempCollections(); if (_useIncremental) { - // create the inc collection and make sure we have index on "0" key - { - Client::WriteContext ctx( _config.incLong ); - string err; - if ( ! userCreateNS( _config.incLong.c_str() , BSON( "autoIndexId" << 0 << "temp" << true ) , err , false ) ) { - uasserted( 13631 , str::stream() << "userCreateNS failed for mr incLong ns: " << _config.incLong << " err: " << err ); - } + // Create the inc collection and make sure we have index on "0" key. + // Intentionally not replicating the inc collection to secondaries. + Client::WriteContext incCtx( _config.incLong ); + Collection* incColl = incCtx.ctx().db()->getCollection( _config.incLong ); + if ( !incColl ) { + CollectionOptions options; + options.setNoIdIndex(); + options.temp = true; + incColl = incCtx.ctx().db()->createCollection( _config.incLong, options ); } - BSONObj sortKey = BSON( "0" << 1 ); - _db.ensureIndex( _config.incLong , sortKey ); + BSONObj indexSpec = BSON( "key" << BSON( "0" << 1 ) << "ns" << _config.incLong + << "name" << "_temp_0" ); + Status status = incColl->getIndexCatalog()->createIndex( indexSpec, false ); + if ( !status.isOK() ) { + uasserted( 17305 , str::stream() << "createIndex failed for mr incLong ns: " << + _config.incLong << " err: " << status.code() ); + } } - // create temp collection + vector indexesToInsert; + { - Client::WriteContext ctx( _config.tempNamespace.c_str() ); - string errmsg; - if ( ! userCreateNS( _config.tempNamespace.c_str() , BSON("temp" << true) , errmsg , true ) ) { - uasserted(13630, str::stream() << "userCreateNS failed for mr tempLong ns: " - << _config.tempNamespace << " err: " << errmsg ); + // copy indexes into temporary storage + Client::WriteContext finalCtx( _config.outputOptions.finalNamespace ); + Collection* finalColl = + finalCtx.ctx().db()->getCollection( _config.outputOptions.finalNamespace ); + if ( finalColl ) { + IndexCatalog::IndexIterator ii = + finalColl->getIndexCatalog()->getIndexIterator( true ); + // Iterate over finalColl's indexes. + while ( ii.more() ) { + IndexDescriptor* currIndex = ii.next(); + BSONObjBuilder b; + b.append( "ns" , _config.tempNamespace ); + + // Copy over contents of the index descriptor's infoObj. + BSONObjIterator j( currIndex->infoObj() ); + while ( j.more() ) { + BSONElement e = j.next(); + if ( str::equals( e.fieldName() , "_id" ) || + str::equals( e.fieldName() , "ns" ) ) + continue; + b.append( e ); + } + indexesToInsert.push_back( b.obj() ); + } } } { - // copy indexes - auto_ptr idx = _db.getIndexes(_config.outputOptions.finalNamespace); - while ( idx->more() ) { - BSONObj i = idx->next(); - - BSONObjBuilder b( i.objsize() + 16 ); - b.append( "ns" , _config.tempNamespace ); - BSONObjIterator j( i ); - while ( j.more() ) { - BSONElement e = j.next(); - if ( str::equals( e.fieldName() , "_id" ) || - str::equals( e.fieldName() , "ns" ) ) - continue; - - b.append( e ); - } - - BSONObj indexToInsert = b.obj(); - Namespace tempNamespace(_config.tempNamespace.c_str()); - insert(tempNamespace.getSisterNS("system.indexes").c_str(), indexToInsert); + // create temp collection and insert the indexes from temporary storage + Client::WriteContext tempCtx( _config.tempNamespace ); + uassert(10001, "no longer master", isMasterNs(_config.tempNamespace.c_str())); + Collection* tempColl = tempCtx.ctx().db()->getCollection( _config.tempNamespace ); + if ( !tempColl ) { + CollectionOptions options; + options.temp = true; + tempColl = tempCtx.ctx().db()->createCollection( _config.tempNamespace, options ); + + // Log the createCollection operation. + BSONObjBuilder b; + b.append( "create", nsToCollectionSubstring( _config.tempNamespace )); + b.appendElements( options.toBSON() ); + string logNs = nsToDatabase( _config.tempNamespace ) + ".$cmd"; + logOp( "c", logNs.c_str(), b.obj() ); } + for ( vector::iterator it = indexesToInsert.begin(); + it != indexesToInsert.end(); ++it ) { + tempColl->getIndexCatalog()->createIndex( *it, false ); + // Log the createIndex operation. + string logNs = nsToDatabase( _config.tempNamespace ) + ".system.indexes"; + logOp( "i", logNs.c_str(), *it ); + } } } @@ -505,7 +564,7 @@ namespace mongo { auto_ptr cursor = _db.query( _config.tempNamespace , BSONObj() ); while ( cursor->more() ) { Lock::DBWrite lock( _config.outputOptions.finalNamespace ); - BSONObj o = cursor->next(); + BSONObj o = cursor->nextSafe(); Helpers::upsert( _config.outputOptions.finalNamespace , o ); getDur().commitIfNeeded(); pm.hit(); @@ -523,7 +582,7 @@ namespace mongo { auto_ptr cursor = _db.query( _config.tempNamespace , BSONObj() ); while ( cursor->more() ) { Lock::GlobalWrite lock; // TODO(erh) why global? - BSONObj temp = cursor->next(); + BSONObj temp = cursor->nextSafe(); BSONObj old; bool found; @@ -557,22 +616,35 @@ namespace mongo { } /** - * Insert doc in collection + * Insert doc in collection. This should be replicated. */ void State::insert( const string& ns , const BSONObj& o ) { verify( _onDisk ); Client::WriteContext ctx( ns ); + uassert(10004, "no longer master", isMasterNs(ns.c_str())); + Collection* coll = getCollectionOrUassert(ctx.ctx().db(), ns); + + BSONObjBuilder b; + if ( !o.hasField( "_id" ) ) { + b.appendOID( "_id", NULL, true ); + } + b.appendElements(o); + BSONObj bo = b.obj(); - theDataFileMgr.insertAndLog( ns.c_str() , o , false ); + uassertStatusOK( coll->insertDocument( bo, true ).getStatus() ); + logOp( "i", ns.c_str(), bo ); } /** - * Insert doc into the inc collection + * Insert doc into the inc collection. This should not be replicated. */ void State::_insertToInc( BSONObj& o ) { verify( _onDisk ); - theDataFileMgr.insertWithObjMod( _config.incLong.c_str(), o, false, true ); + + Client::WriteContext ctx( _config.incLong ); + Collection* coll = getCollectionOrUassert(ctx.ctx().db(), _config.incLong); + uassertStatusOK( coll->insertDocument( o, true ).getStatus() ); getDur().commitIfNeeded(); } @@ -622,7 +694,10 @@ namespace mongo { */ void State::init() { // setup js - _scope.reset(globalScriptEngine->getPooledScope( _config.dbname, "mapreduce" ).release() ); + const string userToken = ClientBasic::getCurrent()->getAuthorizationSession() + ->getAuthenticatedUserNamesToken(); + _scope.reset(globalScriptEngine->getPooledScope( + _config.dbname, "mapreduce" + userToken).release()); if ( ! _config.scopeSetup.isEmpty() ) _scope->init( &_config.scopeSetup ); @@ -764,6 +839,13 @@ namespace mongo { _config.reducer->numReduces = _scope->getNumberInt("_redCt"); } + Collection* State::getCollectionOrUassert(Database* db, const StringData& ns) { + Collection* out = db ? db->getCollection(ns) : NULL; + uassert(18697, "Collection unexpectedly disappeared: " + ns.toString(), + out); + return out; + } + /** * Applies last reduce and finalize on a list of tuples (key, val) * Inserts single result {_id: key, value: val} into temp collection @@ -835,12 +917,18 @@ namespace mongo { // use index on "0" to pull sorted data verify( _temp->size() == 0 ); BSONObj sortKey = BSON( "0" << 1 ); + { - bool foundIndex = false; + Client::WriteContext incCtx( _config.incLong ); + Collection* incColl = getCollectionOrUassert(incCtx.ctx().db(), _config.incLong ); - auto_ptr idx = _db.getIndexes( _config.incLong ); - while ( idx.get() && idx->more() ) { - BSONObj x = idx->next(); + bool foundIndex = false; + IndexCatalog::IndexIterator ii = + incColl->getIndexCatalog()->getIndexIterator( true ); + // Iterate over incColl's indexes. + while ( ii.more() ) { + IndexDescriptor* currIndex = ii.next(); + BSONObj x = currIndex->infoObj(); if ( sortKey.woCompare( x["key"].embeddedObject() ) == 0 ) { foundIndex = true; break; @@ -859,48 +947,52 @@ namespace mongo { "M/R: (3/3) Final Reduce Progress", _db.count(_config.incLong, BSONObj(), QueryOption_SlaveOk))); - shared_ptr temp = - NamespaceDetailsTransient::bestGuessCursor(_config.incLong.c_str(), - BSONObj(), - sortKey); - ClientCursor::Holder cursor(new ClientCursor(QueryOption_NoCursorTimeout, - temp, - _config.incLong.c_str())); - // iterate over all sorted objects - while ( cursor->ok() ) { - BSONObj o = cursor->current().getOwned(); - cursor->advance(); + CanonicalQuery* cq; + verify(CanonicalQuery::canonicalize(_config.incLong, BSONObj(), sortKey, BSONObj(), &cq).isOK()); + Runner* rawRunner; + verify(getRunner(cq, &rawRunner, QueryPlannerParams::NO_TABLE_SCAN).isOK()); + + auto_ptr runner(rawRunner); + const ScopedRunnerRegistration safety(runner.get()); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + // iterate over all sorted objects + BSONObj o; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&o, NULL))) { + o = o.getOwned(); // we will be accessing outside of the lock pm.hit(); if ( o.woSortOrder( prev , sortKey ) == 0 ) { // object is same as previous, add to array all.push_back( o ); if ( pm->hits() % 100 == 0 ) { - if ( ! cursor->yield() ) { - break; - } killCurrentOp.checkForInterrupt(); } continue; } - ClientCursor::YieldLock yield (cursor.get()); + runner->saveState(); + + // can't be a smart pointer since it needs throw annotation on destructor + dbtempreleasecond* yield = new dbtempreleasecond(); try { // reduce a finalize array finalReduce( all ); } catch (...) { - yield.relock(); + delete yield; // if throws, replaces current exception rather than terminating. throw; } + delete yield; all.clear(); prev = o; all.push_back( o ); - if ( ! yield.stillOk() ) { + if (!runner->restoreState()) { break; } @@ -910,7 +1002,7 @@ namespace mongo { { dbtempreleasecond tl; if ( ! tl.unlocked() ) - LOG( LL_WARNING ) << "map/reduce can't temp release" << endl; + warning() << "map/reduce can't temp release" << endl; // reduce and finalize last array finalReduce( all ); } @@ -936,25 +1028,23 @@ namespace mongo { _dupCount = 0; for ( InMemory::iterator i=_temp->begin(); i!=_temp->end(); ++i ) { - BSONObj key = i->first; BSONList& all = i->second; if ( all.size() == 1 ) { // only 1 value for this key if ( _onDisk ) { // this key has low cardinality, so just write to collection - Client::WriteContext ctx(_config.incLong.c_str()); _insertToInc( *(all.begin()) ); } else { // add to new map - _add( n.get() , all[0] , nSize ); + nSize += _add(n.get(), all[0]); } } else if ( all.size() > 1 ) { // several values, reduce and add to map BSONObj res = _config.reducer->reduce( all ); - _add( n.get() , res , nSize ); + nSize += _add(n.get(), res); } } @@ -971,7 +1061,6 @@ namespace mongo { return; Lock::DBWrite kl(_config.incLong); - Client::Context ctx(_config.incLong); for ( InMemory::iterator i=_temp->begin(); i!=_temp->end(); i++ ) { BSONList& all = i->second; @@ -991,21 +1080,25 @@ namespace mongo { */ void State::emit( const BSONObj& a ) { _numEmits++; - _add( _temp.get() , a , _size ); + _size += _add(_temp.get(), a); } - void State::_add( InMemory* im, const BSONObj& a , long& size ) { + int State::_add(InMemory* im, const BSONObj& a) { BSONList& all = (*im)[a]; all.push_back( a ); - size += a.objsize() + 16; - if (all.size() > 1) + if (all.size() > 1) { ++_dupCount; + } + + return a.objsize() + 16; } - /** - * this method checks the size of in memory map and potentially flushes to disk - */ - void State::checkSize() { + void State::reduceAndSpillInMemoryStateIfNeeded() { + // Make sure no DB read locks are held, because we might try to acquire write lock and + // upgrade is not supported. + // + dassert(!cc().lockState().hasAnyReadLock()); + if (_jsMode) { // try to reduce if it is beneficial int dupCt = _scope->getNumberInt("_dupCt"); @@ -1104,7 +1197,7 @@ namespace mongo { virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { - addPrivilegesRequiredForMapReduce(dbname, cmdObj, out); + addPrivilegesRequiredForMapReduce(this, dbname, cmdObj, out); } bool run(const string& dbname , BSONObj& cmd, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { @@ -1118,32 +1211,25 @@ namespace mongo { uassert( 16149 , "cannot run map reduce without the js engine", globalScriptEngine ); - ClientCursor::Holder holdCursor; - ShardChunkManagerPtr chunkManager; + CollectionMetadataPtr collMetadata; + // Prevent sharding state from changing during the MR. + auto_ptr rangePreserver; { - // Get chunk manager before we check our version, to make sure it doesn't increment - // in the meantime - if ( shardingState.needShardChunkManager( config.ns ) ) { - chunkManager = shardingState.getShardChunkManager( config.ns ); + Client::ReadContext ctx(config.ns); + Collection* collection = ctx.ctx().db()->getCollection( config.ns ); + if ( collection ) + rangePreserver.reset(new RangePreserver(collection)); + + // Get metadata before we check our version, to make sure it doesn't increment + // in the meantime. Need to do this in the same lock scope as the block. + if (shardingState.needCollectionMetadata(config.ns)) { + collMetadata = shardingState.getCollectionMetadata( config.ns ); } - - // Check our version immediately, to avoid migrations happening in the meantime while we do prep - Client::ReadContext ctx( config.ns ); - - // Get a very basic cursor, prevents deletion of migrated data while we m/r - shared_ptr temp = NamespaceDetailsTransient::getCursor( config.ns.c_str(), BSONObj(), BSONObj() ); - uassert( 15876, str::stream() << "could not create cursor over " << config.ns << " to hold data while prepping m/r", temp.get() ); - holdCursor.reset( new ClientCursor( QueryOption_NoCursorTimeout , temp , config.ns.c_str() ) ); - uassert( 15877, str::stream() << "could not create m/r holding client cursor over " << config.ns, holdCursor ); - } bool shouldHaveData = false; - long long num = 0; - long long inReduce = 0; - BSONObjBuilder countsBuilder; BSONObjBuilder timingBuilder; State state( config ); @@ -1165,88 +1251,110 @@ namespace mongo { state.init(); state.prepTempCollection(); ON_BLOCK_EXIT_OBJ(state, &State::dropTempCollections); - ProgressMeterHolder pm(op->setMessage("m/r: (1/3) emit phase", - "M/R: (1/3) Emit Progress", - state.incomingDocuments())); + + int progressTotal = 0; + bool showTotal = true; + if ( state.config().filter.isEmpty() ) { + progressTotal = state.incomingDocuments(); + } + else { + showTotal = false; + // Set an arbitrary total > 0 so the meter will be activated. + progressTotal = 1; + } + + ProgressMeter& progress( op->setMessage("m/r: (1/3) emit phase", + "M/R: (1/3) Emit Progress", + progressTotal )); + progress.showTotal(showTotal); + ProgressMeterHolder pm(progress); wassert( config.limit < 0x4000000 ); // see case on next line to 32 bit unsigned + long long mapTime = 0; + long long reduceTime = 0; + long long numInputs = 0; { // We've got a cursor preventing migrations off, now re-establish our useful cursor // Need lock and context to use it Lock::DBRead lock( config.ns ); + // This context does no version check, safe b/c we checked earlier and have an // open cursor - Client::Context ctx(config.ns, dbpath, false); - - // obtain full cursor on data to apply mr to - shared_ptr temp = NamespaceDetailsTransient::getCursor( config.ns.c_str(), config.filter, config.sort ); - uassert( 16052, str::stream() << "could not create cursor over " << config.ns << " for query : " << config.filter << " sort : " << config.sort, temp.get() ); - ClientCursor::Holder cursor(new ClientCursor(QueryOption_NoCursorTimeout, - temp, - config.ns.c_str())); - uassert( 16053, str::stream() << "could not create client cursor over " << config.ns << " for query : " << config.filter << " sort : " << config.sort, cursor.get() ); + Client::Context ctx(config.ns, storageGlobalParams.dbpath, false); - Timer mt; - // go through each doc - while ( cursor->ok() ) { - if ( ! cursor->yieldSometimes( ClientCursor::WillNeed ) ) { - cursor.release(); - break; - } + CanonicalQuery* cq; + if (!CanonicalQuery::canonicalize(config.ns, config.filter, config.sort, BSONObj(), &cq).isOK()) { + uasserted(17238, "Can't canonicalize query " + config.filter.toString()); + return 0; + } - if ( ! cursor->currentMatches() ) { - cursor->advance(); - continue; - } + Runner* rawRunner; + if (!getRunner(cq, &rawRunner).isOK()) { + uasserted(17239, "Can't get runner for query " + config.filter.toString()); + return 0; + } - // make sure we dont process duplicates in case data gets moved around during map - // TODO This won't actually help when data gets moved, it's to handle multikeys. - if ( cursor->currentIsDup() ) { - cursor->advance(); - continue; - } + auto_ptr runner(rawRunner); + const ScopedRunnerRegistration safety(runner.get()); + runner->setYieldPolicy(Runner::YIELD_AUTO); - BSONObj o = cursor->current(); - cursor->advance(); + Timer mt; + // go through each doc + BSONObj o; + while (Runner::RUNNER_ADVANCED == runner->getNext(&o, NULL)) { // check to see if this is a new object we don't own yet // because of a chunk migration - if ( chunkManager && ! chunkManager->belongsToMe( o ) ) - continue; + if ( collMetadata ) { + KeyPattern kp( collMetadata->getKeyPattern() ); + if ( !collMetadata->keyBelongsToMe( kp.extractSingleKey( o ) ) ) { + continue; + } + } // do map if ( config.verbose ) mt.reset(); config.mapper->map( o ); if ( config.verbose ) mapTime += mt.micros(); - num++; - if ( num % 100 == 0 ) { - // try to yield lock regularly - ClientCursor::YieldLock yield (cursor.get()); + // Check if the state accumulated so far needs to be written to a + // collection. This may yield the DB lock temporarily and then + // acquire it again. + // + numInputs++; + if (numInputs % 100 == 0) { Timer t; - // check if map needs to be dumped to disk - state.checkSize(); - inReduce += t.micros(); - if ( ! yield.stillOk() ) { - break; + // TODO: As an optimization, we might want to do the save/restore + // state and yield inside the reduceAndSpillInMemoryState method, + // so it only happens if necessary. + // + runner->saveState(); + { + dbtemprelease unlock; + state.reduceAndSpillInMemoryStateIfNeeded(); } + runner->restoreState(); + + reduceTime += t.micros(); killCurrentOp.checkForInterrupt(); } + pm.hit(); - if ( config.limit && num >= config.limit ) + if (config.limit && numInputs >= config.limit) break; } } pm.finished(); killCurrentOp.checkForInterrupt(); + // update counters - countsBuilder.appendNumber( "input" , num ); + countsBuilder.appendNumber("input", numInputs); countsBuilder.appendNumber( "emit" , state.numEmits() ); if ( state.numEmits() ) shouldHaveData = true; @@ -1264,9 +1372,9 @@ namespace mongo { state.dumpToInc(); // final reduce state.finalReduce( op , pm ); - inReduce += rt.micros(); + reduceTime += rt.micros(); countsBuilder.appendNumber( "reduce" , state.numReduces() ); - timingBuilder.appendNumber( "reduceTime" , inReduce / 1000 ); + timingBuilder.appendNumber("reduceTime", reduceTime / 1000); timingBuilder.append( "mode" , state.jsMode() ? "js" : "mixed" ); long long finalCount = state.postProcessCollection(op, pm); @@ -1313,6 +1421,7 @@ namespace mongo { */ class MapReduceFinishCommand : public Command { public: + void help(stringstream& h) const { h << "internal"; } MapReduceFinishCommand() : Command( "mapreduce.shardedfinish" ) {} virtual bool slaveOk() const { return !replSet; } virtual bool slaveOverrideOk() const { return true; } @@ -1321,8 +1430,8 @@ namespace mongo { const BSONObj& cmdObj, std::vector* out) { ActionSet actions; - actions.addAction(ActionType::mapReduceShardedFinish); - out->push_back(Privilege(dbname, actions)); + actions.addAction(ActionType::internal); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { ShardedConnectionInfo::addHook(); @@ -1354,7 +1463,6 @@ namespace mongo { ProgressMeterHolder pm(op->setMessage("m/r: merge sort and reduce", "M/R Merge Sort and Reduce Progress")); set servers; - vector< auto_ptr > shardCursors; { // parse per shard results @@ -1457,6 +1565,9 @@ namespace mongo { break; } + // Forget temporary input collection, if output is sharded collection + ShardConnection::forgetNS( inputNS ); + result.append( "chunkSizes" , chunkSizes.arr() ); long long outputCount = state.postProcessCollection(op, pm); diff --git a/src/mongo/db/commands/mr.h b/src/mongo/db/commands/mr.h index 68b329cf26e..5c3a550cbfe 100644 --- a/src/mongo/db/commands/mr.h +++ b/src/mongo/db/commands/mr.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -30,6 +42,9 @@ namespace mongo { + class Collection; + class Database; + namespace mr { typedef vector BSONList; @@ -240,10 +255,14 @@ namespace mongo { void emit( const BSONObj& a ); /** - * if size is big, run a reduce - * if its still big, dump to temp collection - */ - void checkSize(); + * Checks the size of the transient in-memory results accumulated so far and potentially + * runs reduce in order to compact them. If the data is still too large, it will be + * spilled to the output collection. + * + * NOTE: Make sure that no DB locks are held, when calling this function, because it may + * try to acquire write DB lock for the write to the output collection. + */ + void reduceAndSpillInMemoryStateIfNeeded(); /** * run reduce on _temp @@ -308,13 +327,21 @@ namespace mongo { void switchMode(bool jsMode); void bailFromJS(); + Collection* getCollectionOrUassert(Database* db, const StringData& ns); + const Config& _config; DBDirectClient _db; bool _useIncremental; // use an incremental collection protected: - void _add( InMemory* im , const BSONObj& a , long& size ); + /** + * Appends a new document to the in-memory list of tuples, which are under that + * document's key. + * + * @return estimated in-memory size occupied by the newly added document. + */ + int _add(InMemory* im , const BSONObj& a); scoped_ptr _scope; bool _onDisk; // if the end result of this map reduce is disk or not @@ -335,7 +362,8 @@ namespace mongo { BSONObj fast_emit( const BSONObj& args, void* data ); BSONObj _bailFromJS( const BSONObj& args, void* data ); - void addPrivilegesRequiredForMapReduce(const std::string& dbname, + void addPrivilegesRequiredForMapReduce(Command* commandTemplate, + const std::string& dbname, const BSONObj& cmdObj, std::vector* out); } // end mr namespace diff --git a/src/mongo/db/commands/mr_common.cpp b/src/mongo/db/commands/mr_common.cpp index 0fe6db47121..b0d6c9059da 100644 --- a/src/mongo/db/commands/mr_common.cpp +++ b/src/mongo/db/commands/mr_common.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/db/commands/mr.h" @@ -22,6 +34,7 @@ #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" #include "mongo/db/jsobj.h" #include "mongo/util/mongoutils/str.h" @@ -89,17 +102,20 @@ namespace mongo { return outputOptions; } - void addPrivilegesRequiredForMapReduce(const std::string& dbname, + void addPrivilegesRequiredForMapReduce(Command* commandTemplate, + const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { Config::OutputOptions outputOptions = Config::parseOutputOptions(dbname, cmdObj); - ActionSet inputActions, outputActions; - inputActions.addAction(ActionType::find); - std::string inputNs = dbname + '.' + cmdObj.firstElement().valuestr(); - out->push_back(Privilege(inputNs, inputActions)); + ResourcePattern inputResource(commandTemplate->parseResourcePattern(dbname, cmdObj)); + uassert(17142, mongoutils::str::stream() << + "Invalid input resource " << inputResource.toString(), + inputResource.isExactNamespacePattern()); + out->push_back(Privilege(inputResource, ActionType::find)); if (outputOptions.outType != Config::INMEMORY) { + ActionSet outputActions; outputActions.addAction(ActionType::insert); if (outputOptions.outType == Config::REPLACE) { outputActions.addAction(ActionType::remove); @@ -108,9 +124,15 @@ namespace mongo { outputActions.addAction(ActionType::update); } - std::string outputNs = outputOptions.finalNamespace; + ResourcePattern outputResource( + ResourcePattern::forExactNamespace( + NamespaceString(outputOptions.finalNamespace))); + uassert(17143, mongoutils::str::stream() << "Invalid target namespace " << + outputResource.ns().ns(), + outputResource.ns().isValid()); + // TODO: check if outputNs exists and add createCollection privilege if not - out->push_back(Privilege(outputNs, outputActions)); + out->push_back(Privilege(outputResource, outputActions)); } } } diff --git a/src/mongo/db/commands/oplog_note.cpp b/src/mongo/db/commands/oplog_note.cpp new file mode 100644 index 00000000000..a1d280b3755 --- /dev/null +++ b/src/mongo/db/commands/oplog_note.cpp @@ -0,0 +1,82 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include + +#include "mongo/bson/util/bson_extract.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/commands.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/replication_server_status.h" + +namespace mongo { + class AppendOplogNoteCmd : public Command { + public: + AppendOplogNoteCmd() : Command( "appendOplogNote" ) {} + virtual bool slaveOk() const { return false; } + virtual bool adminOnly() const { return true; } + virtual LockType locktype() const { return NONE; } + virtual void help( stringstream &help ) const { + help << "Adds a no-op entry to the oplog"; + } + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::appendOplogNote)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + return Status::OK(); + } + virtual bool run(const string& dbname, + BSONObj& cmdObj, + int, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + if (!replSettings.master) { + return appendCommandStatus(result, Status( + ErrorCodes::NoReplicationEnabled, + "Must have replication set up to run \"appendOplogNote\"")); + } + BSONElement dataElement; + Status status = bsonExtractTypedField(cmdObj, "data", Object, &dataElement); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + logOpComment(dataElement.Obj()); + return true; + } + + } appendOplogNoteCmd; + +} // namespace mongo diff --git a/src/mongo/db/commands/parallel_collection_scan.cpp b/src/mongo/db/commands/parallel_collection_scan.cpp new file mode 100644 index 00000000000..fe2b01b850e --- /dev/null +++ b/src/mongo/db/commands/parallel_collection_scan.cpp @@ -0,0 +1,254 @@ +// parallel_collection_scan.cpp + +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" + +namespace mongo { + + + class ParallelCollectionScanCmd : public Command { + public: + + struct ExtentInfo { + ExtentInfo( DiskLoc dl, size_t s ) + : diskLoc(dl), size(s) { + } + DiskLoc diskLoc; + size_t size; + }; + + class ExtentRunner : public Runner { + public: + ExtentRunner( const StringData& ns, + Database* db, + Collection* collection, + const vector& extents ) + : _ns( ns.toString() ), + _collection( collection ), + _extents( extents ), + _extentManager( db->getExtentManager() ) { + + invariant( _extents.size() > 0 ); + + _currentExtent = 0; + _currentRecord = _getExtent( _currentExtent )->firstRecord; + if ( _currentRecord.isNull() ) + _advance(); + } + ~ExtentRunner() { + } + + virtual RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut) { + if ( _collection == NULL ) + return RUNNER_DEAD; + if ( _currentRecord.isNull() ) + return RUNNER_EOF; + + if ( objOut ) + *objOut = _collection->docFor( _currentRecord ); + if ( dlOut ) + *dlOut = _currentRecord; + _advance(); + return RUNNER_ADVANCED; + } + + virtual bool isEOF() { + return _collection == NULL || _currentRecord.isNull(); + } + virtual void kill() { + _collection = NULL; + } + virtual void setYieldPolicy(YieldPolicy policy) { + invariant( false ); + } + virtual void saveState() {} + virtual bool restoreState() { return true;} + virtual const string& ns() { return _ns; } + virtual void invalidate(const DiskLoc& dl, InvalidationType type) { + switch ( type ) { + case INVALIDATION_DELETION: + if ( dl == _currentRecord ) + _advance(); + break; + case INVALIDATION_MUTATION: + // no-op + break; + } + } + virtual const Collection* collection() { + return _collection; + } + virtual Status getInfo(TypeExplain** explain, PlanInfo** planInfo) const { + return Status( ErrorCodes::InternalError, "no" ); + } + private: + + /** + * @return if more data + */ + bool _advance() { + + while ( _currentRecord.isNull() ) { + // need to move to next extent + if ( _currentExtent + 1 >= _extents.size() ) + return false; + _currentExtent++; + _currentRecord = _getExtent( _currentExtent )->firstRecord; + if ( !_currentRecord.isNull() ) + return true; + // if we're here, the extent was empty, keep looking + } + + // we're in an extent, advance + _currentRecord = _extentManager.getNextRecordInExtent( _currentRecord ); + if ( _currentRecord.isNull() ) { + // finished this extent, need to move to the next one + return _advance(); + } + return true; + } + + Extent* _getExtent( size_t offset ) { + DiskLoc dl = _extents[offset].diskLoc; + return _extentManager.getExtent( dl ); + } + + string _ns; + Collection* _collection; + vector _extents; + ExtentManager& _extentManager; + + size_t _currentExtent; + DiskLoc _currentRecord; + }; + + // ------------------------------------------------ + + ParallelCollectionScanCmd() : Command( "parallelCollectionScan" ){} + + virtual LockType locktype() const { return READ; } + virtual bool logTheOp() { return false; } + virtual bool slaveOk() const { return true; } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + ActionSet actions; + actions.addAction(ActionType::find); + Privilege p(parseResourcePattern(dbname, cmdObj), actions); + if ( client->getAuthorizationSession()->isAuthorizedForPrivilege(p) ) + return Status::OK(); + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + virtual bool run( const string& dbname, BSONObj& cmdObj, int options, + string& errmsg, BSONObjBuilder& result, + bool fromRepl = false ) { + + NamespaceString ns( dbname, cmdObj[name].String() ); + + Database* db = cc().database(); + Collection* collection = db->getCollection( ns ); + + if ( !collection ) + return appendCommandStatus( result, + Status( ErrorCodes::NamespaceNotFound, + str::stream() << + "ns does not exist: " << ns.ns() ) ); + + size_t numCursors = static_cast( cmdObj["numCursors"].numberInt() ); + + if ( numCursors == 0 || numCursors > 10000 ) + return appendCommandStatus( result, + Status( ErrorCodes::BadValue, + str::stream() << + "numCursors has to be between 1 and 10000" << + " was: " << numCursors ) ); + + vector< vector > buckets; + + const ExtentManager& extentManager = db->getExtentManager(); + + { + DiskLoc extentDiskLoc = collection->details()->firstExtent(); + int extentNumber = 0; + while (!extentDiskLoc.isNull()) { + + Extent* thisExtent = extentManager.getExtent( extentDiskLoc ); + ExtentInfo info( extentDiskLoc, thisExtent->length ); + if ( buckets.size() < numCursors ) { + vector v; + v.push_back( info ); + buckets.push_back( v ); + } + else { + buckets[ extentNumber % buckets.size() ].push_back( info ); + } + + extentDiskLoc = thisExtent->xnext; + extentNumber++; + } + + BSONArrayBuilder bucketsBuilder; + for ( size_t i = 0; i < buckets.size(); i++ ) { + + auto_ptr runner( new ExtentRunner( ns.ns(), + db, + collection, + buckets[i] ) ); + ClientCursor* cc = new ClientCursor( collection, runner.release() ); + + // we are mimicking the aggregation cursor output here + // that is why there are ns, ok and empty firstBatch + BSONObjBuilder threadResult; + { + BSONObjBuilder cursor; + cursor.appendArray( "firstBatch", BSONObj() ); + cursor.append( "ns", ns ); + cursor.append( "id", cc->cursorid() ); + threadResult.append( "cursor", cursor.obj() ); + } + threadResult.appendBool( "ok", 1 ); + + bucketsBuilder.append( threadResult.obj() ); + } + result.appendArray( "cursors", bucketsBuilder.obj() ); + } + + return true; + + } + } parallelCollectionScanCmd; + +} diff --git a/src/mongo/db/commands/parameters.cpp b/src/mongo/db/commands/parameters.cpp index 3a4c49f4096..a1ad06d4f69 100644 --- a/src/mongo/db/commands/parameters.cpp +++ b/src/mongo/db/commands/parameters.cpp @@ -14,13 +14,32 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" + +#include "mongo/client/replica_set_monitor.h" +#include "mongo/client/sasl_client_authenticate.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/security_key.h" #include "mongo/db/commands.h" -#include "mongo/db/cmdline.h" -#include "mongo/client/dbclient_rs.h" #include "mongo/db/server_parameters.h" +#include "mongo/db/storage_options.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/net/ssl_manager.h" +#include "mongo/util/net/ssl_options.h" namespace mongo { @@ -45,7 +64,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::getParameter); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } virtual void help( stringstream &help ) const { help << "get administrative option(s)\nexample:\n"; @@ -58,6 +77,22 @@ namespace mongo { int before = result.len(); + // TODO: convert to ServerParameters -- SERVER-10515 + + if (isJournalingEnabled() && (all || cmdObj.hasElement("journalCommitInterval")) && + !isMongos()) { + result.append("journalCommitInterval", + getJournalCommitInterval()); + } + if( all || cmdObj.hasElement( "traceExceptions" ) ) { + result.append("traceExceptions", + DBException::traceExceptions); + } + if( all || cmdObj.hasElement( "replMonitorMaxFailedChecks" ) ) { + result.append("replMonitorMaxFailedChecks", + ReplicaSetMonitor::maxConsecutiveFailedChecks); + } + const ServerParameter::Map& m = ServerParameterSet::getGlobal()->getMap(); for ( ServerParameter::Map::const_iterator i = m.begin(); i != m.end(); ++i ) { if ( all || cmdObj.hasElement( i->first.c_str() ) ) { @@ -84,7 +119,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::setParameter); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } virtual void help( stringstream &help ) const { help << "set administrative option(s)\n"; @@ -95,16 +130,20 @@ namespace mongo { int s = 0; bool found = false; - // TODO: remove these manual things + // TODO: convert to ServerParameters -- SERVER-10515 if( cmdObj.hasElement("journalCommitInterval") ) { - if( !cmdLine.dur ) { + if (isMongos()) { + errmsg = "cannot set journalCommitInterval on a mongos"; + return false; + } + if(!isJournalingEnabled()) { errmsg = "journaling is off"; return false; } int x = (int) cmdObj["journalCommitInterval"].Number(); verify( x > 1 && x < 500 ); - cmdLine.journalCommitInterval = x; + setJournalCommitInterval(x); log() << "setParameter journalCommitInterval=" << x << endl; s++; } @@ -114,9 +153,9 @@ namespace mongo { s++; } if( cmdObj.hasElement( "replMonitorMaxFailedChecks" ) ) { - if( s == 0 ) result.append( "was", ReplicaSetMonitor::getMaxFailedChecks() ); - ReplicaSetMonitor::setMaxFailedChecks( - cmdObj["replMonitorMaxFailedChecks"].numberInt() ); + if( s == 0 ) result.append( "was", ReplicaSetMonitor::maxConsecutiveFailedChecks ); + ReplicaSetMonitor::maxConsecutiveFailedChecks = + cmdObj["replMonitorMaxFailedChecks"].numberInt(); s++; } @@ -160,29 +199,199 @@ namespace mongo { } cmdSet; namespace { - ExportedServerParameter LogLevelSetting( ServerParameterSet::getGlobal(), - "logLevel", - &logLevel, - true, - true ); - - ExportedServerParameter NoTableScanSetting( ServerParameterSet::getGlobal(), - "notablescan", - &cmdLine.noTableScan, - true, - true ); + class LogLevelSetting : public ServerParameter { + public: + LogLevelSetting() : ServerParameter(ServerParameterSet::getGlobal(), "logLevel") {} + + virtual void append(BSONObjBuilder& b, const std::string& name) { + b << name << logger::globalLogDomain()->getMinimumLogSeverity().toInt(); + } + + virtual Status set(const BSONElement& newValueElement) { + typedef logger::LogSeverity LogSeverity; + int newValue; + if (!newValueElement.coerce(&newValue) || newValue < 0) + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Invalid value for logLevel: " << newValueElement); + LogSeverity newSeverity = (newValue > 0) ? LogSeverity::Debug(newValue) : + LogSeverity::Log(); + logger::globalLogDomain()->setMinimumLoggedSeverity(newSeverity); + return Status::OK(); + } + + virtual Status setFromString(const std::string& str) { + typedef logger::LogSeverity LogSeverity; + int newValue; + Status status = parseNumberFromString(str, &newValue); + if (!status.isOK()) + return status; + if (newValue < 0) + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Invalid value for logLevel: " << newValue); + LogSeverity newSeverity = (newValue > 0) ? LogSeverity::Debug(newValue) : + LogSeverity::Log(); + logger::globalLogDomain()->setMinimumLoggedSeverity(newSeverity); + return Status::OK(); + } + } logLevelSetting; + + class SSLModeSetting : public ServerParameter { + public: + SSLModeSetting() : ServerParameter(ServerParameterSet::getGlobal(), "sslMode", + false, // allowedToChangeAtStartup + true // allowedToChangeAtRuntime + ) {} + + std::string sslModeStr() { + switch (sslGlobalParams.sslMode.load()) { + case SSLGlobalParams::SSLMode_disabled: + return "disabled"; + case SSLGlobalParams::SSLMode_allowSSL: + return "allowSSL"; + case SSLGlobalParams::SSLMode_preferSSL: + return "preferSSL"; + case SSLGlobalParams::SSLMode_requireSSL: + return "requireSSL"; + default: + return "undefined"; + } + } + + virtual void append(BSONObjBuilder& b, const std::string& name) { + b << name << sslModeStr(); + } + + virtual Status set(const BSONElement& newValueElement) { + try { + return setFromString(newValueElement.String()); + } + catch (MsgAssertionException msg) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Invalid value for sslMode via setParameter command: " + << newValueElement); + } + + } + + virtual Status setFromString(const std::string& str) { +#ifndef MONGO_SSL + return Status(ErrorCodes::IllegalOperation, mongoutils::str::stream() << + "Unable to set sslMode, SSL support is not compiled into server"); +#endif + if (str != "disabled" && str != "allowSSL" && + str != "preferSSL" && str != "requireSSL") { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Invalid value for sslMode via setParameter command: " + << str); + } + + int oldMode = sslGlobalParams.sslMode.load(); + if (str == "preferSSL" && oldMode == SSLGlobalParams::SSLMode_allowSSL) { + sslGlobalParams.sslMode.store(SSLGlobalParams::SSLMode_preferSSL); + } + else if (str == "requireSSL" && oldMode == SSLGlobalParams::SSLMode_preferSSL) { + sslGlobalParams.sslMode.store(SSLGlobalParams::SSLMode_requireSSL); + } + else { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Illegal state transition for sslMode, attempt to change from " + << sslModeStr() << " to " << str); + } + return Status::OK(); + } + } sslModeSetting; + + class ClusterAuthModeSetting : public ServerParameter { + public: + ClusterAuthModeSetting() : + ServerParameter(ServerParameterSet::getGlobal(), "clusterAuthMode", + false, // allowedToChangeAtStartup + true // allowedToChangeAtRuntime + ) {} + + std::string clusterAuthModeStr() { + switch (serverGlobalParams.clusterAuthMode.load()) { + case ServerGlobalParams::ClusterAuthMode_keyFile: + return "keyFile"; + case ServerGlobalParams::ClusterAuthMode_sendKeyFile: + return "sendKeyFile"; + case ServerGlobalParams::ClusterAuthMode_sendX509: + return "sendX509"; + case ServerGlobalParams::ClusterAuthMode_x509: + return "x509"; + default: + return "undefined"; + } + } + + virtual void append(BSONObjBuilder& b, const std::string& name) { + b << name << clusterAuthModeStr(); + } + + virtual Status set(const BSONElement& newValueElement) { + try { + return setFromString(newValueElement.String()); + } + catch (MsgAssertionException msg) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Invalid value for clusterAuthMode via setParameter command: " + << newValueElement); + } + + } + + virtual Status setFromString(const std::string& str) { +#ifndef MONGO_SSL + return Status(ErrorCodes::IllegalOperation, mongoutils::str::stream() << + "Unable to set clusterAuthMode, " << + "SSL support is not compiled into server"); +#endif + if (str != "keyFile" && str != "sendKeyFile" && + str != "sendX509" && str != "x509") { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Invalid value for clusterAuthMode via setParameter command: " + << str); + } + + int oldMode = serverGlobalParams.clusterAuthMode.load(); + int sslMode = sslGlobalParams.sslMode.load(); + if (str == "sendX509" && + oldMode == ServerGlobalParams::ClusterAuthMode_sendKeyFile) { + if (sslMode == SSLGlobalParams::SSLMode_disabled || + sslMode == SSLGlobalParams::SSLMode_allowSSL) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Illegal state transition for clusterAuthMode, " << + "need to enable SSL for outgoing connections"); + } + serverGlobalParams.clusterAuthMode.store + (ServerGlobalParams::ClusterAuthMode_sendX509); +#ifdef MONGO_SSL + setInternalUserAuthParams(BSON(saslCommandMechanismFieldName << + "MONGODB-X509" << + saslCommandUserDBFieldName << "$external" << + saslCommandUserFieldName << + getSSLManager()->getClientSubjectName())); +#endif + } + else if (str == "x509" && + oldMode == ServerGlobalParams::ClusterAuthMode_sendX509) { + serverGlobalParams.clusterAuthMode.store + (ServerGlobalParams::ClusterAuthMode_x509); + } + else { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "Illegal state transition for clusterAuthMode, change from " + << clusterAuthModeStr() << " to " << str); + } + return Status::OK(); + } + } clusterAuthModeSetting; ExportedServerParameter QuietSetting( ServerParameterSet::getGlobal(), "quiet", - &cmdLine.quiet, + &serverGlobalParams.quiet, true, true ); - - ExportedServerParameter SyncdelaySetting( ServerParameterSet::getGlobal(), - "syncdelay", - &cmdLine.syncdelay, - true, - true ); } } diff --git a/src/mongo/db/commands/pipeline_command.cpp b/src/mongo/db/commands/pipeline_command.cpp index d5905325bd2..8763fda4b7b 100644 --- a/src/mongo/db/commands/pipeline_command.cpp +++ b/src/mongo/db/commands/pipeline_command.cpp @@ -12,15 +12,31 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" +#include #include #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/curop.h" #include "mongo/db/commands.h" #include "mongo/db/interrupt_status_mongod.h" #include "mongo/db/pipeline/accumulator.h" @@ -30,9 +46,210 @@ #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/pipeline_d.h" #include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/query/find_constants.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/storage_options.h" namespace mongo { +namespace { + + /** + * This is a Runner implementation backed by an aggregation pipeline. + */ + class PipelineRunner : public Runner { + public: + PipelineRunner(intrusive_ptr pipeline, const boost::shared_ptr& child) + : _pipeline(pipeline) + , _includeMetaData(_pipeline->getContext()->inShard) // send metadata to merger + , _childRunner(child) + {} + + virtual RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut) { + if (!objOut || dlOut) + return RUNNER_ERROR; + + if (!_stash.empty()) { + *objOut = _stash.back(); + _stash.pop_back(); + return RUNNER_ADVANCED; + } + + if (boost::optional next = getNextBson()) { + *objOut = *next; + return RUNNER_ADVANCED; + } + + return RUNNER_EOF; + } + virtual bool isEOF() { + if (!_stash.empty()) + return false; + + if (boost::optional next = getNextBson()) { + _stash.push_back(*next); + return false; + } + + return true; + } + virtual const string& ns() { + return _pipeline->getContext()->ns.ns(); + } + + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const { + // This should never get called in practice anyway. + return Status(ErrorCodes::InternalError, + "PipelineCursor doesn't implement getExplainPlan"); + } + + // propagate to child runner if still in use + virtual void invalidate(const DiskLoc& dl, InvalidationType type) { + if (boost::shared_ptr runner = _childRunner.lock()) { + runner->invalidate(dl, type); + } + } + virtual void kill() { + if (boost::shared_ptr runner = _childRunner.lock()) { + runner->kill(); + } + } + + // These are all no-ops for PipelineRunners + virtual void setYieldPolicy(YieldPolicy policy) {} + virtual void saveState() {} + virtual bool restoreState() { return true; } + virtual const Collection* collection() { return NULL; } + + /** + * Make obj the next object returned by getNext(). + */ + void pushBack(const BSONObj& obj) { + _stash.push_back(obj); + } + + private: + boost::optional getNextBson() { + if (boost::optional next = _pipeline->output()->getNext()) { + if (_includeMetaData) { + return next->toBsonWithMetaData(); + } + else { + return next->toBson(); + } + } + + return boost::none; + } + + // Things in the _stash sould be returned before pulling items from _pipeline. + const intrusive_ptr _pipeline; + vector _stash; + const bool _includeMetaData; + boost::weak_ptr _childRunner; + }; +} + + static bool isCursorCommand(BSONObj cmdObj) { + BSONElement cursorElem = cmdObj["cursor"]; + if (cursorElem.eoo()) + return false; + + uassert(16954, "cursor field must be missing or an object", + cursorElem.type() == Object); + + BSONObj cursor = cursorElem.embeddedObject(); + BSONElement batchSizeElem = cursor["batchSize"]; + if (batchSizeElem.eoo()) { + uassert(16955, "cursor object can't contain fields other than batchSize", + cursor.isEmpty()); + } + else { + uassert(16956, "cursor.batchSize must be a number", + batchSizeElem.isNumber()); + + // This can change in the future, but for now all negatives are reserved. + uassert(16957, "Cursor batchSize must not be negative", + batchSizeElem.numberLong() >= 0); + } + + return true; + } + + static void handleCursorCommand(const string& ns, + ClientCursorPin* pin, + PipelineRunner* runner, + const BSONObj& cmdObj, + BSONObjBuilder& result) { + + ClientCursor* cursor = pin ? pin->c() : NULL; + if (pin) { + invariant(cursor); + invariant(cursor->getRunner() == runner); + invariant(cursor->isAggCursor); + } + + BSONElement batchSizeElem = cmdObj.getFieldDotted("cursor.batchSize"); + const long long batchSize = batchSizeElem.isNumber() + ? batchSizeElem.numberLong() + : 101; // same as query + + // can't use result BSONObjBuilder directly since it won't handle exceptions correctly. + BSONArrayBuilder resultsArray; + const int byteLimit = MaxBytesToReturnToClientAtOnce; + BSONObj next; + for (int objCount = 0; objCount < batchSize; objCount++) { + // The initial getNext() on a PipelineRunner may be very expensive so we don't + // do it when batchSize is 0 since that indicates a desire for a fast return. + if (runner->getNext(&next, NULL) != Runner::RUNNER_ADVANCED) { + if (pin) pin->deleteUnderlying(); + // make it an obvious error to use cursor or runner after this point + cursor = NULL; + runner = NULL; + break; + } + + if (resultsArray.len() + next.objsize() > byteLimit) { + // too big. next will be the first doc in the second batch + runner->pushBack(next); + break; + } + + resultsArray.append(next); + } + + // NOTE: runner->isEOF() can have side effects such as writing by $out. However, it should + // be relatively quick since if there was no pin then the input is empty. Also, this + // violates the contract for batchSize==0. Sharding requires a cursor to be returned in that + // case. This is ok for now however, since you can't have a sharded collection that doesn't + // exist. + const bool canReturnMoreBatches = pin; + if (!canReturnMoreBatches && runner && !runner->isEOF()) { + // msgasserting since this shouldn't be possible to trigger from today's aggregation + // language. The wording assumes that the only reason pin would be null is if the + // collection doesn't exist. + msgasserted(17391, str::stream() + << "Aggregation has more results than fit in initial batch, but can't " + << "create cursor since collection " << ns << " doesn't exist"); + } + + if (cursor) { + // If a time limit was set on the pipeline, remaining time is "rolled over" to the + // cursor (for use by future getmore ops). + cursor->setLeftoverMaxTimeMicros( cc().curop()->getRemainingMaxTimeMicros() ); + + cc().curop()->debug().cursorid = cursor->cursorid(); + } + + BSONObjBuilder cursorObj(result.subobjStart("cursor")); + cursorObj.append("id", cursor ? cursor->cursorid() : 0LL); + cursorObj.append("ns", ns); + cursorObj.append("firstBatch", resultsArray.arr()); + cursorObj.done(); + } + + class PipelineCommand : public Command { public: @@ -40,143 +257,121 @@ namespace mongo { // Locks are managed manually, in particular by DocumentSourceCursor. virtual LockType locktype() const { return NONE; } - virtual bool slaveOk() const { return true; } + virtual bool slaveOk() const { return false; } + virtual bool slaveOverrideOk() const { return true; } virtual void help(stringstream &help) const { - help << "{ pipeline : [ { : {...}}, ... ] }"; + help << "{ pipeline: [ { $operator: {...}}, ... ]" + << ", explain: " + << ", allowDiskUse: " + << ", cursor: {batchSize: }" + << " }" + << endl + << "See http://dochub.mongodb.org/core/aggregation for more details." + ; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + Pipeline::addRequiredPrivileges(this, dbname, cmdObj, out); } virtual bool run(const string &db, BSONObj &cmdObj, int options, string &errmsg, BSONObjBuilder &result, bool fromRepl) { + string ns = parseNs(db, cmdObj); + intrusive_ptr pCtx = - ExpressionContext::create(&InterruptStatusMongod::status); + new ExpressionContext(InterruptStatusMongod::status, NamespaceString(ns)); + pCtx->tempDir = storageGlobalParams.dbpath + "/_tmp"; /* try to parse the command; if this fails, then we didn't run */ intrusive_ptr pPipeline = Pipeline::parseCommand(errmsg, cmdObj, pCtx); if (!pPipeline.get()) return false; - string ns = parseNs(db, cmdObj); - - if (pPipeline->getSplitMongodPipeline()) { - // This is only used in testing - return executeSplitPipeline(result, errmsg, ns, db, pPipeline, pCtx); - } - #if _DEBUG // This is outside of the if block to keep the object alive until the pipeline is finished. BSONObj parsed; - if (!pPipeline->isExplain() && !pCtx->getInShard()) { + if (!pPipeline->isExplain() && !pCtx->inShard) { // Make sure all operations round-trip through Pipeline::toBson() // correctly by reparsing every command on DEBUG builds. This is // important because sharded aggregations rely on this ability. // Skipping when inShard because this has already been through the // transformation (and this unsets pCtx->inShard). - BSONObjBuilder bb; - pPipeline->toBson(&bb); - parsed = bb.obj(); + parsed = pPipeline->serialize().toBson(); pPipeline = Pipeline::parseCommand(errmsg, parsed, pCtx); verify(pPipeline); } #endif - // This does the mongod-specific stuff like creating a cursor - PipelineD::prepareCursorSource(pPipeline, nsToDatabase(ns), pCtx); - return pPipeline->run(result, errmsg); - } + PipelineRunner* runner = NULL; + scoped_ptr pin; // either this OR the runnerHolder will be non-null + auto_ptr runnerHolder; + { + // This will throw if the sharding version for this connection is out of date. The + // lock must be held continuously from now until we have we created both the output + // ClientCursor and the input Runner. This ensures that both are using the same + // sharding version that we synchronize on here. This is also why we always need to + // create a ClientCursor even when we aren't outputting to a cursor. See the comment + // on ShardFilterStage for more details. + Client::ReadContext ctx(ns); - private: - /* - Execute the pipeline for the explain. This is common to both the - locked and unlocked code path. However, the results are different. - For an explain, with no lock, it really outputs the pipeline - chain rather than fetching the data. - */ - bool executeSplitPipeline(BSONObjBuilder& result, string& errmsg, - const string& ns, const string& db, - intrusive_ptr& pPipeline, - intrusive_ptr& pCtx) { - /* setup as if we're in the router */ - pCtx->setInRouter(true); - - /* - Here, we'll split the pipeline in the same way we would for sharding, - for testing purposes. - - Run the shard pipeline first, then feed the results into the remains - of the existing pipeline. - - Start by splitting the pipeline. - */ - intrusive_ptr pShardSplit = pPipeline->splitForSharded(); - - /* - Write the split pipeline as we would in order to transmit it to - the shard servers. - */ - BSONObjBuilder shardBuilder; - pShardSplit->toBson(&shardBuilder); - BSONObj shardBson(shardBuilder.done()); - - DEV (log() << "\n---- shardBson\n" << - shardBson.jsonString(Strict, 1) << "\n----\n").flush(); - - /* for debugging purposes, show what the pipeline now looks like */ - DEV { - BSONObjBuilder pipelineBuilder; - pPipeline->toBson(&pipelineBuilder); - BSONObj pipelineBson(pipelineBuilder.done()); - (log() << "\n---- pipelineBson\n" << - pipelineBson.jsonString(Strict, 1) << "\n----\n").flush(); - } + Collection* collection = ctx.ctx().db()->getCollection(ns); - /* on the shard servers, create the local pipeline */ - intrusive_ptr pShardCtx( - ExpressionContext::create(&InterruptStatusMongod::status)); - intrusive_ptr pShardPipeline( - Pipeline::parseCommand(errmsg, shardBson, pShardCtx)); - if (!pShardPipeline.get()) { - return false; + // This does mongod-specific stuff like creating the input Runner and adding to the + // front of the pipeline if needed. + boost::shared_ptr input = PipelineD::prepareCursorSource(pPipeline, pCtx); + pPipeline->stitch(); + + runnerHolder.reset(new PipelineRunner(pPipeline, input)); + runner = runnerHolder.get(); + + if (!collection && input) { + // If we don't have a collection, we won't be able to register any Runners, so + // make sure that the input Runner (likely an EOFRunner) doesn't need to be + // registered. + invariant(!input->collection()); + } + + if (collection) { + ClientCursor* cursor = new ClientCursor(collection, + runnerHolder.release(), + 0, /* queryOptions */ + cmdObj.getOwned()); + + cursor->isAggCursor = true; // enable special locking behavior + pin.reset(new ClientCursorPin(collection, cursor->cursorid())); + // Don't add any code between here and the start of the try block. + } } - PipelineD::prepareCursorSource(pShardPipeline, nsToDatabase(ns), pCtx); - - /* run the shard pipeline */ - BSONObjBuilder shardResultBuilder; - string shardErrmsg; - pShardPipeline->run(shardResultBuilder, shardErrmsg); - BSONObj shardResult(shardResultBuilder.done()); - - /* pick out the shard result, and prepare to read it */ - intrusive_ptr pShardSource; - BSONObjIterator shardIter(shardResult); - while(shardIter.more()) { - BSONElement shardElement(shardIter.next()); - const char *pFieldName = shardElement.fieldName(); - - if ((strcmp(pFieldName, "result") == 0) || - (strcmp(pFieldName, "serverPipeline") == 0)) { - pPipeline->addInitialSource(DocumentSourceBsonArray::create(&shardElement, pCtx)); - - /* - Connect the output of the shard pipeline with the mongos - pipeline that will merge the results. - */ - return pPipeline->run(result, errmsg); + try { + // Unless set to true, the ClientCursor created above will be deleted on block exit. + bool keepCursor = false; + + // If both explain and cursor are specified, explain wins. + if (pPipeline->isExplain()) { + result << "stages" << Value(pPipeline->writeExplainOps()); } + else if (isCursorCommand(cmdObj)) { + handleCursorCommand(ns, pin.get(), runner, cmdObj, result); + keepCursor = true; + } + else { + pPipeline->run(result); + } + + if (!keepCursor && pin) pin->deleteUnderlying(); } + catch (...) { + // Clean up cursor on way out of scope. + if (pin) pin->deleteUnderlying(); + throw; + } + // Any code that needs the cursor pinned must be inside the try block, above. - /* NOTREACHED */ - verify(false); - return false; + return true; } } cmdPipeline; diff --git a/src/mongo/db/commands/plan_cache_commands.cpp b/src/mongo/db/commands/plan_cache_commands.cpp new file mode 100644 index 00000000000..f1e44f0a0ab --- /dev/null +++ b/src/mongo/db/commands/plan_cache_commands.cpp @@ -0,0 +1,419 @@ +/** +* Copyright (C) 2013 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include +#include + +#include "mongo/base/init.h" +#include "mongo/base/status.h" +#include "mongo/db/client.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/commands/plan_cache_commands.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/plan_ranker.h" + +namespace { + + using std::string; + using namespace mongo; + + /** + * Utility function to extract error code and message from status + * and append to BSON results. + */ + void addStatus(const Status& status, BSONObjBuilder& builder) { + builder.append("ok", status.isOK() ? 1.0 : 0.0); + if (!status.isOK()) { + builder.append("code", status.code()); + } + if (!status.reason().empty()) { + builder.append("errmsg", status.reason()); + } + } + + /** + * Retrieves a collection's plan cache from the database. + */ + Status getPlanCache(Database* db, const string& ns, PlanCache** planCacheOut) { + invariant(db); + + Collection* collection = db->getCollection(ns); + if (NULL == collection) { + return Status(ErrorCodes::BadValue, "no such collection"); + } + + CollectionInfoCache* infoCache = collection->infoCache(); + invariant(infoCache); + + PlanCache* planCache = infoCache->getPlanCache(); + invariant(planCache); + + *planCacheOut = planCache; + return Status::OK(); + } + + // + // Command instances. + // Registers commands with the command system and make commands + // available to the client. + // + + MONGO_INITIALIZER_WITH_PREREQUISITES(SetupPlanCacheCommands, MONGO_NO_PREREQUISITES)( + InitializerContext* context) { + + // PlanCacheCommand constructors refer to static ActionType instances. + // Registering commands in a mongo static initializer ensures that + // the ActionType construction will be completed first. + new PlanCacheListQueryShapes(); + new PlanCacheClear(); + new PlanCacheListPlans(); + + return Status::OK(); + } + +} // namespace + +namespace mongo { + + using std::string; + using std::stringstream; + using std::vector; + using boost::scoped_ptr; + + PlanCacheCommand::PlanCacheCommand(const string& name, const string& helpText, + ActionType actionType) + : Command(name), + helpText(helpText), + actionType(actionType) { } + + bool PlanCacheCommand::run(const string& dbname, BSONObj& cmdObj, int options, + string& errmsg, BSONObjBuilder& result, bool fromRepl) { + string ns = parseNs(dbname, cmdObj); + + Status status = runPlanCacheCommand(ns, cmdObj, &result); + + if (!status.isOK()) { + addStatus(status, result); + return false; + } + + return true; + } + + Command::LockType PlanCacheCommand::locktype() const { + return NONE; + } + + bool PlanCacheCommand::slaveOk() const { + return false; + } + + bool PlanCacheCommand::slaveOverrideOk() const { + return true; + } + + void PlanCacheCommand::help(stringstream& ss) const { + ss << helpText; + } + + Status PlanCacheCommand::checkAuthForCommand(ClientBasic* client, const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + ResourcePattern pattern = parseResourcePattern(dbname, cmdObj); + + if (authzSession->isAuthorizedForActionsOnResource(pattern, actionType)) { + return Status::OK(); + } + + return Status(ErrorCodes::Unauthorized, "unauthorized"); + } + + // static + Status PlanCacheCommand::canonicalize(const string& ns, const BSONObj& cmdObj, + CanonicalQuery** canonicalQueryOut) { + // query - required + BSONElement queryElt = cmdObj.getField("query"); + if (queryElt.eoo()) { + return Status(ErrorCodes::BadValue, "required field query missing"); + } + if (!queryElt.isABSONObj()) { + return Status(ErrorCodes::BadValue, "required field query must be an object"); + } + if (queryElt.eoo()) { + return Status(ErrorCodes::BadValue, "required field query missing"); + } + BSONObj queryObj = queryElt.Obj(); + + // sort - optional + BSONElement sortElt = cmdObj.getField("sort"); + BSONObj sortObj; + if (!sortElt.eoo()) { + if (!sortElt.isABSONObj()) { + return Status(ErrorCodes::BadValue, "optional field sort must be an object"); + } + sortObj = sortElt.Obj(); + } + + // projection - optional + BSONElement projElt = cmdObj.getField("projection"); + BSONObj projObj; + if (!projElt.eoo()) { + if (!projElt.isABSONObj()) { + return Status(ErrorCodes::BadValue, "optional field projection must be an object"); + } + projObj = projElt.Obj(); + } + + // Create canonical query + CanonicalQuery* cqRaw; + Status result = CanonicalQuery::canonicalize(ns, queryObj, sortObj, projObj, &cqRaw); + if (!result.isOK()) { + return result; + } + + *canonicalQueryOut = cqRaw; + return Status::OK(); + } + + PlanCacheListQueryShapes::PlanCacheListQueryShapes() : PlanCacheCommand("planCacheListQueryShapes", + "Displays all query shapes in a collection.", + ActionType::planCacheRead) { } + + Status PlanCacheListQueryShapes::runPlanCacheCommand(const string& ns, BSONObj& cmdObj, + BSONObjBuilder* bob) { + // This is a read lock. The query cache is owned by the collection. + Client::ReadContext readCtx(ns); + Client::Context& ctx = readCtx.ctx(); + PlanCache* planCache; + Status status = getPlanCache(ctx.db(), ns, &planCache); + if (!status.isOK()) { + // No collection - return results with empty shapes array. + BSONArrayBuilder arrayBuilder(bob->subarrayStart("shapes")); + arrayBuilder.doneFast(); + return Status::OK(); + } + return list(*planCache, bob); + } + + // static + Status PlanCacheListQueryShapes::list(const PlanCache& planCache, BSONObjBuilder* bob) { + invariant(bob); + + // Fetch all cached solutions from plan cache. + vector solutions = planCache.getAllEntries(); + + BSONArrayBuilder arrayBuilder(bob->subarrayStart("shapes")); + for (vector::const_iterator i = solutions.begin(); i != solutions.end(); i++) { + PlanCacheEntry* entry = *i; + invariant(entry); + + BSONObjBuilder shapeBuilder(arrayBuilder.subobjStart()); + shapeBuilder.append("query", entry->query); + shapeBuilder.append("sort", entry->sort); + shapeBuilder.append("projection", entry->projection); + shapeBuilder.doneFast(); + + // Release resources for cached solution after extracting query shape. + delete entry; + } + arrayBuilder.doneFast(); + + return Status::OK(); + } + + PlanCacheClear::PlanCacheClear() : PlanCacheCommand("planCacheClear", + "Drops one or all cached queries in a collection.", + ActionType::planCacheWrite) { } + + Status PlanCacheClear::runPlanCacheCommand(const string& ns, BSONObj& cmdObj, + BSONObjBuilder* bob) { + // This is a read lock. The query cache is owned by the collection. + Client::ReadContext readCtx(ns); + Client::Context& ctx = readCtx.ctx(); + PlanCache* planCache; + Status status = getPlanCache(ctx.db(), ns, &planCache); + if (!status.isOK()) { + // No collection - nothing to do. Return OK status. + return Status::OK(); + } + return clear(planCache, ns, cmdObj); + } + + // static + Status PlanCacheClear::clear(PlanCache* planCache, const string& ns, const BSONObj& cmdObj) { + invariant(planCache); + + // According to the specification, the planCacheClear command runs in two modes: + // - clear all query shapes; or + // - clear plans for single query shape when a query shape is described in the + // command arguments. + if (cmdObj.hasField("query")) { + CanonicalQuery* cqRaw; + Status status = PlanCacheCommand::canonicalize(ns, cmdObj, &cqRaw); + if (!status.isOK()) { + return status; + } + + scoped_ptr cq(cqRaw); + + if (!planCache->contains(*cq)) { + // Log if asked to clear non-existent query shape. + LOG(1) << ns << ": query shape doesn't exist in PlanCache - " + << cq->getQueryObj().toString() + << "(sort: " << cq->getParsed().getSort() + << "; projection: " << cq->getParsed().getProj() << ")"; + return Status::OK(); + } + + Status result = planCache->remove(*cq); + if (!result.isOK()) { + return result; + } + + LOG(1) << ns << ": removed plan cache entry - " << cq->getQueryObj().toString() + << "(sort: " << cq->getParsed().getSort() + << "; projection: " << cq->getParsed().getProj() << ")"; + + return Status::OK(); + } + + // If query is not provided, make sure sort and projection are not in arguments. + // We do not want to clear the entire cache inadvertently when the user + // forgets to provide a value for "query". + if (cmdObj.hasField("sort") || cmdObj.hasField("projection")) { + return Status(ErrorCodes::BadValue, "sort or projection provided without query"); + } + + planCache->clear(); + + LOG(1) << ns << ": cleared plan cache"; + + return Status::OK(); + } + + PlanCacheListPlans::PlanCacheListPlans() : PlanCacheCommand("planCacheListPlans", + "Displays the cached plans for a query shape.", + ActionType::planCacheRead) { } + + Status PlanCacheListPlans::runPlanCacheCommand(const string& ns, BSONObj& cmdObj, + BSONObjBuilder* bob) { + Client::ReadContext readCtx(ns); + Client::Context& ctx = readCtx.ctx(); + PlanCache* planCache; + Status status = getPlanCache(ctx.db(), ns, &planCache); + if (!status.isOK()) { + // No collection - return empty plans array. + BSONArrayBuilder plansBuilder(bob->subarrayStart("plans")); + plansBuilder.doneFast(); + return Status::OK(); + } + return list(*planCache, ns, cmdObj, bob); + } + + // static + Status PlanCacheListPlans::list(const PlanCache& planCache, const std::string& ns, + const BSONObj& cmdObj, BSONObjBuilder* bob) { + CanonicalQuery* cqRaw; + Status status = canonicalize(ns, cmdObj, &cqRaw); + if (!status.isOK()) { + return status; + } + + scoped_ptr cq(cqRaw); + + if (!planCache.contains(*cq)) { + // Return empty plans in results if query shape does not + // exist in plan cache. + BSONArrayBuilder plansBuilder(bob->subarrayStart("plans")); + plansBuilder.doneFast(); + return Status::OK(); + } + + PlanCacheEntry* entryRaw; + Status result = planCache.getEntry(*cq, &entryRaw); + if (!result.isOK()) { + return result; + } + scoped_ptr entry(entryRaw); + + BSONArrayBuilder plansBuilder(bob->subarrayStart("plans")); + size_t numPlans = entry->plannerData.size(); + invariant(numPlans == entry->decision->stats.size()); + invariant(numPlans == entry->decision->scores.size()); + for (size_t i = 0; i < numPlans; ++i) { + BSONObjBuilder planBob(plansBuilder.subobjStart()); + + // Create plan details field. + // Currently, simple string representationg of + // SolutionCacheData. Need to revisit format when we + // need to parse user-provided plan details for planCacheAddPlan. + SolutionCacheData* scd = entry->plannerData[i]; + BSONObjBuilder detailsBob(planBob.subobjStart("details")); + detailsBob.append("solution", scd->toString()); + detailsBob.doneFast(); + + // reason is comprised of score and initial stats provided by + // multi plan runner. + BSONObjBuilder reasonBob(planBob.subobjStart("reason")); + reasonBob.append("score", entry->decision->scores[i]); + BSONObjBuilder statsBob(reasonBob.subobjStart("stats")); + PlanStageStats* stats = entry->decision->stats.vector()[i]; + if (stats) { + statsToBSON(*stats, &statsBob); + } + statsBob.doneFast(); + reasonBob.doneFast(); + + // BSON object for 'feedback' field is created from query executions + // and shows number of executions since this cached solution was + // created as well as score data (average and standard deviation). + BSONObjBuilder feedbackBob(planBob.subobjStart("feedback")); + if (i == 0U) { + feedbackBob.append("nfeedback", int(entry->feedback.size())); + feedbackBob.append("averageScore", entry->averageScore.get_value_or(0)); + feedbackBob.append("stdDevScore",entry->stddevScore.get_value_or(0)); + BSONArrayBuilder scoresBob(feedbackBob.subarrayStart("scores")); + for (size_t i = 0; i < entry->feedback.size(); ++i) { + BSONObjBuilder scoreBob(scoresBob.subobjStart()); + scoreBob.append("score", entry->feedback[i]->score); + } + scoresBob.doneFast(); + } + feedbackBob.doneFast(); + + planBob.append("filterSet", scd->indexFilterApplied); + } + plansBuilder.doneFast(); + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/commands/plan_cache_commands.h b/src/mongo/db/commands/plan_cache_commands.h new file mode 100644 index 00000000000..df7d655c416 --- /dev/null +++ b/src/mongo/db/commands/plan_cache_commands.h @@ -0,0 +1,169 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/commands.h" +#include "mongo/db/query/plan_cache.h" + +namespace mongo { + + /** + * DB commands for plan cache. + * These are in a header to facilitate unit testing. See plan_cache_commands_test.cpp. + */ + + /** + * PlanCacheCommand + * Defines common attributes for all plan cache related commands + * such as slaveOk and locktype. + */ + class PlanCacheCommand : public Command { + public: + PlanCacheCommand(const std::string& name, const std::string& helpText, + ActionType actionType); + + /** + * Entry point from command subsystem. + * Implementation provides standardization of error handling + * such as adding error code and message to BSON result. + * + * Do not override in derived classes. + * Override runPlanCacheCommands instead to + * implement plan cache command functionality. + */ + + bool run(const std::string& dbname, BSONObj& cmdObj, int options, + std::string& errmsg, BSONObjBuilder& result, bool fromRepl); + + /** + * It's fine to return NONE here because plan cache commands + * create explicit read context to access collection info cache. + * Refer to dbcommands.cpp on how locktype() is handled. + */ + virtual LockType locktype() const; + + virtual bool slaveOk() const; + + virtual bool slaveOverrideOk() const; + + virtual void help(std::stringstream& ss) const; + + /** + * Two action types defined for plan cache commands: + * - planCacheRead + * - planCacheWrite + */ + virtual Status checkAuthForCommand(ClientBasic* client, const std::string& dbname, + const BSONObj& cmdObj); + /** + * Subset of command arguments used by plan cache commands + * Override to provide command functionality. + * Should contain just enough logic to invoke run*Command() function + * in plan_cache.h + */ + virtual Status runPlanCacheCommand(const std::string& ns, BSONObj& cmdObj, + BSONObjBuilder* bob) = 0; + + /** + * Validatess query shape from command object and returns canonical query. + */ + static Status canonicalize(const std::string& ns, const BSONObj& cmdObj, + CanonicalQuery** canonicalQueryOut); + + private: + std::string helpText; + ActionType actionType; + }; + + /** + * planCacheListQueryShapes + * + * { planCacheListQueryShapes: } + * + */ + class PlanCacheListQueryShapes : public PlanCacheCommand { + public: + PlanCacheListQueryShapes(); + virtual Status runPlanCacheCommand(const std::string& ns, BSONObj& cmdObj, BSONObjBuilder* bob); + + /** + * Looks up cache keys for collection's plan cache. + * Inserts keys for query into BSON builder. + */ + static Status list(const PlanCache& planCache, BSONObjBuilder* bob); + }; + + /** + * planCacheClear + * + * { + * planCacheClear: , + * query: , + * sort: , + * projection: + * } + * + */ + class PlanCacheClear : public PlanCacheCommand { + public: + PlanCacheClear(); + virtual Status runPlanCacheCommand(const std::string& ns, BSONObj& cmdObj, BSONObjBuilder* bob); + + /** + * Clears collection's plan cache. + * If query shape is provided, clears plans for that single query shape only. + */ + static Status clear(PlanCache* planCache, const std::string& ns, const BSONObj& cmdObj); + }; + + /** + * planCacheListPlans + * + * { + * planCacheListPlans: , + * query: , + * sort: , + * projection: + * } + * + */ + class PlanCacheListPlans : public PlanCacheCommand { + public: + PlanCacheListPlans(); + virtual Status runPlanCacheCommand(const std::string& ns, BSONObj& cmdObj, + BSONObjBuilder* bob); + + /** + * Displays the cached plans for a query shape. + */ + static Status list(const PlanCache& planCache, const std::string& ns, + const BSONObj& cmdObj, BSONObjBuilder* bob); + }; + +} // namespace mongo diff --git a/src/mongo/db/commands/plan_cache_commands_test.cpp b/src/mongo/db/commands/plan_cache_commands_test.cpp new file mode 100644 index 00000000000..45088250fd9 --- /dev/null +++ b/src/mongo/db/commands/plan_cache_commands_test.cpp @@ -0,0 +1,383 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/commands/plan_cache_commands.h + */ + +#include "mongo/db/commands/plan_cache_commands.h" + +#include +#include "mongo/db/json.h" +#include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/mongoutils/str.h" + +using namespace mongo; + +namespace { + + using std::string; + using std::vector; + + static const char* ns = "somebogusns"; + + /** + * Tests for planCacheListQueryShapes + */ + + /** + * Utility function to get list of keys in the cache. + */ + std::vector getShapes(const PlanCache& planCache) { + BSONObjBuilder bob; + ASSERT_OK(PlanCacheListQueryShapes::list(planCache, &bob)); + BSONObj resultObj = bob.obj(); + BSONElement shapesElt = resultObj.getField("shapes"); + ASSERT_EQUALS(shapesElt.type(), mongo::Array); + vector shapesEltArray = shapesElt.Array(); + vector shapes; + for (vector::const_iterator i = shapesEltArray.begin(); + i != shapesEltArray.end(); ++i) { + const BSONElement& elt = *i; + + ASSERT_TRUE(elt.isABSONObj()); + BSONObj obj = elt.Obj(); + + // Check required fields. + // query + BSONElement queryElt = obj.getField("query"); + ASSERT_TRUE(queryElt.isABSONObj()); + + // sort + BSONElement sortElt = obj.getField("sort"); + ASSERT_TRUE(sortElt.isABSONObj()); + + // projection + BSONElement projectionElt = obj.getField("projection"); + ASSERT_TRUE(projectionElt.isABSONObj()); + + // All fields OK. Append to vector. + shapes.push_back(obj.getOwned()); + } + return shapes; + } + + /** + * Utility function to create a SolutionCacheData + */ + SolutionCacheData* createSolutionCacheData() { + auto_ptr scd(new SolutionCacheData()); + scd->tree.reset(new PlanCacheIndexTree()); + return scd.release(); + } + + /** + * Utility function to create a PlanRankingDecision + */ + PlanRankingDecision* createDecision(size_t numPlans) { + auto_ptr why(new PlanRankingDecision()); + for (size_t i = 0; i < numPlans; ++i) { + auto_ptr stats(new PlanStageStats(CommonStats(), STAGE_COLLSCAN)); + stats->specific.reset(new CollectionScanStats()); + why->stats.mutableVector().push_back(stats.release()); + why->scores.push_back(0U); + why->candidateOrder.push_back(i); + } + return why.release(); + } + + TEST(PlanCacheCommandsTest, planCacheListQueryShapesEmpty) { + PlanCache empty; + vector shapes = getShapes(empty); + ASSERT_TRUE(shapes.empty()); + } + + TEST(PlanCacheCommandsTest, planCacheListQueryShapesOneKey) { + // Create a canonical query + CanonicalQuery* cqRaw; + ASSERT_OK(CanonicalQuery::canonicalize(ns, fromjson("{a: 1}"), &cqRaw)); + auto_ptr cq(cqRaw); + + // Plan cache with one entry + PlanCache planCache; + QuerySolution qs; + qs.cacheData.reset(createSolutionCacheData()); + std::vector solns; + solns.push_back(&qs); + planCache.add(*cq, solns, createDecision(1U)); + + vector shapes = getShapes(planCache); + ASSERT_EQUALS(shapes.size(), 1U); + ASSERT_EQUALS(shapes[0].getObjectField("query"), cq->getQueryObj()); + ASSERT_EQUALS(shapes[0].getObjectField("sort"), cq->getParsed().getSort()); + ASSERT_EQUALS(shapes[0].getObjectField("projection"), cq->getParsed().getProj()); + } + + /** + * Tests for planCacheClear + */ + + TEST(PlanCacheCommandsTest, planCacheClearAllShapes) { + // Create a canonical query + CanonicalQuery* cqRaw; + ASSERT_OK(CanonicalQuery::canonicalize(ns, fromjson("{a: 1}"), &cqRaw)); + auto_ptr cq(cqRaw); + + // Plan cache with one entry + PlanCache planCache; + QuerySolution qs; + qs.cacheData.reset(createSolutionCacheData()); + std::vector solns; + solns.push_back(&qs); + planCache.add(*cq, solns, createDecision(1U)); + ASSERT_EQUALS(getShapes(planCache).size(), 1U); + + // Clear cache and confirm number of keys afterwards. + ASSERT_OK(PlanCacheClear::clear(&planCache, ns, BSONObj())); + ASSERT_EQUALS(getShapes(planCache).size(), 0U); + } + + /** + * Tests for PlanCacheCommand::makeCacheKey + * Mostly validation on the input parameters + */ + + TEST(PlanCacheCommandsTest, Canonicalize) { + // Invalid parameters + CanonicalQuery* cqRaw; + // Missing query field + ASSERT_NOT_OK(PlanCacheCommand::canonicalize(ns, fromjson("{}"), &cqRaw)); + // Query needs to be an object + ASSERT_NOT_OK(PlanCacheCommand::canonicalize(ns, fromjson("{query: 1}"), &cqRaw)); + // Sort needs to be an object + ASSERT_NOT_OK(PlanCacheCommand::canonicalize(ns, fromjson("{query: {}, sort: 1}"), + &cqRaw)); + // Bad query (invalid sort order) + ASSERT_NOT_OK(PlanCacheCommand::canonicalize(ns, fromjson("{query: {}, sort: {a: 0}}"), + &cqRaw)); + + // Valid parameters + ASSERT_OK(PlanCacheCommand::canonicalize(ns, fromjson("{query: {a: 1, b: 1}}"), &cqRaw)); + scoped_ptr query(cqRaw); + + + // Equivalent query should generate same key. + ASSERT_OK(PlanCacheCommand::canonicalize(ns, fromjson("{query: {b: 1, a: 1}}"), &cqRaw)); + scoped_ptr equivQuery(cqRaw); + ASSERT_EQUALS(query->getPlanCacheKey(), equivQuery->getPlanCacheKey()); + + // Sort query should generate different key from unsorted query. + ASSERT_OK(PlanCacheCommand::canonicalize(ns, + fromjson("{query: {a: 1, b: 1}, sort: {a: 1}}"), &cqRaw)); + scoped_ptr sortQuery(cqRaw); + ASSERT_NOT_EQUALS(query->getPlanCacheKey(), sortQuery->getPlanCacheKey()); + + // Projected query should generate different key from unprojected query. + ASSERT_OK(PlanCacheCommand::canonicalize(ns, + fromjson("{query: {a: 1, b: 1}, projection: {_id: 0, a: 1}}"), &cqRaw)); + scoped_ptr projectionQuery(cqRaw); + ASSERT_NOT_EQUALS(query->getPlanCacheKey(), projectionQuery->getPlanCacheKey()); + } + + /** + * Tests for planCacheClear (single query shape) + */ + + TEST(PlanCacheCommandsTest, planCacheClearInvalidParameter) { + PlanCache planCache; + // Query field type must be BSON object. + ASSERT_NOT_OK(PlanCacheClear::clear(&planCache, ns, fromjson("{query: 12345}"))); + ASSERT_NOT_OK(PlanCacheClear::clear(&planCache, ns, fromjson("{query: /keyisnotregex/}"))); + // Query must pass canonicalization. + ASSERT_NOT_OK(PlanCacheClear::clear(&planCache, ns, + fromjson("{query: {a: {$no_such_op: 1}}}"))); + // Sort present without query is an error. + ASSERT_NOT_OK(PlanCacheClear::clear(&planCache, ns, fromjson("{sort: {a: 1}}"))); + // Projection present without query is an error. + ASSERT_NOT_OK(PlanCacheClear::clear(&planCache, ns, + fromjson("{projection: {_id: 0, a: 1}}"))); + } + + TEST(PlanCacheCommandsTest, planCacheClearUnknownKey) { + PlanCache planCache; + ASSERT_OK(PlanCacheClear::clear(&planCache, ns, fromjson("{query: {a: 1}}"))); + } + + TEST(PlanCacheCommandsTest, planCacheClearOneKey) { + // Create 2 canonical queries. + CanonicalQuery* cqRaw; + ASSERT_OK(CanonicalQuery::canonicalize(ns, fromjson("{a: 1}"), &cqRaw)); + auto_ptr cqA(cqRaw); + ASSERT_OK(CanonicalQuery::canonicalize(ns, fromjson("{b: 1}"), &cqRaw)); + auto_ptr cqB(cqRaw); + + // Create plan cache with 2 entries. + PlanCache planCache; + QuerySolution qs; + qs.cacheData.reset(createSolutionCacheData()); + std::vector solns; + solns.push_back(&qs); + planCache.add(*cqA, solns, createDecision(1U)); + planCache.add(*cqB, solns, createDecision(1U)); + + // Check keys in cache before dropping {b: 1} + vector shapesBefore = getShapes(planCache); + ASSERT_EQUALS(shapesBefore.size(), 2U); + BSONObj shapeA = BSON("query" << cqA->getQueryObj() << "sort" << cqA->getParsed().getSort() + << "projection" << cqA->getParsed().getProj()); + BSONObj shapeB = BSON("query" << cqB->getQueryObj() << "sort" << cqB->getParsed().getSort() + << "projection" << cqB->getParsed().getProj()); + ASSERT_TRUE(std::find(shapesBefore.begin(), shapesBefore.end(), shapeA) != shapesBefore.end()); + ASSERT_TRUE(std::find(shapesBefore.begin(), shapesBefore.end(), shapeB) != shapesBefore.end()); + + // Drop {b: 1} from cache. Make sure {a: 1} is still in cache afterwards. + BSONObjBuilder bob; + ASSERT_OK(PlanCacheClear::clear(&planCache, ns, BSON("query" << cqB->getQueryObj()))); + vector shapesAfter = getShapes(planCache); + ASSERT_EQUALS(shapesAfter.size(), 1U); + ASSERT_EQUALS(shapesAfter[0], shapeA); + } + + /** + * Tests for planCacheListPlans + */ + + /** + * Function to extract plan ID from BSON element. + * Validates planID during extraction. + * Each BSON element contains an embedded BSON object with the following layout: + * { + * plan: , + * details: , + * reason: , + * feedback: , + * source: + * } + * Compilation note: GCC 4.4 has issues with getPlan() declared as a function object. + */ + BSONObj getPlan(const BSONElement& elt) { + ASSERT_TRUE(elt.isABSONObj()); + BSONObj obj = elt.Obj(); + + // Check required fields. + // details + BSONElement detailsElt = obj.getField("details"); + ASSERT_TRUE(detailsElt.isABSONObj()); + + // reason + BSONElement reasonElt = obj.getField("reason"); + ASSERT_TRUE(reasonElt.isABSONObj()); + + // feedback + BSONElement feedbackElt = obj.getField("feedback"); + ASSERT_TRUE(feedbackElt.isABSONObj()); + + return obj.getOwned(); + } + + /** + * Utility function to get list of plan IDs for a query in the cache. + */ + vector getPlans(const PlanCache& planCache, const BSONObj& query, + const BSONObj& sort, const BSONObj& projection) { + BSONObjBuilder bob; + BSONObj cmdObj = BSON("query" << query << "sort" << sort << "projection" << projection); + ASSERT_OK(PlanCacheListPlans::list(planCache, ns, cmdObj, &bob)); + BSONObj resultObj = bob.obj(); + BSONElement plansElt = resultObj.getField("plans"); + ASSERT_EQUALS(plansElt.type(), mongo::Array); + vector planEltArray = plansElt.Array(); + ASSERT_FALSE(planEltArray.empty()); + vector plans(planEltArray.size()); + std::transform(planEltArray.begin(), planEltArray.end(), plans.begin(), getPlan); + return plans; + } + + TEST(PlanCacheCommandsTest, planCacheListPlansInvalidParameter) { + PlanCache planCache; + BSONObjBuilder ignored; + // Missing query field is not ok. + ASSERT_NOT_OK(PlanCacheListPlans::list(planCache, ns, BSONObj(), &ignored)); + // Query field type must be BSON object. + ASSERT_NOT_OK(PlanCacheListPlans::list(planCache, ns, fromjson("{query: 12345}"), + &ignored)); + ASSERT_NOT_OK(PlanCacheListPlans::list(planCache, ns, fromjson("{query: /keyisnotregex/}"), + &ignored)); + } + + TEST(PlanCacheCommandsTest, planCacheListPlansUnknownKey) { + // Leave the plan cache empty. + PlanCache planCache; + + BSONObjBuilder ignored; + ASSERT_OK(PlanCacheListPlans::list(planCache, ns, fromjson("{query: {a: 1}}"), &ignored)); + } + + TEST(PlanCacheCommandsTest, planCacheListPlansOnlyOneSolutionTrue) { + // Create a canonical query + CanonicalQuery* cqRaw; + ASSERT_OK(CanonicalQuery::canonicalize(ns, fromjson("{a: 1}"), &cqRaw)); + auto_ptr cq(cqRaw); + + // Plan cache with one entry + PlanCache planCache; + QuerySolution qs; + qs.cacheData.reset(createSolutionCacheData()); + std::vector solns; + solns.push_back(&qs); + planCache.add(*cq, solns, createDecision(1U)); + + vector plans = getPlans(planCache, cq->getQueryObj(), + cq->getParsed().getSort(), cq->getParsed().getProj()); + ASSERT_EQUALS(plans.size(), 1U); + } + + TEST(PlanCacheCommandsTest, planCacheListPlansOnlyOneSolutionFalse) { + // Create a canonical query + CanonicalQuery* cqRaw; + ASSERT_OK(CanonicalQuery::canonicalize(ns, fromjson("{a: 1}"), &cqRaw)); + auto_ptr cq(cqRaw); + + // Plan cache with one entry + PlanCache planCache; + QuerySolution qs; + qs.cacheData.reset(createSolutionCacheData()); + // Add cache entry with 2 solutions. + std::vector solns; + solns.push_back(&qs); + solns.push_back(&qs); + planCache.add(*cq, solns, createDecision(2U)); + + vector plans = getPlans(planCache, cq->getQueryObj(), + cq->getParsed().getSort(), cq->getParsed().getProj()); + ASSERT_EQUALS(plans.size(), 2U); + } + +} // namespace diff --git a/src/mongo/db/commands/rename_collection.cpp b/src/mongo/db/commands/rename_collection.cpp new file mode 100644 index 00000000000..b695b88b023 --- /dev/null +++ b/src/mongo/db/commands/rename_collection.cpp @@ -0,0 +1,313 @@ +// rename_collection.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/client/dbclientcursor.h" +#include "mongo/db/background.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/commands.h" +#include "mongo/db/commands/rename_collection.h" +#include "mongo/db/dbhelpers.h" +#include "mongo/db/index_builder.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/instance.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/ops/insert.h" +#include "mongo/db/structure/collection_iterator.h" + +namespace mongo { + + class CmdRenameCollection : public Command { + public: + CmdRenameCollection() : Command( "renameCollection" ) {} + virtual bool adminOnly() const { + return true; + } + virtual bool slaveOk() const { + return false; + } + virtual LockType locktype() const { return WRITE; } + virtual bool lockGlobally() const { return true; } + virtual bool logTheOp() { + return true; // can't log steps when doing fast rename within a db, so always log the op rather than individual steps comprising it. + } + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + return rename_collection::checkAuthForRenameCollectionCommand(client, dbname, cmdObj); + } + virtual void help( stringstream &help ) const { + help << " example: { renameCollection: foo.a, to: bar.b }"; + } + + virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + string source = cmdObj.getStringField( name.c_str() ); + string target = cmdObj.getStringField( "to" ); + + if ( !NamespaceString::validCollectionComponent(target.c_str()) ) { + errmsg = "invalid collection name: " + target; + return false; + } + if ( source.empty() || target.empty() ) { + errmsg = "invalid command syntax"; + return false; + } + + if (!fromRepl) { // If it got through on the master, need to allow it here too + Status sourceStatus = userAllowedWriteNS(source); + if (!sourceStatus.isOK()) { + errmsg = "error with source namespace: " + sourceStatus.reason(); + return false; + } + + Status targetStatus = userAllowedWriteNS(target); + if (!targetStatus.isOK()) { + errmsg = "error with target namespace: " + targetStatus.reason(); + return false; + } + } + + BackgroundOperation::assertNoBgOpInProgForNs( source ); + + string sourceDB = nsToDatabase(source); + string targetDB = nsToDatabase(target); + + bool capped = false; + long long size = 0; + std::vector indexesInProg; + + { + Client::Context srcCtx( source ); + Collection* sourceColl = srcCtx.db()->getCollection( source ); + + if ( !sourceColl ) { + errmsg = "source namespace does not exist"; + return false; + } + + // Ensure that collection name does not exceed maximum length. + // Ensure that index names do not push the length over the max. + // Iterator includes unfinished indexes. + IndexCatalog::IndexIterator sourceIndIt = + sourceColl->getIndexCatalog()->getIndexIterator( true ); + int longestIndexNameLength = 0; + while ( sourceIndIt.more() ) { + int thisLength = sourceIndIt.next()->indexName().length(); + if ( thisLength > longestIndexNameLength ) + longestIndexNameLength = thisLength; + } + + unsigned int longestAllowed = + min(int(Namespace::MaxNsColletionLen), + int(Namespace::MaxNsLen) - 2/*strlen(".$")*/ - longestIndexNameLength); + if (target.size() > longestAllowed) { + StringBuilder sb; + sb << "collection name length of " << target.size() + << " exceeds maximum length of " << longestAllowed + << ", allowing for index names"; + errmsg = sb.str(); + return false; + } + + { + const NamespaceDetails *nsd = nsdetails( source ); + capped = nsd->isCapped(); + if ( capped ) + for( DiskLoc i = nsd->firstExtent(); !i.isNull(); i = i.ext()->xnext ) + size += i.ext()->length; + } + } + + { + Client::Context ctx( target ); + + // Check if the target namespace exists and if dropTarget is true. + // If target exists and dropTarget is not true, return false. + if ( ctx.db()->getCollection( target ) ) { + if ( !cmdObj["dropTarget"].trueValue() ) { + errmsg = "target namespace exists"; + return false; + } + + Status s = cc().database()->dropCollection( target ); + if ( !s.isOK() ) { + errmsg = s.toString(); + return false; + } + } + + // If we are renaming in the same database, just + // rename the namespace and we're done. + if ( sourceDB == targetDB ) { + Status s = ctx.db()->renameCollection( source, target, + cmdObj["stayTemp"].trueValue() ); + if ( !s.isOK() ) { + errmsg = s.toString(); + return false; + } + return true; + } + + // Otherwise, we are enaming across databases, so we must copy all + // the data and then remove the source collection. + + // Create the target collection. + Collection* targetColl = NULL; + if ( capped ) { + BSONObjBuilder spec; + spec.appendBool( "capped", true ); + spec.append( "size", double( size ) ); + spec.appendBool( "autoIndexId", false ); + userCreateNS( target.c_str(), spec.obj(), errmsg, false ); + targetColl = ctx.db()->getCollection( target ); + } + else { + CollectionOptions options; + options.setNoIdIndex(); + // No logOp necessary because the entire renameCollection command is one logOp. + targetColl = ctx.db()->createCollection( target, options ); + } + if ( !targetColl ) { + errmsg = "Failed to create target collection."; + return false; + } + } + + // Copy over all the data from source collection to target collection. + bool insertSuccessful = true; + boost::scoped_ptr sourceIt; + + { + Client::Context srcCtx( source ); + Collection* sourceColl = srcCtx.db()->getCollection( source ); + sourceIt.reset( sourceColl->getIterator( DiskLoc(), false, CollectionScanParams::FORWARD ) ); + } + + Collection* targetColl = NULL; + while ( !sourceIt->isEOF() ) { + BSONObj o; + { + Client::Context srcCtx( source ); + o = sourceIt->getNext().obj(); + } + // Insert and check return status of insert. + { + Client::Context ctx( target ); + if ( !targetColl ) + targetColl = ctx.db()->getCollection( target ); + // No logOp necessary because the entire renameCollection command is one logOp. + Status s = targetColl->insertDocument( o, true ).getStatus(); + if ( !s.isOK() ) { + insertSuccessful = false; + errmsg = s.toString(); + break; + } + getDur().commitIfNeeded(); + } + } + + // If inserts were unsuccessful, drop the target collection and return false. + if ( !insertSuccessful ) { + Client::Context ctx( target ); + Status s = ctx.db()->dropCollection( target ); + if ( !s.isOK() ) + errmsg = s.toString(); + return false; + } + + // Copy over the indexes to temp storage and then to the target.. + vector copiedIndexes; + bool indexSuccessful = true; + { + Client::Context srcCtx( source ); + Collection* sourceColl = srcCtx.db()->getCollection( source ); + IndexCatalog::IndexIterator sourceIndIt = + sourceColl->getIndexCatalog()->getIndexIterator( true ); + + while ( sourceIndIt.more() ) { + BSONObj currIndex = sourceIndIt.next()->infoObj(); + + // Process the source index. + BSONObjBuilder b; + BSONObjIterator i( currIndex ); + while( i.moreWithEOO() ) { + BSONElement e = i.next(); + if ( e.eoo() ) + break; + else if ( strcmp( e.fieldName(), "ns" ) == 0 ) + b.append( "ns", target ); + else + b.append( e ); + } + + BSONObj newIndex = b.obj(); + copiedIndexes.push_back( newIndex ); + } + } + + { + Client::Context ctx( target ); + if ( !targetColl ) + targetColl = ctx.db()->getCollection( target ); + + for ( vector::iterator it = copiedIndexes.begin(); + it != copiedIndexes.end(); ++it ) { + Status s = targetColl->getIndexCatalog()->createIndex( *it, true ); + if ( !s.isOK() ) { + indexSuccessful = false; + errmsg = s.toString(); + break; + } + } + + // If indexes were unsuccessful, drop the target collection and return false. + if ( !indexSuccessful ) { + Status s = ctx.db()->dropCollection( target ); + if ( !s.isOK() ) + errmsg = s.toString(); + return false; + } + } + + // Drop the source collection. + { + Client::Context srcCtx( source ); + Status s = srcCtx.db()->dropCollection( source ); + if ( !s.isOK() ) { + errmsg = s.toString(); + return false; + } + } + + return true; + } + } cmdrenamecollection; + +} diff --git a/src/mongo/db/commands/rename_collection.h b/src/mongo/db/commands/rename_collection.h index 9e593cdd742..f8651bccd4c 100644 --- a/src/mongo/db/commands/rename_collection.h +++ b/src/mongo/db/commands/rename_collection.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -23,10 +35,14 @@ #include "mongo/db/jsobj.h" namespace mongo { + + class ClientBasic; + namespace rename_collection { - void addPrivilegesRequiredForRenameCollection(const BSONObj& cmdObj, - std::vector* out); + Status checkAuthForRenameCollectionCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj); } // namespace rename_collection } // namespace mongo diff --git a/src/mongo/db/commands/rename_collection_common.cpp b/src/mongo/db/commands/rename_collection_common.cpp index 51ca4bbe6fa..fba1daffc0b 100644 --- a/src/mongo/db/commands/rename_collection_common.cpp +++ b/src/mongo/db/commands/rename_collection_common.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/commands/rename_collection.h" @@ -21,33 +33,71 @@ #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/client_basic.h" #include "mongo/db/jsobj.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" namespace mongo { namespace rename_collection { - void addPrivilegesRequiredForRenameCollection(const BSONObj& cmdObj, - std::vector* out) { + Status checkAuthForRenameCollectionCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { NamespaceString sourceNS = NamespaceString(cmdObj.getStringField("renameCollection")); NamespaceString targetNS = NamespaceString(cmdObj.getStringField("to")); - ActionSet sourceActions; - ActionSet targetActions; - - if (sourceNS.db == targetNS.db) { - sourceActions.addAction(ActionType::renameCollectionSameDB); - targetActions.addAction(ActionType::renameCollectionSameDB); - } else { - sourceActions.addAction(ActionType::cloneCollectionLocalSource); - sourceActions.addAction(ActionType::dropCollection); - targetActions.addAction(ActionType::createCollection); - targetActions.addAction(ActionType::cloneCollectionTarget); - targetActions.addAction(ActionType::ensureIndex); + bool dropTarget = cmdObj["dropTarget"].trueValue(); + + if (sourceNS.db() == targetNS.db() && !sourceNS.isSystem() && !targetNS.isSystem()) { + // If renaming within the same database, then if you have renameCollectionSameDB and + // either can read both of source and dest collections or *can't* read either of source + // or dest collection, then you get can do the rename, even without insert on the + // destination collection. + bool canRename = client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(sourceNS.db()), + ActionType::renameCollectionSameDB); + + bool canDropTargetIfNeeded = true; + if (dropTarget) { + canDropTargetIfNeeded = + client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(targetNS), + ActionType::dropCollection); + } + + bool canReadSrc = client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(sourceNS), ActionType::find); + bool canReadDest = client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(targetNS), ActionType::find); + + if (canRename && canDropTargetIfNeeded && (canReadSrc || !canReadDest)) { + return Status::OK(); + } + } + + // Check privileges on source collection + ActionSet actions; + actions.addAction(ActionType::find); + actions.addAction(ActionType::dropCollection); + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(sourceNS), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + // Check privileges on dest collection + actions.removeAllActions(); + actions.addAction(ActionType::insert); + actions.addAction(ActionType::createIndex); + if (dropTarget) { + actions.addAction(ActionType::dropCollection); + } + if (!client->getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(targetNS), actions)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); } - out->push_back(Privilege(sourceNS.ns(), sourceActions)); - out->push_back(Privilege(targetNS.ns(), targetActions)); + return Status::OK(); } } // namespace rename_collection diff --git a/src/mongo/db/commands/server_status.cpp b/src/mongo/db/commands/server_status.cpp index 1515514e7f3..a6d951d5bf6 100644 --- a/src/mongo/db/commands/server_status.cpp +++ b/src/mongo/db/commands/server_status.cpp @@ -14,18 +14,32 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/client_basic.h" -#include "mongo/db/cmdline.h" #include "mongo/db/commands.h" #include "mongo/db/commands/server_status.h" #include "mongo/db/stats/counters.h" +#include "mongo/platform/process_id.h" #include "mongo/util/net/listen.h" #include "mongo/util/processinfo.h" #include "mongo/util/ramlog.h" @@ -72,7 +86,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::serverStatus); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { @@ -82,15 +96,15 @@ namespace mongo { BSONObjBuilder timeBuilder(256); const ClientBasic* myClientBasic = ClientBasic::getCurrent(); - AuthorizationManager* authManager = myClientBasic->getAuthorizationManager(); + AuthorizationSession* authSession = myClientBasic->getAuthorizationSession(); // --- basic fields that are global result.append("host", prettyHostName() ); result.append("version", versionString); - result.append("process",cmdLine.binaryName); - result.append("pid", (int)getpid()); - result.append("uptime",(double) (time(0)-cmdLine.started)); + result.append("process", serverGlobalParams.binaryName); + result.append("pid", ProcessId::getCurrent().asLongLong()); + result.append("uptime", (double) (time(0) - serverGlobalParams.started)); result.append("uptimeMillis", (long long)(curTimeMillis64()-_started)); result.append("uptimeEstimate",(double) (start/1000)); result.appendDate( "localTime" , jsTime() ); @@ -104,7 +118,7 @@ namespace mongo { std::vector requiredPrivileges; section->addRequiredPrivileges(&requiredPrivileges); - if (!authManager->checkAuthForPrivileges(requiredPrivileges).isOK()) + if (!authSession->isAuthorizedForPrivileges(requiredPrivileges)) continue; bool include = section->includeByDefault(); @@ -138,20 +152,16 @@ namespace mongo { // --- some hard coded global things hard to pull out { - RamLog* rl = RamLog::get( "warnings" ); - massert(15880, "no ram log for warnings?" , rl); - - if (rl->lastWrite() >= time(0)-(10*60)){ // only show warnings from last 10 minutes - vector lines; - rl->get( lines ); - - BSONArrayBuilder arr( result.subarrayStart( "warnings" ) ); - for ( unsigned i=std::max(0,(int)lines.size()-10); i= time(0)-(10*60)){ // only show warnings from last 10 minutes + BSONArrayBuilder arr(result.subarrayStart("warnings")); + while (rl.more()) { + arr.append(rl.next()); + } arr.done(); } } - + timeBuilder.appendNumber( "at end" , Listener::getElapsedTimeMillis() - start ); if ( Listener::getElapsedTimeMillis() - start > 1000 ) { BSONObj t = timeBuilder.obj(); diff --git a/src/mongo/db/commands/server_status.h b/src/mongo/db/commands/server_status.h index dedfd0dadf6..ade476ec064 100644 --- a/src/mongo/db/commands/server_status.h +++ b/src/mongo/db/commands/server_status.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once diff --git a/src/mongo/db/commands/shutdown.cpp b/src/mongo/db/commands/shutdown.cpp new file mode 100644 index 00000000000..07024583845 --- /dev/null +++ b/src/mongo/db/commands/shutdown.cpp @@ -0,0 +1,52 @@ +/** +* Copyright (C) 2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/commands/shutdown.h" + +#include +#include + +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + void CmdShutdown::addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::shutdown); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); + } + +} // namespace mongo + diff --git a/src/mongo/db/commands/shutdown.h b/src/mongo/db/commands/shutdown.h new file mode 100644 index 00000000000..2df03f3ce59 --- /dev/null +++ b/src/mongo/db/commands/shutdown.h @@ -0,0 +1,66 @@ +/** +* Copyright (C) 2009 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include +#include + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class CmdShutdown : public Command { + public: + virtual bool requiresAuth() { return true; } + virtual bool adminOnly() const { return true; } + virtual bool localHostOnlyIfNoAuth(const BSONObj& cmdObj) { return true; } + virtual bool logTheOp() { + return false; + } + virtual bool slaveOk() const { + return true; + } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out); + virtual LockType locktype() const { return NONE; } + virtual void help( stringstream& help ) const; + CmdShutdown() : Command("shutdown") {} + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl); + private: + bool shutdownHelper(); + }; + +} // namespace mongo + diff --git a/src/mongo/db/commands/storage_details.cpp b/src/mongo/db/commands/storage_details.cpp index 9197ce2e6df..374f353d010 100644 --- a/src/mongo/db/commands/storage_details.cpp +++ b/src/mongo/db/commands/storage_details.cpp @@ -26,7 +26,8 @@ #include "mongo/db/db.h" #include "mongo/db/jsobj.h" #include "mongo/db/kill_current_op.h" -#include "mongo/db/namespace_details.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/catalog/collection.h" #include "mongo/util/processinfo.h" #include "mongo/util/mongoutils/str.h" @@ -315,7 +316,7 @@ namespace { std::vector* out) { ActionSet actions; actions.addAction(ActionType::storageDetails); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } private: @@ -328,7 +329,7 @@ namespace { }; MONGO_INITIALIZER(StorageDetailsCmd)(InitializerContext* context) { - if (cmdLine.experimental.storageDetailsCmdEnabled) { + if (serverGlobalParams.experimental.storageDetailsCmdEnabled) { // Leaked intentionally: a Command registers itself when constructed. new StorageDetailsCmd(); } @@ -366,11 +367,11 @@ namespace { /** * @return the requested extent if it exists, otherwise NULL */ - const Extent* getNthExtent(int extentNum, const NamespaceDetails* nsd) { + const Extent* getNthExtent(Database* db, int extentNum, const NamespaceDetails* nsd) { int curExtent = 0; - for (Extent* ex = DataFileMgr::getExtent(nsd->firstExtent); + for (Extent* ex = db->getExtentManager().getExtent(nsd->firstExtent()); ex != NULL; - ex = ex->getNextExtent()) { + ex = db->getExtentManager().getNextExtent(ex)) { if (curExtent == extentNum) return ex; curExtent++; @@ -543,8 +544,11 @@ namespace { recordsArrayBuilder.reset(new BSONArrayBuilder(result.subarrayStart("records"))); } + Database* db = cc().database(); + ExtentManager& extentManager = db->getExtentManager(); + DiskLoc prevDl = ex->firstRecord; - for (DiskLoc dl = ex->firstRecord; ! dl.isNull(); dl = r->nextInExtent(dl)) { + for (DiskLoc dl = ex->firstRecord; !dl.isNull(); dl = extentManager.getNextRecordInExtent(dl)) { r = dl.rec(); processRecord(dl, prevDl, r, extentOfs, params, sliceData, recordsArrayBuilder.get()); @@ -564,7 +568,7 @@ namespace { if (processingDeletedRecords) { for (int bucketNum = 0; bucketNum < mongo::Buckets; bucketNum++) { - DiskLoc dl = nsd->deletedList[bucketNum]; + DiskLoc dl = nsd->deletedListEntry(bucketNum); while (!dl.isNull()) { DeletedRecord* dr = dl.drec(); processDeletedRecord(dl, dr, ex, params, bucketNum, sliceData, @@ -704,9 +708,11 @@ namespace { /** * @param ex requested extent; if NULL analyze entire namespace */ - bool runInternal(const NamespaceDetails* nsd, const Extent* ex, SubCommand subCommand, - AnalyzeParams& globalParams, string& errmsg, BSONObjBuilder& result) { - + bool runInternal(const Database* db, const Collection* collection, const Extent* ex, + SubCommand subCommand, AnalyzeParams& globalParams, + string& errmsg, BSONObjBuilder& result) { + const NamespaceDetails* nsd = collection->details(); + const ExtentManager& em = db->getExtentManager(); BSONObjBuilder outputBuilder; // temporary builder to avoid output corruption in case of // failure bool success = false; @@ -714,22 +720,22 @@ namespace { success = analyzeExtent(nsd, ex, subCommand, globalParams, errmsg, outputBuilder); } else { - const DiskLoc dl = nsd->firstExtent; + const DiskLoc dl = nsd->firstExtent(); if (dl.isNull()) { errmsg = "no extents in namespace"; return false; } - long long storageSize = nsd->storageSize(NULL, NULL); + long long storageSize = collection->storageSize(NULL, NULL); if (globalParams.numberOfSlices != 0) { globalParams.granularity = ceilingDiv(storageSize, globalParams.numberOfSlices); } BSONArrayBuilder extentsArrayBuilder(outputBuilder.subarrayStart("extents")); - for (Extent* curExtent = dl.ext(); + for (Extent* curExtent = em.getExtent(dl); curExtent != NULL; - curExtent = curExtent->getNextExtent()) { + curExtent = em.getNextExtent(curExtent)) { AnalyzeParams extentParams(globalParams); extentParams.numberOfSlices = 0; // use the specified or calculated granularity; @@ -775,14 +781,18 @@ namespace { } const string ns = dbname + "." + cmdObj.firstElement().valuestrsafe(); - const NamespaceDetails* nsd = nsdetails(ns); - if (!cmdLine.quiet) { - tlog() << "CMD: storageDetails " << ns << ", analyze " << subCommandStr << endl; + + if (!serverGlobalParams.quiet) { + MONGO_TLOG(0) << "CMD: storageDetails " << ns << ", analyze " << subCommandStr << endl; } - if (!nsd) { + + Database* db = cc().database(); + const Collection* collection = db->getCollection( ns ); + if (!collection) { errmsg = "ns not found"; return false; } + const NamespaceDetails* nsd = collection->details(); const Extent* extent = NULL; @@ -794,7 +804,7 @@ namespace { return false; } int extentNum = extentElm.numberInt(); - extent = getNthExtent(extentNum, nsd); + extent = getNthExtent(db, extentNum, nsd); if (extent == NULL) { errmsg = str::stream() << "extent " << extentNum << " does not exist"; return false; @@ -850,7 +860,7 @@ namespace { params.showRecords = cmdObj["showRecords"].trueValue(); - return runInternal(nsd, extent, subCommand, params, errmsg, result); + return runInternal(db, collection, extent, subCommand, params, errmsg, result); } } // namespace diff --git a/src/mongo/db/commands/test_commands.cpp b/src/mongo/db/commands/test_commands.cpp new file mode 100644 index 00000000000..4667f6b7ecc --- /dev/null +++ b/src/mongo/db/commands/test_commands.cpp @@ -0,0 +1,213 @@ +// test_commands.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/base/init.h" +#include "mongo/base/initializer_context.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/index_builder.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + /* For testing only, not for general use. Enabled via command-line */ + class GodInsert : public Command { + public: + GodInsert() : Command( "godinsert" ) { } + virtual bool adminOnly() const { return false; } + virtual bool logTheOp() { return false; } + virtual bool slaveOk() const { return true; } + virtual LockType locktype() const { return NONE; } + // No auth needed because it only works when enabled via command line. + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) {} + virtual void help( stringstream &help ) const { + help << "internal. for testing only."; + } + virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { + string coll = cmdObj[ "godinsert" ].valuestrsafe(); + log() << "test only command godinsert invoked coll:" << coll << endl; + uassert( 13049, "godinsert must specify a collection", !coll.empty() ); + string ns = dbname + "." + coll; + BSONObj obj = cmdObj[ "obj" ].embeddedObjectUserCheck(); + + Lock::DBWrite lk(ns); + Client::Context ctx( ns ); + Database* db = ctx.db(); + Collection* collection = db->getCollection( ns ); + if ( !collection ) { + collection = db->createCollection( ns ); + if ( !collection ) { + errmsg = "could not create collection"; + return false; + } + } + StatusWith res = collection->insertDocument( obj, false ); + return appendCommandStatus( result, res.getStatus() ); + } + }; + + /* for diagnostic / testing purposes. Enabled via command line. */ + class CmdSleep : public Command { + public: + virtual LockType locktype() const { return NONE; } + virtual bool adminOnly() const { return true; } + virtual bool logTheOp() { return false; } + virtual bool slaveOk() const { return true; } + virtual void help( stringstream& help ) const { + help << "internal testing command. Makes db block (in a read lock) for 100 seconds\n"; + help << "w:true write lock. secs:"; + } + // No auth needed because it only works when enabled via command line. + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) {} + CmdSleep() : Command("sleep") { } + bool run(const string& ns, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + log() << "test only command sleep invoked" << endl; + long long millis = 10 * 1000; + + if (cmdObj["secs"].isNumber() && cmdObj["millis"].isNumber()) { + millis = cmdObj["secs"].numberLong() * 1000 + cmdObj["millis"].numberLong(); + } + else if (cmdObj["secs"].isNumber()) { + millis = cmdObj["secs"].numberLong() * 1000; + } + else if (cmdObj["millis"].isNumber()) { + millis = cmdObj["millis"].numberLong(); + } + + if(cmdObj.getBoolField("w")) { + Lock::GlobalWrite lk; + sleepmillis(millis); + } + else { + Lock::GlobalRead lk; + sleepmillis(millis); + } + + // Interrupt point for testing (e.g. maxTimeMS). + killCurrentOp.checkForInterrupt(); + + return true; + } + }; + + // Testing only, enabled via command-line. + class CapTrunc : public Command { + public: + CapTrunc() : Command( "captrunc" ) {} + virtual bool slaveOk() const { return false; } + virtual LockType locktype() const { return WRITE; } + // No auth needed because it only works when enabled via command line. + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) {} + virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { + string coll = cmdObj[ "captrunc" ].valuestrsafe(); + uassert( 13416, "captrunc must specify a collection", !coll.empty() ); + string ns = dbname + "." + coll; + int n = cmdObj.getIntField( "n" ); + + // inclusive range? + bool inc = cmdObj.getBoolField( "inc" ); + NamespaceDetails *nsd = nsdetails( ns ); + massert( 13417, "captrunc collection not found or empty", nsd); + + boost::scoped_ptr runner(InternalPlanner::collectionScan(ns, InternalPlanner::BACKWARD)); + DiskLoc end; + // We remove 'n' elements so the start is one past that + for( int i = 0; i < n + 1; ++i ) { + Runner::RunnerState state = runner->getNext(NULL, &end); + massert( 13418, "captrunc invalid n", Runner::RUNNER_ADVANCED == state); + } + nsd->cappedTruncateAfter( ns.c_str(), end, inc ); + return true; + } + }; + + // Testing-only, enabled via command line. + class EmptyCapped : public Command { + public: + EmptyCapped() : Command( "emptycapped" ) {} + virtual bool slaveOk() const { return false; } + virtual LockType locktype() const { return WRITE; } + virtual bool logTheOp() { return true; } + // No auth needed because it only works when enabled via command line. + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) {} + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + std::string coll = cmdObj[ "emptycapped" ].valuestrsafe(); + std::string ns = db->name() + '.' + coll; + + IndexCatalog::IndexKillCriteria criteria; + criteria.ns = ns; + return IndexBuilder::killMatchingIndexBuilds(db->getCollection(ns), criteria); + } + + virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { + string coll = cmdObj[ "emptycapped" ].valuestrsafe(); + uassert( 13428, "emptycapped must specify a collection", !coll.empty() ); + string ns = dbname + "." + coll; + NamespaceDetails *nsd = nsdetails( ns ); + massert( 13429, "emptycapped no such collection", nsd ); + + std::vector indexes = stopIndexBuilds(cc().database(), cmdObj); + + nsd->emptyCappedCollection( ns.c_str() ); + + IndexBuilder::restoreIndexes(indexes); + + return true; + } + }; + + // ---------------------------- + + MONGO_INITIALIZER(RegisterEmptyCappedCmd)(InitializerContext* context) { + if (Command::testCommandsEnabled) { + // Leaked intentionally: a Command registers itself when constructed. + new CapTrunc(); + new CmdSleep(); + new EmptyCapped(); + new GodInsert(); + } + return Status::OK(); + } + + +} diff --git a/src/mongo/db/commands/touch.cpp b/src/mongo/db/commands/touch.cpp index 30785f7f4ba..4e22c72d6f9 100644 --- a/src/mongo/db/commands/touch.cpp +++ b/src/mongo/db/commands/touch.cpp @@ -16,13 +16,26 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include #include +#include "mongo/db/kill_current_op.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" @@ -30,14 +43,60 @@ #include "mongo/db/commands.h" #include "mongo/db/d_concurrency.h" #include "mongo/db/curop-inl.h" -#include "mongo/db/index.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/structure/catalog/index_details.h" #include "mongo/db/jsobj.h" #include "mongo/db/pdfile.h" +#include "mongo/db/catalog/collection.h" #include "mongo/util/timer.h" #include "mongo/util/touch_pages.h" namespace mongo { + struct touch_location { + const char* root; + size_t length; + }; + + /** @return numRanges touched */ + int touchNs( const std::string& ns ) { + std::vector< touch_location > ranges; + boost::scoped_ptr mongoFilesLock; + { + Client::ReadContext ctx(ns); + + Database* db = ctx.ctx().db(); + ExtentManager& em = db->getExtentManager(); + + Collection* collection = db->getCollection( ns ); + uassert( 16154, "namespace does not exist", collection ); + + Extent* ext = em.getExtent( collection->details()->firstExtent() ); + while ( ext ) { + touch_location tl; + tl.root = reinterpret_cast(ext); + tl.length = ext->length; + ranges.push_back(tl); + ext = em.getNextExtent( ext ); + } + mongoFilesLock.reset(new LockMongoFilesShared()); + } + // DB read lock is dropped; no longer needed after this point. + + std::string progress_msg = "touch " + ns + " extents"; + ProgressMeterHolder pm(cc().curop()->setMessage(progress_msg.c_str(), + "Touch Progress", + ranges.size())); + for ( std::vector< touch_location >::iterator it = ranges.begin(); it != ranges.end(); ++it ) { + touch_pages( it->root, it->length ); + pm.hit(); + killCurrentOp.checkForInterrupt(); + } + pm.finished(); + + return static_cast( ranges.size() ); + } + class TouchCmd : public Command { public: virtual LockType locktype() const { return NONE; } @@ -51,13 +110,12 @@ namespace mongo { "{ touch : , [data : true] , [index : true] }\n" " at least one of data or index must be true; default is both are false\n"; } - virtual bool requiresAuth() { return true; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { ActionSet actions; actions.addAction(ActionType::touch); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } TouchCmd() : Command("touch") { } @@ -98,29 +156,41 @@ namespace mongo { if (touch_data) { log() << "touching namespace " << ns << endl; - touchNs( ns ); + Timer t; + int numRanges = touchNs( ns ); + result.append( "data", BSON( "numRanges" << numRanges << + "millis" << t.millis() ) ); log() << "touching namespace " << ns << " complete" << endl; } if (touch_indexes) { + Timer t; // enumerate indexes std::vector< std::string > indexes; { Client::ReadContext ctx(ns); NamespaceDetails *nsd = nsdetails(ns); massert( 16153, "namespace does not exist", nsd ); - + NamespaceDetails::IndexIterator ii = nsd->ii(); while ( ii.more() ) { IndexDetails& idx = ii.next(); indexes.push_back( idx.indexNamespace() ); } } + + int numRanges = 0; + for ( std::vector::const_iterator it = indexes.begin(); it != indexes.end(); it++ ) { - touchNs( *it ); + numRanges += touchNs( *it ); } + + result.append( "indexes", BSON( "num" << static_cast(indexes.size()) << + "numRanges" << numRanges << + "millis" << t.millis() ) ); + } return true; } diff --git a/src/mongo/db/commands/user_management_commands.cpp b/src/mongo/db/commands/user_management_commands.cpp new file mode 100644 index 00000000000..61cb9f7326f --- /dev/null +++ b/src/mongo/db/commands/user_management_commands.cpp @@ -0,0 +1,3025 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/platform/basic.h" + +#include "mongo/db/commands/user_management_commands.h" + +#include +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/bson/util/bson_extract.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/audit.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authz_documents_update_guard.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/resource_pattern.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_document_parser.h" +#include "mongo/db/auth/user_management_commands_parser.h" +#include "mongo/db/commands.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/unordered_set.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/net/ssl_manager.h" +#include "mongo/util/sequence_util.h" + +namespace mongo { + + namespace str = mongoutils::str; + + static void redactPasswordData(mutablebson::Element parent) { + namespace mmb = mutablebson; + const StringData pwdFieldName("pwd", StringData::LiteralTag()); + for (mmb::Element pwdElement = mmb::findFirstChildNamed(parent, pwdFieldName); + pwdElement.ok(); + pwdElement = mmb::findElementNamed(pwdElement.rightSibling(), pwdFieldName)) { + + pwdElement.setValueString("xxx"); + } + } + + static BSONArray roleSetToBSONArray(const unordered_set& roles) { + BSONArrayBuilder rolesArrayBuilder; + for (unordered_set::const_iterator it = roles.begin(); it != roles.end(); ++it) { + const RoleName& role = *it; + rolesArrayBuilder.append( + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << role.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << role.getDB())); + } + return rolesArrayBuilder.arr(); + } + + static BSONArray rolesVectorToBSONArray(const std::vector& roles) { + BSONArrayBuilder rolesArrayBuilder; + for (std::vector::const_iterator it = roles.begin(); it != roles.end(); ++it) { + const RoleName& role = *it; + rolesArrayBuilder.append( + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << role.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << role.getDB())); + } + return rolesArrayBuilder.arr(); + } + + static Status privilegeVectorToBSONArray(const PrivilegeVector& privileges, BSONArray* result) { + BSONArrayBuilder arrBuilder; + for (PrivilegeVector::const_iterator it = privileges.begin(); + it != privileges.end(); ++it) { + const Privilege& privilege = *it; + + ParsedPrivilege parsedPrivilege; + std::string errmsg; + if (!ParsedPrivilege::privilegeToParsedPrivilege(privilege, + &parsedPrivilege, + &errmsg)) { + return Status(ErrorCodes::FailedToParse, errmsg); + } + if (!parsedPrivilege.isValid(&errmsg)) { + return Status(ErrorCodes::FailedToParse, errmsg); + } + arrBuilder.append(parsedPrivilege.toBSON()); + } + *result = arrBuilder.arr(); + return Status::OK(); + } + + static Status getCurrentUserRoles(AuthorizationManager* authzManager, + const UserName& userName, + unordered_set* roles) { + User* user; + authzManager->invalidateUserByName(userName); // Need to make sure cache entry is up to date + Status status = authzManager->acquireUser(userName, &user); + if (!status.isOK()) { + return status; + } + RoleNameIterator rolesIt = user->getRoles(); + while (rolesIt.more()) { + roles->insert(rolesIt.next()); + } + authzManager->releaseUser(user); + return Status::OK(); + } + + static Status checkAuthorizedToGrantRoles(AuthorizationSession* authzSession, + const std::vector& roles) { + for (size_t i = 0; i < roles.size(); ++i) { + if (!authzSession->isAuthorizedToGrantRole(roles[i])) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to grant role: " << + roles[i].getFullName()); + } + } + return Status::OK(); + } + + static Status checkAuthorizedToRevokeRoles(AuthorizationSession* authzSession, + const std::vector& roles) { + for (size_t i = 0; i < roles.size(); ++i) { + if (!authzSession->isAuthorizedToRevokeRole(roles[i])) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to revoke role: " << + roles[i].getFullName()); + } + } + return Status::OK(); + } + + static Status checkAuthorizedToGrantPrivileges(AuthorizationSession* authzSession, + const PrivilegeVector& privileges) { + for (PrivilegeVector::const_iterator it = privileges.begin(); + it != privileges.end(); ++it) { + Status status = authzSession->checkAuthorizedToGrantPrivilege(*it); + if (!status.isOK()) { + return status; + } + } + + return Status::OK(); + } + + static Status checkAuthorizedToRevokePrivileges(AuthorizationSession* authzSession, + const PrivilegeVector& privileges) { + for (PrivilegeVector::const_iterator it = privileges.begin(); + it != privileges.end(); ++it) { + Status status = authzSession->checkAuthorizedToRevokePrivilege(*it); + if (!status.isOK()) { + return status; + } + } + + return Status::OK(); + } + + /* + * Checks that every role in "rolesToAdd" exists, that adding each of those roles to "role" + * will not result in a cycle to the role graph, and that every role being added comes from the + * same database as the role it is being added to (or that the role being added to is from the + * "admin" database. + */ + static Status checkOkayToGrantRolesToRole(const RoleName& role, + const std::vector rolesToAdd, + AuthorizationManager* authzManager) { + for (vector::const_iterator it = rolesToAdd.begin(); + it != rolesToAdd.end(); ++it) { + const RoleName& roleToAdd = *it; + if (roleToAdd == role) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Cannot grant role " << + role.getFullName() << " to itself."); + } + + if (role.getDB() != "admin" && roleToAdd.getDB() != role.getDB()) { + return Status(ErrorCodes::InvalidRoleModification, + str::stream() << "Roles on the \'" << role.getDB() << + "\' database cannot be granted roles from other databases"); + } + + BSONObj roleToAddDoc; + Status status = authzManager->getRoleDescription(roleToAdd, false, &roleToAddDoc); + if (status == ErrorCodes::RoleNotFound) { + return Status(ErrorCodes::RoleNotFound, + "Cannot grant nonexistent role " + roleToAdd.toString()); + } + if (!status.isOK()) { + return status; + } + std::vector indirectRoles; + status = auth::parseRoleNamesFromBSONArray( + BSONArray(roleToAddDoc["inheritedRoles"].Obj()), + role.getDB(), + &indirectRoles); + if (!status.isOK()) { + return status; + } + + if (sequenceContains(indirectRoles, role)) { + return Status(ErrorCodes::InvalidRoleModification, + mongoutils::str::stream() << "Granting " << + roleToAdd.getFullName() << " to " << role.getFullName() + << " would introduce a cycle in the role graph."); + } + } + return Status::OK(); + } + + /** + * Checks that every privilege being granted targets just the database the role is from, or that + * the role is from the "admin" db. + */ + static Status checkOkayToGrantPrivilegesToRole(const RoleName& role, + const PrivilegeVector& privileges) { + + if (role.getDB() == "admin") { + return Status::OK(); + } + + for (PrivilegeVector::const_iterator it = privileges.begin(); + it != privileges.end(); ++it) { + const ResourcePattern& resource = (*it).getResourcePattern(); + if ((resource.isDatabasePattern() || resource.isExactNamespacePattern()) && + (resource.databaseToMatch() == role.getDB())) { + continue; + } + + return Status(ErrorCodes::InvalidRoleModification, + str::stream() << "Roles on the \'" << role.getDB() << + "\' database cannot be granted privileges that target other " + "databases or the cluster"); + } + + return Status::OK(); + } + + static Status requireAuthSchemaVersion26Final(AuthorizationManager* authzManager) { + int foundSchemaVersion; + Status status = authzManager->getAuthorizationVersion(&foundSchemaVersion); + if (!status.isOK()) { + return status; + } + + if (foundSchemaVersion != AuthorizationManager::schemaVersion26Final) { + return Status( + ErrorCodes::AuthSchemaIncompatible, + str::stream() << "User and role management commands require auth data to have " + "schema version " << AuthorizationManager::schemaVersion26Final << + " but found " << foundSchemaVersion); + } + return authzManager->writeAuthSchemaVersionIfNeeded(); + } + + static Status requireAuthSchemaVersion26UpgradeOrFinal(AuthorizationManager* authzManager) { + int foundSchemaVersion; + Status status = authzManager->getAuthorizationVersion(&foundSchemaVersion); + if (!status.isOK()) { + return status; + } + + if (foundSchemaVersion != AuthorizationManager::schemaVersion26Final && + foundSchemaVersion != AuthorizationManager::schemaVersion26Upgrade) { + return Status( + ErrorCodes::AuthSchemaIncompatible, + str::stream() << "The usersInfo and rolesInfo commands require auth data to " + "have schema version " << AuthorizationManager::schemaVersion26Final << + " or " << AuthorizationManager::schemaVersion26Upgrade << + " but found " << foundSchemaVersion); + } + return Status::OK(); + } + + class CmdCreateUser : public Command { + public: + + CmdCreateUser() : Command("createUser") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Adds a user to the system" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + auth::CreateOrUpdateUserArgs args; + Status status = auth::parseCreateOrUpdateUserCommands(cmdObj, + "createUser", + dbname, + &args); + if (!status.isOK()) { + return status; + } + + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(args.userName.getDB()), + ActionType::createUser)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to create users on db: " << + args.userName.getDB()); + } + + return checkAuthorizedToGrantRoles(authzSession, args.roles); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + auth::CreateOrUpdateUserArgs args; + Status status = auth::parseCreateOrUpdateUserCommands(cmdObj, + "createUser", + dbname, + &args); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (args.userName.getDB() == "local") { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, "Cannot create users in the local database")); + } + + if (!args.hasHashedPassword && args.userName.getDB() != "$external") { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Must provide a 'pwd' field for all user documents, except those" + " with '$external' as the user's source db")); + } + + if (args.hasHashedPassword && args.userName.getDB() == "$external") { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Cannot set the password for users defined on the '$external' " + "database")); + } + + if (!args.hasRoles) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "\"createUser\" command requires a \"roles\" array")); + } + +#ifdef MONGO_SSL + if (args.userName.getDB() == "$external" && + getSSLManager() && + getSSLManager()->getServerSubjectName() == args.userName.getUser()) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Cannot create an x.509 user with the same " + "subjectname as the server")); + } +#endif + + BSONObjBuilder userObjBuilder; + userObjBuilder.append("_id", + str::stream() << args.userName.getDB() << "." << + args.userName.getUser()); + userObjBuilder.append(AuthorizationManager::USER_NAME_FIELD_NAME, + args.userName.getUser()); + userObjBuilder.append(AuthorizationManager::USER_DB_FIELD_NAME, + args.userName.getDB()); + if (args.hasHashedPassword) { + userObjBuilder.append("credentials", BSON("MONGODB-CR" << args.hashedPassword)); + } else { + // Must be an external user + userObjBuilder.append("credentials", BSON("external" << true)); + } + if (args.hasCustomData) { + userObjBuilder.append("customData", args.customData); + } + userObjBuilder.append("roles", rolesVectorToBSONArray(args.roles)); + + BSONObj userObj = userObjBuilder.obj(); + V2UserDocumentParser parser; + status = parser.checkValidUserDocument(userObj); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Create user")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Role existence has to be checked after acquiring the update lock + for (size_t i = 0; i < args.roles.size(); ++i) { + BSONObj ignored; + status = authzManager->getRoleDescription(args.roles[i], false, &ignored); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + } + + audit::logCreateUser(ClientBasic::getCurrent(), + args.userName, + args.hasHashedPassword, + args.hasCustomData? &args.customData : NULL, + args.roles); + status = authzManager->insertPrivilegeDocument(dbname, + userObj, + args.writeConcern); + return appendCommandStatus(result, status); + } + + virtual void redactForLogging(mutablebson::Document* cmdObj) { + redactPasswordData(cmdObj->root()); + } + + } cmdCreateUser; + + class CmdUpdateUser : public Command { + public: + + CmdUpdateUser() : Command("updateUser") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Used to update a user, for example to change its password" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + auth::CreateOrUpdateUserArgs args; + Status status = auth::parseCreateOrUpdateUserCommands(cmdObj, + "updateUser", + dbname, + &args); + if (!status.isOK()) { + return status; + } + + if (args.hasHashedPassword) { + if (!authzSession->isAuthorizedToChangeOwnPasswordAsUser(args.userName) && + !authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(args.userName.getDB()), + ActionType::changePassword)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to change password of user: " << + args.userName.getFullName()); + } + } + + if (args.hasCustomData) { + if (!authzSession->isAuthorizedToChangeOwnCustomDataAsUser(args.userName) && + !authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(args.userName.getDB()), + ActionType::changeCustomData)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to change customData of user: " + << args.userName.getFullName()); + } + } + + if (args.hasRoles) { + // You don't know what roles you might be revoking, so require the ability to + // revoke any role in the system. + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forAnyNormalResource(), ActionType::revokeRole)) { + return Status(ErrorCodes::Unauthorized, + "In order to use updateUser to set roles array, must be " + "authorized to revoke any role in the system"); + } + + return checkAuthorizedToGrantRoles(authzSession, args.roles); + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + auth::CreateOrUpdateUserArgs args; + Status status = auth::parseCreateOrUpdateUserCommands(cmdObj, + "updateUser", + dbname, + &args); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (!args.hasHashedPassword && !args.hasCustomData && !args.hasRoles) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Must specify at least one field to update in updateUser")); + } + + if (args.hasHashedPassword && args.userName.getDB() == "$external") { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Cannot set the password for users defined on the '$external' " + "database")); + } + + BSONObjBuilder updateSetBuilder; + if (args.hasHashedPassword) { + updateSetBuilder.append("credentials.MONGODB-CR", args.hashedPassword); + } + if (args.hasCustomData) { + updateSetBuilder.append("customData", args.customData); + } + if (args.hasRoles) { + updateSetBuilder.append("roles", rolesVectorToBSONArray(args.roles)); + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Update user")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + + // Role existence has to be checked after acquiring the update lock + if (args.hasRoles) { + for (size_t i = 0; i < args.roles.size(); ++i) { + BSONObj ignored; + status = authzManager->getRoleDescription(args.roles[i], false, &ignored); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + } + } + + audit::logUpdateUser(ClientBasic::getCurrent(), + args.userName, + args.hasHashedPassword, + args.hasCustomData? &args.customData : NULL, + args.hasRoles? &args.roles : NULL); + + status = authzManager->updatePrivilegeDocument(args.userName, + BSON("$set" << updateSetBuilder.done()), + args.writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserByName(args.userName); + return appendCommandStatus(result, status); + } + + virtual void redactForLogging(mutablebson::Document* cmdObj) { + redactPasswordData(cmdObj->root()); + } + + } cmdUpdateUser; + + class CmdDropUser : public Command { + public: + + CmdDropUser() : Command("dropUser") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Drops a single user." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + UserName userName; + BSONObj unusedWriteConcern; + Status status = auth::parseAndValidateDropUserCommand(cmdObj, + dbname, + &userName, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(userName.getDB()), ActionType::dropUser)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to drop users from the " << + userName.getDB() << " database"); + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Drop user")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + + UserName userName; + BSONObj writeConcern; + status = auth::parseAndValidateDropUserCommand(cmdObj, + dbname, + &userName, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + int nMatched; + + audit::logDropUser(ClientBasic::getCurrent(), userName); + + status = authzManager->removePrivilegeDocuments( + BSON(AuthorizationManager::USER_NAME_FIELD_NAME << userName.getUser() << + AuthorizationManager::USER_DB_FIELD_NAME << userName.getDB()), + writeConcern, + &nMatched); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserByName(userName); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (nMatched == 0) { + return appendCommandStatus( + result, + Status(ErrorCodes::UserNotFound, + str::stream() << "User '" << userName.getFullName() << + "' not found")); + } + + return true; + } + + } cmdDropUser; + + class CmdDropAllUsersFromDatabase : public Command { + public: + + CmdDropAllUsersFromDatabase() : Command("dropAllUsersFromDatabase") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Drops all users for a single database." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(dbname), ActionType::dropUser)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to drop users from the " << + dbname << " database"); + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Drop all users from database")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + BSONObj writeConcern; + status = auth::parseAndValidateDropAllUsersFromDatabaseCommand(cmdObj, + dbname, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + int numRemoved; + + audit::logDropAllUsersFromDatabase(ClientBasic::getCurrent(), dbname); + + status = authzManager->removePrivilegeDocuments( + BSON(AuthorizationManager::USER_DB_FIELD_NAME << dbname), + writeConcern, + &numRemoved); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUsersFromDB(dbname); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + result.append("n", numRemoved); + return true; + } + + } cmdDropAllUsersFromDatabase; + + class CmdGrantRolesToUser: public Command { + public: + + CmdGrantRolesToUser() : Command("grantRolesToUser") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Grants roles to a user." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + std::vector roles; + std::string unusedUserNameString; + BSONObj unusedWriteConcern; + Status status = auth::parseRolePossessionManipulationCommands(cmdObj, + "grantRolesToUser", + dbname, + &unusedUserNameString, + &roles, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToGrantRoles(authzSession, roles); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Grant roles to user")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + std::string userNameString; + std::vector roles; + BSONObj writeConcern; + status = auth::parseRolePossessionManipulationCommands(cmdObj, + "grantRolesToUser", + dbname, + &userNameString, + &roles, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + UserName userName(userNameString, dbname); + unordered_set userRoles; + status = getCurrentUserRoles(authzManager, userName, &userRoles); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + for (vector::iterator it = roles.begin(); it != roles.end(); ++it) { + RoleName& roleName = *it; + BSONObj roleDoc; + status = authzManager->getRoleDescription(roleName, false, &roleDoc); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + userRoles.insert(roleName); + } + + audit::logGrantRolesToUser(ClientBasic::getCurrent(), + userName, + roles); + BSONArray newRolesBSONArray = roleSetToBSONArray(userRoles); + status = authzManager->updatePrivilegeDocument( + userName, BSON("$set" << BSON("roles" << newRolesBSONArray)), writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserByName(userName); + return appendCommandStatus(result, status); + } + + } cmdGrantRolesToUser; + + class CmdRevokeRolesFromUser: public Command { + public: + + CmdRevokeRolesFromUser() : Command("revokeRolesFromUser") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Revokes roles from a user." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + std::vector roles; + std::string unusedUserNameString; + BSONObj unusedWriteConcern; + Status status = auth::parseRolePossessionManipulationCommands(cmdObj, + "revokeRolesFromUser", + dbname, + &unusedUserNameString, + &roles, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToRevokeRoles(authzSession, roles); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Revoke roles from user")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + std::string userNameString; + std::vector roles; + BSONObj writeConcern; + status = auth::parseRolePossessionManipulationCommands(cmdObj, + "revokeRolesFromUser", + dbname, + &userNameString, + &roles, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + UserName userName(userNameString, dbname); + unordered_set userRoles; + status = getCurrentUserRoles(authzManager, userName, &userRoles); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + for (vector::iterator it = roles.begin(); it != roles.end(); ++it) { + RoleName& roleName = *it; + BSONObj roleDoc; + status = authzManager->getRoleDescription(roleName, false, &roleDoc); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + userRoles.erase(roleName); + } + + audit::logRevokeRolesFromUser(ClientBasic::getCurrent(), + userName, + roles); + BSONArray newRolesBSONArray = roleSetToBSONArray(userRoles); + status = authzManager->updatePrivilegeDocument( + userName, BSON("$set" << BSON("roles" << newRolesBSONArray)), writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserByName(userName); + return appendCommandStatus(result, status); + } + + } cmdRevokeRolesFromUser; + + class CmdUsersInfo: public Command { + public: + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual bool slaveOverrideOk() const { + return true; + } + + virtual LockType locktype() const { + return NONE; + } + + CmdUsersInfo() : Command("usersInfo") {} + + virtual void help(stringstream& ss) const { + ss << "Returns information about users." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + auth::UsersInfoArgs args; + Status status = auth::parseUsersInfoCommand(cmdObj, dbname, &args); + if (!status.isOK()) { + return status; + } + + if (args.allForDB) { + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(dbname), ActionType::viewUser)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to view users from the " << + dbname << " database"); + } + } else { + for (size_t i = 0; i < args.userNames.size(); ++i) { + if (authzSession->lookupUser(args.userNames[i])) { + continue; // Can always view users you are logged in as + } + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(args.userNames[i].getDB()), + ActionType::viewUser)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to view users from the " << + dbname << " database"); + } + } + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + + auth::UsersInfoArgs args; + Status status = auth::parseUsersInfoCommand(cmdObj, dbname, &args); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + status = requireAuthSchemaVersion26UpgradeOrFinal(getGlobalAuthorizationManager()); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (args.allForDB && args.showPrivileges) { + return appendCommandStatus( + result, + Status(ErrorCodes::IllegalOperation, + "Can only get privilege details on exact-match usersInfo " + "queries.")); + } + + BSONArrayBuilder usersArrayBuilder; + if (args.showPrivileges) { + // If you want privileges you need to call getUserDescription on each user. + for (size_t i = 0; i < args.userNames.size(); ++i) { + BSONObj userDetails; + status = getGlobalAuthorizationManager()->getUserDescription( + args.userNames[i], &userDetails); + if (status.code() == ErrorCodes::UserNotFound) { + continue; + } + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + if (!args.showCredentials) { + // getUserDescription always includes credentials, need to strip it out + BSONObjBuilder userWithoutCredentials(usersArrayBuilder.subobjStart()); + for (BSONObjIterator it(userDetails); it.more(); ) { + BSONElement e = it.next(); + if (e.fieldNameStringData() != "credentials") + userWithoutCredentials.append(e); + } + userWithoutCredentials.doneFast(); + } else { + usersArrayBuilder.append(userDetails); + } + } + } else { + // If you don't need privileges, you can just do a regular query on system.users + BSONObjBuilder queryBuilder; + if (args.allForDB) { + queryBuilder.append(AuthorizationManager::USER_DB_FIELD_NAME, dbname); + } else { + BSONArrayBuilder usersMatchArray; + for (size_t i = 0; i < args.userNames.size(); ++i) { + usersMatchArray.append(BSON(AuthorizationManager::USER_NAME_FIELD_NAME << + args.userNames[i].getUser() << + AuthorizationManager::USER_DB_FIELD_NAME << + args.userNames[i].getDB())); + } + queryBuilder.append("$or", usersMatchArray.arr()); + + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + int authzVersion; + Status status = authzManager->getAuthorizationVersion(&authzVersion); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + NamespaceString usersNamespace = + authzVersion== AuthorizationManager::schemaVersion26Final ? + AuthorizationManager::usersCollectionNamespace : + AuthorizationManager::usersAltCollectionNamespace; + BSONObjBuilder projection; + if (!args.showCredentials) { + projection.append("credentials", 0); + } + BSONArrayBuilder& (BSONArrayBuilder::* appendBSONObj) (const BSONObj&) = + &BSONArrayBuilder::append; + const boost::function function = + boost::bind(appendBSONObj, &usersArrayBuilder, _1); + authzManager->queryAuthzDocument(usersNamespace, + queryBuilder.done(), + projection.done(), + function); + } + result.append("users", usersArrayBuilder.arr()); + return true; + } + + } cmdUsersInfo; + + class CmdCreateRole: public Command { + public: + + CmdCreateRole() : Command("createRole") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Adds a role to the system" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + auth::CreateOrUpdateRoleArgs args; + Status status = auth::parseCreateOrUpdateRoleCommands(cmdObj, + "createRole", + dbname, + &args); + if (!status.isOK()) { + return status; + } + + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(args.roleName.getDB()), + ActionType::createRole)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to create roles on db: " << + args.roleName.getDB()); + } + + status = checkAuthorizedToGrantRoles(authzSession, args.roles); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToGrantPrivileges(authzSession, args.privileges); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + auth::CreateOrUpdateRoleArgs args; + Status status = auth::parseCreateOrUpdateRoleCommands(cmdObj, + "createRole", + dbname, + &args); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (args.roleName.getRole().empty()) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, "Role name must be non-empty")); + } + + if (args.roleName.getDB() == "local") { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, "Cannot create roles in the local database")); + } + + if (args.roleName.getDB() == "$external") { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Cannot create roles in the $external database")); + } + + if (!args.hasRoles) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "\"createRole\" command requires a \"roles\" array")); + } + + if (!args.hasPrivileges) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "\"createRole\" command requires a \"privileges\" array")); + } + + BSONObjBuilder roleObjBuilder; + + roleObjBuilder.append("_id", str::stream() << args.roleName.getDB() << "." << + args.roleName.getRole()); + roleObjBuilder.append(AuthorizationManager::ROLE_NAME_FIELD_NAME, + args.roleName.getRole()); + roleObjBuilder.append(AuthorizationManager::ROLE_SOURCE_FIELD_NAME, + args.roleName.getDB()); + + BSONArray privileges; + status = privilegeVectorToBSONArray(args.privileges, &privileges); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + roleObjBuilder.append("privileges", privileges); + + roleObjBuilder.append("roles", rolesVectorToBSONArray(args.roles)); + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Create role")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Role existence has to be checked after acquiring the update lock + status = checkOkayToGrantRolesToRole(args.roleName, args.roles, authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + status = checkOkayToGrantPrivilegesToRole(args.roleName, args.privileges); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + audit::logCreateRole(ClientBasic::getCurrent(), + args.roleName, + args.roles, + args.privileges); + + status = authzManager->insertRoleDocument(roleObjBuilder.done(), args.writeConcern); + return appendCommandStatus(result, status); + } + + } cmdCreateRole; + + class CmdUpdateRole: public Command { + public: + + CmdUpdateRole() : Command("updateRole") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Used to update a role" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + auth::CreateOrUpdateRoleArgs args; + Status status = auth::parseCreateOrUpdateRoleCommands(cmdObj, + "updateRole", + dbname, + &args); + if (!status.isOK()) { + return status; + } + + // You don't know what roles or privileges you might be revoking, so require the ability + // to revoke any role (or privilege) in the system. + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forAnyNormalResource(), ActionType::revokeRole)) { + return Status(ErrorCodes::Unauthorized, + "updateRole command required the ability to revoke any role in the " + "system"); + } + + status = checkAuthorizedToGrantRoles(authzSession, args.roles); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToGrantPrivileges(authzSession, args.privileges); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + auth::CreateOrUpdateRoleArgs args; + Status status = auth::parseCreateOrUpdateRoleCommands(cmdObj, + "updateRole", + dbname, + &args); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (!args.hasPrivileges && !args.hasRoles) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Must specify at least one field to update in updateRole")); + } + + BSONObjBuilder updateSetBuilder; + + if (args.hasPrivileges) { + BSONArray privileges; + status = privilegeVectorToBSONArray(args.privileges, &privileges); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + updateSetBuilder.append("privileges", privileges); + } + + if (args.hasRoles) { + updateSetBuilder.append("roles", rolesVectorToBSONArray(args.roles)); + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Update role")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Role existence has to be checked after acquiring the update lock + BSONObj ignored; + status = authzManager->getRoleDescription(args.roleName, false, &ignored); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (args.hasRoles) { + status = checkOkayToGrantRolesToRole(args.roleName, args.roles, authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + } + + if (args.hasPrivileges) { + status = checkOkayToGrantPrivilegesToRole(args.roleName, args.privileges); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + } + + audit::logUpdateRole(ClientBasic::getCurrent(), + args.roleName, + args.hasRoles? &args.roles : NULL, + args.hasPrivileges? &args.privileges : NULL); + + status = authzManager->updateRoleDocument(args.roleName, + BSON("$set" << updateSetBuilder.done()), + args.writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + return appendCommandStatus(result, status); + } + } cmdUpdateRole; + + class CmdGrantPrivilegesToRole: public Command { + public: + + CmdGrantPrivilegesToRole() : Command("grantPrivilegesToRole") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Grants privileges to a role" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + PrivilegeVector privileges; + RoleName unusedRoleName; + BSONObj unusedWriteConcern; + Status status = auth::parseAndValidateRolePrivilegeManipulationCommands( + cmdObj, + "grantPrivilegesToRole", + dbname, + &unusedRoleName, + &privileges, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToGrantPrivileges(authzSession, privileges); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Grant privileges to role")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + RoleName roleName; + PrivilegeVector privilegesToAdd; + BSONObj writeConcern; + status = auth::parseAndValidateRolePrivilegeManipulationCommands( + cmdObj, + "grantPrivilegesToRole", + dbname, + &roleName, + &privilegesToAdd, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (RoleGraph::isBuiltinRole(roleName)) { + return appendCommandStatus( + result, + Status(ErrorCodes::InvalidRoleModification, + str::stream() << roleName.getFullName() << + " is a built-in role and cannot be modified.")); + } + + status = checkOkayToGrantPrivilegesToRole(roleName, privilegesToAdd); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + BSONObj roleDoc; + status = authzManager->getRoleDescription(roleName, true, &roleDoc); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + PrivilegeVector privileges; + status = auth::parseAndValidatePrivilegeArray(BSONArray(roleDoc["privileges"].Obj()), + &privileges); + + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + for (PrivilegeVector::iterator it = privilegesToAdd.begin(); + it != privilegesToAdd.end(); ++it) { + Privilege::addPrivilegeToPrivilegeVector(&privileges, *it); + } + + // Build up update modifier object to $set privileges. + mutablebson::Document updateObj; + mutablebson::Element setElement = updateObj.makeElementObject("$set"); + status = updateObj.root().pushBack(setElement); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + mutablebson::Element privilegesElement = updateObj.makeElementArray("privileges"); + status = setElement.pushBack(privilegesElement); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + status = authzManager->getBSONForPrivileges(privileges, privilegesElement); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + BSONObjBuilder updateBSONBuilder; + updateObj.writeTo(&updateBSONBuilder); + + audit::logGrantPrivilegesToRole(ClientBasic::getCurrent(), + roleName, + privilegesToAdd); + + status = authzManager->updateRoleDocument( + roleName, + updateBSONBuilder.done(), + writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + return appendCommandStatus(result, status); + } + + } cmdGrantPrivilegesToRole; + + class CmdRevokePrivilegesFromRole: public Command { + public: + + CmdRevokePrivilegesFromRole() : Command("revokePrivilegesFromRole") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Revokes privileges from a role" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + PrivilegeVector privileges; + RoleName unusedRoleName; + BSONObj unusedWriteConcern; + Status status = auth::parseAndValidateRolePrivilegeManipulationCommands( + cmdObj, + "revokePrivilegesFromRole", + dbname, + &unusedRoleName, + &privileges, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToRevokePrivileges(authzSession, privileges); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Revoke privileges from role")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + RoleName roleName; + PrivilegeVector privilegesToRemove; + BSONObj writeConcern; + status = auth::parseAndValidateRolePrivilegeManipulationCommands( + cmdObj, + "revokePrivilegesFromRole", + dbname, + &roleName, + &privilegesToRemove, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (RoleGraph::isBuiltinRole(roleName)) { + return appendCommandStatus( + result, + Status(ErrorCodes::InvalidRoleModification, + str::stream() << roleName.getFullName() << + " is a built-in role and cannot be modified.")); + } + + BSONObj roleDoc; + status = authzManager->getRoleDescription(roleName, true, &roleDoc); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + PrivilegeVector privileges; + status = auth::parseAndValidatePrivilegeArray(BSONArray(roleDoc["privileges"].Obj()), + &privileges); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + for (PrivilegeVector::iterator itToRm = privilegesToRemove.begin(); + itToRm != privilegesToRemove.end(); ++itToRm) { + for (PrivilegeVector::iterator curIt = privileges.begin(); + curIt != privileges.end(); ++curIt) { + if (curIt->getResourcePattern() == itToRm->getResourcePattern()) { + curIt->removeActions(itToRm->getActions()); + if (curIt->getActions().empty()) { + privileges.erase(curIt); + } + break; + } + } + } + + // Build up update modifier object to $set privileges. + mutablebson::Document updateObj; + mutablebson::Element setElement = updateObj.makeElementObject("$set"); + status = updateObj.root().pushBack(setElement); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + mutablebson::Element privilegesElement = updateObj.makeElementArray("privileges"); + status = setElement.pushBack(privilegesElement); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + status = authzManager->getBSONForPrivileges(privileges, privilegesElement); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + audit::logRevokePrivilegesFromRole(ClientBasic::getCurrent(), + roleName, + privilegesToRemove); + + BSONObjBuilder updateBSONBuilder; + updateObj.writeTo(&updateBSONBuilder); + status = authzManager->updateRoleDocument( + roleName, + updateBSONBuilder.done(), + writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + return appendCommandStatus(result, status); + } + + } cmdRevokePrivilegesFromRole; + + class CmdGrantRolesToRole: public Command { + public: + + CmdGrantRolesToRole() : Command("grantRolesToRole") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Grants roles to another role." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + std::vector roles; + std::string unusedUserNameString; + BSONObj unusedWriteConcern; + Status status = auth::parseRolePossessionManipulationCommands(cmdObj, + "grantRolesToRole", + dbname, + &unusedUserNameString, + &roles, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToGrantRoles(authzSession, roles); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + std::string roleNameString; + std::vector rolesToAdd; + BSONObj writeConcern; + Status status = auth::parseRolePossessionManipulationCommands( + cmdObj, + "grantRolesToRole", + dbname, + &roleNameString, + &rolesToAdd, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + RoleName roleName(roleNameString, dbname); + if (RoleGraph::isBuiltinRole(roleName)) { + return appendCommandStatus( + result, + Status(ErrorCodes::InvalidRoleModification, + str::stream() << roleName.getFullName() << + " is a built-in role and cannot be modified.")); + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Grant roles to role")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Role existence has to be checked after acquiring the update lock + BSONObj roleDoc; + status = authzManager->getRoleDescription(roleName, false, &roleDoc); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Check for cycles + status = checkOkayToGrantRolesToRole(roleName, rolesToAdd, authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Add new roles to existing roles + std::vector directRoles; + status = auth::parseRoleNamesFromBSONArray(BSONArray(roleDoc["roles"].Obj()), + roleName.getDB(), + &directRoles); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + for (vector::iterator it = rolesToAdd.begin(); it != rolesToAdd.end(); ++it) { + const RoleName& roleToAdd = *it; + if (!sequenceContains(directRoles, roleToAdd)) // Don't double-add role + directRoles.push_back(*it); + } + + audit::logGrantRolesToRole(ClientBasic::getCurrent(), + roleName, + rolesToAdd); + + status = authzManager->updateRoleDocument( + roleName, + BSON("$set" << BSON("roles" << rolesVectorToBSONArray(directRoles))), + writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + return appendCommandStatus(result, status); + } + + } cmdGrantRolesToRole; + + class CmdRevokeRolesFromRole: public Command { + public: + + CmdRevokeRolesFromRole() : Command("revokeRolesFromRole") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Revokes roles from another role." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + std::vector roles; + std::string unusedUserNameString; + BSONObj unusedWriteConcern; + Status status = auth::parseRolePossessionManipulationCommands(cmdObj, + "revokeRolesFromRole", + dbname, + &unusedUserNameString, + &roles, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + return checkAuthorizedToRevokeRoles(authzSession, roles); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Revoke roles from role")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + std::string roleNameString; + std::vector rolesToRemove; + BSONObj writeConcern; + status = auth::parseRolePossessionManipulationCommands(cmdObj, + "revokeRolesFromRole", + dbname, + &roleNameString, + &rolesToRemove, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + RoleName roleName(roleNameString, dbname); + if (RoleGraph::isBuiltinRole(roleName)) { + return appendCommandStatus( + result, + Status(ErrorCodes::InvalidRoleModification, + str::stream() << roleName.getFullName() << + " is a built-in role and cannot be modified.")); + } + + BSONObj roleDoc; + status = authzManager->getRoleDescription(roleName, false, &roleDoc); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + std::vector roles; + status = auth::parseRoleNamesFromBSONArray(BSONArray(roleDoc["roles"].Obj()), + roleName.getDB(), + &roles); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + for (vector::const_iterator it = rolesToRemove.begin(); + it != rolesToRemove.end(); ++it) { + vector::iterator itToRm = std::find(roles.begin(), roles.end(), *it); + if (itToRm != roles.end()) { + roles.erase(itToRm); + } + } + + audit::logRevokeRolesFromRole(ClientBasic::getCurrent(), + roleName, + rolesToRemove); + + status = authzManager->updateRoleDocument( + roleName, + BSON("$set" << BSON("roles" << rolesVectorToBSONArray(roles))), + writeConcern); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + return appendCommandStatus(result, status); + } + + } cmdRevokeRolesFromRole; + + class CmdDropRole: public Command { + public: + + CmdDropRole() : Command("dropRole") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Drops a single role. Before deleting the role completely it must remove it " + "from any users or roles that reference it. If any errors occur in the middle " + "of that process it's possible to be left in a state where the role has been " + "removed from some user/roles but otherwise still exists."<< endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + RoleName roleName; + BSONObj unusedWriteConcern; + Status status = auth::parseDropRoleCommand(cmdObj, + dbname, + &roleName, + &unusedWriteConcern); + if (!status.isOK()) { + return status; + } + + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(roleName.getDB()), ActionType::dropRole)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to drop roles from the " << + roleName.getDB() << " database"); + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Drop role")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + Status status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + RoleName roleName; + BSONObj writeConcern; + status = auth::parseDropRoleCommand(cmdObj, + dbname, + &roleName, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (RoleGraph::isBuiltinRole(roleName)) { + return appendCommandStatus( + result, + Status(ErrorCodes::InvalidRoleModification, + str::stream() << roleName.getFullName() << + " is a built-in role and cannot be modified.")); + } + + BSONObj roleDoc; + status = authzManager->getRoleDescription(roleName, false, &roleDoc); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Remove this role from all users + int nMatched; + status = authzManager->updateAuthzDocuments( + NamespaceString("admin.system.users"), + BSON("roles" << BSON("$elemMatch" << + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << + roleName.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + roleName.getDB()))), + BSON("$pull" << BSON("roles" << + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << + roleName.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + roleName.getDB()))), + false, + true, + writeConcern, + &nMatched); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + if (!status.isOK()) { + ErrorCodes::Error code = status.code() == ErrorCodes::UnknownError ? + ErrorCodes::UserModificationFailed : status.code(); + return appendCommandStatus( + result, + Status(code, + str::stream() << "Failed to remove role " << roleName.getFullName() + << " from all users: " << status.reason())); + } + + // Remove this role from all other roles + status = authzManager->updateAuthzDocuments( + NamespaceString("admin.system.roles"), + BSON("roles" << BSON("$elemMatch" << + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << + roleName.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + roleName.getDB()))), + BSON("$pull" << BSON("roles" << + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << + roleName.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + roleName.getDB()))), + false, + true, + writeConcern, + &nMatched); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + if (!status.isOK()) { + ErrorCodes::Error code = status.code() == ErrorCodes::UnknownError ? + ErrorCodes::RoleModificationFailed : status.code(); + return appendCommandStatus( + result, + Status(code, + str::stream() << "Removed role " << roleName.getFullName() << + " from all users but failed to remove from all roles: " << + status.reason())); + } + + audit::logDropRole(ClientBasic::getCurrent(), + roleName); + // Finally, remove the actual role document + status = authzManager->removeRoleDocuments( + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << roleName.getRole() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << roleName.getDB()), + writeConcern, + &nMatched); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + if (!status.isOK()) { + return appendCommandStatus( + result, + Status(status.code(), + str::stream() << "Removed role " << roleName.getFullName() << + " from all users and roles but failed to actually delete" + " the role itself: " << status.reason())); + } + + dassert(nMatched == 0 || nMatched == 1); + if (nMatched == 0) { + return appendCommandStatus( + result, + Status(ErrorCodes::RoleNotFound, + str::stream() << "Role '" << roleName.getFullName() << + "' not found")); + } + + return true; + } + + } cmdDropRole; + + class CmdDropAllRolesFromDatabase: public Command { + public: + + CmdDropAllRolesFromDatabase() : Command("dropAllRolesFromDatabase") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual void help(stringstream& ss) const { + ss << "Drops all roles from the given database. Before deleting the roles completely " + "it must remove them from any users or other roles that reference them. If any " + "errors occur in the middle of that process it's possible to be left in a state " + "where the roles have been removed from some user/roles but otherwise still " + "exist." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(dbname), ActionType::dropRole)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to drop roles from the " << + dbname << " database"); + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + BSONObj writeConcern; + Status status = auth::parseDropAllRolesFromDatabaseCommand(cmdObj, + dbname, + &writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("Drop roles from database")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + // Remove these roles from all users + int nMatched; + status = authzManager->updateAuthzDocuments( + AuthorizationManager::usersCollectionNamespace, + BSON("roles" << BSON(AuthorizationManager::ROLE_SOURCE_FIELD_NAME << dbname)), + BSON("$pull" << BSON("roles" << + BSON(AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + dbname))), + false, + true, + writeConcern, + &nMatched); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + if (!status.isOK()) { + ErrorCodes::Error code = status.code() == ErrorCodes::UnknownError ? + ErrorCodes::UserModificationFailed : status.code(); + return appendCommandStatus( + result, + Status(code, + str::stream() << "Failed to remove roles from \"" << dbname + << "\" db from all users: " << status.reason())); + } + + // Remove these roles from all other roles + std::string sourceFieldName = + str::stream() << "roles." << AuthorizationManager::ROLE_SOURCE_FIELD_NAME; + status = authzManager->updateAuthzDocuments( + AuthorizationManager::rolesCollectionNamespace, + BSON(sourceFieldName << dbname), + BSON("$pull" << BSON("roles" << + BSON(AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + dbname))), + false, + true, + writeConcern, + &nMatched); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + if (!status.isOK()) { + ErrorCodes::Error code = status.code() == ErrorCodes::UnknownError ? + ErrorCodes::RoleModificationFailed : status.code(); + return appendCommandStatus( + result, + Status(code, + str::stream() << "Failed to remove roles from \"" << dbname + << "\" db from all roles: " << status.reason())); + } + + audit::logDropAllRolesFromDatabase(ClientBasic::getCurrent(), dbname); + // Finally, remove the actual role documents + status = authzManager->removeRoleDocuments( + BSON(AuthorizationManager::ROLE_SOURCE_FIELD_NAME << dbname), + writeConcern, + &nMatched); + // Must invalidate even on bad status - what if the write succeeded but the GLE failed? + authzManager->invalidateUserCache(); + if (!status.isOK()) { + return appendCommandStatus( + result, + Status(status.code(), + str::stream() << "Removed roles from \"" << dbname << "\" db " + " from all users and roles but failed to actually delete" + " those roles themselves: " << status.reason())); + } + + result.append("n", nMatched); + + return true; + } + + } cmdDropAllRolesFromDatabase; + + class CmdRolesInfo: public Command { + public: + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual bool slaveOverrideOk() const { + return true; + } + + virtual LockType locktype() const { + return NONE; + } + + CmdRolesInfo() : Command("rolesInfo") {} + + virtual void help(stringstream& ss) const { + ss << "Returns information about roles." << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + auth::RolesInfoArgs args; + Status status = auth::parseRolesInfoCommand(cmdObj, dbname, &args); + if (!status.isOK()) { + return status; + } + + if (args.allForDB) { + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(dbname), ActionType::viewRole)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to view roles from the " << + dbname << " database"); + } + } else { + for (size_t i = 0; i < args.roleNames.size(); ++i) { + if (authzSession->isAuthenticatedAsUserWithRole(args.roleNames[i])) { + continue; // Can always see roles that you are a member of + } + + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(args.roleNames[i].getDB()), + ActionType::viewRole)) { + return Status(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to view roles from the " << + args.roleNames[i].getDB() << " database"); + } + } + } + + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + + auth::RolesInfoArgs args; + Status status = auth::parseRolesInfoCommand(cmdObj, dbname, &args); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + status = requireAuthSchemaVersion26UpgradeOrFinal(getGlobalAuthorizationManager()); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + BSONArrayBuilder rolesArrayBuilder; + if (args.allForDB) { + std::vector rolesDocs; + status = getGlobalAuthorizationManager()->getRoleDescriptionsForDB( + dbname, args.showPrivileges, args.showBuiltinRoles, &rolesDocs); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + for (size_t i = 0; i < rolesDocs.size(); ++i) { + rolesArrayBuilder.append(rolesDocs[i]); + } + } else { + for (size_t i = 0; i < args.roleNames.size(); ++i) { + BSONObj roleDetails; + status = getGlobalAuthorizationManager()->getRoleDescription( + args.roleNames[i], args.showPrivileges, &roleDetails); + if (status.code() == ErrorCodes::RoleNotFound) { + continue; + } + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + rolesArrayBuilder.append(roleDetails); + } + } + result.append("roles", rolesArrayBuilder.arr()); + return true; + } + + } cmdRolesInfo; + + class CmdInvalidateUserCache: public Command { + public: + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return true; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual bool adminOnly() const { + return true; + } + + virtual bool isWriteCommandForConfigServer() const { return false; } + + CmdInvalidateUserCache() : Command("invalidateUserCache") {} + + virtual void help(stringstream& ss) const { + ss << "Invalidates the in-memory cache of user information" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::invalidateUserCache)) { + return Status(ErrorCodes::Unauthorized, "Not authorized to invalidate user cache"); + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + authzManager->invalidateUserCache(); + return true; + } + + } cmdInvalidateUserCache; + + class CmdGetCacheGeneration: public Command { + public: + + virtual bool slaveOk() const { + return true; + } + + virtual bool adminOnly() const { + return true; + } + + virtual LockType locktype() const { return NONE; } + + virtual bool isWriteCommandForConfigServer() const { return false; } + + CmdGetCacheGeneration() : Command("_getUserCacheGeneration") {} + + virtual void help(stringstream& ss) const { + ss << "internal" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::internal)) { + return Status(ErrorCodes::Unauthorized, "Not authorized to get cache generation"); + } + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + result.append("cacheGeneration", authzManager->getCacheGeneration()); + return true; + } + + } CmdGetCacheGeneration; + + /** + * This command is used only by mongorestore to handle restoring users/roles. We do this so + * that mongorestore doesn't do direct inserts into the admin.system.users and + * admin.system.roles, which would bypass the authzUpdateLock and allow multiple concurrent + * modifications to users/roles. What mongorestore now does instead is it inserts all user/role + * definitions it wants to restore into temporary collections, then this command moves those + * user/role definitions into their proper place in admin.system.users and admin.system.roles. + * It either adds the users/roles to the existing ones or replaces the existing ones, depending + * on whether the "drop" argument is true or false. + */ + class CmdMergeAuthzCollections : public Command { + public: + + CmdMergeAuthzCollections() : Command("_mergeAuthzCollections") {} + + virtual bool logTheOp() { + return false; + } + + virtual bool slaveOk() const { + return false; + } + + virtual LockType locktype() const { + return NONE; + } + + virtual bool adminOnly() const { + return true; + } + + virtual void help(stringstream& ss) const { + ss << "Internal command used by mongorestore for updating user/role data" << endl; + } + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + auth::MergeAuthzCollectionsArgs args; + Status status = auth::parseMergeAuthzCollectionsCommand(cmdObj, &args); + if (!status.isOK()) { + return status; + } + + AuthorizationSession* authzSession = client->getAuthorizationSession(); + ActionSet actions; + actions.addAction(ActionType::createUser); + actions.addAction(ActionType::createRole); + actions.addAction(ActionType::grantRole); + actions.addAction(ActionType::revokeRole); + if (args.drop) { + actions.addAction(ActionType::dropUser); + actions.addAction(ActionType::dropRole); + } + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forAnyNormalResource(), actions)) { + return Status(ErrorCodes::Unauthorized, + "Not authorized to update user/role data using _mergeAuthzCollections" + " command"); + } + if (!args.usersCollName.empty() && + !authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString(args.usersCollName)), + ActionType::find)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "Not authorized to read " << + args.usersCollName); + } + if (!args.rolesCollName.empty() && + !authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString(args.rolesCollName)), + ActionType::find)) { + return Status(ErrorCodes::Unauthorized, + mongoutils::str::stream() << "Not authorized to read " << + args.rolesCollName); + } + return Status::OK(); + } + + static UserName extractUserNameFromBSON(const BSONObj& userObj) { + std::string name; + std::string db; + Status status = bsonExtractStringField(userObj, + AuthorizationManager::USER_NAME_FIELD_NAME, + &name); + uassertStatusOK(status); + status = bsonExtractStringField(userObj, + AuthorizationManager::USER_DB_FIELD_NAME, + &db); + uassertStatusOK(status); + return UserName(name, db); + } + + /** + * Extracts the UserName from the user document and adds it to set of existing users. + * This function is written so it can used with boost::bind over the result set of a query + * on admin.system.users to add the user names of all existing users to the "usersToDrop" + * set used in the command body. + */ + static void extractAndInsertUserName(unordered_set* existingUsers, + const BSONObj& userObj) { + + UserName userName = extractUserNameFromBSON(userObj); + existingUsers->insert(userName); + } + + static RoleName extractRoleNameFromBSON(const BSONObj& roleObj) { + std::string name; + std::string db; + Status status = bsonExtractStringField(roleObj, + AuthorizationManager::ROLE_NAME_FIELD_NAME, + &name); + uassertStatusOK(status); + status = bsonExtractStringField(roleObj, + AuthorizationManager::ROLE_SOURCE_FIELD_NAME, + &db); + uassertStatusOK(status); + return RoleName(name, db); + } + + /** + * Extracts the RoleName from the role document and adds it to set of existing roles. + * This function is written so it can used with boost::bind over the result set of a query + * on admin.system.roles to add the role names of all existing roles to the "rolesToDrop" + * set used in the command body. + */ + static void extractAndInsertRoleName(unordered_set* existingRoles, + const BSONObj& roleObj) { + RoleName roleName = extractRoleNameFromBSON(roleObj); + existingRoles->insert(roleName); + } + + /** + * Audits the fact that we are creating or updating the user described by userObj. + */ + static void auditCreateOrUpdateUser(const BSONObj& userObj, bool create) { + UserName userName = extractUserNameFromBSON(userObj); + std::vector roles; + uassertStatusOK(auth::parseRoleNamesFromBSONArray(BSONArray(userObj["roles"].Obj()), + userName.getDB(), + &roles)); + BSONObj customData; + if (userObj.hasField("customData")) { + customData = userObj["customData"].Obj(); + } + + if (create) { + audit::logCreateUser(ClientBasic::getCurrent(), + userName, + userObj["credentials"].Obj().hasField("MONGODB-CR"), + userObj.hasField("customData") ? &customData : NULL, + roles); + } else { + audit::logUpdateUser(ClientBasic::getCurrent(), + userName, + userObj["credentials"].Obj().hasField("MONGODB-CR"), + userObj.hasField("customData") ? &customData : NULL, + &roles); + + } + } + + /** + * Audits the fact that we are creating or updating the role described by roleObj. + */ + static void auditCreateOrUpdateRole(const BSONObj& roleObj, bool create) { + RoleName roleName = extractRoleNameFromBSON(roleObj); + std::vector roles; + std::vector privileges; + uassertStatusOK(auth::parseRoleNamesFromBSONArray(BSONArray(roleObj["roles"].Obj()), + roleName.getDB(), + &roles)); + uassertStatusOK(auth::parseAndValidatePrivilegeArray( + BSONArray(roleObj["privileges"].Obj()), &privileges)); + if (create) { + audit::logCreateRole(ClientBasic::getCurrent(), roleName, roles, privileges); + } else { + audit::logUpdateRole(ClientBasic::getCurrent(), roleName, &roles, &privileges); + } + } + + /** + * Designed to be used with boost::bind to be called on every user object in the result + * set of a query over the tempUsersCollection provided to the command. For each user + * in the temp collection that is defined on the given db, adds that user to the actual + * admin.system.users collection. + * Also removes any users it encounters from the usersToDrop set. + */ + static void addUser(AuthorizationManager* authzManager, + const StringData& db, + bool update, + const BSONObj& writeConcern, + unordered_set* usersToDrop, + const BSONObj& userObj) { + UserName userName = extractUserNameFromBSON(userObj); + if (!db.empty() && userName.getDB() != db) { + return; + } + + if (update && usersToDrop->count(userName)) { + auditCreateOrUpdateUser(userObj, false); + Status status = authzManager->updatePrivilegeDocument(userName, + userObj, + writeConcern); + if (!status.isOK()) { + // Match the behavior of mongorestore to continue on failure + warning() << "Could not update user " << userName << + " in _mergeAuthzCollections command: " << status << endl; + } + } else { + auditCreateOrUpdateUser(userObj, true); + Status status = authzManager->insertPrivilegeDocument(userName.getDB().toString(), + userObj, + writeConcern); + if (!status.isOK()) { + // Match the behavior of mongorestore to continue on failure + warning() << "Could not insert user " << userName << + " in _mergeAuthzCollections command: " << status << endl; + } + } + usersToDrop->erase(userName); + } + + /** + * Designed to be used with boost::bind to be called on every role object in the result + * set of a query over the tempRolesCollection provided to the command. For each role + * in the temp collection that is defined on the given db, adds that role to the actual + * admin.system.roles collection. + * Also removes any roles it encounters from the rolesToDrop set. + */ + static void addRole(AuthorizationManager* authzManager, + const StringData& db, + bool update, + const BSONObj& writeConcern, + unordered_set* rolesToDrop, + const BSONObj roleObj) { + RoleName roleName = extractRoleNameFromBSON(roleObj); + if (!db.empty() && roleName.getDB() != db) { + return; + } + + if (update && rolesToDrop->count(roleName)) { + auditCreateOrUpdateRole(roleObj, false); + Status status = authzManager->updateRoleDocument(roleName, + roleObj, + writeConcern); + if (!status.isOK()) { + // Match the behavior of mongorestore to continue on failure + warning() << "Could not update role " << roleName << + " in _mergeAuthzCollections command: " << status << endl; + } + } else { + auditCreateOrUpdateRole(roleObj, true); + Status status = authzManager->insertRoleDocument(roleObj, writeConcern); + if (!status.isOK()) { + // Match the behavior of mongorestore to continue on failure + warning() << "Could not insert role " << roleName << + " in _mergeAuthzCollections command: " << status << endl; + } + } + rolesToDrop->erase(roleName); + } + + /** + * Moves all user objects from usersCollName into admin.system.users. If drop is true, + * removes any users that were in admin.system.users but not in usersCollName. + */ + Status processUsers(AuthorizationManager* authzManager, + const StringData& usersCollName, + const StringData& db, + bool drop, + const BSONObj& writeConcern) { + // When the "drop" argument has been provided, we use this set to store the users + // that are currently in the system, and remove from it as we encounter + // same-named users in the collection we are restoring from. Once we've fully + // moved over the temp users collection into its final location, we drop + // any users that previously existed there but weren't in the temp collection. + // This is so that we can completely replace the system.users + // collection with the users from the temp collection, without removing all + // users at the beginning and thus potentially locking ourselves out by having + // no users in the whole system for a time. + unordered_set usersToDrop; + + if (drop) { + // Create map of the users currently in the DB + BSONObj query = db.empty() ? + BSONObj() : BSON(AuthorizationManager::USER_DB_FIELD_NAME << db); + BSONObj fields = BSON(AuthorizationManager::USER_NAME_FIELD_NAME << 1 << + AuthorizationManager::USER_DB_FIELD_NAME << 1); + + Status status = authzManager->queryAuthzDocument( + AuthorizationManager::usersCollectionNamespace, + query, + fields, + boost::bind(&CmdMergeAuthzCollections::extractAndInsertUserName, + &usersToDrop, + _1)); + if (!status.isOK()) { + return status; + } + } + + Status status = authzManager->queryAuthzDocument( + NamespaceString(usersCollName), + db.empty() ? BSONObj() : BSON(AuthorizationManager::USER_DB_FIELD_NAME << db), + BSONObj(), + boost::bind(&CmdMergeAuthzCollections::addUser, + authzManager, + db, + drop, + writeConcern, + &usersToDrop, + _1)); + if (!status.isOK()) { + return status; + } + + if (drop) { + int numRemoved; + for (unordered_set::iterator it = usersToDrop.begin(); + it != usersToDrop.end(); ++it) { + const UserName& userName = *it; + audit::logDropUser(ClientBasic::getCurrent(), userName); + status = authzManager->removePrivilegeDocuments( + BSON(AuthorizationManager::USER_NAME_FIELD_NAME << + userName.getUser().toString() << + AuthorizationManager::USER_DB_FIELD_NAME << + userName.getDB().toString() + ), + writeConcern, + &numRemoved); + if (!status.isOK()) { + return status; + } + dassert(numRemoved == 1); + } + } + + return Status::OK(); + } + + /** + * Moves all user objects from usersCollName into admin.system.users. If drop is true, + * removes any users that were in admin.system.users but not in usersCollName. + */ + Status processRoles(AuthorizationManager* authzManager, + const StringData& rolesCollName, + const StringData& db, + bool drop, + const BSONObj& writeConcern) { + // When the "drop" argument has been provided, we use this set to store the roles + // that are currently in the system, and remove from it as we encounter + // same-named roles in the collection we are restoring from. Once we've fully + // moved over the temp roles collection into its final location, we drop + // any roles that previously existed there but weren't in the temp collection. + // This is so that we can completely replace the system.roles + // collection with the roles from the temp collection, without removing all + // roles at the beginning and thus potentially locking ourselves out. + unordered_set rolesToDrop; + + if (drop) { + // Create map of the roles currently in the DB + BSONObj query = db.empty() ? + BSONObj() : BSON(AuthorizationManager::ROLE_SOURCE_FIELD_NAME << db); + BSONObj fields = BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << 1 << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << 1); + + Status status = authzManager->queryAuthzDocument( + AuthorizationManager::rolesCollectionNamespace, + query, + fields, + boost::bind(&CmdMergeAuthzCollections::extractAndInsertRoleName, + &rolesToDrop, + _1)); + if (!status.isOK()) { + return status; + } + } + + Status status = authzManager->queryAuthzDocument( + NamespaceString(rolesCollName), + db.empty() ? + BSONObj() : BSON(AuthorizationManager::ROLE_SOURCE_FIELD_NAME << db), + BSONObj(), + boost::bind(&CmdMergeAuthzCollections::addRole, + authzManager, + db, + drop, + writeConcern, + &rolesToDrop, + _1)); + if (!status.isOK()) { + return status; + } + + if (drop) { + int numRemoved; + for (unordered_set::iterator it = rolesToDrop.begin(); + it != rolesToDrop.end(); ++it) { + const RoleName& roleName = *it; + audit::logDropRole(ClientBasic::getCurrent(), roleName); + status = authzManager->removeRoleDocuments( + BSON(AuthorizationManager::ROLE_NAME_FIELD_NAME << + roleName.getRole().toString() << + AuthorizationManager::ROLE_SOURCE_FIELD_NAME << + roleName.getDB().toString() + ), + writeConcern, + &numRemoved); + if (!status.isOK()) { + return status; + } + dassert(numRemoved == 1); + } + } + + return Status::OK(); + } + + bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl) { + + auth::MergeAuthzCollectionsArgs args; + Status status = auth::parseMergeAuthzCollectionsCommand(cmdObj, &args); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (args.usersCollName.empty() && args.rolesCollName.empty()) { + return appendCommandStatus( + result, Status(ErrorCodes::BadValue, + "Must provide at least one of \"tempUsersCollection\" and " + "\"tempRolescollection\"")); + } + + AuthorizationManager* authzManager = getGlobalAuthorizationManager(); + AuthzDocumentsUpdateGuard updateGuard(authzManager); + if (!updateGuard.tryLock("_mergeAuthzCollections")) { + return appendCommandStatus( + result, + Status(ErrorCodes::LockBusy, "Could not lock auth data update lock.")); + } + + status = requireAuthSchemaVersion26Final(authzManager); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (!args.usersCollName.empty()) { + Status status = processUsers(authzManager, + args.usersCollName, + args.db, + args.drop, + args.writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + } + + if (!args.rolesCollName.empty()) { + Status status = processRoles(authzManager, + args.rolesCollName, + args.db, + args.drop, + args.writeConcern); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + } + + return true; + } + + } cmdMergeAuthzCollections; + + CmdAuthSchemaUpgrade::CmdAuthSchemaUpgrade() : Command("authSchemaUpgrade") {} + CmdAuthSchemaUpgrade::~CmdAuthSchemaUpgrade() {} + + bool CmdAuthSchemaUpgrade::slaveOk() const { return false; } + bool CmdAuthSchemaUpgrade::adminOnly() const { return true; } + Command::LockType CmdAuthSchemaUpgrade::locktype() const { return NONE; } + + void CmdAuthSchemaUpgrade::help(stringstream& ss) const { + ss << "Upgrades the auth data storage schema"; + } + + Status CmdAuthSchemaUpgrade::checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + + AuthorizationSession* authzSession = client->getAuthorizationSession(); + if (!authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::authSchemaUpgrade)) { + return Status(ErrorCodes::Unauthorized, + "Not authorized to run authSchemaUpgrade command."); + } + return Status::OK(); + } +} diff --git a/src/mongo/db/commands/user_management_commands.h b/src/mongo/db/commands/user_management_commands.h new file mode 100644 index 00000000000..39b74cab4b3 --- /dev/null +++ b/src/mongo/db/commands/user_management_commands.h @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include "mongo/db/commands.h" + +namespace mongo { + + class CmdAuthSchemaUpgrade : public Command { + public: + CmdAuthSchemaUpgrade(); + virtual ~CmdAuthSchemaUpgrade(); + + virtual bool slaveOk() const; + virtual bool adminOnly() const; + virtual LockType locktype() const; + virtual void help(stringstream& ss) const; + + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj); + }; + +} // namespace mongo diff --git a/src/mongo/db/commands/validate.cpp b/src/mongo/db/commands/validate.cpp new file mode 100644 index 00000000000..ed63031b2be --- /dev/null +++ b/src/mongo/db/commands/validate.cpp @@ -0,0 +1,422 @@ +// validate.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/commands.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/query/runner.h" +#include "mongo/db/storage/extent.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + class ValidateCmd : public Command { + public: + ValidateCmd() : Command( "validate" ) {} + + virtual bool slaveOk() const { + return true; + } + + virtual void help(stringstream& h) const { h << "Validate contents of a namespace by scanning its data structures for correctness. Slow.\n" + "Add full:true option to do a more thorough check"; } + + virtual LockType locktype() const { return READ; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::validate); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); + } + //{ validate: "collectionnamewithoutthedbpart" [, scandata: ] [, full: } */ + + bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { + string ns = dbname + "." + cmdObj.firstElement().valuestrsafe(); + + NamespaceString ns_string(ns); + if ( !ns_string.isNormal() && cmdObj["full"].trueValue() ) { + errmsg = "Can only run full validate on a regular collection"; + return false; + } + + if (!serverGlobalParams.quiet) { + MONGO_TLOG(0) << "CMD: validate " << ns << endl; + } + + Database* db = cc().database(); + if ( !db ) { + errmsg = "ns nout found"; + return false; + } + + Collection* collection = db->getCollection( ns ); + if ( !collection ) { + errmsg = "ns not found"; + return false; + } + + result.append( "ns", ns ); + validateNS( ns , collection, cmdObj, result); + return true; + } + + private: + void validateNS(const string& ns, + Collection* collection, + const BSONObj& cmdObj, + BSONObjBuilder& result) { + + const bool full = cmdObj["full"].trueValue(); + const bool scanData = full || cmdObj["scandata"].trueValue(); + + NamespaceDetails* nsd = collection->details(); + + bool valid = true; + BSONArrayBuilder errors; // explanation(s) for why valid = false + if ( collection->isCapped() ){ + result.append("capped", nsd->isCapped()); + result.appendNumber("max", nsd->maxCappedDocs()); + } + + if ( nsd->firstExtent().isNull() ) + result.append( "firstExtent", "null" ); + else + result.append( "firstExtent", str::stream() << nsd->firstExtent().toString() + << " ns:" << nsd->firstExtent().ext()->nsDiagnostic.toString()); + if ( nsd->lastExtent().isNull() ) + result.append( "lastExtent", "null" ); + else + result.append( "lastExtent", str::stream() << nsd->lastExtent().toString() + << " ns:" << nsd->lastExtent().ext()->nsDiagnostic.toString()); + + BSONArrayBuilder extentData; + int extentCount = 0; + try { + + if ( !nsd->firstExtent().isNull() ) { + nsd->firstExtent().ext()->assertOk(); + nsd->lastExtent().ext()->assertOk(); + } + + DiskLoc extentDiskLoc = nsd->firstExtent(); + while (!extentDiskLoc.isNull()) { + Extent* thisExtent = extentDiskLoc.ext(); + if (full) { + extentData << thisExtent->dump(); + } + if (!thisExtent->validates(extentDiskLoc, &errors)) { + valid = false; + } + DiskLoc nextDiskLoc = thisExtent->xnext; + if (extentCount > 0 && !nextDiskLoc.isNull() + && nextDiskLoc.ext()->xprev != extentDiskLoc) { + StringBuilder sb; + sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString() + << " in extent " << nextDiskLoc.toString() + << " does not point to extent " << extentDiskLoc.toString(); + errors << sb.str(); + valid = false; + } + if (nextDiskLoc.isNull() && extentDiskLoc != nsd->lastExtent()) { + StringBuilder sb; + sb << "'lastExtent' pointer " << nsd->lastExtent().toString() + << " does not point to last extent in list " << extentDiskLoc.toString(); + errors << sb.str(); + valid = false; + } + extentDiskLoc = nextDiskLoc; + extentCount++; + killCurrentOp.checkForInterrupt(); + } + } + catch (const DBException& e) { + StringBuilder sb; + sb << "exception validating extent " << extentCount + << ": " << e.what(); + errors << sb.str(); + valid = false; + } + result.append("extentCount", extentCount); + + if ( full ) + result.appendArray( "extents" , extentData.arr() ); + + result.appendNumber("datasize", nsd->dataSize()); + result.appendNumber("nrecords", nsd->numRecords()); + result.appendNumber("lastExtentSize", nsd->lastExtentSize()); + result.appendNumber("padding", nsd->paddingFactor()); + + try { + + bool testingLastExtent = false; + try { + if (nsd->firstExtent().isNull()) { + // this is ok + } + else { + result.append("firstExtentDetails", nsd->firstExtent().ext()->dump()); + if (!nsd->firstExtent().ext()->xprev.isNull()) { + StringBuilder sb; + sb << "'xprev' pointer in 'firstExtent' " << nsd->firstExtent().toString() + << " is " << nsd->firstExtent().ext()->xprev.toString() + << ", should be null"; + errors << sb.str(); + valid=false; + } + } + testingLastExtent = true; + if (nsd->lastExtent().isNull()) { + // this is ok + } + else { + if (nsd->firstExtent() != nsd->lastExtent()) { + result.append("lastExtentDetails", nsd->lastExtent().ext()->dump()); + if (!nsd->lastExtent().ext()->xnext.isNull()) { + StringBuilder sb; + sb << "'xnext' pointer in 'lastExtent' " << nsd->lastExtent().toString() + << " is " << nsd->lastExtent().ext()->xnext.toString() + << ", should be null"; + errors << sb.str(); + valid = false; + } + } + } + } + catch (const DBException& e) { + StringBuilder sb; + sb << "exception processing '" + << (testingLastExtent ? "lastExtent" : "firstExtent") + << "': " << e.what(); + errors << sb.str(); + valid = false; + } + + set recs; + if( scanData ) { + int n = 0; + int nInvalid = 0; + long long nQuantizedSize = 0; + long long nPowerOf2QuantizedSize = 0; + long long len = 0; + long long nlen = 0; + long long bsonLen = 0; + int outOfOrder = 0; + DiskLoc cl_last; + + DiskLoc cl; + Runner::RunnerState state; + auto_ptr runner(InternalPlanner::collectionScan(ns)); + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &cl))) { + n++; + + if ( n < 1000000 ) + recs.insert(cl); + if ( nsd->isCapped() ) { + if ( cl < cl_last ) + outOfOrder++; + cl_last = cl; + } + + Record *r = cl.rec(); + len += r->lengthWithHeaders(); + nlen += r->netLength(); + + if ( r->lengthWithHeaders() == + NamespaceDetails::quantizeAllocationSpace + ( r->lengthWithHeaders() ) ) { + // Count the number of records having a size consistent with + // the quantizeAllocationSpace quantization implementation. + ++nQuantizedSize; + } + + if ( r->lengthWithHeaders() == + NamespaceDetails::quantizePowerOf2AllocationSpace + ( r->lengthWithHeaders() - 1 ) ) { + // Count the number of records having a size consistent with the + // quantizePowerOf2AllocationSpace quantization implementation. + // Because of SERVER-8311, power of 2 quantization is not idempotent and + // r->lengthWithHeaders() - 1 must be checked instead of the record + // length itself. + ++nPowerOf2QuantizedSize; + } + + if (full){ + BSONObj obj = BSONObj::make(r); + const Status status = validateBSON(obj.objdata(), obj.objsize()); + if (!status.isOK()) { + valid = false; + if (nInvalid == 0) // only log once; + errors << "invalid bson object detected (see logs for more info)"; + + nInvalid++; + log() << "Invalid bson detected in " << ns + << ": " << status.reason(); + } + else { + bsonLen += obj.objsize(); + } + } + } + if (Runner::RUNNER_EOF != state) { + // TODO: more descriptive logging. + warning() << "Internal error while reading collection " << ns << endl; + } + if ( nsd->isCapped() && !nsd->capLooped() ) { + result.append("cappedOutOfOrder", outOfOrder); + if ( outOfOrder > 1 ) { + valid = false; + errors << "too many out of order records"; + } + } + result.append("objectsFound", n); + + if (full) { + result.append("invalidObjects", nInvalid); + } + + result.appendNumber("nQuantizedSize", nQuantizedSize); + result.appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize); + result.appendNumber("bytesWithHeaders", len); + result.appendNumber("bytesWithoutHeaders", nlen); + + if (full) { + result.appendNumber("bytesBson", bsonLen); + } + } + + BSONArrayBuilder deletedListArray; + for ( int i = 0; i < Buckets; i++ ) { + deletedListArray << nsd->deletedListEntry(i).isNull(); + } + + int ndel = 0; + long long delSize = 0; + BSONArrayBuilder delBucketSizes; + int incorrect = 0; + for ( int i = 0; i < Buckets; i++ ) { + DiskLoc loc = nsd->deletedListEntry(i); + try { + int k = 0; + while ( !loc.isNull() ) { + if ( recs.count(loc) ) + incorrect++; + ndel++; + + if ( loc.questionable() ) { + if( nsd->isCapped() && !loc.isValid() && i == 1 ) { + /* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid + see comments in namespace.h + */ + break; + } + + string err( str::stream() << "bad pointer in deleted record list: " + << loc.toString() + << " bucket: " << i + << " k: " << k ); + errors << err; + valid = false; + break; + } + + DeletedRecord *d = loc.drec(); + delSize += d->lengthWithHeaders(); + loc = d->nextDeleted(); + k++; + killCurrentOp.checkForInterrupt(); + } + delBucketSizes << k; + } + catch (...) { + errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i)); + valid = false; + } + } + result.appendNumber("deletedCount", ndel); + result.appendNumber("deletedSize", delSize); + if ( full ) { + result << "delBucketSizes" << delBucketSizes.arr(); + } + + if ( incorrect ) { + errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list"); + valid = false; + } + + int idxn = 0; + try { + IndexCatalog* indexCatalog = collection->getIndexCatalog(); + + result.append("nIndexes", indexCatalog->numIndexesReady() ); + BSONObjBuilder indexes; // not using subObjStart to be exception safe + IndexCatalog::IndexIterator i = indexCatalog->getIndexIterator(false); + while( i.more() ) { + IndexDescriptor* descriptor = i.next(); + log() << "validating index " << descriptor->indexNamespace() << endl; + IndexAccessMethod* iam = indexCatalog->getIndex( descriptor ); + verify( iam ); + + int64_t keys; + iam->validate(&keys); + indexes.appendNumber(descriptor->indexNamespace(), + static_cast(keys)); + idxn++; + } + result.append("keysPerIndex", indexes.done()); + } + catch (...) { + errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn)); + valid=false; + } + + } + catch (AssertionException) { + errors << "exception during validate"; + valid = false; + } + + result.appendBool("valid", valid); + result.append("errors", errors.arr()); + + if ( !full ){ + result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan."); + } + + if ( !valid ) { + result.append("advice", "ns corrupt. See http://dochub.mongodb.org/core/data-recovery"); + } + + } + } validateCmd; + +} diff --git a/src/mongo/db/commands/write_commands/batch_executor.cpp b/src/mongo/db/commands/write_commands/batch_executor.cpp new file mode 100644 index 00000000000..eb66b396d17 --- /dev/null +++ b/src/mongo/db/commands/write_commands/batch_executor.cpp @@ -0,0 +1,1272 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/commands/write_commands/batch_executor.h" + +#include + +#include "mongo/base/error_codes.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/commands.h" +#include "mongo/db/instance.h" +#include "mongo/db/introspect.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/lasterror.h" +#include "mongo/db/ops/delete_executor.h" +#include "mongo/db/ops/delete_request.h" +#include "mongo/db/ops/insert.h" +#include "mongo/db/ops/update_executor.h" +#include "mongo/db/ops/update_lifecycle_impl.h" +#include "mongo/db/ops/update_request.h" +#include "mongo/db/pagefault.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/replication_server_status.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/server_parameters.h" +#include "mongo/db/stats/counters.h" +#include "mongo/db/write_concern.h" +#include "mongo/s/collection_metadata.h" +#include "mongo/s/d_logic.h" +#include "mongo/s/shard_key_pattern.h" +#include "mongo/s/stale_exception.h" +#include "mongo/s/write_ops/batched_upsert_detail.h" +#include "mongo/s/write_ops/write_error_detail.h" +#include "mongo/util/elapsed_tracker.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace { + + /** + * Data structure to safely hold and clean up results of single write operations. + */ + class WriteOpResult { + MONGO_DISALLOW_COPYING(WriteOpResult); + public: + WriteOpResult() {} + + WriteOpStats& getStats() { return _stats; } + + WriteErrorDetail* getError() { return _error.get(); } + WriteErrorDetail* releaseError() { return _error.release(); } + void setError(WriteErrorDetail* error) { _error.reset(error); } + + private: + WriteOpStats _stats; + std::auto_ptr _error; + }; + + } // namespace + + // TODO: Determine queueing behavior we want here + MONGO_EXPORT_SERVER_PARAMETER( queueForMigrationCommit, bool, true ); + + using mongoutils::str::stream; + + WriteBatchExecutor::WriteBatchExecutor( const BSONObj& wc, + Client* client, + OpCounters* opCounters, + LastError* le ) : + _defaultWriteConcern( wc ), + _client( client ), + _opCounters( opCounters ), + _le( le ), + _stats( new WriteBatchStats ) { + } + + static WCErrorDetail* toWriteConcernError( const Status& wcStatus, + const WriteConcernResult& wcResult ) { + + WCErrorDetail* wcError = new WCErrorDetail; + + wcError->setErrCode( wcStatus.code() ); + wcError->setErrMessage( wcStatus.reason() ); + if ( wcResult.wTimedOut ) + wcError->setErrInfo( BSON( "wtimeout" << true ) ); + + return wcError; + } + + static WriteErrorDetail* toWriteError( const Status& status ) { + + WriteErrorDetail* error = new WriteErrorDetail; + + // TODO: Complex transform here? + error->setErrCode( status.code() ); + error->setErrMessage( status.reason() ); + + return error; + } + + static void toBatchError( const Status& status, BatchedCommandResponse* response ) { + response->clear(); + response->setErrCode( status.code() ); + response->setErrMessage( status.reason() ); + response->setOk( false ); + dassert( response->isValid(NULL) ); + } + + static void noteInCriticalSection( WriteErrorDetail* staleError ) { + BSONObjBuilder builder; + if ( staleError->isErrInfoSet() ) + builder.appendElements( staleError->getErrInfo() ); + builder.append( "inCriticalSection", true ); + staleError->setErrInfo( builder.obj() ); + } + + void WriteBatchExecutor::executeBatch( const BatchedCommandRequest& request, + BatchedCommandResponse* response ) { + + // Validate namespace + const NamespaceString nss = NamespaceString( request.getNS() ); + if ( !nss.isValid() ) { + toBatchError( Status( ErrorCodes::InvalidNamespace, + nss.ns() + " is not a valid namespace" ), + response ); + return; + } + + // Make sure we can write to the namespace + Status allowedStatus = userAllowedWriteNS( nss ); + if ( !allowedStatus.isOK() ) { + toBatchError( allowedStatus, response ); + return; + } + + // Validate insert index requests + // TODO: Push insert index requests through createIndex once all upgrade paths support it + string errMsg; + if ( request.isInsertIndexRequest() && !request.isValidIndexRequest( &errMsg ) ) { + toBatchError( Status( ErrorCodes::InvalidOptions, errMsg ), response ); + return; + } + + // Validate write concern + // TODO: Lift write concern parsing out of this entirely + WriteConcernOptions writeConcern; + + BSONObj wcDoc; + if ( request.isWriteConcernSet() ) { + wcDoc = request.getWriteConcern(); + } + + Status wcStatus = Status::OK(); + if ( wcDoc.isEmpty() ) { + + // The default write concern if empty is w : 1 + // Specifying w : 0 is/was allowed, but is interpreted identically to w : 1 + + wcStatus = writeConcern.parse( + _defaultWriteConcern.isEmpty() ? + WriteConcernOptions::Acknowledged : _defaultWriteConcern ); + + if ( writeConcern.wNumNodes == 0 && writeConcern.wMode.empty() ) { + writeConcern.wNumNodes = 1; + } + } + else { + wcStatus = writeConcern.parse( wcDoc ); + } + + if ( wcStatus.isOK() ) { + wcStatus = validateWriteConcern( writeConcern ); + } + + if ( !wcStatus.isOK() ) { + toBatchError( wcStatus, response ); + return; + } + + if ( request.sizeWriteOps() == 0u ) { + toBatchError( Status( ErrorCodes::InvalidLength, + "no write ops were included in the batch" ), + response ); + return; + } + + // Validate batch size + if ( request.sizeWriteOps() > BatchedCommandRequest::kMaxWriteBatchSize ) { + toBatchError( Status( ErrorCodes::InvalidLength, + stream() << "exceeded maximum write batch size of " + << BatchedCommandRequest::kMaxWriteBatchSize ), + response ); + return; + } + + // + // End validation + // + + bool silentWC = writeConcern.wMode.empty() && writeConcern.wNumNodes == 0 + && writeConcern.syncMode == WriteConcernOptions::NONE; + + Timer commandTimer; + + OwnedPointerVector writeErrorsOwned; + vector& writeErrors = writeErrorsOwned.mutableVector(); + + OwnedPointerVector upsertedOwned; + vector& upserted = upsertedOwned.mutableVector(); + + // + // Apply each batch item, possibly bulking some items together in the write lock. + // Stops on error if batch is ordered. + // + + bulkExecute( request, &upserted, &writeErrors ); + + // + // Try to enforce the write concern if everything succeeded (unordered or ordered) + // OR if something succeeded and we're unordered. + // + + auto_ptr wcError; + bool needToEnforceWC = writeErrors.empty() + || ( !request.getOrdered() + && writeErrors.size() < request.sizeWriteOps() ); + + if ( needToEnforceWC ) { + + _client->curop()->setMessage( "waiting for write concern" ); + + WriteConcernResult res; + Status status = waitForWriteConcern( writeConcern, _client->getLastOp(), &res ); + + if ( !status.isOK() ) { + wcError.reset( toWriteConcernError( status, res ) ); + } + } + + // + // Refresh metadata if needed + // + + bool staleBatch = !writeErrors.empty() + && writeErrors.back()->getErrCode() == ErrorCodes::StaleShardVersion; + + if ( staleBatch ) { + + const BatchedRequestMetadata* requestMetadata = request.getMetadata(); + dassert( requestMetadata ); + + // Make sure our shard name is set or is the same as what was set previously + if ( shardingState.setShardName( requestMetadata->getShardName() ) ) { + + // + // First, we refresh metadata if we need to based on the requested version. + // + + ChunkVersion latestShardVersion; + shardingState.refreshMetadataIfNeeded( request.getTargetingNS(), + requestMetadata->getShardVersion(), + &latestShardVersion ); + + // Report if we're still changing our metadata + // TODO: Better reporting per-collection + if ( shardingState.inCriticalMigrateSection() ) { + noteInCriticalSection( writeErrors.back() ); + } + + if ( queueForMigrationCommit ) { + + // + // Queue up for migration to end - this allows us to be sure that clients will + // not repeatedly try to refresh metadata that is not yet written to the config + // server. Not necessary for correctness. + // Exposed as optional parameter to allow testing of queuing behavior with + // different network timings. + // + + const ChunkVersion& requestShardVersion = requestMetadata->getShardVersion(); + + // + // Only wait if we're an older version (in the current collection epoch) and + // we're not write compatible, implying that the current migration is affecting + // writes. + // + + if ( requestShardVersion.isOlderThan( latestShardVersion ) && + !requestShardVersion.isWriteCompatibleWith( latestShardVersion ) ) { + + while ( shardingState.inCriticalMigrateSection() ) { + + log() << "write request to old shard version " + << requestMetadata->getShardVersion().toString() + << " waiting for migration commit" << endl; + + shardingState.waitTillNotInCriticalSection( 10 /* secs */); + } + } + } + } + else { + // If our shard name is stale, our version must have been stale as well + dassert( writeErrors.size() == request.sizeWriteOps() ); + } + } + + // + // Construct response + // + + response->setOk( true ); + + if ( !silentWC ) { + + if ( upserted.size() ) { + response->setUpsertDetails( upserted ); + } + + if ( writeErrors.size() ) { + response->setErrDetails( writeErrors ); + } + + if ( wcError.get() ) { + response->setWriteConcernError( wcError.release() ); + } + + if ( anyReplEnabled() ) { + response->setLastOp( _client->getLastOp() ); + if (theReplSet) { + response->setElectionId( theReplSet->getElectionId() ); + } + } + + // Set the stats for the response + response->setN( _stats->numInserted + _stats->numUpserted + _stats->numMatched + + _stats->numDeleted ); + if ( request.getBatchType() == BatchedCommandRequest::BatchType_Update ) + response->setNModified( _stats->numModified ); + } + + dassert( response->isValid( NULL ) ); + } + + // Translates write item type to wire protocol op code. + // Helper for WriteBatchExecutor::applyWriteItem(). + static int getOpCode( BatchedCommandRequest::BatchType writeType ) { + switch ( writeType ) { + case BatchedCommandRequest::BatchType_Insert: + return dbInsert; + case BatchedCommandRequest::BatchType_Update: + return dbUpdate; + default: + dassert( writeType == BatchedCommandRequest::BatchType_Delete ); + return dbDelete; + } + return 0; + } + + static void buildStaleError( const ChunkVersion& shardVersionRecvd, + const ChunkVersion& shardVersionWanted, + WriteErrorDetail* error ) { + + // Write stale error to results + error->setErrCode( ErrorCodes::StaleShardVersion ); + + BSONObjBuilder infoB; + shardVersionWanted.addToBSON( infoB, "vWanted" ); + error->setErrInfo( infoB.obj() ); + + string errMsg = stream() << "stale shard version detected before write, received " + << shardVersionRecvd.toString() << " but local version is " + << shardVersionWanted.toString(); + error->setErrMessage( errMsg ); + } + + static bool checkShardVersion(ShardingState* shardingState, + const BatchedCommandRequest& request, + WriteOpResult* result) { + + const NamespaceString nss( request.getTargetingNS() ); + Lock::assertWriteLocked( nss.ns() ); + + ChunkVersion requestShardVersion = + request.isMetadataSet() && request.getMetadata()->isShardVersionSet() ? + request.getMetadata()->getShardVersion() : ChunkVersion::IGNORED(); + + if ( shardingState->enabled() ) { + + CollectionMetadataPtr metadata = shardingState->getCollectionMetadata( nss.ns() ); + + if ( !ChunkVersion::isIgnoredVersion( requestShardVersion ) ) { + + ChunkVersion shardVersion = + metadata ? metadata->getShardVersion() : ChunkVersion::UNSHARDED(); + + if ( !requestShardVersion.isWriteCompatibleWith( shardVersion ) ) { + result->setError(new WriteErrorDetail); + buildStaleError(requestShardVersion, shardVersion, result->getError()); + return false; + } + } + } + + return true; + } + + static bool checkIsMasterForCollection(const std::string& ns, WriteOpResult* result) { + if (!isMasterNs(ns.c_str())) { + WriteErrorDetail* errorDetail = new WriteErrorDetail; + result->setError(errorDetail); + errorDetail->setErrCode(ErrorCodes::NotMaster); + errorDetail->setErrMessage("Not primary while writing to " + ns); + return false; + } + return true; + } + + static void buildUniqueIndexError( const BSONObj& keyPattern, + const BSONObj& indexPattern, + WriteErrorDetail* error ) { + error->setErrCode( ErrorCodes::CannotCreateIndex ); + string errMsg = stream() << "cannot create unique index over " << indexPattern + << " with shard key pattern " << keyPattern; + error->setErrMessage( errMsg ); + } + + static bool checkIndexConstraints(ShardingState* shardingState, + const BatchedCommandRequest& request, + WriteOpResult* result) { + + const NamespaceString nss( request.getTargetingNS() ); + Lock::assertWriteLocked( nss.ns() ); + + if ( !request.isUniqueIndexRequest() ) + return true; + + if ( shardingState->enabled() ) { + + CollectionMetadataPtr metadata = shardingState->getCollectionMetadata( nss.ns() ); + + if ( metadata ) { + if ( !isUniqueIndexCompatible( metadata->getKeyPattern(), + request.getIndexKeyPattern() ) ) { + + result->setError(new WriteErrorDetail); + buildUniqueIndexError(metadata->getKeyPattern(), + request.getIndexKeyPattern(), + result->getError()); + + return false; + } + } + } + + return true; + } + + // + // HELPERS FOR CUROP MANAGEMENT AND GLOBAL STATS + // + + static CurOp* beginCurrentOp( Client* client, const BatchItemRef& currWrite ) { + + // Execute the write item as a child operation of the current operation. + auto_ptr currentOp( new CurOp( client, client->curop() ) ); + + // Set up the child op with more info + HostAndPort remote = + client->hasRemote() ? client->getRemote() : HostAndPort( "0.0.0.0", 0 ); + // TODO Modify CurOp "wrapped" constructor to take an opcode, so calling .reset() + // is unneeded + currentOp->reset( remote, getOpCode( currWrite.getRequest()->getBatchType() ) ); + currentOp->ensureStarted(); + currentOp->setNS( currWrite.getRequest()->getNS() ); + + currentOp->debug().ns = currentOp->getNS(); + currentOp->debug().op = currentOp->getOp(); + + if ( currWrite.getOpType() == BatchedCommandRequest::BatchType_Insert ) { + currentOp->setQuery( currWrite.getDocument() ); + currentOp->debug().query = currWrite.getDocument(); + currentOp->debug().ninserted = 0; + } + else if ( currWrite.getOpType() == BatchedCommandRequest::BatchType_Update ) { + currentOp->setQuery( currWrite.getUpdate()->getQuery() ); + currentOp->debug().query = currWrite.getUpdate()->getQuery(); + currentOp->debug().updateobj = currWrite.getUpdate()->getUpdateExpr(); + // Note: debug().nMatched, nModified and nmoved are set internally in update + } + else { + dassert( currWrite.getOpType() == BatchedCommandRequest::BatchType_Delete ); + currentOp->setQuery( currWrite.getDelete()->getQuery() ); + currentOp->debug().query = currWrite.getDelete()->getQuery(); + currentOp->debug().ndeleted = 0; + } + + return currentOp.release(); + } + + void WriteBatchExecutor::incOpStats( const BatchItemRef& currWrite ) { + + if ( currWrite.getOpType() == BatchedCommandRequest::BatchType_Insert ) { + _opCounters->gotInsert(); + } + else if ( currWrite.getOpType() == BatchedCommandRequest::BatchType_Update ) { + _opCounters->gotUpdate(); + } + else { + dassert( currWrite.getOpType() == BatchedCommandRequest::BatchType_Delete ); + _opCounters->gotDelete(); + } + } + + void WriteBatchExecutor::incWriteStats( const BatchItemRef& currWrite, + const WriteOpStats& stats, + const WriteErrorDetail* error, + CurOp* currentOp ) { + + if ( currWrite.getOpType() == BatchedCommandRequest::BatchType_Insert ) { + _stats->numInserted += stats.n; + _le->nObjects = stats.n; + currentOp->debug().ninserted += stats.n; + } + else if ( currWrite.getOpType() == BatchedCommandRequest::BatchType_Update ) { + if ( stats.upsertedID.isEmpty() ) { + _stats->numMatched += stats.n; + _stats->numModified += stats.nModified; + } + else { + ++_stats->numUpserted; + } + + if ( !error ) { + _le->recordUpdate( stats.upsertedID.isEmpty() && stats.n > 0, + stats.n, + stats.upsertedID ); + } + } + else { + dassert( currWrite.getOpType() == BatchedCommandRequest::BatchType_Delete ); + _stats->numDeleted += stats.n; + if ( !error ) { + _le->recordDelete( stats.n ); + } + currentOp->debug().ndeleted += stats.n; + } + + if (error && !_le->disabled) { + _le->raiseError(error->getErrCode(), error->getErrMessage().c_str()); + } + } + + static void finishCurrentOp( Client* client, CurOp* currentOp, WriteErrorDetail* opError ) { + + currentOp->done(); + int executionTime = currentOp->debug().executionTime = currentOp->totalTimeMillis(); + currentOp->debug().recordStats(); + + if ( opError ) { + currentOp->debug().exceptionInfo = ExceptionInfo( opError->getErrMessage(), + opError->getErrCode() ); + + MONGO_TLOG(3) << " Caught Assertion in " << opToString( currentOp->getOp() ) + << ", continuing " << causedBy( opError->getErrMessage() ) << endl; + } + + bool logAll = logger::globalLogDomain()->shouldLog( logger::LogSeverity::Debug( 1 ) ); + bool logSlow = executionTime + > ( serverGlobalParams.slowMS + currentOp->getExpectedLatencyMs() ); + + if ( logAll || logSlow ) { + MONGO_TLOG(0) << currentOp->debug().report( *currentOp ) << endl; + } + + if ( currentOp->shouldDBProfile( executionTime ) ) { + profile( *client, currentOp->getOp(), *currentOp ); + } + } + + // END HELPERS + + // + // CORE WRITE OPERATIONS (declaration) + // These functions write to the database and return stats and zero or one of: + // - page fault + // - error + // + + static void singleInsert( const BSONObj& docToInsert, + Collection* collection, + const PregeneratedKeys* pregen, + WriteOpResult* result ); + + static void singleCreateIndex( const BSONObj& indexDesc, + Collection* collection, + WriteOpResult* result ); + + static void multiUpdate( const BatchItemRef& updateItem, + WriteOpResult* result ); + + static void multiRemove( const BatchItemRef& removeItem, WriteOpResult* result ); + + // + // WRITE EXECUTION + // In general, the exec* operations manage db lock state and stats before dispatching to the + // core write operations, which are *only* responsible for performing a write and reporting + // success or failure. + // + + /** + * Representation of the execution state of execInserts. Used by a single + * execution of execInserts in a single thread. + */ + class WriteBatchExecutor::ExecInsertsState { + MONGO_DISALLOW_COPYING(ExecInsertsState); + public: + /** + * Constructs a new instance, for performing inserts described in "aRequest". + */ + explicit ExecInsertsState(const BatchedCommandRequest* aRequest); + + /** + * Acquires the write lock and client context needed to perform the current write operation. + * Returns true on success, after which it is safe to use the "context" and "collection" + * members. It is safe to call this function if this instance already holds the write lock. + * + * On failure, writeLock, context and collection will be NULL/clear. + */ + bool lockAndCheck(WriteOpResult* result); + + /** + * Releases the client context and write lock acquired by lockAndCheck. Safe to call + * regardless of whether or not this state object currently owns the lock. + */ + void unlock(); + + /** + * Returns true if this executor has the lock on the target database. + */ + bool hasLock() { return _writeLock.get(); } + + /** + * Gets the lock-holding object. Only valid if hasLock(). + */ + Lock::DBWrite& getLock() { return *_writeLock; } + + /** + * Gets the target collection for the batch operation. Value is undefined + * unless hasLock() is true. + */ + Collection* getCollection() { return _collection; } + + // Request object describing the inserts. + const BatchedCommandRequest* request; + + // Index of the current insert operation to perform. + size_t currIndex; + + // Translation of insert documents in "request" into insert-ready forms. This vector has a + // correspondence with elements of the "request", and "currIndex" is used to + // index both. + std::vector > normalizedInserts; + + // We generate these outside of any locks + std::vector pregeneratedKeys; + + private: + bool _lockAndCheckImpl(WriteOpResult* result); + + // Guard object for the write lock on the target database. + scoped_ptr _writeLock; + + // Context object on the target database. Must appear after writeLock, so that it is + // destroyed in proper order. + scoped_ptr _context; + + // Target collection. + Collection* _collection; + }; + + void WriteBatchExecutor::bulkExecute( const BatchedCommandRequest& request, + std::vector* upsertedIds, + std::vector* errors ) { + + if ( request.getBatchType() == BatchedCommandRequest::BatchType_Insert ) { + execInserts( request, errors ); + } + else if ( request.getBatchType() == BatchedCommandRequest::BatchType_Update ) { + for ( size_t i = 0; i < request.sizeWriteOps(); i++ ) { + + WriteErrorDetail* error = NULL; + BSONObj upsertedId; + execUpdate( BatchItemRef( &request, i ), &upsertedId, &error ); + + if ( !upsertedId.isEmpty() ) { + BatchedUpsertDetail* batchUpsertedId = new BatchedUpsertDetail; + batchUpsertedId->setIndex( i ); + batchUpsertedId->setUpsertedID( upsertedId ); + upsertedIds->push_back( batchUpsertedId ); + } + + if ( error ) { + errors->push_back( error ); + if ( request.getOrdered() ) + break; + } + } + } + else { + dassert( request.getBatchType() == BatchedCommandRequest::BatchType_Delete ); + for ( size_t i = 0; i < request.sizeWriteOps(); i++ ) { + + WriteErrorDetail* error = NULL; + execRemove( BatchItemRef( &request, i ), &error ); + + if ( error ) { + errors->push_back( error ); + if ( request.getOrdered() ) + break; + } + } + } + + // Fill in stale version errors for unordered batches (update/delete can't do this on own) + if ( !errors->empty() && !request.getOrdered() ) { + + const WriteErrorDetail* finalError = errors->back(); + + if ( finalError->getErrCode() == ErrorCodes::StaleShardVersion ) { + for ( size_t i = finalError->getIndex() + 1; i < request.sizeWriteOps(); i++ ) { + WriteErrorDetail* dupStaleError = new WriteErrorDetail; + finalError->cloneTo( dupStaleError ); + errors->push_back( dupStaleError ); + } + } + } + } + + // Goes over the request and preprocesses normalized versions of all the inserts in the request + static void normalizeInserts( const BatchedCommandRequest& request, + vector >* normalizedInserts, + vector* pregen ) { + + normalizedInserts->reserve(request.sizeWriteOps()); + for ( size_t i = 0; i < request.sizeWriteOps(); ++i ) { + BSONObj insertDoc = request.getInsertRequest()->getDocumentsAt( i ); + StatusWith normalInsert = fixDocumentForInsert( insertDoc ); + normalizedInserts->push_back( normalInsert ); + if ( request.getOrdered() && !normalInsert.isOK() ) + break; + + if ( !normalInsert.getValue().isEmpty() ) + insertDoc = normalInsert.getValue(); + + pregen->push_back( PregeneratedKeys() ); + GeneratorHolder::getInstance()->prepare( request.getTargetingNS(), + insertDoc, + &pregen->back() ); + } + } + + void WriteBatchExecutor::execInserts( const BatchedCommandRequest& request, + std::vector* errors ) { + + // Theory of operation: + // + // Instantiates an ExecInsertsState, which represents all of the state involved in the batch + // insert execution algorithm. Most importantly, encapsulates the lock state. + // + // Every iteration of the loop in execInserts() processes one document insertion, by calling + // insertOne() exactly once for a given value of state.currIndex. + // + // If the ExecInsertsState indicates that the requisite write locks are not held, insertOne + // acquires them and performs lock-acquisition-time checks. However, on non-error + // execution, it does not release the locks. Therefore, the yielding logic in the while + // loop in execInserts() is solely responsible for lock release in the non-error case. + // + // Internally, insertOne loops performing the single insert until it completes without a + // PageFaultException, or until it fails with some kind of error. Errors are mostly + // propagated via the request->error field, but DBExceptions or std::exceptions may escape, + // particularly on operation interruption. These kinds of errors necessarily prevent + // further insertOne calls, and stop the batch. As a result, the only expected source of + // such exceptions are interruptions. + ExecInsertsState state(&request); + normalizeInserts(request, &state.normalizedInserts, &state.pregeneratedKeys); + + ElapsedTracker elapsedTracker(128, 10); // 128 hits or 10 ms, matching RunnerYieldPolicy's + + ShardedConnectionInfo* info = ShardedConnectionInfo::get(false); + if (info) { + if (request.isMetadataSet() && request.getMetadata()->isShardVersionSet()) { + info->setVersion(request.getTargetingNS(), + request.getMetadata()->getShardVersion()); + } + else { + info->setVersion(request.getTargetingNS(), ChunkVersion::IGNORED()); + } + } + + for (state.currIndex = 0; + state.currIndex < state.request->sizeWriteOps(); + ++state.currIndex) { + + if (elapsedTracker.intervalHasElapsed()) { + // Consider yielding between inserts. + + if (state.hasLock()) { + int micros = ClientCursor::suggestYieldMicros(); + if (micros > 0) { + state.unlock(); + killCurrentOp.checkForInterrupt(); + sleepmicros(micros); + } + } + killCurrentOp.checkForInterrupt(); + elapsedTracker.resetLastTime(); + } + + WriteErrorDetail* error = NULL; + execOneInsert(&state, &error); + if (error) { + errors->push_back(error); + error->setIndex(state.currIndex); + if (request.getOrdered()) + return; + } + } + } + + void WriteBatchExecutor::execUpdate( const BatchItemRef& updateItem, + BSONObj* upsertedId, + WriteErrorDetail** error ) { + + // BEGIN CURRENT OP + scoped_ptr currentOp( beginCurrentOp( _client, updateItem ) ); + incOpStats( updateItem ); + + ShardedConnectionInfo* info = ShardedConnectionInfo::get(false); + if (info) { + const BatchedCommandRequest* rootRequest = updateItem.getRequest(); + if (!updateItem.getUpdate()->getMulti() && + rootRequest->isMetadataSet() && + rootRequest->getMetadata()->isShardVersionSet()) { + info->setVersion(rootRequest->getTargetingNS(), + rootRequest->getMetadata()->getShardVersion()); + } + else { + info->setVersion(rootRequest->getTargetingNS(), ChunkVersion::IGNORED()); + } + } + + WriteOpResult result; + multiUpdate( updateItem, &result ); + + if ( !result.getStats().upsertedID.isEmpty() ) { + *upsertedId = result.getStats().upsertedID; + } + + // END CURRENT OP + incWriteStats( updateItem, result.getStats(), result.getError(), currentOp.get() ); + finishCurrentOp( _client, currentOp.get(), result.getError() ); + + if ( result.getError() ) { + result.getError()->setIndex( updateItem.getItemIndex() ); + *error = result.releaseError(); + } + } + + void WriteBatchExecutor::execRemove( const BatchItemRef& removeItem, + WriteErrorDetail** error ) { + + // Removes are similar to updates, but page faults are handled externally + + // BEGIN CURRENT OP + scoped_ptr currentOp( beginCurrentOp( _client, removeItem ) ); + incOpStats( removeItem ); + + ShardedConnectionInfo* info = ShardedConnectionInfo::get(false); + if (info) { + const BatchedCommandRequest* rootRequest = removeItem.getRequest(); + if (removeItem.getDelete()->getLimit() == 1 && + rootRequest->isMetadataSet() && + rootRequest->getMetadata()->isShardVersionSet()) { + info->setVersion(rootRequest->getTargetingNS(), + rootRequest->getMetadata()->getShardVersion()); + } + else { + info->setVersion(rootRequest->getTargetingNS(), ChunkVersion::IGNORED()); + } + } + + WriteOpResult result; + + // NOTE: Deletes will not fault outside the lock once any data has been written + { + PageFaultRetryableSection pageFaultSection; + while ( true ) { + try { + multiRemove( removeItem, &result ); + break; + } + catch (PageFaultException& pfe) { + pfe.touch(); + invariant(!result.getError()); + continue; + } + fassertFailed(17429); + } + } + + // END CURRENT OP + incWriteStats( removeItem, result.getStats(), result.getError(), currentOp.get() ); + finishCurrentOp( _client, currentOp.get(), result.getError() ); + + if ( result.getError() ) { + result.getError()->setIndex( removeItem.getItemIndex() ); + *error = result.releaseError(); + } + } + + // + // IN-DB-LOCK CORE OPERATIONS + // + + WriteBatchExecutor::ExecInsertsState::ExecInsertsState(const BatchedCommandRequest* aRequest) : + request(aRequest), + currIndex(0), + _collection(NULL) { + } + + bool WriteBatchExecutor::ExecInsertsState::_lockAndCheckImpl(WriteOpResult* result) { + if (hasLock()) { + cc().curop()->enter(_context.get()); + return true; + } + + invariant(!_context.get()); + _writeLock.reset(new Lock::DBWrite(request->getNS())); + if (!checkIsMasterForCollection(request->getNS(), result)) { + return false; + } + if (!checkShardVersion(&shardingState, *request, result)) { + return false; + } + if (!checkIndexConstraints(&shardingState, *request, result)) { + return false; + } + _context.reset(new Client::Context(request->getNS(), + storageGlobalParams.dbpath)); + Database* database = _context->db(); + dassert(database); + _collection = database->getCollection(request->getTargetingNS()); + if (!_collection) { + // Implicitly create if it doesn't exist + _collection = database->createCollection(request->getTargetingNS()); + if (!_collection) { + result->setError( + toWriteError(Status(ErrorCodes::InternalError, + "could not create collection " + + request->getTargetingNS()))); + return false; + } + } + return true; + } + + bool WriteBatchExecutor::ExecInsertsState::lockAndCheck(WriteOpResult* result) { + if (_lockAndCheckImpl(result)) + return true; + unlock(); + return false; + } + + void WriteBatchExecutor::ExecInsertsState::unlock() { + _collection = NULL; + _context.reset(); + _writeLock.reset(); + } + + static void insertOne(WriteBatchExecutor::ExecInsertsState* state, WriteOpResult* result) { + invariant(state->currIndex < state->normalizedInserts.size()); + const StatusWith& normalizedInsert(state->normalizedInserts[state->currIndex]); + + if (!normalizedInsert.isOK()) { + result->setError(toWriteError(normalizedInsert.getStatus())); + return; + } + + const BSONObj& insertDoc = normalizedInsert.getValue().isEmpty() ? + state->request->getInsertRequest()->getDocumentsAt( state->currIndex ) : + normalizedInsert.getValue(); + + cc().clearHasWrittenThisOperation(); + { + PageFaultRetryableSection pageFaultSection; + while (true) { + try { + if (!state->lockAndCheck(result)) { + break; + } + + if (!state->request->isInsertIndexRequest()) { + const PregeneratedKeys* pregen = NULL; + if ( state->pregeneratedKeys.size() > state->currIndex ) + pregen = &state->pregeneratedKeys[state->currIndex]; + singleInsert(insertDoc, state->getCollection(), pregen, result); + } + else { + singleCreateIndex(insertDoc, state->getCollection(), result); + } + break; + } + catch (const StaleConfigException& staleExcep) { + result->setError(new WriteErrorDetail); + result->getError()->setErrCode(ErrorCodes::StaleShardVersion); + buildStaleError(staleExcep.getVersionReceived(), + staleExcep.getVersionWanted(), + result->getError()); + break; + } + catch (const DBException& ex) { + Status status(ex.toStatus()); + if (ErrorCodes::isInterruption(status.code())) + throw; + result->setError(toWriteError(status)); + break; + } + catch (PageFaultException& pfe) { + state->unlock(); + pfe.touch(); + continue; // Try the operation again. + } + fassertFailed(17430); + } + } // end PageFaultRetryableSection + + // Errors release the write lock, as a matter of policy. + if (result->getError()) + state->unlock(); + } + + void WriteBatchExecutor::execOneInsert(ExecInsertsState* state, WriteErrorDetail** error) { + BatchItemRef currInsertItem(state->request, state->currIndex); + scoped_ptr currentOp(beginCurrentOp(_client, currInsertItem)); + incOpStats(currInsertItem); + + WriteOpResult result; + insertOne(state, &result); + + if (state->hasLock()) { + // Normally, unlocking records lock time stats on the active CurOp. However, + // insertOne() may not release the lock. In that case, record time by hand. + state->getLock().recordTime(); + // If we deschedule here, there could be substantial unaccounted locked time. + // Any time from here will be attributed to the next insert in the batch, or + // not attributed to any operation if this is the last op in the batch. + state->getLock().resetTime(); + } + + incWriteStats(currInsertItem, + result.getStats(), + result.getError(), + currentOp.get()); + finishCurrentOp(_client, currentOp.get(), result.getError()); + + if (result.getError()) { + *error = result.releaseError(); + } + } + + /** + * Perform a single insert into a collection. Requires the insert be preprocessed and the + * collection already has been created. + * + * Might fault or error, otherwise populates the result. + */ + static void singleInsert( const BSONObj& docToInsert, + Collection* collection, + const PregeneratedKeys* pregen, + WriteOpResult* result ) { + + const string& insertNS = collection->ns().ns(); + + Lock::assertWriteLocked( insertNS ); + + StatusWith status = collection->insertDocument( docToInsert, true, pregen ); + + if ( !status.isOK() ) { + result->setError(toWriteError(status.getStatus())); + } + else { + logOp( "i", insertNS.c_str(), docToInsert ); + getDur().commitIfNeeded(); + result->getStats().n = 1; + } + } + + /** + * Perform a single index insert into a collection. Requires the index descriptor be + * preprocessed and the collection already has been created. + * + * Might fault or error, otherwise populates the result. + */ + static void singleCreateIndex( const BSONObj& indexDesc, + Collection* collection, + WriteOpResult* result ) { + + const string indexNS = collection->ns().getSystemIndexesCollection(); + + Lock::assertWriteLocked( indexNS ); + + Status status = collection->getIndexCatalog()->createIndex( indexDesc, true ); + + if ( status.code() == ErrorCodes::IndexAlreadyExists ) { + result->getStats().n = 0; + } + else if ( !status.isOK() ) { + result->setError(toWriteError(status)); + } + else { + logOp( "i", indexNS.c_str(), indexDesc ); + result->getStats().n = 1; + } + } + + static void multiUpdate( const BatchItemRef& updateItem, + WriteOpResult* result ) { + + const NamespaceString nsString(updateItem.getRequest()->getNS()); + UpdateRequest request(nsString); + request.setQuery(updateItem.getUpdate()->getQuery()); + request.setUpdates(updateItem.getUpdate()->getUpdateExpr()); + request.setMulti(updateItem.getUpdate()->getMulti()); + request.setUpsert(updateItem.getUpdate()->getUpsert()); + request.setUpdateOpLog(true); + UpdateLifecycleImpl updateLifecycle(true, request.getNamespaceString()); + request.setLifecycle(&updateLifecycle); + + UpdateExecutor executor(&request, &cc().curop()->debug()); + Status status = executor.prepare(); + if (!status.isOK()) { + result->setError(toWriteError(status)); + return; + } + + /////////////////////////////////////////// + Lock::DBWrite writeLock( nsString.ns() ); + /////////////////////////////////////////// + + if ( !checkShardVersion( &shardingState, *updateItem.getRequest(), result ) ) + return; + + Client::Context ctx(nsString.ns(), + storageGlobalParams.dbpath); + + try { + UpdateResult res = executor.execute(); + + const long long numDocsModified = res.numDocsModified; + const long long numMatched = res.numMatched; + const BSONObj resUpsertedID = res.upserted; + + // We have an _id from an insert + const bool didInsert = !resUpsertedID.isEmpty(); + + result->getStats().nModified = didInsert ? 0 : numDocsModified; + result->getStats().n = didInsert ? 1 : numMatched; + result->getStats().upsertedID = resUpsertedID; + } + catch (const StaleConfigException& staleExcep) { + result->setError(new WriteErrorDetail); + result->getError()->setErrCode(ErrorCodes::StaleShardVersion); + buildStaleError(staleExcep.getVersionReceived(), + staleExcep.getVersionWanted(), + result->getError()); + } + catch (const DBException& ex) { + status = ex.toStatus(); + if (ErrorCodes::isInterruption(status.code())) { + throw; + } + result->setError(toWriteError(status)); + } + } + + /** + * Perform a remove operation, which might remove multiple documents. Dispatches to remove code + * currently to do most of this. + * + * Might fault or error, otherwise populates the result. + */ + static void multiRemove( const BatchItemRef& removeItem, + WriteOpResult* result ) { + + const NamespaceString nss( removeItem.getRequest()->getNS() ); + DeleteRequest request( nss ); + request.setQuery( removeItem.getDelete()->getQuery() ); + request.setMulti( removeItem.getDelete()->getLimit() != 1 ); + request.setUpdateOpLog(true); + request.setGod( false ); + DeleteExecutor executor( &request ); + Status status = executor.prepare(); + if ( !status.isOK() ) { + result->setError(toWriteError(status)); + return; + } + + /////////////////////////////////////////// + Lock::DBWrite writeLock( nss.ns() ); + /////////////////////////////////////////// + + // Check version once we're locked + + if ( !checkShardVersion( &shardingState, *removeItem.getRequest(), result ) ) { + // Version error + return; + } + + // Context once we're locked, to set more details in currentOp() + // TODO: better constructor? + Client::Context writeContext(nss.ns(), + storageGlobalParams.dbpath); + + try { + result->getStats().n = executor.execute(); + } + catch (const StaleConfigException& staleExcep) { + result->setError(new WriteErrorDetail); + result->getError()->setErrCode(ErrorCodes::StaleShardVersion); + buildStaleError(staleExcep.getVersionReceived(), + staleExcep.getVersionWanted(), + result->getError()); + return; + } + catch ( const DBException& ex ) { + status = ex.toStatus(); + if (ErrorCodes::isInterruption(status.code())) { + throw; + } + result->setError(toWriteError(status)); + } + } + +} // namespace mongo diff --git a/src/mongo/db/commands/write_commands/batch_executor.h b/src/mongo/db/commands/write_commands/batch_executor.h new file mode 100644 index 00000000000..c45443231ef --- /dev/null +++ b/src/mongo/db/commands/write_commands/batch_executor.h @@ -0,0 +1,197 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/client.h" +#include "mongo/s/write_ops/batched_command_request.h" +#include "mongo/s/write_ops/batched_command_response.h" +#include "mongo/s/write_ops/batched_delete_document.h" +#include "mongo/s/write_ops/batched_update_document.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + class BSONObjBuilder; + class CurOp; + class OpCounters; + struct LastError; + + struct WriteOpStats; + class WriteBatchStats; + + /** + * An instance of WriteBatchExecutor is an object capable of issuing a write batch. + */ + class WriteBatchExecutor { + MONGO_DISALLOW_COPYING(WriteBatchExecutor); + public: + + // State object used by private execInserts. TODO: Do not expose this type. + class ExecInsertsState; + + WriteBatchExecutor( const BSONObj& defaultWriteConcern, + Client* client, + OpCounters* opCounters, + LastError* le ); + + /** + * Issues writes with requested write concern. Fills response with errors if problems + * occur. + */ + void executeBatch( const BatchedCommandRequest& request, BatchedCommandResponse* response ); + + const WriteBatchStats& getStats() const; + + private: + /** + * Executes the writes in the batch and returns upserted _ids and write errors. + * Dispatches to one of the three functions below for DBLock, CurOp, and stats management. + */ + void bulkExecute( const BatchedCommandRequest& request, + std::vector* upsertedIds, + std::vector* errors ); + + /** + * Executes the inserts of an insert batch and returns the write errors. + * + * Internally uses the DBLock of the request namespace. + * May execute multiple inserts inside the same DBLock, and/or take the DBLock multiple + * times. + */ + void execInserts( const BatchedCommandRequest& request, + std::vector* errors ); + + /** + * Executes a single insert from a batch, described in the opaque "state" object. + */ + void execOneInsert( ExecInsertsState* state, WriteErrorDetail** error ); + + /** + * Executes an update item (which may update many documents or upsert), and returns the + * upserted _id on upsert or error on failure. + * + * Internally uses the DBLock of the update namespace. + * May take the DBLock multiple times. + */ + void execUpdate( const BatchItemRef& updateItem, + BSONObj* upsertedId, + WriteErrorDetail** error ); + + /** + * Executes a delete item (which may remove many documents) and returns an error on failure. + * + * Internally uses the DBLock of the delete namespace. + * May take the DBLock multiple times. + */ + void execRemove( const BatchItemRef& removeItem, WriteErrorDetail** error ); + + /** + * Helper for incrementing stats on the next CurOp. + * + * No lock requirements. + */ + void incOpStats( const BatchItemRef& currWrite ); + + /** + * Helper for incrementing stats after each individual write op. + * + * No lock requirements (though usually done inside write lock to make stats update look + * atomic). + */ + void incWriteStats( const BatchItemRef& currWrite, + const WriteOpStats& stats, + const WriteErrorDetail* error, + CurOp* currentOp ); + + // Default write concern, if one isn't provide in the batches. + const BSONObj _defaultWriteConcern; + + // Client object to issue writes on behalf of. + // Not owned here. + Client* _client; + + // OpCounters object to update - needed for stats reporting + // Not owned here. + OpCounters* _opCounters; + + // LastError object to use for preparing write results - needed for stats reporting + // Not owned here. + LastError* _le; + + // Stats + scoped_ptr _stats; + }; + + /** + * Holds information about the result of a single write operation. + */ + struct WriteOpStats { + + WriteOpStats() : + n( 0 ), nModified( 0 ) { + } + + void reset() { + n = 0; + nModified = 0; + upsertedID = BSONObj(); + } + + // Num docs logically affected by this operation. + int n; + + // Num docs actually modified by this operation, if applicable (update) + int nModified; + + // _id of newly upserted document, if applicable (update) + BSONObj upsertedID; + }; + + /** + * Full stats accumulated by a write batch execution. Note that these stats do not directly + * correspond to the stats accumulated in opCounters and LastError. + */ + class WriteBatchStats { + public: + + WriteBatchStats() : + numInserted( 0 ), numUpserted( 0 ), numMatched( 0 ), numModified( 0 ), numDeleted( 0 ) { + } + + int numInserted; + int numUpserted; + int numMatched; + int numModified; + int numDeleted; + }; + +} // namespace mongo diff --git a/src/mongo/db/commands/write_commands/write_commands.cpp b/src/mongo/db/commands/write_commands/write_commands.cpp new file mode 100644 index 00000000000..501dbd43dd9 --- /dev/null +++ b/src/mongo/db/commands/write_commands/write_commands.cpp @@ -0,0 +1,188 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/commands/write_commands/write_commands.h" + +#include "mongo/base/init.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/commands/write_commands/batch_executor.h" +#include "mongo/db/commands/write_commands/write_commands_common.h" +#include "mongo/db/curop.h" +#include "mongo/db/json.h" +#include "mongo/db/lasterror.h" +#include "mongo/db/server_parameters.h" +#include "mongo/db/stats/counters.h" + +namespace mongo { + + namespace { + + MONGO_INITIALIZER(RegisterWriteCommands)(InitializerContext* context) { + // Leaked intentionally: a Command registers itself when constructed. + new CmdInsert(); + new CmdUpdate(); + new CmdDelete(); + return Status::OK(); + } + + } // namespace + + // This is set in rs.cpp when the replica set is initialized, and is stored in dbcommands.cpp + // for now. See dbcommands.cpp + extern BSONObj* getLastErrorDefault; + + WriteCmd::WriteCmd( const StringData& name, BatchedCommandRequest::BatchType writeType ) : + Command( name ), _writeType( writeType ) { + } + + void WriteCmd::redactTooLongLog( mutablebson::Document* cmdObj, const StringData& fieldName ) { + namespace mmb = mutablebson; + mmb::Element root = cmdObj->root(); + mmb::Element field = root.findFirstChildNamed( fieldName ); + + // If the cmdObj is too large, it will be a "too big" message given by CachedBSONObj.get() + if ( !field.ok() ) { + return; + } + + // Redact the log if there are more than one documents or operations. + if ( field.countChildren() > 1 ) { + field.setValueInt( field.countChildren() ); + } + } + + // Write commands are fanned out in oplog as single writes. + bool WriteCmd::logTheOp() { return false; } + + // Slaves can't perform writes. + bool WriteCmd::slaveOk() const { return false; } + + // Write commands acquire write lock, but not for entire length of execution. + Command::LockType WriteCmd::locktype() const { return NONE; } + + Status WriteCmd::checkAuthForCommand( ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj ) { + + Status status( auth::checkAuthForWriteCommand( client->getAuthorizationSession(), + _writeType, + NamespaceString( parseNs( dbname, cmdObj ) ), + cmdObj )); + + // TODO: Remove this when we standardize GLE reporting from commands + if ( !status.isOK() ) { + setLastError( status.code(), status.reason().c_str() ); + } + + return status; + } + + // Write commands are counted towards their corresponding opcounters, not command opcounters. + bool WriteCmd::shouldAffectCommandCounter() const { return false; } + + bool WriteCmd::run(const string& dbName, + BSONObj& cmdObj, + int options, + string& errMsg, + BSONObjBuilder& result, + bool fromRepl) { + + // Can't be run on secondaries (logTheOp() == false, slaveOk() == false). + dassert( !fromRepl ); + BatchedCommandRequest request( _writeType ); + BatchedCommandResponse response; + + if ( !request.parseBSON( cmdObj, &errMsg ) || !request.isValid( &errMsg ) ) { + return appendCommandStatus( result, Status( ErrorCodes::FailedToParse, errMsg ) ); + } + + // Note that this is a runCommmand, and therefore, the database and the collection name + // are in different parts of the grammar for the command. But it's more convenient to + // work with a NamespaceString. We built it here and replace it in the parsed command. + // Internally, everything work with the namespace string as opposed to just the + // collection name. + NamespaceString nss(dbName, request.getNS()); + request.setNS(nss.ns()); + + BSONObj defaultWriteConcern; + // This is really bad - it's only safe because we leak the defaults by overriding them with + // new defaults and because we never reset to an empty default. + // TODO: fix this for sane behavior where we query repl set object + if ( getLastErrorDefault ) defaultWriteConcern = *getLastErrorDefault; + + cc().setIsWriteCmd(true); + + WriteBatchExecutor writeBatchExecutor(defaultWriteConcern, + &cc(), + &globalOpCounters, + lastError.get()); + + writeBatchExecutor.executeBatch( request, &response ); + + result.appendElements( response.toBSON() ); + return response.getOk(); + } + + CmdInsert::CmdInsert() : + WriteCmd( "insert", BatchedCommandRequest::BatchType_Insert ) { + } + + void CmdInsert::redactForLogging( mutablebson::Document* cmdObj ) { + redactTooLongLog( cmdObj, StringData( "documents", StringData::LiteralTag() ) ); + } + + void CmdInsert::help( stringstream& help ) const { + help << "insert documents"; + } + + CmdUpdate::CmdUpdate() : + WriteCmd( "update", BatchedCommandRequest::BatchType_Update ) { + } + + void CmdUpdate::redactForLogging( mutablebson::Document* cmdObj ) { + redactTooLongLog( cmdObj, StringData( "updates", StringData::LiteralTag() ) ); + } + + void CmdUpdate::help( stringstream& help ) const { + help << "update documents"; + } + + CmdDelete::CmdDelete() : + WriteCmd( "delete", BatchedCommandRequest::BatchType_Delete ) { + } + + void CmdDelete::redactForLogging( mutablebson::Document* cmdObj ) { + redactTooLongLog( cmdObj, StringData( "deletes", StringData::LiteralTag() ) ); + } + + void CmdDelete::help( stringstream& help ) const { + help << "delete documents"; + } + +} // namespace mongo diff --git a/src/mongo/db/commands/write_commands/write_commands.h b/src/mongo/db/commands/write_commands/write_commands.h new file mode 100644 index 00000000000..7be9cc4935a --- /dev/null +++ b/src/mongo/db/commands/write_commands/write_commands.h @@ -0,0 +1,118 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/commands.h" +#include "mongo/db/client_basic.h" +#include "mongo/s/write_ops/batched_command_request.h" + +namespace mongo { + + /** + * Base class for write commands. Write commands support batch writes and write concern, + * and return per-item error information. All write commands use the (non-virtual) entry + * point WriteCmd::run(). + * + * Command parsing is performed by the WriteBatch class (command syntax documented there), + * and command execution is performed by the WriteBatchExecutor class. + */ + class WriteCmd : public Command { + MONGO_DISALLOW_COPYING(WriteCmd); + public: + virtual ~WriteCmd() {} + + protected: + + /** + * Instantiates a command that can be invoked by "name", which will be capable of issuing + * write batches of type "writeType", and will require privilege "action" to run. + */ + WriteCmd( const StringData& name, BatchedCommandRequest::BatchType writeType ); + + // Full log of write command can be quite large. + static void redactTooLongLog( mutablebson::Document* cmdObj, const StringData& fieldName ); + + private: + virtual bool logTheOp(); + + virtual bool slaveOk() const; + + virtual LockType locktype() const; + + virtual Status checkAuthForCommand( ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj ); + + virtual bool shouldAffectCommandCounter() const; + + // Write command entry point. + virtual bool run(const string& dbname, + BSONObj& cmdObj, + int options, + string& errmsg, + BSONObjBuilder& result, + bool fromRepl); + + // Type of batch (e.g. insert). + BatchedCommandRequest::BatchType _writeType; + }; + + class CmdInsert : public WriteCmd { + MONGO_DISALLOW_COPYING(CmdInsert); + public: + CmdInsert(); + void redactForLogging(mutablebson::Document* cmdObj); + + private: + virtual void help(stringstream& help) const; + }; + + class CmdUpdate : public WriteCmd { + MONGO_DISALLOW_COPYING(CmdUpdate); + public: + CmdUpdate(); + void redactForLogging(mutablebson::Document* cmdObj); + + private: + virtual void help(stringstream& help) const; + }; + + class CmdDelete : public WriteCmd { + MONGO_DISALLOW_COPYING(CmdDelete); + public: + CmdDelete(); + void redactForLogging(mutablebson::Document* cmdObj); + + private: + virtual void help(stringstream& help) const; + }; + +} // namespace mongo diff --git a/src/mongo/db/commands/write_commands/write_commands_common.cpp b/src/mongo/db/commands/write_commands/write_commands_common.cpp new file mode 100644 index 00000000000..72c132ae1c0 --- /dev/null +++ b/src/mongo/db/commands/write_commands/write_commands_common.cpp @@ -0,0 +1,99 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/commands/write_commands/write_commands_common.h" + +#include +#include + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +namespace auth { + + using std::string; + using std::vector; + + Status checkAuthForWriteCommand( AuthorizationSession* authzSession, + BatchedCommandRequest::BatchType cmdType, + const NamespaceString& cmdNSS, + const BSONObj& cmdObj ) { + + vector privileges; + + if ( cmdType == BatchedCommandRequest::BatchType_Insert ) { + + if ( !cmdNSS.isSystemDotIndexes() ) { + privileges.push_back( Privilege( ResourcePattern::forExactNamespace( cmdNSS ), + ActionType::insert ) ); + } + else { + // Special-case indexes until we have a command + string nsToIndex, errMsg; + if ( !BatchedCommandRequest::getIndexedNS( cmdObj, &nsToIndex, &errMsg ) ) { + return Status( ErrorCodes::FailedToParse, errMsg ); + } + + NamespaceString nssToIndex( nsToIndex ); + privileges.push_back( Privilege( ResourcePattern::forExactNamespace( nssToIndex ), + ActionType::createIndex ) ); + } + } + else if ( cmdType == BatchedCommandRequest::BatchType_Update ) { + + ActionSet actions; + actions.addAction( ActionType::update ); + + // Upsert also requires insert privs + if ( BatchedCommandRequest::containsUpserts( cmdObj ) ) { + actions.addAction( ActionType::insert ); + } + + privileges.push_back( Privilege( ResourcePattern::forExactNamespace( cmdNSS ), + actions ) ); + + } + else { + fassert( 17251, cmdType == BatchedCommandRequest::BatchType_Delete ); + privileges.push_back( Privilege( ResourcePattern::forExactNamespace( cmdNSS ), + ActionType::remove ) ); + } + + if ( authzSession->isAuthorizedForPrivileges( privileges ) ) + return Status::OK(); + + return Status( ErrorCodes::Unauthorized, "unauthorized" ); + } + +} +} diff --git a/src/mongo/db/commands/write_commands/write_commands_common.h b/src/mongo/db/commands/write_commands/write_commands_common.h new file mode 100644 index 00000000000..a1fe6bc9772 --- /dev/null +++ b/src/mongo/db/commands/write_commands/write_commands_common.h @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/s/write_ops/batched_command_request.h" + +/** + * Contains common functionality shared between the batch write commands in mongos and mongod. + */ + +namespace mongo { +namespace auth { + + Status checkAuthForWriteCommand( AuthorizationSession* authzSession, + BatchedCommandRequest::BatchType cmdType, + const NamespaceString& cmdNSS, + const BSONObj& cmdObj ); + +} +} diff --git a/src/mongo/db/common.cpp b/src/mongo/db/common.cpp deleted file mode 100644 index 970b8386fde..00000000000 --- a/src/mongo/db/common.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/** @file common.cpp - Common code for server binaries (mongos, mongod, test). - Nothing used by driver should be here. - */ - -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "jsobjmanipulator.h" - -/** - * this just has globals - */ -namespace mongo { - - NOINLINE_DECL OpTime OpTime::skewed() { - bool toLog = false; - ONCE toLog = true; - RARELY toLog = true; - last.i++; - if ( last.i & 0x80000000 ) - toLog = true; - if ( toLog ) { - log() << "clock skew detected prev: " << last.secs << " now: " << (unsigned) time(0) << endl; - } - if ( last.i & 0x80000000 ) { - log() << "error large clock skew detected, shutting down" << endl; - throw ClockSkewException(); - } - return last; - } - -} diff --git a/src/mongo/db/compact.cpp b/src/mongo/db/compact.cpp deleted file mode 100644 index b5ea97bc043..00000000000 --- a/src/mongo/db/compact.cpp +++ /dev/null @@ -1,425 +0,0 @@ -/** @file compact.cpp - compaction of deleted space in pdfiles (datafiles) -*/ - -/** -* Copyright (C) 2010 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful,b -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" - -#include -#include - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/background.h" -#include "mongo/db/commands.h" -#include "mongo/db/d_concurrency.h" -#include "mongo/db/curop-inl.h" -#include "mongo/db/extsort.h" -#include "mongo/db/index.h" -#include "mongo/db/index_update.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/kill_current_op.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/sort_phase_one.h" -#include "mongo/util/concurrency/task.h" -#include "mongo/util/timer.h" -#include "mongo/util/touch_pages.h" - -namespace mongo { - - void freeExtents(DiskLoc firstExt, DiskLoc lastExt); - - /* this should be done in alloc record not here, but doing here for now. - really dumb; it's a start. - */ - unsigned quantizeMask(unsigned x) { - if( x > 4096 * 20 ) - return ~4095; - if( x >= 512 ) - return ~63; - return ~0; - } - - /** @return number of skipped (invalid) documents */ - unsigned compactExtent(const char *ns, NamespaceDetails *d, const DiskLoc diskloc, int n, - const scoped_array &indexSpecs, - scoped_array& phase1, int nidx, bool validate, - double pf, int pb) - { - log() << "compact begin extent #" << n << " for namespace " << ns << endl; - unsigned oldObjSize = 0; // we'll report what the old padding was - unsigned oldObjSizeWithPadding = 0; - - Extent *e = diskloc.ext(); - e->assertOk(); - verify( e->validates(diskloc) ); - unsigned skipped = 0; - - { - // the next/prev pointers within the extent might not be in order so we first page the whole thing in - // sequentially - log() << "compact paging in len=" << e->length/1000000.0 << "MB" << endl; - Timer t; - MongoDataFile* mdf = cc().database()->getFile( diskloc.a() ); - HANDLE fd = mdf->getFd(); - int offset = diskloc.getOfs(); - Extent* ext = diskloc.ext(); - size_t length = ext->length; - - touch_pages(fd, offset, length, ext); - int ms = t.millis(); - if( ms > 1000 ) - log() << "compact end paging in " << ms << "ms " << e->length/1000000.0/ms << "MB/sec" << endl; - } - - { - log() << "compact copying records" << endl; - long long datasize = 0; - long long nrecords = 0; - DiskLoc L = e->firstRecord; - if( !L.isNull() ) { - while( 1 ) { - Record *recOld = L.rec(); - L = recOld->nextInExtent(L); - BSONObj objOld = BSONObj::make(recOld); - - if( !validate || objOld.valid() ) { - nrecords++; - unsigned sz = objOld.objsize(); - - oldObjSize += sz; - oldObjSizeWithPadding += recOld->netLength(); - - unsigned lenWHdr = sz + Record::HeaderSize; - unsigned lenWPadding = lenWHdr; - { - lenWPadding = static_cast(pf*lenWPadding); - lenWPadding += pb; - lenWPadding = lenWPadding & quantizeMask(lenWPadding); - if( lenWPadding < lenWHdr || lenWPadding > BSONObjMaxUserSize / 2 ) { - lenWPadding = lenWHdr; - } - } - DiskLoc loc = allocateSpaceForANewRecord(ns, d, lenWPadding, false); - uassert(14024, "compact error out of space during compaction", !loc.isNull()); - Record *recNew = loc.rec(); - datasize += recNew->netLength(); - recNew = (Record *) getDur().writingPtr(recNew, lenWHdr); - addRecordToRecListInExtent(recNew, loc); - memcpy(recNew->data(), objOld.objdata(), sz); - - { - // extract keys for all indexes we will be rebuilding - for( int x = 0; x < nidx; x++ ) { - phase1[x].addKeys(indexSpecs[x], objOld, loc, false); - } - } - } - else { - if( ++skipped <= 10 ) - log() << "compact skipping invalid object" << endl; - } - - if( L.isNull() ) { - // we just did the very last record from the old extent. it's still pointed to - // by the old extent ext, but that will be fixed below after this loop - break; - } - - // remove the old records (orphan them) periodically so our commit block doesn't get too large - bool stopping = false; - RARELY stopping = *killCurrentOp.checkForInterruptNoAssert() != 0; - if( stopping || getDur().aCommitIsNeeded() ) { - e->firstRecord.writing() = L; - Record *r = L.rec(); - getDur().writingInt(r->prevOfs()) = DiskLoc::NullOfs; - getDur().commitIfNeeded(); - killCurrentOp.checkForInterrupt(false); - } - } - } // if !L.isNull() - - verify( d->firstExtent == diskloc ); - verify( d->lastExtent != diskloc ); - DiskLoc newFirst = e->xnext; - d->firstExtent.writing() = newFirst; - newFirst.ext()->xprev.writing().Null(); - getDur().writing(e)->markEmpty(); - freeExtents( diskloc, diskloc ); - // update datasize/record count for this namespace's extent - { - NamespaceDetails::Stats *s = getDur().writing(&d->stats); - s->datasize += datasize; - s->nrecords += nrecords; - } - - getDur().commitIfNeeded(); - - { - double op = 1.0; - if( oldObjSize ) - op = static_cast(oldObjSizeWithPadding)/oldObjSize; - log() << "compact finished extent #" << n << " containing " << nrecords << " documents (" << datasize/1000000.0 << "MB)" - << " oldPadding: " << op << ' ' << static_cast(op*100.0)/100 - << endl; - } - } - - return skipped; - } - - bool _compact(const char *ns, NamespaceDetails *d, string& errmsg, bool validate, BSONObjBuilder& result, double pf, int pb) { - // this is a big job, so might as well make things tidy before we start just to be nice. - getDur().commitIfNeeded(); - - list extents; - for( DiskLoc L = d->firstExtent; !L.isNull(); L = L.ext()->xnext ) - extents.push_back(L); - log() << "compact " << extents.size() << " extents" << endl; - - ProgressMeterHolder pm(cc().curop()->setMessage("compact extent", - "Extent Compacting Progress", - extents.size())); - - // same data, but might perform a little different after compact? - NamespaceDetailsTransient::get(ns).clearQueryCache(); - - int nidx = d->nIndexes; - scoped_array indexSpecs( new IndexSpec[nidx] ); - scoped_array phase1( new SortPhaseOne[nidx] ); - { - NamespaceDetails::IndexIterator ii = d->ii(); - // For each existing index... - for( int idxNo = 0; ii.more(); ++idxNo ) { - // Build a new index spec based on the old index spec. - BSONObjBuilder b; - BSONObj::iterator i(ii.next().info.obj()); - while( i.more() ) { - BSONElement e = i.next(); - if ( str::equals( e.fieldName(), "v" ) ) { - // Drop any preexisting index version spec. The default index version will - // be used instead for the new index. - continue; - } - if ( str::equals( e.fieldName(), "background" ) ) { - // Create the new index in the foreground. - continue; - } - // Pass the element through to the new index spec. - b.append(e); - } - // Add the new index spec to 'indexSpecs'. - BSONObj o = b.obj().getOwned(); - indexSpecs[idxNo].reset(o); - // Create an external sorter. - phase1[idxNo].sorter.reset - ( new BSONObjExternalSorter - // Use the default index interface, since the new index will be created - // with the default index version. - ( IndexInterface::defaultVersion(), - o.getObjectField("key") ) ); - phase1[idxNo].sorter->hintNumObjects( d->stats.nrecords ); - } - } - - log() << "compact orphan deleted lists" << endl; - for( int i = 0; i < Buckets; i++ ) { - d->deletedList[i].writing().Null(); - } - - - - // Start over from scratch with our extent sizing and growth - d->lastExtentSize=0; - - // before dropping indexes, at least make sure we can allocate one extent! - uassert(14025, "compact error no space available to allocate", !allocateSpaceForANewRecord(ns, d, Record::HeaderSize+1, false).isNull()); - - // note that the drop indexes call also invalidates all clientcursors for the namespace, which is important and wanted here - log() << "compact dropping indexes" << endl; - BSONObjBuilder b; - if( !dropIndexes(d, ns, "*", errmsg, b, true) ) { - errmsg = "compact drop indexes failed"; - log() << errmsg << endl; - return false; - } - - getDur().commitIfNeeded(); - - long long skipped = 0; - int n = 0; - - // reset data size and record counts to 0 for this namespace - // as we're about to tally them up again for each new extent - { - NamespaceDetails::Stats *s = getDur().writing(&d->stats); - s->datasize = 0; - s->nrecords = 0; - } - - for( list::iterator i = extents.begin(); i != extents.end(); i++ ) { - skipped += compactExtent(ns, d, *i, n++, indexSpecs, phase1, nidx, validate, pf, pb); - pm.hit(); - } - - if( skipped ) { - result.append("invalidObjects", skipped); - } - - verify( d->firstExtent.ext()->xprev.isNull() ); - - // indexes will do their own progress meter? - pm.finished(); - - // build indexes - NamespaceString s(ns); - string si = s.db + ".system.indexes"; - for( int i = 0; i < nidx; i++ ) { - killCurrentOp.checkForInterrupt(false); - BSONObj info = indexSpecs[i].info; - log() << "compact create index " << info["key"].Obj().toString() << endl; - scoped_lock precalcLock(theDataFileMgr._precalcedMutex); - try { - theDataFileMgr.setPrecalced(&phase1[i]); - theDataFileMgr.insert(si.c_str(), info.objdata(), info.objsize()); - } - catch(...) { - theDataFileMgr.setPrecalced(NULL); - throw; - } - theDataFileMgr.setPrecalced(NULL); - } - - return true; - } - - bool compact(const string& ns, string &errmsg, bool validate, BSONObjBuilder& result, double pf, int pb) { - massert( 14028, "bad ns", NamespaceString::normal(ns.c_str()) ); - massert( 14027, "can't compact a system namespace", !str::contains(ns, ".system.") ); // items in system.indexes cannot be moved there are pointers to those disklocs in NamespaceDetails - - bool ok; - { - Lock::DBWrite lk(ns); - BackgroundOperation::assertNoBgOpInProgForNs(ns.c_str()); - Client::Context ctx(ns); - NamespaceDetails *d = nsdetails(ns); - massert( 13660, str::stream() << "namespace " << ns << " does not exist", d ); - massert( 13661, "cannot compact capped collection", !d->isCapped() ); - log() << "compact " << ns << " begin" << endl; - if( pf != 0 || pb != 0 ) { - log() << "paddingFactor:" << pf << " paddingBytes:" << pb << endl; - } - try { - ok = _compact(ns.c_str(), d, errmsg, validate, result, pf, pb); - } - catch(...) { - log() << "compact " << ns << " end (with error)" << endl; - throw; - } - log() << "compact " << ns << " end" << endl; - } - return ok; - } - - bool isCurrentlyAReplSetPrimary(); - - class CompactCmd : public Command { - public: - virtual LockType locktype() const { return NONE; } - virtual bool adminOnly() const { return false; } - virtual bool slaveOk() const { return true; } - virtual bool maintenanceMode() const { return true; } - virtual bool logTheOp() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::compact); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - virtual void help( stringstream& help ) const { - help << "compact collection\n" - "warning: this operation blocks the server and is slow. you can cancel with cancelOp()\n" - "{ compact : , [force:], [validate:],\n" - " [paddingFactor:], [paddingBytes:] }\n" - " force - allows to run on a replica set primary\n" - " validate - check records are noncorrupt before adding to newly compacting extents. slower but safer (defaults to true in this version)\n"; - } - virtual bool requiresAuth() { return true; } - CompactCmd() : Command("compact") { } - - virtual bool run(const string& db, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - string coll = cmdObj.firstElement().valuestr(); - if( coll.empty() || db.empty() ) { - errmsg = "no collection name specified"; - return false; - } - - if( isCurrentlyAReplSetPrimary() && !cmdObj["force"].trueValue() ) { - errmsg = "will not run compact on an active replica set primary as this is a slow blocking operation. use force:true to force"; - return false; - } - - string ns = db + '.' + coll; - if ( ! NamespaceString::normal(ns.c_str()) ) { - errmsg = "bad namespace name"; - return false; - } - - // parameter validation to avoid triggering assertions in compact() - if ( str::contains(ns, ".system.") ) { - errmsg = "can't compact a system namespace"; - return false; - } - - { - Lock::DBWrite lk(ns); - Client::Context ctx(ns); - NamespaceDetails *d = nsdetails(ns); - if( ! d ) { - errmsg = "namespace does not exist"; - return false; - } - - if ( d->isCapped() ) { - errmsg = "cannot compact a capped collection"; - return false; - } - } - - double pf = 1.0; - int pb = 0; - if( cmdObj.hasElement("paddingFactor") ) { - pf = cmdObj["paddingFactor"].Number(); - verify( pf >= 1.0 && pf <= 4.0 ); - } - if( cmdObj.hasElement("paddingBytes") ) { - pb = (int) cmdObj["paddingBytes"].Number(); - verify( pb >= 0 && pb <= 1024 * 1024 ); - } - - bool validate = !cmdObj.hasElement("validate") || cmdObj["validate"].trueValue(); // default is true at the moment - bool ok = compact(ns, errmsg, validate, result, pf, pb); - return ok; - } - }; - static CompactCmd compactCmd; - -} diff --git a/src/mongo/db/conn_pool_options.cpp b/src/mongo/db/conn_pool_options.cpp new file mode 100644 index 00000000000..0ac2c985920 --- /dev/null +++ b/src/mongo/db/conn_pool_options.cpp @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/conn_pool_options.h" + +#include "mongo/base/init.h" +#include "mongo/db/server_parameters.h" +#include "mongo/client/connpool.h" +#include "mongo/s/shard.h" + +namespace mongo { + + int ConnPoolOptions::maxConnsPerHost(200); + int ConnPoolOptions::maxShardedConnsPerHost(200); + + namespace { + + ExportedServerParameter // + maxConnsPerHostParameter(ServerParameterSet::getGlobal(), + "connPoolMaxConnsPerHost", + &ConnPoolOptions::maxConnsPerHost, + true, + false /* can't change at runtime */); + + ExportedServerParameter // + maxShardedConnsPerHostParameter(ServerParameterSet::getGlobal(), + "connPoolMaxShardedConnsPerHost", + &ConnPoolOptions::maxShardedConnsPerHost, + true, + false /* can't change at runtime */); + + MONGO_INITIALIZER(InitializeConnectionPools)(InitializerContext* context) { + + // Initialize the sharded and unsharded outgoing connection pools + // NOTES: + // - All mongods and mongoses have both pools + // - The connection hooks for sharding are added on startup (mongos) or on first sharded + // operation (mongod) + + pool.setName("connection pool"); + pool.setMaxPoolSize(ConnPoolOptions::maxConnsPerHost); + + shardConnectionPool.setName("sharded connection pool"); + shardConnectionPool.setMaxPoolSize(ConnPoolOptions::maxShardedConnsPerHost); + + return Status::OK(); + } + } + +} diff --git a/src/mongo/db/conn_pool_options.h b/src/mongo/db/conn_pool_options.h new file mode 100644 index 00000000000..a5402c6a367 --- /dev/null +++ b/src/mongo/db/conn_pool_options.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { + + // NOTE: + // The connection pools themselves are placed in different files and are currently hard to move + // due to spaghetti dependencies. + // TODO: Extract conn pools from driver files and shardconnection.cpp + + /** + * Struct namespace for connection pool options on mongos and mongod + */ + struct ConnPoolOptions { + + /** + * Maximum connections per host the connection pool should use + */ + static int maxConnsPerHost; + + /** + * Maximum connections per host the sharded conn pool should use + */ + static int maxShardedConnsPerHost; + }; + +} diff --git a/src/mongo/db/connection_factory.cpp b/src/mongo/db/connection_factory.cpp deleted file mode 100644 index 0d5a10070fe..00000000000 --- a/src/mongo/db/connection_factory.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// connection_factory.cpp - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/client/connpool.h" -#include "mongo/db/client.h" - -// This file contains the server-only (mongod and mongos) implementation of the factory functions -// for getting ScopedDbConnections. Handles setting authentication info on the underlying -// connection as needed. -namespace mongo { - - ScopedDbConnection* ScopedDbConnection::getScopedDbConnection() { - ScopedDbConnection* conn = new ScopedDbConnection(); - return conn; - } - - ScopedDbConnection* ScopedDbConnection::getScopedDbConnection(const string& host, - double socketTimeout) { - ScopedDbConnection* conn = new ScopedDbConnection(host, socketTimeout); - return conn; - } - - ScopedDbConnection* ScopedDbConnection::getScopedDbConnection(const ConnectionString& host, - double socketTimeout) { - return getScopedDbConnection(host.toString()); - } - - - ScopedDbConnection* ScopedDbConnection::getInternalScopedDbConnection() { - ScopedDbConnection* conn = new ScopedDbConnection(); - return conn; - } - - ScopedDbConnection* ScopedDbConnection::getInternalScopedDbConnection(const string& host, - double socketTimeout) { - ScopedDbConnection* conn = new ScopedDbConnection(host, socketTimeout); - return conn; - } - - ScopedDbConnection* ScopedDbConnection::getInternalScopedDbConnection(const ConnectionString& host, - double socketTimeout) { - return getInternalScopedDbConnection(host.toString()); - } - -} diff --git a/src/mongo/db/curop-inl.h b/src/mongo/db/curop-inl.h index c029aa5a51d..dba3454c70a 100644 --- a/src/mongo/db/curop-inl.h +++ b/src/mongo/db/curop-inl.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "curop.h" +#include "mongo/db/curop.h" diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index 68e24e2b171..a199f313ddf 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -19,11 +31,26 @@ #include "mongo/base/counter.h" #include "mongo/db/commands/server_status.h" #include "mongo/db/curop.h" -#include "mongo/db/database.h" +#include "mongo/db/catalog/database.h" #include "mongo/db/kill_current_op.h" +#include "mongo/db/matcher.h" +#include "mongo/util/fail_point_service.h" namespace mongo { + // Enabling the maxTimeAlwaysTimeOut fail point will cause any query or command run with a valid + // non-zero max time to fail immediately. Any getmore operation on a cursor already created + // with a valid non-zero max time will also fail immediately. + // + // This fail point cannot be used with the maxTimeNeverTimeOut fail point. + MONGO_FP_DECLARE(maxTimeAlwaysTimeOut); + + // Enabling the maxTimeNeverTimeOut fail point will cause the server to never time out any + // query, command, or getmore operation, regardless of whether a max time is set. + // + // This fail point cannot be used with the maxTimeAlwaysTimeOut fail point. + MONGO_FP_DECLARE(maxTimeNeverTimeOut); + // todo : move more here CurOp::CurOp( Client * client , CurOp * wrapped ) : @@ -37,6 +64,7 @@ namespace mongo { _reset(); _op = 0; _opNum = _nextOpNum++; + _command = NULL; // These addresses should never be written to again. The zeroes are // placed here as a precaution because currentOp may be accessed // without the db mutex. @@ -45,9 +73,11 @@ namespace mongo { void CurOp::_reset() { _suppressFromCurop = false; - _command = false; + _isCommand = false; _dbprofile = 0; _end = 0; + _maxTimeMicros = 0; + _maxTimeTracker.reset(); _message = ""; _progressMeter.finished(); _killPending.store(0); @@ -67,6 +97,44 @@ namespace mongo { _active = true; // this should be last for ui clarity } + CurOp* CurOp::getOp(const BSONObj& criteria) { + // Regarding Matcher: This is not quite the right hammer to use here. + // Future: use an actual property of CurOp to flag index builds + // and use that to filter. + // This will probably need refactoring once we change index builds + // to be a real command instead of an insert into system.indexes + Matcher matcher(criteria); + + Client& me = cc(); + + scoped_lock client_lock(Client::clientsMutex); + for (std::set::iterator it = Client::clients.begin(); + it != Client::clients.end(); + it++) { + + Client *client = *it; + verify(client); + + CurOp* curop = client->curop(); + if (client == &me || curop == NULL) { + continue; + } + + if ( !curop->active() ) + continue; + + if ( curop->killPendingStrict() ) + continue; + + BSONObj info = curop->description(); + if (matcher.matches(info)) { + return curop; + } + } + + return NULL; + } + void CurOp::reset( const HostAndPort& remote, int op ) { reset(); if( _remote != remote ) { @@ -105,9 +173,22 @@ namespace mongo { _client = 0; } + void CurOp::setNS( const StringData& ns ) { + ns.substr( 0, Namespace::MaxNsLen ).copyTo( _ns, true ); + } + + void CurOp::ensureStarted() { - if ( _start == 0 ) + if ( _start == 0 ) { _start = curTimeMicros64(); + + // If ensureStarted() is invoked after setMaxTimeMicros(), then time limit tracking will + // start here. This is because time limit tracking can only commence after the + // operation is assigned a start time. + if (_maxTimeMicros > 0) { + _maxTimeTracker.setTimeLimit(_start, _maxTimeMicros); + } + } } void CurOp::enter( Client::Context * context ) { @@ -126,7 +207,7 @@ namespace mongo { if ( _client ) { const LockState& ls = _client->lockState(); verify( ls.threadState() ); - Top::global.record( _ns , _op , ls.hasAnyWriteLock() ? 1 : -1 , micros , _command ); + Top::global.record( _ns , _op , ls.hasAnyWriteLock() ? 1 : -1 , micros , _isCommand ); } } @@ -138,6 +219,7 @@ namespace mongo { if( a ) { b.append("secs_running", elapsedSeconds() ); + b.append("microsecs_running", static_cast(elapsedMicros()) ); } b.append( "op" , opToString( _op ) ); @@ -151,6 +233,10 @@ namespace mongo { _query.append(b , "query"); } + if ( !debug().planSummary.empty() ) { + b.append( "planSummary" , debug().planSummary.toString() ); + } + if( !_remote.empty() ) { b.append("client", _remote.toString()); } @@ -188,6 +274,23 @@ namespace mongo { return b.obj(); } + BSONObj CurOp::description() { + BSONObjBuilder bob; + bool a = _active && _start; + bob.append("active", a); + bob.append( "op" , opToString( _op ) ); + bob.append("ns", _ns); + if (_op == dbInsert) { + _query.append(bob, "insert"); + } + else { + _query.append(bob, "query"); + } + if( killPending() ) + bob.append("killPending", true); + return bob.obj(); + } + void CurOp::setKillWaiterFlags() { for (size_t i = 0; i < _notifyList.size(); ++i) *(_notifyList[i]) = true; @@ -201,6 +304,37 @@ namespace mongo { } } + void CurOp::setMaxTimeMicros(uint64_t maxTimeMicros) { + _maxTimeMicros = maxTimeMicros; + + if (_maxTimeMicros == 0) { + // 0 is "allow to run indefinitely". + return; + } + + // If the operation has a start time, then enable the tracker. + // + // If the operation has no start time yet, then ensureStarted() will take responsibility for + // enabling the tracker. + if (isStarted()) { + _maxTimeTracker.setTimeLimit(startTime(), _maxTimeMicros); + } + } + + bool CurOp::maxTimeHasExpired() { + if (MONGO_FAIL_POINT(maxTimeNeverTimeOut)) { + return false; + } + if (_maxTimeMicros > 0 && MONGO_FAIL_POINT(maxTimeAlwaysTimeOut)) { + return true; + } + return _maxTimeTracker.checkTimeLimit(); + } + + uint64_t CurOp::getRemainingMaxTimeMicros() const { + return _maxTimeTracker.getRemainingMicros(); + } + AtomicUInt CurOp::_nextOpNum; static Counter64 returnedCounter; @@ -208,12 +342,15 @@ namespace mongo { static Counter64 updatedCounter; static Counter64 deletedCounter; static Counter64 scannedCounter; + static Counter64 scannedObjectCounter; static ServerStatusMetricField displayReturned( "document.returned", &returnedCounter ); static ServerStatusMetricField displayUpdated( "document.updated", &updatedCounter ); static ServerStatusMetricField displayInserted( "document.inserted", &insertedCounter ); static ServerStatusMetricField displayDeleted( "document.deleted", &deletedCounter ); static ServerStatusMetricField displayScanned( "queryExecutor.scanned", &scannedCounter ); + static ServerStatusMetricField displayScannedObjects( "queryExecutor.scannedObjects", + &scannedObjectCounter ); static Counter64 idhackCounter; static Counter64 scanAndOrderCounter; @@ -228,12 +365,14 @@ namespace mongo { returnedCounter.increment( nreturned ); if ( ninserted > 0 ) insertedCounter.increment( ninserted ); - if ( nupdated > 0 ) - updatedCounter.increment( nupdated ); + if ( nMatched > 0 ) + updatedCounter.increment( nMatched ); if ( ndeleted > 0 ) deletedCounter.increment( ndeleted ); if ( nscanned > 0 ) scannedCounter.increment( nscanned ); + if ( nscannedObjects > 0 ) + scannedObjectCounter.increment( nscannedObjects ); if ( idhack ) idhackCounter.increment(); @@ -242,4 +381,76 @@ namespace mongo { if ( fastmod ) fastmodCounter.increment(); } + + CurOp::MaxTimeTracker::MaxTimeTracker() { + reset(); + } + + void CurOp::MaxTimeTracker::reset() { + _enabled = false; + _targetEpochMicros = 0; + _approxTargetServerMillis = 0; + } + + void CurOp::MaxTimeTracker::setTimeLimit(uint64_t startEpochMicros, uint64_t durationMicros) { + dassert(durationMicros != 0); + + _enabled = true; + + _targetEpochMicros = startEpochMicros + durationMicros; + + uint64_t now = curTimeMicros64(); + // If our accurate time source thinks time is not up yet, calculate the next target for + // our approximate time source. + if (_targetEpochMicros > now) { + _approxTargetServerMillis = Listener::getElapsedTimeMillis() + + static_cast((_targetEpochMicros - now) / 1000); + } + // Otherwise, set our approximate time source target such that it thinks time is already + // up. + else { + _approxTargetServerMillis = Listener::getElapsedTimeMillis(); + } + } + + bool CurOp::MaxTimeTracker::checkTimeLimit() { + if (!_enabled) { + return false; + } + + // Does our approximate time source think time is not up yet? If so, return early. + if (_approxTargetServerMillis > Listener::getElapsedTimeMillis()) { + return false; + } + + uint64_t now = curTimeMicros64(); + // Does our accurate time source think time is not up yet? If so, readjust the target for + // our approximate time source and return early. + if (_targetEpochMicros > now) { + _approxTargetServerMillis = Listener::getElapsedTimeMillis() + + static_cast((_targetEpochMicros - now) / 1000); + return false; + } + + // Otherwise, time is up. + return true; + } + + uint64_t CurOp::MaxTimeTracker::getRemainingMicros() const { + if (!_enabled) { + // 0 is "allow to run indefinitely". + return 0; + } + + // Does our accurate time source think time is up? If so, claim there is 1 microsecond + // left for this operation. + uint64_t now = curTimeMicros64(); + if (_targetEpochMicros <= now) { + return 1; + } + + // Otherwise, calculate remaining time. + return _targetEpochMicros - now; + } + } diff --git a/src/mongo/db/curop.h b/src/mongo/db/curop.h index 30d94d2a63e..d7caa765f5e 100644 --- a/src/mongo/db/curop.h +++ b/src/mongo/db/curop.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ @@ -23,7 +35,7 @@ #include "mongo/bson/util/atomic_int.h" #include "mongo/db/client.h" -#include "mongo/db/namespace-inl.h" +#include "mongo/db/structure/catalog/namespace.h" #include "mongo/util/concurrency/spin_lock.h" #include "mongo/util/net/hostandport.h" #include "mongo/util/progress_meter.h" @@ -33,10 +45,81 @@ namespace mongo { class CurOp; + /** + * stores a copy of a bson obj in a fixed size buffer + * if its too big for the buffer, says "too big" + * useful for keeping a copy around indefinitely without wasting a lot of space or doing malloc + */ + class CachedBSONObjBase { + public: + static BSONObj _tooBig; // { $msg : "query not recording (too large)" } + }; + + template + class CachedBSONObj : public CachedBSONObjBase { + public: + enum { TOO_BIG_SENTINEL = 1 } ; + + CachedBSONObj() { + _size = (int*)_buf; + reset(); + } + + void reset( int sz = 0 ) { + _lock.lock(); + _reset( sz ); + _lock.unlock(); + } + + void set( const BSONObj& o ) { + scoped_spinlock lk(_lock); + size_t sz = o.objsize(); + if ( sz > sizeof(_buf) ) { + _reset(TOO_BIG_SENTINEL); + } + else { + memcpy(_buf, o.objdata(), sz ); + } + } + + int size() const { return *_size; } + bool have() const { return size() > 0; } + bool tooBig() const { return size() == TOO_BIG_SENTINEL; } + + BSONObj get() const { + scoped_spinlock lk(_lock); + return _get(); + } + + void append( BSONObjBuilder& b , const StringData& name ) const { + scoped_spinlock lk(_lock); + BSONObj temp = _get(); + b.append( name , temp ); + } + + private: + /** you have to be locked when you call this */ + BSONObj _get() const { + int sz = size(); + if ( sz == 0 ) + return BSONObj(); + if ( sz == TOO_BIG_SENTINEL ) + return _tooBig; + return BSONObj( _buf ).copy(); + } + + /** you have to be locked when you call this */ + void _reset( int sz ) { _size[0] = sz; } + + mutable SpinLock _lock; + int * _size; + char _buf[BUFFER_SIZE]; + }; + /* lifespan is different than CurOp because of recursives with DBDirectClient */ class OpDebug { public: - OpDebug() : ns(""){ reset(); } + OpDebug() : ns(""), planSummary(2048) { reset(); } void reset(); @@ -78,9 +161,11 @@ namespace mongo { // debugging/profile info long long nscanned; + long long nscannedObjects; bool idhack; // indicates short circuited code path on an update to make the update faster bool scanAndOrder; // scanandorder query plan aspect was used - long long nupdated; // number of records updated + long long nMatched; // number of records that match the query + long long nModified; // number of records written (no no-ops) long long nmoved; // updates resulted in a move (moves are expensive) long long ninserted; long long ndeleted; @@ -88,6 +173,11 @@ namespace mongo { bool fastmodinsert; // upsert of an $operation. builds a default object bool upsert; // true if the update actually did an insert int keyUpdates; + ThreadSafeString planSummary; // a brief string describing the query solution + + // New Query Framework debugging/profiling info + // TODO: should this really be an opaque BSONObj? Not sure. + CachedBSONObj<4096> execStats; // error handling ExceptionInfo exceptionInfo; @@ -98,71 +188,6 @@ namespace mongo { int responseLength; }; - /** - * stores a copy of a bson obj in a fixed size buffer - * if its too big for the buffer, says "too big" - * useful for keeping a copy around indefinitely without wasting a lot of space or doing malloc - */ - class CachedBSONObj { - public: - enum { TOO_BIG_SENTINEL = 1 } ; - static BSONObj _tooBig; // { $msg : "query not recording (too large)" } - - CachedBSONObj() { - _size = (int*)_buf; - reset(); - } - - void reset( int sz = 0 ) { - _lock.lock(); - _reset( sz ); - _lock.unlock(); - } - - void set( const BSONObj& o ) { - scoped_spinlock lk(_lock); - size_t sz = o.objsize(); - if ( sz > sizeof(_buf) ) { - _reset(TOO_BIG_SENTINEL); - } - else { - memcpy(_buf, o.objdata(), sz ); - } - } - - int size() const { return *_size; } - bool have() const { return size() > 0; } - - BSONObj get() const { - scoped_spinlock lk(_lock); - return _get(); - } - - void append( BSONObjBuilder& b , const StringData& name ) const { - scoped_spinlock lk(_lock); - BSONObj temp = _get(); - b.append( name , temp ); - } - - private: - /** you have to be locked when you call this */ - BSONObj _get() const { - int sz = size(); - if ( sz == 0 ) - return BSONObj(); - if ( sz == TOO_BIG_SENTINEL ) - return _tooBig; - return BSONObj( _buf ).copy(); - } - - /** you have to be locked when you call this */ - void _reset( int sz ) { _size[0] = sz; } - - mutable SpinLock _lock; - int * _size; - char _buf[512]; - }; - /* Current operation (for the current Client). an embedded member of Client class, and typically used from within the mutex there. */ @@ -175,13 +200,11 @@ namespace mongo { BSONObj query() const { return _query.get(); } void appendQuery( BSONObjBuilder& b , const StringData& name ) const { _query.append( b , name ); } - void ensureStarted(); - bool isStarted() const { return _start > 0; } void enter( Client::Context * context ); void leave( Client::Context * context ); void reset(); void reset( const HostAndPort& remote, int op ); - void markCommand() { _command = true; } + void markCommand() { _isCommand = true; } OpDebug& debug() { return _debug; } int profileLevel() const { return _dbprofile; } const char * getNS() const { return _ns; } @@ -190,7 +213,7 @@ namespace mongo { if ( _dbprofile <= 0 ) return false; - return _dbprofile >= 2 || ms >= cmdLine.slowMS; + return _dbprofile >= 2 || ms >= serverGlobalParams.slowMS; } AtomicUInt opNum() const { return _opNum; } @@ -200,6 +223,41 @@ namespace mongo { bool displayInCurop() const { return _active && ! _suppressFromCurop; } int getOp() const { return _op; } + + // + // Methods for controlling CurOp "max time". + // + + /** + * Sets the amount of time operation this should be allowed to run, units of microseconds. + * The special value 0 is "allow to run indefinitely". + */ + void setMaxTimeMicros(uint64_t maxTimeMicros); + + /** + * Checks whether this operation has been running longer than its time limit. Returns + * false if not, or if the operation has no time limit. + * + * Note that KillCurrentOp objects are responsible for interrupting CurOp objects that + * have exceeded their allotted time; CurOp objects do not interrupt themselves. + */ + bool maxTimeHasExpired(); + + /** + * Returns the number of microseconds remaining for this operation's time limit, or the + * special value 0 if the operation has no time limit. + * + * Calling this method is more expensive than calling its sibling "maxTimeHasExpired()", + * since an accurate measure of remaining time needs to be calculated. + */ + uint64_t getRemainingMaxTimeMicros() const; + + // + // Methods for getting/setting elapsed time. + // + + void ensureStarted(); + bool isStarted() const { return _start > 0; } unsigned long long startTime() { // micros ensureStarted(); return _start; @@ -208,19 +266,31 @@ namespace mongo { _active = false; _end = curTimeMicros64(); } + unsigned long long totalTimeMicros() { massert( 12601 , "CurOp not marked done yet" , ! _active ); return _end - startTime(); } int totalTimeMillis() { return (int) (totalTimeMicros() / 1000); } + unsigned long long elapsedMicros() { + return curTimeMicros64() - startTime(); + } int elapsedMillis() { - unsigned long long total = curTimeMicros64() - startTime(); - return (int) (total / 1000); + return (int) (elapsedMicros() / 1000); } int elapsedSeconds() { return elapsedMillis() / 1000; } + void setQuery(const BSONObj& query) { _query.set( query ); } Client * getClient() const { return _client; } + + Command * getCommand() const { return _command; } + void setCommand(Command* command) { _command = command; } + BSONObj info(); + + // Fetches less information than "info()"; used to search for ops with certain criteria + BSONObj description(); + string getRemoteString( bool includePort = true ) { return _remote.toString(includePort); } ProgressMeter& setMessage(const char * msg, std::string name = "Progress", @@ -245,6 +315,22 @@ namespace mongo { LockStat& lockStat() { return _lockStat; } void setKillWaiterFlags(); + + /** + * this should be used very sparingly + * generally the Context should set this up + * but sometimes you want to do it ahead of time + */ + void setNS( const StringData& ns ); + + /** + * Find a currently running operation matching the given criteria. This assumes that you're + * going to kill the operation, so it must be called multiple times to get multiple matching + * operations. + * @param criteria the search to do against the infoNoauth() BSONObj + * @return a pointer to a matching op or NULL if no ops match + */ + static CurOp* getOp(const BSONObj& criteria); private: friend class Client; void _reset(); @@ -252,17 +338,18 @@ namespace mongo { static AtomicUInt _nextOpNum; Client * _client; CurOp * _wrapped; + Command * _command; unsigned long long _start; unsigned long long _end; bool _active; bool _suppressFromCurop; // unless $all is set int _op; - bool _command; + bool _isCommand; int _dbprofile; // 0=off, 1=slow, 2=all AtomicUInt _opNum; // todo: simple being "unsigned" may make more sense here char _ns[Namespace::MaxNsLen+2]; HostAndPort _remote; // CAREFUL here with thread safety - CachedBSONObj _query; // CachedBSONObj is thread safe + CachedBSONObj<512> _query; // CachedBSONObj is thread safe OpDebug _debug; ThreadSafeString _message; ProgressMeter _progressMeter; @@ -276,5 +363,57 @@ namespace mongo { // a writebacklisten for example will block for 30s // so this should be 30000 in that case long long _expectedLatencyMs; + + // Time limit for this operation. 0 if the operation has no time limit. + uint64_t _maxTimeMicros; + + /** Nested class that implements tracking of a time limit for a CurOp object. */ + class MaxTimeTracker { + MONGO_DISALLOW_COPYING(MaxTimeTracker); + public: + /** Newly-constructed MaxTimeTracker objects have the time limit disabled. */ + MaxTimeTracker(); + + /** Disables the time tracker. */ + void reset(); + + /** Returns whether or not time tracking is enabled. */ + bool isEnabled() const { return _enabled; } + + /** + * Enables time tracking. The time limit is set to be "durationMicros" microseconds + * from "startEpochMicros" (units of microseconds since the epoch). + * + * "durationMicros" must be nonzero. + */ + void setTimeLimit(uint64_t startEpochMicros, uint64_t durationMicros); + + /** + * Checks whether the time limit has been hit. Returns false if not, or if time + * tracking is disabled. + */ + bool checkTimeLimit(); + + /** + * Returns the number of microseconds remaining for the time limit, or the special + * value 0 if time tracking is disabled. + * + * Calling this method is more expensive than calling its sibling "checkInterval()", + * since an accurate measure of remaining time needs to be calculated. + */ + uint64_t getRemainingMicros() const; + private: + // Whether or not time tracking is enabled for this operation. + bool _enabled; + + // Point in time at which the time limit is hit. Units of microseconds since the + // epoch. + uint64_t _targetEpochMicros; + + // Approximate point in time at which the time limit is hit. Units of milliseconds + // since the server process was started. + int64_t _approxTargetServerMillis; + } _maxTimeTracker; + }; } diff --git a/src/mongo/db/curop_test.cpp b/src/mongo/db/curop_test.cpp new file mode 100644 index 00000000000..392c53a6509 --- /dev/null +++ b/src/mongo/db/curop_test.cpp @@ -0,0 +1,91 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include + +#include "mongo/base/init.h" +#include "mongo/db/curop.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { + + namespace { + + const long long intervalLong = 2000 * 1000; // 2s in micros + const long long intervalShort = 10 * 1000; // 10ms in micros + + // + // Before executing the TimeHasExpired suite, spawn a dummy listener thread to be the + // process time tracker (the tests rely on Listener::_timeTracker being available). + // + + class TestListener : public Listener { + public: + TestListener() : Listener("test", "", 0) {} // port 0 => any available high port + virtual void acceptedMP(MessagingPort *mp) {} + }; + + void timeTrackerSetup() { + TestListener listener; + listener.setAsTimeTracker(); + listener.setupSockets(); + listener.initAndListen(); + } + + MONGO_INITIALIZER(CurOpTest)(InitializerContext* context) { + boost::thread t(timeTrackerSetup); + + // Wait for listener thread to start tracking time. + while (Listener::getElapsedTimeMillis() == 0) { + sleepmillis(10); + } + + return Status::OK(); + } + + // Long operation + short timeout => time should expire. + TEST(TimeHasExpired, PosSimple) { + CurOp curOp(NULL); + curOp.setMaxTimeMicros(intervalShort); + curOp.ensureStarted(); + sleepmicros(intervalLong); + ASSERT_TRUE(curOp.maxTimeHasExpired()); + } + + // Short operation + long timeout => time should not expire. + TEST(TimeHasExpired, NegSimple) { + CurOp curOp(NULL); + curOp.setMaxTimeMicros(intervalLong); + curOp.ensureStarted(); + sleepmicros(intervalShort); + ASSERT_FALSE(curOp.maxTimeHasExpired()); + } + + } // namespace + +} // namespace mongo diff --git a/src/mongo/db/cursor.cpp b/src/mongo/db/cursor.cpp deleted file mode 100644 index 1ce5da1b36a..00000000000 --- a/src/mongo/db/cursor.cpp +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Copyright (C) 2008 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "mongo/pch.h" - -#include "mongo/db/curop-inl.h" -#include "mongo/db/kill_current_op.h" -#include "mongo/db/pdfile.h" - -namespace mongo { - - bool BasicCursor::advance() { - killCurrentOp.checkForInterrupt(); - if ( eof() ) { - if ( tailable_ && !last.isNull() ) { - curr = s->next( last ); - } - else { - return false; - } - } - else { - last = curr; - curr = s->next( curr ); - } - incNscanned(); - return ok(); - } - - /* these will be used outside of mutexes - really functors - thus the const */ - class Forward : public AdvanceStrategy { - virtual DiskLoc next( const DiskLoc &prev ) const { - return prev.rec()->getNext( prev ); - } - } _forward; - - class Reverse : public AdvanceStrategy { - virtual DiskLoc next( const DiskLoc &prev ) const { - return prev.rec()->getPrev( prev ); - } - } _reverse; - - const AdvanceStrategy *forward() { - return &_forward; - } - const AdvanceStrategy *reverse() { - return &_reverse; - } - - DiskLoc nextLoop( NamespaceDetails *nsd, const DiskLoc &prev ) { - verify( nsd->capLooped() ); - DiskLoc next = forward()->next( prev ); - if ( !next.isNull() ) - return next; - return nsd->firstRecord(); - } - - DiskLoc prevLoop( NamespaceDetails *nsd, const DiskLoc &curr ) { - verify( nsd->capLooped() ); - DiskLoc prev = reverse()->next( curr ); - if ( !prev.isNull() ) - return prev; - return nsd->lastRecord(); - } - - ForwardCappedCursor* ForwardCappedCursor::make( NamespaceDetails* nsd, - const DiskLoc& startLoc ) { - auto_ptr ret( new ForwardCappedCursor( nsd ) ); - ret->init( startLoc ); - return ret.release(); - } - - ForwardCappedCursor::ForwardCappedCursor( NamespaceDetails* _nsd ) : - nsd( _nsd ) { - } - - void ForwardCappedCursor::init( const DiskLoc& startLoc ) { - if ( !nsd ) - return; - DiskLoc start = startLoc; - if ( start.isNull() ) { - if ( !nsd->capLooped() ) - start = nsd->firstRecord(); - else { - start = nsd->capExtent.ext()->firstRecord; - if ( !start.isNull() && start == nsd->capFirstNewRecord ) { - start = nsd->capExtent.ext()->lastRecord; - start = nextLoop( nsd, start ); - } - } - } - curr = start; - s = this; - incNscanned(); - } - - DiskLoc ForwardCappedCursor::next( const DiskLoc &prev ) const { - verify( nsd ); - if ( !nsd->capLooped() ) - return forward()->next( prev ); - - DiskLoc i = prev; - // Last record - if ( i == nsd->capExtent.ext()->lastRecord ) - return DiskLoc(); - i = nextLoop( nsd, i ); - // If we become capFirstNewRecord from same extent, advance to next extent. - if ( i == nsd->capFirstNewRecord && - i != nsd->capExtent.ext()->firstRecord ) - i = nextLoop( nsd, nsd->capExtent.ext()->lastRecord ); - // If we have just gotten to beginning of capExtent, skip to capFirstNewRecord - if ( i == nsd->capExtent.ext()->firstRecord ) - i = nsd->capFirstNewRecord; - return i; - } - - ReverseCappedCursor::ReverseCappedCursor( NamespaceDetails *_nsd, const DiskLoc &startLoc ) : - nsd( _nsd ) { - if ( !nsd ) - return; - DiskLoc start = startLoc; - if ( start.isNull() ) { - if ( !nsd->capLooped() ) { - start = nsd->lastRecord(); - } - else { - start = nsd->capExtent.ext()->lastRecord; - } - } - curr = start; - s = this; - incNscanned(); - } - - DiskLoc ReverseCappedCursor::next( const DiskLoc &prev ) const { - verify( nsd ); - if ( !nsd->capLooped() ) - return reverse()->next( prev ); - - DiskLoc i = prev; - // Last record - if ( nsd->capFirstNewRecord == nsd->capExtent.ext()->firstRecord ) { - if ( i == nextLoop( nsd, nsd->capExtent.ext()->lastRecord ) ) { - return DiskLoc(); - } - } - else { - if ( i == nsd->capExtent.ext()->firstRecord ) { - return DiskLoc(); - } - } - // If we are capFirstNewRecord, advance to prev extent, otherwise just get prev. - if ( i == nsd->capFirstNewRecord ) - i = prevLoop( nsd, nsd->capExtent.ext()->firstRecord ); - else - i = prevLoop( nsd, i ); - // If we just became last in cap extent, advance past capFirstNewRecord - // (We know capExtent.ext()->firstRecord != capFirstNewRecord, since would - // have returned DiskLoc() earlier otherwise.) - if ( i == nsd->capExtent.ext()->lastRecord ) - i = reverse()->next( nsd->capFirstNewRecord ); - - return i; - } -} // namespace mongo diff --git a/src/mongo/db/cursor.h b/src/mongo/db/cursor.h deleted file mode 100644 index 6d92d614b81..00000000000 --- a/src/mongo/db/cursor.h +++ /dev/null @@ -1,319 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/pch.h" - -#include "jsobj.h" -#include "diskloc.h" -#include "matcher.h" -#include "mongo/db/projection.h" - -namespace mongo { - - class NamespaceDetails; - class Record; - class CoveredIndexMatcher; - - /** - * Query cursors, base class. This is for our internal cursors. "ClientCursor" is a separate - * concept and is for the user's cursor. - * - * WARNING concurrency: the vfunctions below are called back from within a - * ClientCursor::ccmutex. Don't cause a deadlock, you've been warned. - * - * Two general techniques may be used to ensure a Cursor is in a consistent state after a write. - * - The Cursor may be advanced before the document at its current position is deleted. - * - The Cursor may record its position and then relocate this position. - * A particular Cursor may potentially utilize only one of the above techniques, but a client - * that is Cursor subclass agnostic must implement a pattern handling both techniques. - * - * When the document at a Cursor's current position is deleted (or moved to a new location) the - * following pattern is used: - * DiskLoc toDelete = cursor->currLoc(); - * while( cursor->ok() && cursor->currLoc() == toDelete ) { - * cursor->advance(); - * } - * cursor->prepareToTouchEarlierIterate(); - * delete( toDelete ); - * cursor->recoverFromTouchingEarlierIterate(); - * - * When a cursor yields, the following pattern is used: - * cursor->prepareToYield(); - * while( Op theOp = nextOp() ) { - * if ( theOp.type() == INSERT || theOp.type() == UPDATE_IN_PLACE ) { - * theOp.run(); - * } - * else if ( theOp.type() == DELETE ) { - * if ( cursor->refLoc() == theOp.toDelete() ) { - * cursor->recoverFromYield(); - * while ( cursor->ok() && cursor->refLoc() == theOp.toDelete() ) { - * cursor->advance(); - * } - * cursor->prepareToYield(); - * } - * theOp.run(); - * } - * } - * cursor->recoverFromYield(); - * - * The break before a getMore request is typically treated as a yield, but if a Cursor supports - * getMore but not yield the following pattern is currently used: - * cursor->noteLocation(); - * runOtherOps(); - * cursor->checkLocation(); - * - * But see SERVER-5725. - * - * A Cursor may rely on additional callbacks not listed above to relocate its position after a - * write. - */ - class Cursor : boost::noncopyable { - public: - virtual ~Cursor() {} - virtual bool ok() = 0; - bool eof() { return !ok(); } - virtual Record* _current() = 0; - virtual BSONObj current() = 0; - virtual DiskLoc currLoc() = 0; - virtual bool advance() = 0; /*true=ok*/ - virtual BSONObj currKey() const { return BSONObj(); } - - // DiskLoc the cursor requires for continued operation. Before this - // DiskLoc is deleted, the cursor must be incremented or destroyed. - virtual DiskLoc refLoc() = 0; - - /* Implement these if you want the cursor to be "tailable" */ - - /* Request that the cursor starts tailing after advancing past last record. */ - /* The implementation may or may not honor this request. */ - virtual void setTailable() {} - /* indicates if tailing is enabled. */ - virtual bool tailable() { - return false; - } - - virtual void aboutToDeleteBucket(const DiskLoc& b) { } - - /* optional to implement. if implemented, means 'this' is a prototype */ - virtual Cursor* clone() { - return 0; - } - - virtual BSONObj indexKeyPattern() { - return BSONObj(); - } - - virtual bool supportGetMore() = 0; - - /* called after every query block is iterated -- i.e. between getMore() blocks - so you can note where we are, if necessary. - */ - virtual void noteLocation() { } - - /* called before query getmore block is iterated */ - virtual void checkLocation() { } - - /** - * Called before a document pointed at by an earlier iterate of this cursor is to be - * modified. It is ok if the current iterate also points to the document to be modified. - */ - virtual void prepareToTouchEarlierIterate() { noteLocation(); } - - /** Recover from a previous call to prepareToTouchEarlierIterate(). */ - virtual void recoverFromTouchingEarlierIterate() { checkLocation(); } - - virtual bool supportYields() = 0; - - /** Called before a ClientCursor yield. */ - virtual void prepareToYield() { noteLocation(); } - - /** Called after a ClientCursor yield. Recovers from a previous call to prepareToYield(). */ - virtual void recoverFromYield() { checkLocation(); } - - virtual string toString() { return "abstract?"; } - - /* used for multikey index traversal to avoid sending back dups. see Matcher::matches(). - if a multikey index traversal: - if loc has already been sent, returns true. - otherwise, marks loc as sent. - */ - virtual bool getsetdup(DiskLoc loc) = 0; - - virtual bool isMultiKey() const = 0; - - virtual bool autoDedup() const { return true; } - - /** - * return true if the keys in the index have been modified from the main doc - * if you have { a : 1 , b : [ 1 , 2 ] } - * an index on { a : 1 } would not be modified - * an index on { b : 1 } would be since the values of the array are put in the index - * not the array - */ - virtual bool modifiedKeys() const = 0; - - virtual BSONObj prettyIndexBounds() const { return BSONArray(); } - - /** - * If true, this is an unindexed cursor over a capped collection. Currently such cursors must - * not own a delegate ClientCursor, due to the implementation of ClientCursor::aboutToDelete(). - SERVER-4563 - */ - virtual bool capped() const { return false; } - - virtual long long nscanned() = 0; - - // The implementation may return different matchers depending on the - // position of the cursor. If matcher() is nonzero at the start, - // matcher() should be checked each time advance() is called. - // Implementations which generate their own matcher should return this - // to avoid a matcher being set manually. - // Note that the return values differ subtly here - - // Used when we want fast matcher lookup - virtual CoveredIndexMatcher *matcher() const { return 0; } - - virtual bool currentMatches( MatchDetails *details = 0 ) { - return !matcher() || matcher()->matchesCurrent( this, details ); - } - - // A convenience function for setting the value of matcher() manually - // so it may be accessed later. Implementations which must generate - // their own matcher() should assert here. - virtual void setMatcher( shared_ptr< CoveredIndexMatcher > matcher ) { - massert( 13285, "manual matcher config not allowed", false ); - } - - /** @return the covered index projector for the current iterate, if any. */ - virtual const Projection::KeyOnly *keyFieldsOnly() const { return 0; } - - /** - * Manually set the value of keyFieldsOnly() so it may be accessed later. Implementations - * that generate their own keyFieldsOnly() must assert. - */ - virtual void setKeyFieldsOnly( const shared_ptr &keyFieldsOnly ) { - massert( 16159, "manual keyFieldsOnly config not allowed", false ); - } - - virtual void explainDetails( BSONObjBuilder& b ) { return; } - }; - - // strategy object implementing direction of traversal. - class AdvanceStrategy { - public: - virtual ~AdvanceStrategy() { } - virtual DiskLoc next( const DiskLoc &prev ) const = 0; - }; - - const AdvanceStrategy *forward(); - const AdvanceStrategy *reverse(); - - /** - * table-scan style cursor - * - * A BasicCursor relies on advance() to ensure it is in a consistent state after a write. If - * the document at a BasicCursor's current position will be deleted or relocated, the cursor - * must first be advanced. The same is true of BasicCursor subclasses. - */ - class BasicCursor : public Cursor { - public: - BasicCursor(DiskLoc dl, const AdvanceStrategy *_s = forward()) : curr(dl), s( _s ), _nscanned() { - incNscanned(); - init(); - } - BasicCursor(const AdvanceStrategy *_s = forward()) : s( _s ), _nscanned() { - init(); - } - bool ok() { return !curr.isNull(); } - Record* _current() { - verify( ok() ); - return curr.rec(); - } - BSONObj current() { - Record *r = _current(); - return BSONObj::make(r); - } - virtual DiskLoc currLoc() { return curr; } - virtual DiskLoc refLoc() { return curr.isNull() ? last : curr; } - bool advance(); - virtual string toString() { return "BasicCursor"; } - virtual void setTailable() { - if ( !curr.isNull() || !last.isNull() ) - tailable_ = true; - } - virtual bool tailable() { return tailable_; } - virtual bool getsetdup(DiskLoc loc) { return false; } - virtual bool isMultiKey() const { return false; } - virtual bool modifiedKeys() const { return false; } - virtual bool supportGetMore() { return true; } - virtual bool supportYields() { return true; } - virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); } - virtual void setMatcher( shared_ptr< CoveredIndexMatcher > matcher ) { _matcher = matcher; } - virtual const Projection::KeyOnly *keyFieldsOnly() const { return _keyFieldsOnly.get(); } - virtual void setKeyFieldsOnly( const shared_ptr &keyFieldsOnly ) { - _keyFieldsOnly = keyFieldsOnly; - } - virtual long long nscanned() { return _nscanned; } - - protected: - DiskLoc curr, last; - const AdvanceStrategy *s; - void incNscanned() { if ( !curr.isNull() ) { ++_nscanned; } } - private: - bool tailable_; - shared_ptr< CoveredIndexMatcher > _matcher; - shared_ptr _keyFieldsOnly; - long long _nscanned; - void init() { tailable_ = false; } - }; - - /* used for order { $natural: -1 } */ - class ReverseCursor : public BasicCursor { - public: - ReverseCursor(DiskLoc dl) : BasicCursor( dl, reverse() ) { } - ReverseCursor() : BasicCursor( reverse() ) { } - virtual string toString() { return "ReverseCursor"; } - }; - - class ForwardCappedCursor : public BasicCursor, public AdvanceStrategy { - public: - static ForwardCappedCursor* make( NamespaceDetails* nsd = 0, - const DiskLoc& startLoc = DiskLoc() ); - virtual string toString() { - return "ForwardCappedCursor"; - } - virtual DiskLoc next( const DiskLoc &prev ) const; - virtual bool capped() const { return true; } - private: - ForwardCappedCursor( NamespaceDetails* nsd ); - void init( const DiskLoc& startLoc ); - NamespaceDetails *nsd; - }; - - class ReverseCappedCursor : public BasicCursor, public AdvanceStrategy { - public: - ReverseCappedCursor( NamespaceDetails *nsd = 0, const DiskLoc &startLoc = DiskLoc() ); - virtual string toString() { - return "ReverseCappedCursor"; - } - virtual DiskLoc next( const DiskLoc &prev ) const; - virtual bool capped() const { return true; } - private: - NamespaceDetails *nsd; - }; - -} // namespace mongo diff --git a/src/mongo/db/d_concurrency.cpp b/src/mongo/db/d_concurrency.cpp index 0595b229565..22d40c62dd6 100644 --- a/src/mongo/db/d_concurrency.cpp +++ b/src/mongo/db/d_concurrency.cpp @@ -14,24 +14,38 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "d_concurrency.h" -#include "../util/concurrency/qlock.h" -#include "../util/concurrency/threadlocal.h" -#include "../util/concurrency/rwlock.h" -#include "../util/concurrency/mapsf.h" -#include "../util/assert_util.h" -#include "../util/stacktrace.h" -#include "client.h" -#include "curop.h" -#include "namespacestring.h" -#include "d_globals.h" -#include "server.h" -#include "dur.h" -#include "lockstat.h" +#include "mongo/pch.h" + +#include "mongo/db/d_concurrency.h" + +#include "mongo/db/client.h" #include "mongo/db/commands/server_status.h" +#include "mongo/db/curop.h" +#include "mongo/db/d_globals.h" +#include "mongo/db/dur.h" +#include "mongo/db/lockstat.h" +#include "mongo/db/namespace_string.h" +#include "mongo/server.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/concurrency/mapsf.h" +#include "mongo/util/concurrency/qlock.h" +#include "mongo/util/concurrency/rwlock.h" +#include "mongo/util/concurrency/threadlocal.h" +#include "mongo/util/stacktrace.h" // oplog locking // no top level read locks @@ -70,11 +84,6 @@ namespace mongo { void releasingWriteLock(); } - // e.g. externalobjsortmutex uses hlmutex as it can be locked for very long times - // todo : report HLMutex status in db.currentOp() output - // perhaps move this elsewhere as this could be used in mongos and this file is for mongod - HLMutex::HLMutex(const char *name) : SimpleMutex(name) { } - /* dbname->lock Currently these are never deleted - will linger if db was closed. (that should be fine.) We don't put the lock inside the Database object as those can come and go with open and @@ -300,7 +309,10 @@ namespace mongo { long long acquisitionTime = _timer.micros(); _timer.reset(); _stat = stat; + + // increment the operation level statistics cc().curop()->lockStat().recordAcquireTimeMicros( _type , acquisitionTime ); + return acquisitionTime; } @@ -327,12 +339,10 @@ namespace mongo { void Lock::ScopedLock::relock() { _pbws_lk.relock(); - _relock(); resetTime(); + _relock(); } - - Lock::TempRelease::TempRelease() : cant( Lock::nested() ) { if( cant ) diff --git a/src/mongo/db/d_concurrency.h b/src/mongo/db/d_concurrency.h index 863a60197b8..50760662ced 100644 --- a/src/mongo/db/d_concurrency.h +++ b/src/mongo/db/d_concurrency.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ @@ -236,15 +248,4 @@ namespace mongo { ~writelocktry(); bool got() const { return _got; } }; - - /** a mutex, but reported in curop() - thus a "high level" (HL) one - some overhead so we don't use this for everything. the externalobjsort mutex - uses this, as it can be held for eons. implementation still needed. */ - class HLMutex : public SimpleMutex { - LockStat ls; - public: - HLMutex(const char *name); - }; - - } diff --git a/src/mongo/db/d_globals.cpp b/src/mongo/db/d_globals.cpp index ddd92d5ee6d..97752a57436 100644 --- a/src/mongo/db/d_globals.cpp +++ b/src/mongo/db/d_globals.cpp @@ -14,12 +14,26 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "d_globals.h" -#include "../util/concurrency/rwlock.h" -#include "clientcursor.h" +#include "mongo/pch.h" + +#include "mongo/db/d_globals.h" + +#include "mongo/db/clientcursor.h" +#include "mongo/util/concurrency/rwlock.h" namespace mongo { diff --git a/src/mongo/db/d_globals.h b/src/mongo/db/d_globals.h index e683e083e6a..ec68900f650 100644 --- a/src/mongo/db/d_globals.h +++ b/src/mongo/db/d_globals.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ // diff --git a/src/mongo/db/database.cpp b/src/mongo/db/database.cpp deleted file mode 100644 index 76a2055ab80..00000000000 --- a/src/mongo/db/database.cpp +++ /dev/null @@ -1,502 +0,0 @@ -// database.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/database.h" - -#include - -#include "mongo/db/auth/auth_index_d.h" -#include "mongo/db/clientcursor.h" -#include "mongo/db/databaseholder.h" -#include "mongo/db/instance.h" -#include "mongo/db/introspect.h" -#include "mongo/db/pdfile.h" - -namespace mongo { - - void assertDbAtLeastReadLocked(const Database *db) { - if( db ) { - Lock::assertAtLeastReadLocked(db->name); - } - else { - verify( Lock::isLocked() ); - } - } - - void assertDbWriteLocked(const Database *db) { - if( db ) { - Lock::assertWriteLocked(db->name); - } - else { - verify( Lock::isW() ); - } - } - - Database::~Database() { - verify( Lock::isW() ); - magic = 0; - size_t n = _files.size(); - for ( size_t i = 0; i < n; i++ ) - delete _files[i]; - if( ccByLoc.size() ) { - log() << "\n\n\nWARNING: ccByLoc not empty on database close! " << ccByLoc.size() << ' ' << name << endl; - } - } - - Database::Database(const char *nm, bool& newDb, const string& _path ) - : name(nm), path(_path), namespaceIndex( path, name ), - profileName(name + ".system.profile") - { - try { - { - // check db name is valid - size_t L = strlen(nm); - uassert( 10028 , "db name is empty", L > 0 ); - uassert( 10032 , "db name too long", L < 64 ); - uassert( 10029 , "bad db name [1]", *nm != '.' ); - uassert( 10030 , "bad db name [2]", nm[L-1] != '.' ); - uassert( 10031 , "bad char(s) in db name", strchr(nm, ' ') == 0 ); -#ifdef _WIN32 - static const char* windowsReservedNames[] = { - "con", "prn", "aux", "nul", - "com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8", "com9", - "lpt1", "lpt2", "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9" - }; - for ( size_t i = 0; i < (sizeof(windowsReservedNames) / sizeof(char*)); ++i ) { - if ( strcasecmp( nm, windowsReservedNames[i] ) == 0 ) { - stringstream errorString; - errorString << "db name \"" << nm << "\" is a reserved name"; - uassert( 16185 , errorString.str(), false ); - } - } -#endif - } - newDb = namespaceIndex.exists(); - _profile = cmdLine.defaultProfile; - checkDuplicateUncasedNames(true); - // If already exists, open. Otherwise behave as if empty until - // there's a write, then open. - if (!newDb) { - namespaceIndex.init(); - openAllFiles(); - } - magic = 781231; - } catch(std::exception& e) { - log() << "warning database " << path << " " << nm << " could not be opened" << endl; - DBException* dbe = dynamic_cast(&e); - if ( dbe != 0 ) { - log() << "DBException " << dbe->getCode() << ": " << e.what() << endl; - } - else { - log() << e.what() << endl; - } - // since destructor won't be called: - for ( size_t i = 0; i < _files.size(); i++ ) { - delete _files[i]; - } - _files.clear(); - throw; - } - } - - void Database::checkDuplicateUncasedNames(bool inholderlock) const { - string duplicate = duplicateUncasedName(inholderlock, name, path ); - if ( !duplicate.empty() ) { - stringstream ss; - ss << "db already exists with different case other: [" << duplicate << "] me [" << name << "]"; - uasserted( DatabaseDifferCaseCode , ss.str() ); - } - } - - /*static*/ - string Database::duplicateUncasedName( bool inholderlock, const string &name, const string &path, set< string > *duplicates ) { - Lock::assertAtLeastReadLocked(name); - - if ( duplicates ) { - duplicates->clear(); - } - - vector others; - getDatabaseNames( others , path ); - - set allShortNames; - dbHolder().getAllShortNames( allShortNames ); - - others.insert( others.end(), allShortNames.begin(), allShortNames.end() ); - - for ( unsigned i=0; iinsert( others[i] ); - } else { - return others[i]; - } - } - if ( duplicates ) { - return duplicates->empty() ? "" : *duplicates->begin(); - } - return ""; - } - - boost::filesystem::path Database::fileName( int n ) const { - stringstream ss; - ss << name << '.' << n; - boost::filesystem::path fullName; - fullName = boost::filesystem::path(path); - if ( directoryperdb ) - fullName /= name; - fullName /= ss.str(); - return fullName; - } - - bool Database::openExistingFile( int n ) { - verify(this); - Lock::assertWriteLocked(name); - { - // must not yet be visible to others as we aren't in the db's write lock and - // we will write to _files vector - thus this assert. - bool loaded = dbHolder().__isLoaded(name, path); - verify( !loaded ); - } - // additionally must be in the dbholder mutex (no assert for that yet) - - // todo: why here? that could be bad as we may be read locked only here - namespaceIndex.init(); - - if ( n < 0 || n >= DiskLoc::MaxFiles ) { - massert( 15924 , str::stream() << "getFile(): bad file number value " << n << " (corrupt db?): run repair", false); - } - - { - if( n < (int) _files.size() && _files[n] ) { - dlog(2) << "openExistingFile " << n << " is already open" << endl; - return true; - } - } - - { - boost::filesystem::path fullName = fileName( n ); - string fullNameString = fullName.string(); - MongoDataFile *df = new MongoDataFile(n); - try { - if( !df->openExisting( fullNameString.c_str() ) ) { - delete df; - return false; - } - } - catch ( AssertionException& ) { - delete df; - throw; - } - while ( n >= (int) _files.size() ) { - _files.push_back(0); - } - _files[n] = df; - } - - return true; - } - - // todo : we stop once a datafile dne. - // if one datafile were missing we should keep going for - // repair purposes yet we do not. - void Database::openAllFiles() { - verify(this); - int n = 0; - while( openExistingFile(n) ) { - n++; - } - } - - void Database::clearTmpCollections() { - - Lock::assertWriteLocked( name ); - Client::Context ctx( name ); - - string systemNamespaces = name + ".system.namespaces"; - - // Note: we build up a toDelete vector rather than dropping the collection inside the loop - // to avoid modifying the system.namespaces collection while iterating over it since that - // would corrupt the cursor. - vector toDelete; - shared_ptr cursor = theDataFileMgr.findAll(systemNamespaces); - while ( cursor && cursor->ok() ) { - BSONObj nsObj = cursor->current(); - cursor->advance(); - - BSONElement e = nsObj.getFieldDotted( "options.temp" ); - if ( !e.trueValue() ) - continue; - - string ns = nsObj["name"].String(); - - // Do not attempt to drop indexes - if ( !NamespaceString::normal(ns.c_str()) ) - continue; - - toDelete.push_back(ns); - } - - for (size_t i=0; i < toDelete.size(); i++) { - const string& ns = toDelete[i]; - - string errmsg; - BSONObjBuilder result; - dropCollection(ns, errmsg, result); - - if ( errmsg.size() > 0 ) { - warning() << "could not delete temp collection: " << ns - << " because of: " << errmsg << endl; - } - } - } - - // todo: this is called a lot. streamline the common case - MongoDataFile* Database::getFile( int n, int sizeNeeded , bool preallocateOnly) { - verify(this); - DEV assertDbAtLeastReadLocked(this); - - namespaceIndex.init(); - if ( n < 0 || n >= DiskLoc::MaxFiles ) { - out() << "getFile(): n=" << n << endl; - massert( 10295 , "getFile(): bad file number value (corrupt db?): run repair", false); - } - DEV { - if ( n > 100 ) { - out() << "getFile(): n=" << n << endl; - } - } - MongoDataFile* p = 0; - if ( !preallocateOnly ) { - while ( n >= (int) _files.size() ) { - verify(this); - if( !Lock::isWriteLocked(this->name) ) { - log() << "error: getFile() called in a read lock, yet file to return is not yet open" << endl; - log() << " getFile(" << n << ") _files.size:" <<_files.size() << ' ' << fileName(n).string() << endl; - log() << " context ns: " << cc().ns() << endl; - verify(false); - } - _files.push_back(0); - } - p = _files[n]; - } - if ( p == 0 ) { - assertDbWriteLocked(this); - boost::filesystem::path fullName = fileName( n ); - string fullNameString = fullName.string(); - p = new MongoDataFile(n); - int minSize = 0; - if ( n != 0 && _files[ n - 1 ] ) - minSize = _files[ n - 1 ]->getHeader()->fileLength; - if ( sizeNeeded + DataFileHeader::HeaderSize > minSize ) - minSize = sizeNeeded + DataFileHeader::HeaderSize; - try { - p->open( fullNameString.c_str(), minSize, preallocateOnly ); - } - catch ( AssertionException& ) { - delete p; - throw; - } - if ( preallocateOnly ) - delete p; - else - _files[n] = p; - } - return preallocateOnly ? 0 : p; - } - - MongoDataFile* Database::addAFile( int sizeNeeded, bool preallocateNextFile ) { - assertDbWriteLocked(this); - int n = (int) _files.size(); - MongoDataFile *ret = getFile( n, sizeNeeded ); - if ( preallocateNextFile ) - preallocateAFile(); - return ret; - } - - bool fileIndexExceedsQuota( const char *ns, int fileIndex, bool enforceQuota ) { - return - cmdLine.quota && - enforceQuota && - fileIndex >= cmdLine.quotaFiles && - // we don't enforce the quota on "special" namespaces as that could lead to problems -- e.g. - // rejecting an index insert after inserting the main record. - !NamespaceString::special( ns ) && - NamespaceString( ns ).db != "local"; - } - - MongoDataFile* Database::suitableFile( const char *ns, int sizeNeeded, bool preallocate, bool enforceQuota ) { - - // check existing files - for ( int i=numFiles()-1; i>=0; i-- ) { - MongoDataFile* f = getFile( i ); - if ( f->getHeader()->unusedLength >= sizeNeeded ) { - if ( fileIndexExceedsQuota( ns, i-1, enforceQuota ) ) // NOTE i-1 is the value used historically for this check. - ; - else - return f; - } - } - - if ( fileIndexExceedsQuota( ns, numFiles(), enforceQuota ) ) { - if ( cc().hasWrittenThisPass() ) { - warning() << "quota exceeded, but can't assert, probably going over quota for: " << ns << endl; - } - else { - uasserted(12501, "quota exceeded"); - } - } - - // allocate files until we either get one big enough or hit maxSize - for ( int i = 0; i < 8; i++ ) { - MongoDataFile* f = addAFile( sizeNeeded, preallocate ); - - if ( f->getHeader()->unusedLength >= sizeNeeded ) - return f; - - if ( f->getHeader()->fileLength >= MongoDataFile::maxSize() ) // this is as big as they get so might as well stop - return f; - } - - uasserted(14810, "couldn't allocate space (suitableFile)"); // callers don't check for null return code - return 0; - } - - MongoDataFile* Database::newestFile() { - int n = numFiles(); - if ( n == 0 ) - return 0; - return getFile(n-1); - } - - - Extent* Database::allocExtent( const char *ns, int size, bool capped, bool enforceQuota ) { - // todo: when profiling, these may be worth logging into profile collection - bool fromFreeList = true; - Extent *e = DataFileMgr::allocFromFreeList( ns, size, capped ); - if( e == 0 ) { - fromFreeList = false; - e = suitableFile( ns, size, !capped, enforceQuota )->createExtent( ns, size, capped ); - } - LOG(1) << "allocExtent " << ns << " size " << size << ' ' << fromFreeList << endl; - return e; - } - - - bool Database::setProfilingLevel( int newLevel , string& errmsg ) { - if ( _profile == newLevel ) - return true; - - if ( newLevel < 0 || newLevel > 2 ) { - errmsg = "profiling level has to be >=0 and <= 2"; - return false; - } - - if ( newLevel == 0 ) { - _profile = 0; - return true; - } - - verify( cc().database() == this ); - - if (!getOrCreateProfileCollection(this, true, &errmsg)) - return false; - - _profile = newLevel; - return true; - } - - bool Database::exists(int n) const { - return boost::filesystem::exists( fileName( n ) ); - } - - int Database::numFiles() const { - DEV assertDbAtLeastReadLocked(this); - return (int) _files.size(); - } - - void Database::flushFiles( bool sync ) { - assertDbAtLeastReadLocked(this); - for( vector::iterator i = _files.begin(); i != _files.end(); i++ ) { - MongoDataFile *f = *i; - f->flush(sync); - } - } - - long long Database::fileSize() const { - long long size=0; - for (int n=0; exists(n); n++) - size += boost::filesystem::file_size( fileName(n) ); - return size; - } - - Database* DatabaseHolder::getOrCreate( const string& ns , const string& path , bool& justCreated ) { - string dbname = _todb( ns ); - { - SimpleMutex::scoped_lock lk(_m); - Lock::assertAtLeastReadLocked(ns); - DBs& m = _paths[path]; - { - DBs::iterator i = m.find(dbname); - if( i != m.end() ) { - justCreated = false; - return i->second; - } - } - - // todo: protect against getting sprayed with requests for different db names that DNE - - // that would make the DBs map very large. not clear what to do to handle though, - // perhaps just log it, which is what we do here with the "> 40" : - bool cant = !Lock::isWriteLocked(ns); - if( logLevel >= 1 || m.size() > 40 || cant || DEBUG_BUILD ) { - log() << "opening db: " << (path==dbpath?"":path) << ' ' << dbname << endl; - } - massert(15927, "can't open database in a read lock. if db was just closed, consider retrying the query. might otherwise indicate an internal error", !cant); - } - - // we mark our thread as having done writes now as we do not want any exceptions - // once we start creating a new database - cc().writeHappened(); - - // this locks _m for defensive checks, so we don't want to be locked right here : - Database *db = new Database( dbname.c_str() , justCreated , path ); - - { - SimpleMutex::scoped_lock lk(_m); - DBs& m = _paths[path]; - verify( m[dbname] == 0 ); - m[dbname] = db; - _size++; - } - - authindex::configureSystemIndexes(dbname); - - db->clearTmpCollections(); - - return db; - } - -} // namespace mongo diff --git a/src/mongo/db/database.h b/src/mongo/db/database.h deleted file mode 100644 index 3a49c1e994d..00000000000 --- a/src/mongo/db/database.h +++ /dev/null @@ -1,152 +0,0 @@ -// database.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/db/cc_by_loc.h" -#include "mongo/db/cmdline.h" -#include "mongo/db/namespace_details.h" -#include "mongo/db/record.h" - -namespace mongo { - - class Extent; - class MongoDataFile; - - /** - * Database represents a database database - * Each database database has its own set of files -- dbname.ns, dbname.0, dbname.1, ... - * NOT memory mapped - */ - class Database { - public: - // you probably need to be in dbHolderMutex when constructing this - Database(const char *nm, /*out*/ bool& newDb, const string& _path = dbpath); - private: - ~Database(); // closes files and other cleanup see below. - public: - /* you must use this to close - there is essential code in this method that is not in the ~Database destructor. - thus the destructor is private. this could be cleaned up one day... - */ - static void closeDatabase( const char *db, const string& path ); - - void openAllFiles(); - void clearTmpCollections(); - - /** - * tries to make sure that this hasn't been deleted - */ - bool isOk() const { return magic == 781231; } - - bool isEmpty() { return ! namespaceIndex.allocated(); } - - /** - * total file size of Database in bytes - */ - long long fileSize() const; - - int numFiles() const; - - /** - * returns file valid for file number n - */ - boost::filesystem::path fileName( int n ) const; - - private: - bool exists(int n) const; - bool openExistingFile( int n ); - - public: - /** - * return file n. if it doesn't exist, create it - */ - MongoDataFile* getFile( int n, int sizeNeeded = 0, bool preallocateOnly = false ); - - MongoDataFile* addAFile( int sizeNeeded, bool preallocateNextFile ); - - /** - * makes sure we have an extra file at the end that is empty - * safe to call this multiple times - the implementation will only preallocate one file - */ - void preallocateAFile() { getFile( numFiles() , 0, true ); } - - MongoDataFile* suitableFile( const char *ns, int sizeNeeded, bool preallocate, bool enforceQuota ); - - Extent* allocExtent( const char *ns, int size, bool capped, bool enforceQuota ); - - MongoDataFile* newestFile(); - - /** - * @return true if success. false if bad level or error creating profile ns - */ - bool setProfilingLevel( int newLevel , string& errmsg ); - - void flushFiles( bool sync ); - - /** - * @return true if ns is part of the database - * ns=foo.bar, db=foo returns true - */ - bool ownsNS( const string& ns ) const { - if ( ! startsWith( ns , name ) ) - return false; - return ns[name.size()] == '.'; - } - - const RecordStats& recordStats() const { return _recordStats; } - RecordStats& recordStats() { return _recordStats; } - - private: - /** - * @throws DatabaseDifferCaseCode if the name is a duplicate based on - * case insensitive matching. - */ - void checkDuplicateUncasedNames(bool inholderlockalready) const; - public: - /** - * @return name of an existing database with same text name but different - * casing, if one exists. Otherwise the empty string is returned. If - * 'duplicates' is specified, it is filled with all duplicate names. - */ - static string duplicateUncasedName( bool inholderlockalready, const string &name, const string &path, set< string > *duplicates = 0 ); - - const string name; // "alleyinsider" - const string path; - - private: - - // must be in the dbLock when touching this (and write locked when writing to of course) - // however during Database object construction we aren't, which is ok as it isn't yet visible - // to others and we are in the dbholder lock then. - vector _files; - - public: // this should be private later - - NamespaceIndex namespaceIndex; - const string profileName; // "alleyinsider.system.profile" - CCByLoc ccByLoc; - int magic; // used for making sure the object is still loaded in memory - - int getProfilingLevel() const { return _profile; } - - private: - RecordStats _recordStats; - int _profile; // 0=off. - }; - -} // namespace mongo diff --git a/src/mongo/db/databaseholder.h b/src/mongo/db/databaseholder.h deleted file mode 100644 index dd345e1bb2e..00000000000 --- a/src/mongo/db/databaseholder.h +++ /dev/null @@ -1,128 +0,0 @@ -// @file databaseholder.h - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/db/database.h" -#include "mongo/db/namespacestring.h" - -namespace mongo { - - /** - * path + dbname -> Database - */ - class DatabaseHolder { - typedef map DBs; - typedef map Paths; - // todo: we want something faster than this if called a lot: - mutable SimpleMutex _m; - Paths _paths; - int _size; - public: - DatabaseHolder() : _m("dbholder"),_size(0) { } - - bool __isLoaded( const string& ns , const string& path ) const { - SimpleMutex::scoped_lock lk(_m); - Paths::const_iterator x = _paths.find( path ); - if ( x == _paths.end() ) - return false; - const DBs& m = x->second; - - string db = _todb( ns ); - - DBs::const_iterator it = m.find(db); - return it != m.end(); - } - // must be write locked as otherwise isLoaded could go false->true on you - // in the background and you might not expect that. - bool _isLoaded( const string& ns , const string& path ) const { - Lock::assertWriteLocked(ns); - return __isLoaded(ns,path); - } - - Database * get( const string& ns , const string& path ) const { - SimpleMutex::scoped_lock lk(_m); - Lock::assertAtLeastReadLocked(ns); - Paths::const_iterator x = _paths.find( path ); - if ( x == _paths.end() ) - return 0; - const DBs& m = x->second; - string db = _todb( ns ); - DBs::const_iterator it = m.find(db); - if ( it != m.end() ) - return it->second; - return 0; - } - - Database* getOrCreate( const string& ns , const string& path , bool& justCreated ); - - void erase( const string& ns , const string& path ) { - SimpleMutex::scoped_lock lk(_m); - verify( Lock::isW() ); - DBs& m = _paths[path]; - _size -= (int)m.erase( _todb( ns ) ); - } - - /** @param force - force close even if something underway - use at shutdown */ - bool closeAll( const string& path , BSONObjBuilder& result, bool force ); - - // "info" as this is informational only could change on you if you are not write locked - int sizeInfo() const { return _size; } - - /** - * gets all unique db names, ignoring paths - * need some lock - */ - void getAllShortNames( set& all ) const { - SimpleMutex::scoped_lock lk(_m); - for ( Paths::const_iterator i=_paths.begin(); i!=_paths.end(); i++ ) { - DBs m = i->second; - for( DBs::const_iterator j=m.begin(); j!=m.end(); j++ ) { - all.insert( j->first ); - } - } - } - - private: - static string _todb( const string& ns ) { - string d = __todb( ns ); - uassert( 13280 , (string)"invalid db name: " + ns , NamespaceString::validDBName( d ) ); - return d; - } - static string __todb( const string& ns ) { - size_t i = ns.find( '.' ); - if ( i == string::npos ) { - uassert( 13074 , "db name can't be empty" , ns.size() ); - return ns; - } - uassert( 13075 , "db name can't be empty" , i > 0 ); - return ns.substr( 0 , i ); - } - }; - - DatabaseHolder& dbHolderUnchecked(); - inline const DatabaseHolder& dbHolder() { - dassert( Lock::isLocked() ); - return dbHolderUnchecked(); - } - inline DatabaseHolder& dbHolderW() { - dassert( Lock::isW() ); - return dbHolderUnchecked(); - } - -} diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 2361ec2a945..90a8013dba2 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -14,18 +14,38 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" +#include #include #include #include +#include "mongo/base/init.h" #include "mongo/base/initializer.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/auth_index_d.h" +#include "mongo/db/auth/authz_manager_external_state_d.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/client.h" #include "mongo/db/clientcursor.h" -#include "mongo/db/cmdline.h" #include "mongo/db/commands/server_status.h" #include "mongo/db/d_concurrency.h" #include "mongo/db/d_globals.h" @@ -33,33 +53,48 @@ #include "mongo/db/dbmessage.h" #include "mongo/db/dbwebserver.h" #include "mongo/db/dur.h" +#include "mongo/db/index_names.h" #include "mongo/db/index_rebuilder.h" #include "mongo/db/initialize_server_global_state.h" #include "mongo/db/instance.h" #include "mongo/db/introspect.h" #include "mongo/db/json.h" #include "mongo/db/kill_current_op.h" -#include "mongo/db/module.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/repl.h" +#include "mongo/db/log_process_details.h" +#include "mongo/db/mongod_options.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/range_deleter_service.h" +#include "mongo/db/repair_database.h" +#include "mongo/db/repl/repl_start.h" +#include "mongo/db/repl/replication_server_status.h" #include "mongo/db/repl/rs.h" #include "mongo/db/restapi.h" +#include "mongo/db/startup_warnings.h" #include "mongo/db/stats/counters.h" #include "mongo/db/stats/snapshots.h" +#include "mongo/db/storage/storage_engine_metadata.h" +#include "mongo/db/storage_options.h" #include "mongo/db/ttl.h" +#include "mongo/platform/process_id.h" #include "mongo/s/d_writeback.h" #include "mongo/scripting/engine.h" #include "mongo/util/background.h" +#include "mongo/util/cmdline_utils/censor_cmdline.h" #include "mongo/util/concurrency/task.h" +#include "mongo/util/concurrency/thread_name.h" #include "mongo/util/exception_filter_win32.h" #include "mongo/util/file_allocator.h" #include "mongo/util/net/message_server.h" +#include "mongo/util/net/ssl_manager.h" #include "mongo/util/ntservice.h" +#include "mongo/util/options_parser/startup_options.h" #include "mongo/util/ramlog.h" +#include "mongo/util/scopeguard.h" +#include "mongo/util/signal_handlers.h" #include "mongo/util/stacktrace.h" #include "mongo/util/startup_test.h" #include "mongo/util/text.h" -#include "mongo/util/version.h" +#include "mongo/util/version_reporting.h" #if !defined(_WIN32) # include @@ -67,36 +102,24 @@ namespace mongo { - namespace dur { - extern unsigned long long DataLimitPerJournalFile; - } + void (*snmpInit)() = NULL; /* only off if --nohints */ extern bool useHints; extern int diagLogging; - extern unsigned lenForNewNsFiles; extern int lockFile; - extern string repairpath; - static void setupSignalHandlers(); - void startReplication(); - static void startSignalProcessingThread(); void exitCleanly( ExitCode code ); #ifdef _WIN32 ntservice::NtServiceDefaultStrings defaultServiceStrings = { L"MongoDB", - L"Mongo DB", - L"Mongo DB Server" + L"MongoDB", + L"MongoDB Server" }; #endif - CmdLine cmdLine; - static bool scriptingEnabled = true; - static bool noHttpInterface = false; - bool shouldRepairDatabases = 0; - static bool forceRepair = 0; Timer startupSrandTimer; const char *ourgetns() { @@ -156,19 +179,6 @@ namespace mongo { }; #endif - void sysRuntimeInfo() { - out() << "sysinfo:" << endl; -#if defined(_SC_PAGE_SIZE) - out() << " page size: " << (int) sysconf(_SC_PAGE_SIZE) << endl; -#endif -#if defined(_SC_PHYS_PAGES) - out() << " _SC_PHYS_PAGES: " << sysconf(_SC_PHYS_PAGES) << endl; -#endif -#if defined(_SC_AVPHYS_PAGES) - out() << " _SC_AVPHYS_PAGES: " << sysconf(_SC_AVPHYS_PAGES) << endl; -#endif - } - /* if server is really busy, wait a bit */ void beNice() { sleepmicros( Client::recommendedYieldMicros() ); @@ -232,7 +242,6 @@ namespace mongo { virtual void disconnected( AbstractMessagingPort* p ) { Client * c = currentClient.get(); if( c ) c->shutdown(); - globalScriptEngine->threadDone(); } }; @@ -245,12 +254,10 @@ namespace mongo { toLog.append( "hostname", getHostNameCached() ); toLog.appendTimeT( "startTime", time(0) ); - char buf[64]; - curTimeString( buf ); - toLog.append( "startTimeLocal", buf ); + toLog.append( "startTimeLocal", dateToCtimeString(curTimeMillis64()) ); - toLog.append( "cmdLine", CmdLine::getParsedOpts() ); - toLog.append( "pid", getpid() ); + toLog.append("cmdLine", serverGlobalParams.parsedOpts); + toLog.append( "pid", ProcessId::getCurrent().asLongLong() ); BSONObjBuilder buildinfo( toLog.subobjStart("buildinfo")); @@ -260,7 +267,6 @@ namespace mongo { BSONObj o = toLog.obj(); Lock::GlobalWrite lk; - Client::GodScope gs; DBDirectClient c; const char* name = "local.startup_log"; c.createCollection( name, 10 * 1024 * 1024, true ); @@ -271,15 +277,32 @@ namespace mongo { //testTheDb(); MessageServer::Options options; options.port = port; - options.ipList = cmdLine.bind_ip; + options.ipList = serverGlobalParams.bind_ip; MessageServer * server = createServer( options , new MyMessageHandler() ); server->setAsTimeTracker(); + // We must setupSockets for both the main server and the web server (if enabled) prior to + // calling logStartup() in order to avoid getting too high of a file descriptor for our + // calls to select (see SERVER-17653). + server->setupSockets(); + + boost::shared_ptr dbWebServer; + if (serverGlobalParams.isHttpInterfaceEnabled) { + dbWebServer.reset(new DbWebServer(serverGlobalParams.bind_ip, + serverGlobalParams.port + 1000, + new RestAdminAccess())); + dbWebServer->setupSockets(); + } + logStartup(); startReplication(); - if ( !noHttpInterface ) - boost::thread web( boost::bind(&webServerThread, new RestAdminAccess() /* takes ownership */)); + + if (serverGlobalParams.isHttpInterfaceEnabled) { + invariant(dbWebServer); + boost::thread web(boost::bind(&webServerListenThread, dbWebServer)); + web.detach(); + } #if(TESTEXHAUST) boost::thread thr(testExhaust); @@ -288,7 +311,7 @@ namespace mongo { } - bool doDBUpgrade( const string& dbName , string errmsg , DataFileHeader * h ) { + void doDBUpgrade( const string& dbName, DataFileHeader* h ) { static DBDirectClient db; if ( h->version == 4 && h->versionMinor == 4 ) { @@ -302,24 +325,56 @@ namespace mongo { BSONObj out; bool ok = db.runCommand( dbName , BSON( "reIndex" << c.substr( dbName.size() + 1 ) ) , out ); if ( ! ok ) { - errmsg = "reindex failed"; - log() << "\t\t reindex failed: " << out << endl; - return false; + log() << "\t\t reindex failed: " << out; + fassertFailed( 17393 ); } } getDur().writingInt(h->versionMinor) = 5; - return true; + return; } // do this in the general case - return repairDatabase( dbName.c_str(), errmsg ); + fassert( 17401, repairDatabase( dbName ) ); + } + + void checkForIdIndexes( Database* db ) { + + if ( db->name() == "local") { + // we do not need an _id index on anything in the local database + return; + } + + list collections; + db->namespaceIndex().getNamespaces( collections ); + + // for each collection, ensure there is a $_id_ index + for (list::iterator i = collections.begin(); i != collections.end(); ++i) { + const string& collectionName = *i; + NamespaceString ns( collectionName ); + if ( ns.isSystem() ) + continue; + + Collection* coll = db->getCollection( collectionName ); + if ( !coll ) + continue; + + if ( coll->getIndexCatalog()->findIdIndex() ) + continue; + + log() << "WARNING: the collection '" << *i + << "' lacks a unique index on _id." + << " This index is needed for replication to function properly" + << startupWarningsLog; + log() << "\t To fix this, you need to create a unique index on _id." + << " See http://dochub.mongodb.org/core/build-replica-set-indexes" + << startupWarningsLog; + } } // ran at startup. - static void repairDatabasesAndCheckVersion() { + static void repairDatabasesAndCheckVersion(bool shouldClearNonLocalTmpCollections) { // LastError * le = lastError.get( true ); - Client::GodScope gs; LOG(1) << "enter repairDatabases (to check pdfile version #)" << endl; Lock::GlobalWrite lk; @@ -328,10 +383,20 @@ namespace mongo { for ( vector< string >::iterator i = dbNames.begin(); i != dbNames.end(); ++i ) { string dbName = *i; LOG(1) << "\t" << dbName << endl; + Client::Context ctx( dbName ); - MongoDataFile *p = cc().database()->getFile( 0 ); + DataFile *p = ctx.db()->getExtentManager().getFile( 0 ); DataFileHeader *h = p->getHeader(); - if ( !h->isCurrentVersion() || forceRepair ) { + + if ( replSettings.usingReplSets() ) { + // we only care about the _id index if we are in a replset + checkForIdIndexes(ctx.db()); + } + + if (shouldClearNonLocalTmpCollections || dbName == "local") + ctx.db()->clearTmpCollections(); + + if (!h->isCurrentVersion() || mongodGlobalParams.repair) { if( h->version <= 0 ) { uasserted(14026, @@ -339,37 +404,40 @@ namespace mongo { << " info: " << h->versionMinor << ' ' << h->fileLength); } - log() << "****" << endl; - log() << "****" << endl; - log() << "need to upgrade database " << dbName << " " - << "with pdfile version " << h->version << "." << h->versionMinor << ", " - << "new version: " - << PDFILE_VERSION << "." << PDFILE_VERSION_MINOR_22_AND_OLDER - << endl; - if ( shouldRepairDatabases ) { + if ( !h->isCurrentVersion() ) { + log() << "****" << endl; + log() << "****" << endl; + log() << "need to upgrade database " << dbName << " " + << "with pdfile version " << h->version << "." << h->versionMinor << ", " + << "new version: " + << PDFILE_VERSION << "." << PDFILE_VERSION_MINOR_22_AND_OLDER + << endl; + } + + if (mongodGlobalParams.upgrade) { // QUESTION: Repair even if file format is higher version than code? - log() << "\t starting upgrade" << endl; - string errmsg; - verify( doDBUpgrade( dbName , errmsg , h ) ); + doDBUpgrade( dbName, h ); } else { log() << "\t Not upgrading, exiting" << endl; log() << "\t run --upgrade to upgrade dbs, then start again" << endl; log() << "****" << endl; dbexit( EXIT_NEED_UPGRADE ); - shouldRepairDatabases = 1; + mongodGlobalParams.upgrade = 1; return; } } else { - if (h->versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER) { - const string systemIndexes = cc().database()->name + ".system.indexes"; - shared_ptr cursor(theDataFileMgr.findAll(systemIndexes)); - for ( ; cursor && cursor->ok(); cursor->advance()) { - const BSONObj index = cursor->current(); - const BSONObj key = index.getObjectField("key"); - const string plugin = IndexPlugin::findPluginName(key); - if (IndexPlugin::existedBefore24(plugin)) + const string systemIndexes = cc().database()->name() + ".system.indexes"; + auto_ptr runner(InternalPlanner::collectionScan(systemIndexes)); + BSONObj index; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&index, NULL))) { + const BSONObj key = index.getObjectField("key"); + const string plugin = IndexNames::findPluginName(key); + + if (!h->is24IndexClean()) { + if (IndexNames::existedBefore24(plugin)) continue; log() << "Index " << index << " claims to be of type '" << plugin << "', " @@ -378,14 +446,32 @@ namespace mongo { << "http://dochub.mongodb.org/core/upgrade-2.4" << startupWarningsLog; } + + const Status keyStatus = validateKeyPattern(key); + if (!keyStatus.isOK()) { + log() << "Problem with index " << index << ": " << keyStatus.reason() + << " This index can still be used however it cannot be rebuilt." + << " For more info see" + << " http://dochub.mongodb.org/core/index-validation" + << startupWarningsLog; + } + } + + if (Runner::RUNNER_EOF != state) { + warning() << "Internal error while reading collection " << systemIndexes; } - Database::closeDatabase( dbName.c_str(), dbpath ); + + // 2.4 does not properly drop indexes on collections named "system". Check for + // orphaned indexes and clean up any if found. + ctx.db()->cleanUpOrphanIndexesOnSystemCollection(); + + Database::closeDatabase(dbName.c_str(), storageGlobalParams.dbpath); } } LOG(1) << "done repairDatabases" << endl; - if ( shouldRepairDatabases ) { + if (mongodGlobalParams.upgrade) { log() << "finished checking dbs" << endl; cc().shutdown(); dbexit( EXIT_CLEAN ); @@ -393,7 +479,7 @@ namespace mongo { } void clearTmpFiles() { - boost::filesystem::path path( dbpath ); + boost::filesystem::path path(storageGlobalParams.dbpath); for ( boost::filesystem::directory_iterator i( path ); i != boost::filesystem::directory_iterator(); ++i ) { string fileName = boost::filesystem::path(*i).leaf().string(); @@ -412,8 +498,7 @@ namespace mongo { */ unsigned long long checkIfReplMissingFromCommandLine() { Lock::GlobalWrite lk; // this is helpful for the query below to work as you can't open files when readlocked - if( !cmdLine.usingReplSets() ) { - Client::GodScope gs; + if (!replSettings.usingReplSets()) { DBDirectClient c; return c.count("local.system.replset"); } @@ -438,22 +523,25 @@ namespace mongo { void run() { Client::initThread( name().c_str() ); - if( cmdLine.syncdelay == 0 ) + if (storageGlobalParams.syncdelay == 0) { log() << "warning: --syncdelay 0 is not recommended and can have strange performance" << endl; - else if( cmdLine.syncdelay == 1 ) + } + else if (storageGlobalParams.syncdelay == 1) { log() << "--syncdelay 1" << endl; - else if( cmdLine.syncdelay != 60 ) - LOG(1) << "--syncdelay " << cmdLine.syncdelay << endl; + } + else if (storageGlobalParams.syncdelay != 60) { + LOG(1) << "--syncdelay " << storageGlobalParams.syncdelay << endl; + } int time_flushing = 0; while ( ! inShutdown() ) { _diaglog.flush(); - if ( cmdLine.syncdelay == 0 ) { + if (storageGlobalParams.syncdelay == 0) { // in case at some point we add an option to change at runtime sleepsecs(5); continue; } - sleepmillis( (long long) std::max(0.0, (cmdLine.syncdelay * 1000) - time_flushing) ); + sleepmillis((long long) std::max(0.0, (storageGlobalParams.syncdelay * 1000) - time_flushing)); if ( inShutdown() ) { // occasional issue trying to flush during shutdown when sleep interrupted @@ -466,7 +554,7 @@ namespace mongo { _flushed(time_flushing); - if( logLevel >= 1 || time_flushing >= 10000 ) { + if( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1)) || time_flushing >= 10000 ) { log() << "flushing mmaps took " << time_flushing << "ms " << " for " << numFiles << " files" << endl; } } @@ -507,7 +595,7 @@ namespace mongo { int m = static_cast(MemoryMappedFile::totalMappedLength() / ( 1024 * 1024 )); b.appendNumber( "mapped" , m ); - if ( cmdLine.dur ) { + if (storageGlobalParams.dur) { m *= 2; b.appendNumber( "mappedWithJournal" , m ); } @@ -529,39 +617,47 @@ namespace mongo { /// warn if readahead > 256KB (gridfs chunk size) static void checkReadAhead(const string& dir) { #ifdef __linux__ - const dev_t dev = getPartition(dir); - - // This path handles the case where the filesystem uses the whole device (including LVM) - string path = str::stream() << - "/sys/dev/block/" << major(dev) << ':' << minor(dev) << "/queue/read_ahead_kb"; - - if (!boost::filesystem::exists(path)){ - // This path handles the case where the filesystem is on a partition. - path = str::stream() - << "/sys/dev/block/" << major(dev) << ':' << minor(dev) // this is a symlink - << "/.." // parent directory of a partition is for the whole device - << "/queue/read_ahead_kb"; - } + try { + const dev_t dev = getPartition(dir); + + // This path handles the case where the filesystem uses the whole device (including LVM) + string path = str::stream() << + "/sys/dev/block/" << major(dev) << ':' << minor(dev) << "/queue/read_ahead_kb"; + + if (!boost::filesystem::exists(path)){ + // This path handles the case where the filesystem is on a partition. + path = str::stream() + << "/sys/dev/block/" << major(dev) << ':' << minor(dev) // this is a symlink + << "/.." // parent directory of a partition is for the whole device + << "/queue/read_ahead_kb"; + } - if (boost::filesystem::exists(path)) { - ifstream file (path.c_str()); - if (file.is_open()) { - int kb; - file >> kb; - if (kb > 256) { - log() << startupWarningsLog; + if (boost::filesystem::exists(path)) { + ifstream file (path.c_str()); + if (file.is_open()) { + int kb; + file >> kb; + if (kb > 256) { + log() << startupWarningsLog; - log() << "** WARNING: Readahead for " << dir << " is set to " << kb << "KB" - << startupWarningsLog; + log() << "** WARNING: Readahead for " << dir << " is set to " << kb << "KB" + << startupWarningsLog; - log() << "** We suggest setting it to 256KB (512 sectors) or less" - << startupWarningsLog; + log() << "** We suggest setting it to 256KB (512 sectors) or less" + << startupWarningsLog; - log() << "** http://dochub.mongodb.org/core/readahead" - << startupWarningsLog; + log() << "** http://dochub.mongodb.org/core/readahead" + << startupWarningsLog; + } } } } + catch (const std::exception& e) { + log() << "unable to validate readahead settings due to error: " << e.what() + << startupWarningsLog; + log() << "for more information, see http://dochub.mongodb.org/core/readahead" + << startupWarningsLog; + } #endif // __linux__ } @@ -569,59 +665,64 @@ namespace mongo { Client::initThread("initandlisten"); - Logstream::get().addGlobalTee( new RamLog("global") ); - bool is32bit = sizeof(int*) == 4; { -#if !defined(_WIN32) - pid_t pid = getpid(); -#else - DWORD pid=GetCurrentProcessId(); -#endif - Nullstream& l = log(); - l << "MongoDB starting : pid=" << pid << " port=" << cmdLine.port << " dbpath=" << dbpath; + ProcessId pid = ProcessId::getCurrent(); + LogstreamBuilder l = log(); + l << "MongoDB starting : pid=" << pid + << " port=" << serverGlobalParams.port + << " dbpath=" << storageGlobalParams.dbpath; if( replSettings.master ) l << " master=" << replSettings.master; if( replSettings.slave ) l << " slave=" << (int) replSettings.slave; l << ( is32bit ? " 32" : " 64" ) << "-bit host=" << getHostNameCached() << endl; } DEV log() << "_DEBUG build (which is slower)" << endl; - show_warnings(); - log() << mongodVersion() << endl; - printGitVersion(); - printSysInfo(); - printAllocator(); - printCommandLineOpts(); - + logStartupWarnings(); +#if defined(_WIN32) + printTargetMinOS(); +#endif + logProcessDetails(); { stringstream ss; ss << endl; ss << "*********************************************************************" << endl; - ss << " ERROR: dbpath (" << dbpath << ") does not exist." << endl; + ss << " ERROR: dbpath (" << storageGlobalParams.dbpath << ") does not exist." << endl; ss << " Create this directory or give existing directory in --dbpath." << endl; ss << " See http://dochub.mongodb.org/core/startingandstoppingmongo" << endl; ss << "*********************************************************************" << endl; - uassert( 10296 , ss.str().c_str(), boost::filesystem::exists( dbpath ) ); + uassert(10296, ss.str().c_str(), boost::filesystem::exists(storageGlobalParams.dbpath)); } { stringstream ss; - ss << "repairpath (" << repairpath << ") does not exist"; - uassert( 12590 , ss.str().c_str(), boost::filesystem::exists( repairpath ) ); + ss << "repairpath (" << storageGlobalParams.repairpath << ") does not exist"; + uassert(12590, ss.str().c_str(), + boost::filesystem::exists(storageGlobalParams.repairpath)); } + // Read storage engine metadata file (introduced in 2.8) if present. + // Do not start server if storage engine in metadata is not 'mmapv1'. + StorageEngineMetadata::validate(storageGlobalParams.dbpath, "mmapv1"); + // TODO check non-journal subdirs if using directory-per-db - checkReadAhead(dbpath); + checkReadAhead(storageGlobalParams.dbpath); - acquirePathLock(forceRepair); - boost::filesystem::remove_all( dbpath + "/_tmp/" ); + acquirePathLock(mongodGlobalParams.repair); + boost::filesystem::remove_all(storageGlobalParams.dbpath + "/_tmp/"); FileAllocator::get()->start(); + // TODO: This should go into a MONGO_INITIALIZER once we have figured out the correct + // dependencies. + if (snmpInit) { + snmpInit(); + } + MONGO_ASSERT_ON_EXCEPTION_WITH_MSG( clearTmpFiles(), "clear tmp files" ); dur::startup(); - if( cmdLine.durOptions & CmdLine::DurRecoverOnly ) + if (storageGlobalParams.durOptions & StorageGlobalParams::DurRecoverOnly) return; unsigned long long missingRepl = checkIfReplMissingFromCommandLine(); @@ -636,25 +737,35 @@ namespace mongo { log() << startupWarningsLog; } - Module::initAll(); - - if ( scriptingEnabled ) { + if (mongodGlobalParams.scriptingEnabled) { ScriptEngine::setup(); globalScriptEngine->setCheckInterruptCallback( jsInterruptCallback ); globalScriptEngine->setGetCurrentOpIdCallback( jsGetCurrentOpIdCallback ); } - repairDatabasesAndCheckVersion(); + // On replica set members we only clear temp collections on DBs other than "local" during + // promotion to primary. On pure slaves, they are only cleared when the oplog tells them to. + // The local DB is special because it is not replicated. See SERVER-10927 for more details. + const bool shouldClearNonLocalTmpCollections = !(missingRepl + || replSettings.usingReplSets() + || replSettings.slave == SimpleSlave); + repairDatabasesAndCheckVersion(shouldClearNonLocalTmpCollections); - if ( shouldRepairDatabases ) + if (mongodGlobalParams.upgrade) return; + uassertStatusOK(getGlobalAuthorizationManager()->initialize()); + /* this is for security on certain platforms (nonce generation) */ srand((unsigned) (curTimeMicros() ^ startupSrandTimer.micros())); - snapshotThread.go(); + // The snapshot thread provides historical collection level and lock statistics for use + // by the web interface. Only needed when HTTP is enabled. + if (serverGlobalParams.isHttpInterfaceEnabled) + snapshotThread.go(); + d.clientCursorMonitor.go(); - PeriodicTask::theRunner->go(); + PeriodicTask::startRunningPeriodicTasks(); if (missingRepl) { // a warning was logged earlier } @@ -663,23 +774,28 @@ namespace mongo { } #ifndef _WIN32 - CmdLine::launchOk(); + mongo::signalForkSuccess(); #endif - if( !noauth ) { + if(getGlobalAuthorizationManager()->isAuthEnabled()) { // open admin db in case we need to use it later. TODO this is not the right way to // resolve this. - Client::WriteContext c("admin", dbpath); + Client::WriteContext c("admin", storageGlobalParams.dbpath); } + authindex::configureSystemIndexes("admin"); + + getDeleter()->startWorkers(); + + // Starts a background thread that rebuilds all incomplete indices. + indexRebuilder.go(); + listen(listenPort); // listen() will return when exit code closes its socket. exitCleanly(EXIT_NET_ERROR); } - void testPretouch(); - void initAndListen(int listenPort) { try { _initAndListen(listenPort); @@ -706,7 +822,7 @@ namespace mongo { void initService() { ntservice::reportStatus( SERVICE_RUNNING ); log() << "Service running" << endl; - initAndListen( cmdLine.port ); + initAndListen(serverGlobalParams.port); } #endif @@ -714,547 +830,172 @@ namespace mongo { using namespace mongo; -#include - -namespace po = boost::program_options; - -void show_help_text(po::options_description options) { - cout << options << endl; -}; - static int mongoDbMain(int argc, char* argv[], char** envp); +#if defined(_WIN32) +// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters +// as main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The +// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent +// and makes them available through the argv() and envp() members. This enables mongoDbMain() +// to process UTF-8 encoded arguments and environment variables without regard to platform. +int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) { + WindowsCommandLine wcl(argc, argvW, envpW); + int exitCode = mongoDbMain(argc, wcl.argv(), wcl.envp()); + ::_exit(exitCode); +} +#else int main(int argc, char* argv[], char** envp) { int exitCode = mongoDbMain(argc, argv, envp); ::_exit(exitCode); } - -static void buildOptionsDescriptions(po::options_description *pVisible, - po::options_description *pHidden, - po::positional_options_description *pPositional) { - - po::options_description& visible_options = *pVisible; - po::options_description& hidden_options = *pHidden; - po::positional_options_description& positional_options = *pPositional; - - po::options_description general_options("General options"); -#if defined(_WIN32) - po::options_description windows_scm_options("Windows Service Control Manager options"); #endif - po::options_description ms_options("Master/slave options (old; use replica sets instead)"); - po::options_description rs_options("Replica set options"); - po::options_description replication_options("Replication options"); - po::options_description sharding_options("Sharding options"); - po::options_description hidden_sharding_options("Sharding options"); - po::options_description ssl_options("SSL options"); - - CmdLine::addGlobalOptions( general_options , hidden_options , ssl_options ); - - StringBuilder dbpathBuilder; - dbpathBuilder << "directory for datafiles - defaults to " << dbpath; - - general_options.add_options() - ("auth", "run with security") - ("cpu", "periodically show cpu and iowait utilization") - ("dbpath", po::value() , dbpathBuilder.str().c_str()) - ("diaglog", po::value(), "0=off 1=W 2=R 3=both 7=W+some reads") - ("directoryperdb", "each database will be stored in a separate directory") - ("ipv6", "enable IPv6 support (disabled by default)") - ("journal", "enable journaling") - ("journalCommitInterval", po::value(), "how often to group/batch commit (ms)") - ("journalOptions", po::value(), "journal diagnostic options") - ("jsonp","allow JSONP access via http (has security implications)") - ("noauth", "run without security") - ("nohttpinterface", "disable http interface") - // SERVER-8536 - // ("noIndexBuildRetry", "don't retry any index builds that were interrupted by shutdown") - ("nojournal", "disable journaling (journaling is on by default for 64 bit)") - ("noprealloc", "disable data file preallocation - will often hurt performance") - ("noscripting", "disable scripting engine") - ("notablescan", "do not allow table scans") - ("nssize", po::value()->default_value(16), ".ns file size (in MB) for new databases") - ("profile",po::value(), "0=off 1=slow, 2=all") - ("quota", "limits each database to a certain number of files (8 default)") - ("quotaFiles", po::value(), "number of files allowed per db, requires --quota") - ("repair", "run repair on all dbs") - ("repairpath", po::value() , "root directory for repair files - defaults to dbpath" ) - ("rest","turn on simple rest api") -#if defined(__linux__) - ("shutdown", "kill a running server (for init scripts)") -#endif - ("slowms",po::value(&cmdLine.slowMS)->default_value(100), "value of slow for profile and console log" ) - ("smallfiles", "use a smaller default file size") - ("syncdelay",po::value(&cmdLine.syncdelay)->default_value(60), "seconds between disk syncs (0=never, but not recommended)") - ("sysinfo", "print some diagnostic system information") - ("upgrade", "upgrade db if needed") - ; -#if defined(_WIN32) - CmdLine::addWindowsOptions( windows_scm_options, hidden_options ); -#endif - - replication_options.add_options() - ("oplogSize", po::value(), "size to use (in MB) for replication op log. default is 5% of disk space (i.e. large is good)") - ; - - ms_options.add_options() - ("master", "master mode") - ("slave", "slave mode") - ("source", po::value(), "when slave: specify master as ") - ("only", po::value(), "when slave: specify a single database to replicate") - ("slavedelay", po::value(), "specify delay (in seconds) to be used when applying master ops to slave") - ("autoresync", "automatically resync if slave data is stale") - ; - - rs_options.add_options() - ("replSet", po::value(), "arg is [/]") - ("replIndexPrefetch", po::value(), "specify index prefetching behavior (if secondary) [none|_id_only|all]") - ; - - sharding_options.add_options() - ("configsvr", "declare this is a config db of a cluster; default port 27019; default dir /data/configdb") - ("shardsvr", "declare this is a shard db of a cluster; default port 27018") - ; - - hidden_sharding_options.add_options() - ("noMoveParanoia" , "turn off paranoid saving of data for the moveChunk command; default" ) - ("moveParanoia" , "turn on paranoid saving of data during the moveChunk command (used for internal system diagnostics)" ) - ; - hidden_options.add(hidden_sharding_options); - - hidden_options.add_options() - ("fastsync", "indicate that this instance is starting from a dbpath snapshot of the repl peer") - ("pretouch", po::value(), "n pretouch threads for applying replicationed operations") // experimental - ("command", po::value< vector >(), "command") - ("cacheSize", po::value(), "cache size (in MB) for rec store") - ("nodur", "disable journaling") - // things we don't want people to use - ("nohints", "ignore query hints") - ("nopreallocj", "don't preallocate journal files") - ("dur", "enable journaling") // old name for --journal - ("durOptions", po::value(), "durability diagnostic options") // deprecated name - // deprecated pairing command line options - ("pairwith", "DEPRECATED") - ("arbiter", "DEPRECATED") - ("opIdMem", "DEPRECATED") - ; - - positional_options.add("command", 3); - visible_options.add(general_options); -#if defined(_WIN32) - visible_options.add(windows_scm_options); -#endif - visible_options.add(replication_options); - visible_options.add(ms_options); - visible_options.add(rs_options); - visible_options.add(sharding_options); -#ifdef MONGO_SSL - visible_options.add(ssl_options); -#endif - Module::addOptions( visible_options ); +MONGO_INITIALIZER_GENERAL(ForkServer, + ("EndStartupOptionHandling"), + ("default"))(InitializerContext* context) { + mongo::forkServerOrDie(); + return Status::OK(); } -static void processCommandLineOptions(const std::vector& argv) { - po::options_description visible_options("Allowed options"); - po::options_description hidden_options("Hidden options"); - po::positional_options_description positional_options; - buildOptionsDescriptions(&visible_options, &hidden_options, &positional_options); - - { - po::variables_map params; - - if (!CmdLine::store(argv, - visible_options, - hidden_options, - positional_options, - params)) { - ::_exit(EXIT_FAILURE); - } - - if (params.count("help")) { - show_help_text(visible_options); - ::_exit(EXIT_SUCCESS); - } - if (params.count("version")) { - cout << mongodVersion() << endl; - printGitVersion(); - ::_exit(EXIT_SUCCESS); - } - if (params.count("sysinfo")) { - sysRuntimeInfo(); +/* + * This function should contain the startup "actions" that we take based on the startup config. It + * is intended to separate the actions from "storage" and "validation" of our startup configuration. + */ +static void startupConfigActions(const std::vector& args) { + // The "command" option is deprecated. For backward compatibility, still support the "run" + // and "dbppath" command. The "run" command is the same as just running mongod, so just + // falls through. + if (moe::startupOptionsParsed.count("command")) { + vector command = moe::startupOptionsParsed["command"].as< vector >(); + + if (command[0].compare("dbpath") == 0) { + cout << storageGlobalParams.dbpath << endl; ::_exit(EXIT_SUCCESS); } - if ( params.count( "dbpath" ) ) { - dbpath = params["dbpath"].as(); - if ( params.count( "fork" ) && dbpath[0] != '/' ) { - // we need to change dbpath if we fork since we change - // cwd to "/" - // fork only exists on *nix - // so '/' is safe - dbpath = cmdLine.cwd + "/" + dbpath; - } - } -#ifdef _WIN32 - if (dbpath.size() > 1 && dbpath[dbpath.size()-1] == '/') { - // size() check is for the unlikely possibility of --dbpath "/" - dbpath = dbpath.erase(dbpath.size()-1); + if (command[0].compare("run") != 0) { + cout << "Invalid command: " << command[0] << endl; + printMongodHelp(moe::startupOptions); + ::_exit(EXIT_FAILURE); } -#endif - if ( params.count("directoryperdb")) { - directoryperdb = true; - } - if (params.count("cpu")) { - cmdLine.cpu = true; - } - if (params.count("noauth")) { - noauth = true; - } - if (params.count("auth")) { - noauth = false; - } - if (params.count("quota")) { - cmdLine.quota = true; - } - if (params.count("quotaFiles")) { - cmdLine.quota = true; - cmdLine.quotaFiles = params["quotaFiles"].as() - 1; - } - bool journalExplicit = false; - if( params.count("nodur") || params.count( "nojournal" ) ) { - journalExplicit = true; - cmdLine.dur = false; - } - if( params.count("dur") || params.count( "journal" ) ) { - if (journalExplicit) { - log() << "Can't specify both --journal and --nojournal options." << endl; - ::_exit(EXIT_BADOPTIONS); - } - journalExplicit = true; - cmdLine.dur = true; - } - if (params.count("durOptions")) { - cmdLine.durOptions = params["durOptions"].as(); - } - if( params.count("journalCommitInterval") ) { - // don't check if dur is false here as many will just use the default, and will default to off on win32. - // ie no point making life a little more complex by giving an error on a dev environment. - cmdLine.journalCommitInterval = params["journalCommitInterval"].as(); - if( cmdLine.journalCommitInterval <= 1 || cmdLine.journalCommitInterval > 300 ) { - out() << "--journalCommitInterval out of allowed range (0-300ms)" << endl; - dbexit( EXIT_BADOPTIONS ); - } - } - if (params.count("journalOptions")) { - cmdLine.durOptions = params["journalOptions"].as(); + if (command.size() > 1) { + cout << "Too many parameters to 'run' command" << endl; + printMongodHelp(moe::startupOptions); + ::_exit(EXIT_FAILURE); } - if (params.count("repairpath")) { - repairpath = params["repairpath"].as(); - if (!repairpath.size()) { - out() << "repairpath is empty" << endl; - dbexit( EXIT_BADOPTIONS ); - } + } - if (cmdLine.dur && !str::startsWith(repairpath, dbpath)) { - out() << "You must use a --repairpath that is a subdirectory of --dbpath when using journaling" << endl; - dbexit( EXIT_BADOPTIONS ); - } - } - if (params.count("nohints")) { - useHints = false; - } - if (params.count("nopreallocj")) { - cmdLine.preallocj = false; - } - if (params.count("nohttpinterface")) { - noHttpInterface = true; - } - if (params.count("rest")) { - cmdLine.rest = true; - } - if (params.count("jsonp")) { - cmdLine.jsonp = true; - } - if (params.count("noscripting")) { - scriptingEnabled = false; - } - if (params.count("noprealloc")) { - cmdLine.prealloc = false; - cout << "note: noprealloc may hurt performance in many applications" << endl; - } - if (params.count("smallfiles")) { - cmdLine.smallfiles = true; - verify( dur::DataLimitPerJournalFile >= 128 * 1024 * 1024 ); - dur::DataLimitPerJournalFile = 128 * 1024 * 1024; - } - if (params.count("diaglog")) { - int x = params["diaglog"].as(); - if ( x < 0 || x > 7 ) { - out() << "can't interpret --diaglog setting" << endl; - dbexit( EXIT_BADOPTIONS ); - } - _diaglog.setLevel(x); - } - if (params.count("repair")) { - if (journalExplicit && cmdLine.dur) { - log() << "Can't specify both --journal and --repair options." << endl; - ::_exit(EXIT_BADOPTIONS); - } +#ifdef _WIN32 + ntservice::configureService(initService, + moe::startupOptionsParsed, + defaultServiceStrings, + std::vector(), + args); +#endif // _WIN32 - Record::MemoryTrackingEnabled = false; - shouldRepairDatabases = 1; - forceRepair = 1; - cmdLine.dur = false; - } - if (params.count("upgrade")) { - Record::MemoryTrackingEnabled = false; - shouldRepairDatabases = 1; - } - if (params.count("notablescan")) { - cmdLine.noTableScan = true; - } - if (params.count("master")) { - replSettings.master = true; - } - if (params.count("slave")) { - replSettings.slave = SimpleSlave; - } - if (params.count("slavedelay")) { - replSettings.slavedelay = params["slavedelay"].as(); - } - if (params.count("fastsync")) { - replSettings.fastsync = true; - } - if (params.count("autoresync")) { - replSettings.autoresync = true; - if( params.count("replSet") ) { - out() << "--autoresync is not used with --replSet" << endl; - out() << "see http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember" << endl; - dbexit( EXIT_BADOPTIONS ); - } - } - if (params.count("source")) { - /* specifies what the source in local.sources should be */ - cmdLine.source = params["source"].as().c_str(); - } - if( params.count("pretouch") ) { - cmdLine.pretouch = params["pretouch"].as(); - } - if (params.count("replSet")) { - if (params.count("slavedelay")) { - out() << "--slavedelay cannot be used with --replSet" << endl; - dbexit( EXIT_BADOPTIONS ); - } - else if (params.count("only")) { - out() << "--only cannot be used with --replSet" << endl; - dbexit( EXIT_BADOPTIONS ); - } - /* seed list of hosts for the repl set */ - cmdLine._replSet = params["replSet"].as().c_str(); - } - if (params.count("replIndexPrefetch")) { - cmdLine.rsIndexPrefetch = params["replIndexPrefetch"].as(); - } - if (params.count("noIndexBuildRetry")) { - cmdLine.indexBuildRetry = false; - } - if (params.count("only")) { - cmdLine.only = params["only"].as().c_str(); - } - if( params.count("nssize") ) { - int x = params["nssize"].as(); - if (x <= 0 || x > (0x7fffffff/1024/1024)) { - out() << "bad --nssize arg" << endl; - dbexit( EXIT_BADOPTIONS ); - } - lenForNewNsFiles = x * 1024 * 1024; - verify(lenForNewNsFiles > 0); - } - if (params.count("oplogSize")) { - long long x = params["oplogSize"].as(); - if (x <= 0) { - out() << "bad --oplogSize arg" << endl; - dbexit( EXIT_BADOPTIONS ); - } - // note a small size such as x==1 is ok for an arbiter. - if( x > 1000 && sizeof(void*) == 4 ) { - out() << "--oplogSize of " << x << "MB is too big for 32 bit version. Use 64 bit build instead." << endl; - dbexit( EXIT_BADOPTIONS ); - } - cmdLine.oplogSize = x * 1024 * 1024; - verify(cmdLine.oplogSize > 0); - } - if (params.count("cacheSize")) { - long x = params["cacheSize"].as(); - if (x <= 0) { - out() << "bad --cacheSize arg" << endl; - dbexit( EXIT_BADOPTIONS ); - } - log() << "--cacheSize option not currently supported" << endl; - } - if (params.count("port") == 0 ) { - if( params.count("configsvr") ) { - cmdLine.port = CmdLine::ConfigServerPort; - } - if( params.count("shardsvr") ) { - if( params.count("configsvr") ) { - log() << "can't do --shardsvr and --configsvr at the same time" << endl; - dbexit( EXIT_BADOPTIONS ); - } - cmdLine.port = CmdLine::ShardServerPort; - } - } - else { - if ( cmdLine.port <= 0 || cmdLine.port > 65535 ) { - out() << "bad --port number" << endl; - dbexit( EXIT_BADOPTIONS ); +#ifdef __linux__ + if (moe::startupOptionsParsed.count("shutdown")){ + bool failed = false; + + string name = (boost::filesystem::path(storageGlobalParams.dbpath) / "mongod.lock").string(); + if ( !boost::filesystem::exists( name ) || boost::filesystem::file_size( name ) == 0 ) + failed = true; + + pid_t pid; + string procPath; + if (!failed){ + try { + ifstream f (name.c_str()); + f >> pid; + procPath = (str::stream() << "/proc/" << pid); + if (!boost::filesystem::exists(procPath)) + failed = true; } - } - if ( params.count("configsvr" ) ) { - cmdLine.configsvr = true; - cmdLine.smallfiles = true; // config server implies small files - dur::DataLimitPerJournalFile = 128 * 1024 * 1024; - if (cmdLine.usingReplSets() || replSettings.master || replSettings.slave) { - log() << "replication should not be enabled on a config server" << endl; - ::_exit(-1); + catch (const std::exception& e){ + cerr << "Error reading pid from lock file [" << name << "]: " << e.what() << endl; + failed = true; } - if ( params.count( "nodur" ) == 0 && params.count( "nojournal" ) == 0 ) - cmdLine.dur = true; - if ( params.count( "dbpath" ) == 0 ) - dbpath = "/data/configdb"; - replSettings.master = true; - if ( params.count( "oplogSize" ) == 0 ) - cmdLine.oplogSize = 5 * 1024 * 1024; } - if ( params.count( "profile" ) ) { - cmdLine.defaultProfile = params["profile"].as(); - } - if (params.count("ipv6")) { - enableIPv6(); - } - - if (params.count("noMoveParanoia") > 0 && params.count("moveParanoia") > 0) { - out() << "The moveParanoia and noMoveParanoia flags cannot both be set; please use only one of them." << endl; - ::_exit( EXIT_BADOPTIONS ); - } - - if (params.count("noMoveParanoia")) - cmdLine.moveParanoia = false; - if (params.count("moveParanoia")) - cmdLine.moveParanoia = true; - - if (params.count("pairwith") || params.count("arbiter") || params.count("opIdMem")) { - out() << "****" << endl; - out() << "Replica Pairs have been deprecated. Invalid options: --pairwith, --arbiter, and/or --opIdMem" << endl; - out() << "" << endl; - out() << "****" << endl; - dbexit( EXIT_BADOPTIONS ); + if (failed) { + std::cerr << "There doesn't seem to be a server running with dbpath: " + << storageGlobalParams.dbpath << std::endl; + ::_exit(EXIT_FAILURE); } - // needs to be after things like --configsvr parsing, thus here. - if( repairpath.empty() ) - repairpath = dbpath; - - // The "command" option is deprecated. For backward compatibility, still support the "run" - // and "dbppath" command. The "run" command is the same as just running mongod, so just - // falls through. - if (params.count("command")) { - vector command = params["command"].as< vector >(); - - if (command[0].compare("dbpath") == 0) { - cout << dbpath << endl; - ::_exit(EXIT_SUCCESS); - } - - if (command[0].compare("run") != 0) { - cout << "Invalid command: " << command[0] << endl; - cout << visible_options << endl; - ::_exit(EXIT_FAILURE); - } - - if (command.size() > 1) { - cout << "Too many parameters to 'run' command" << endl; - cout << visible_options << endl; - ::_exit(EXIT_FAILURE); - } + cout << "killing process with pid: " << pid << endl; + int ret = kill(pid, SIGTERM); + if (ret) { + int e = errno; + cerr << "failed to kill process: " << errnoWithDescription(e) << endl; + ::_exit(EXIT_FAILURE); } - if( cmdLine.pretouch ) - log() << "--pretouch " << cmdLine.pretouch << endl; - - if (sizeof(void*) == 4 && !journalExplicit){ - // trying to make this stand out more like startup warnings - log() << endl; - warning() << "32-bit servers don't have journaling enabled by default. Please use --journal if you want durability." << endl; - log() << endl; + while (boost::filesystem::exists(procPath)) { + sleepsecs(1); } - Module::configAll(params); - -#ifdef _WIN32 - ntservice::configureService(initService, - params, - defaultServiceStrings, - std::vector(), - argv); -#endif // _WIN32 - -#ifdef __linux__ - if (params.count("shutdown")){ - bool failed = false; - - string name = ( boost::filesystem::path( dbpath ) / "mongod.lock" ).string(); - if ( !boost::filesystem::exists( name ) || boost::filesystem::file_size( name ) == 0 ) - failed = true; - - pid_t pid; - string procPath; - if (!failed){ - try { - ifstream f (name.c_str()); - f >> pid; - procPath = (str::stream() << "/proc/" << pid); - if (!boost::filesystem::exists(procPath)) - failed = true; - } - catch (const std::exception& e){ - cerr << "Error reading pid from lock file [" << name << "]: " << e.what() << endl; - failed = true; - } - } - - if (failed) { - cerr << "There doesn't seem to be a server running with dbpath: " << dbpath << endl; - ::_exit(EXIT_FAILURE); - } + ::_exit(EXIT_SUCCESS); + } +#endif +} - cout << "killing process with pid: " << pid << endl; - int ret = kill(pid, SIGTERM); - if (ret) { - int e = errno; - cerr << "failed to kill process: " << errnoWithDescription(e) << endl; - ::_exit(EXIT_FAILURE); - } +MONGO_INITIALIZER_GENERAL(CreateAuthorizationManager, + ("SetupInternalSecurityUser"), + MONGO_NO_DEPENDENTS) + (InitializerContext* context) { + AuthorizationManager* authzManager = + new AuthorizationManager(new AuthzManagerExternalStateMongod()); + setGlobalAuthorizationManager(authzManager); + return Status::OK(); +} - while (boost::filesystem::exists(procPath)) { - sleepsecs(1); - } +#ifdef MONGO_SSL +MONGO_INITIALIZER_GENERAL(setSSLManagerType, + MONGO_NO_PREREQUISITES, + ("SSLManager"))(InitializerContext* context) { + isSSLServer = true; + return Status::OK(); +} +#endif - ::_exit(EXIT_SUCCESS); +#if defined(_WIN32) +namespace mongo { + // the hook for mongoAbort + extern void (*reportEventToSystem)(const char *msg); + static void reportEventToSystemImpl(const char *msg) { + static ::HANDLE hEventLog = RegisterEventSource( NULL, TEXT("mongod") ); + if( hEventLog ) { + std::wstring s = toNativeString(msg); + LPCTSTR txt = s.c_str(); + BOOL ok = ReportEvent( + hEventLog, EVENTLOG_ERROR_TYPE, + 0, 0, NULL, + 1, + 0, + &txt, + 0); + wassert(ok); } -#endif } -} +} // namespace mongo +#endif // if defined(_WIN32) static int mongoDbMain(int argc, char* argv[], char **envp) { static StaticObserver staticObserver; +#if defined(_WIN32) + mongo::reportEventToSystem = &mongo::reportEventToSystemImpl; +#endif + getcurns = ourgetns; setupSignalHandlers(); + // Mongod shouldn't lazily kill cursors. It doesn't mix well with connection pooling. + DBClientConnection::setLazyKillCursor(false); + dbExecCommand = argv[0]; srand(curTimeMicros()); @@ -1271,10 +1012,14 @@ static int mongoDbMain(int argc, char* argv[], char **envp) { if( argc == 1 ) cout << dbExecCommand << " --help for help and startup options" << endl; + Status status = mongo::runGlobalInitializers(argc, argv, envp); + if (!status.isOK()) { + severe() << "Failed global initialization: " << status; + ::_exit(EXIT_FAILURE); + } - processCommandLineOptions(std::vector(argv, argv + argc)); - mongo::runGlobalInitializersOrDie(argc, argv, envp); - CmdLine::censor(argc, argv); + startupConfigActions(std::vector(argv, argv + argc)); + cmdline_utils::censorArgvArray(argc, argv); if (!initializeServerGlobalState()) ::_exit(EXIT_FAILURE); @@ -1293,213 +1038,7 @@ static int mongoDbMain(int argc, char* argv[], char **envp) { #endif StartupTest::runTests(); - initAndListen(cmdLine.port); + initAndListen(serverGlobalParams.port); dbexit(EXIT_CLEAN); return 0; } - -namespace mongo { - - string getDbContext(); - -#undef out - - -#if !defined(_WIN32) - -} // namespace mongo - -#include -#include - -namespace mongo { - - void abruptQuit(int x) { - ostringstream ossSig; - ossSig << "Got signal: " << x << " (" << strsignal( x ) << ")." << endl; - rawOut( ossSig.str() ); - - /* - ostringstream ossOp; - ossOp << "Last op: " << currentOp.infoNoauth() << endl; - rawOut( ossOp.str() ); - */ - - ostringstream oss; - oss << "Backtrace:" << endl; - printStackTrace( oss ); - rawOut( oss.str() ); - - // Don't go through normal shutdown procedure. It may make things worse. - ::_exit(EXIT_ABRUPT); - - } - - void abruptQuitWithAddrSignal( int signal, siginfo_t *siginfo, void * ) { - ostringstream oss; - oss << "Invalid"; - if ( signal == SIGSEGV || signal == SIGBUS ) { - oss << " access"; - } else { - oss << " operation"; - } - oss << " at address: " << siginfo->si_addr << " from thread: " << getThreadName() << endl; - rawOut( oss.str() ); - abruptQuit( signal ); - } - - sigset_t asyncSignals; - // The signals in asyncSignals will be processed by this thread only, in order to - // ensure the db and log mutexes aren't held. - void signalProcessingThread() { - while (true) { - int actualSignal = 0; - int status = sigwait( &asyncSignals, &actualSignal ); - fassert(16781, status == 0); - switch (actualSignal) { - case SIGUSR1: - // log rotate signal - fassert(16782, rotateLogs()); - break; - default: - // interrupt/terminate signal - Client::initThread( "signalProcessingThread" ); - log() << "got signal " << actualSignal << " (" << strsignal( actualSignal ) - << "), will terminate after current cmd ends" << endl; - exitCleanly( EXIT_CLEAN ); - break; - } - } - } - - // this will be called in certain c++ error cases, for example if there are two active - // exceptions - void myterminate() { - rawOut( "terminate() called, printing stack (if implemented for platform):" ); - printStackTrace(); - ::abort(); - } - - // this gets called when new fails to allocate memory - void my_new_handler() { - rawOut( "out of memory, printing stack and exiting:" ); - printStackTrace(); - ::_exit(EXIT_ABRUPT); - } - - void setupSignals_ignoreHelper( int signal ) {} - - void setupSignalHandlers() { - setupCoreSignals(); - - struct sigaction addrSignals; - memset( &addrSignals, 0, sizeof( struct sigaction ) ); - addrSignals.sa_sigaction = abruptQuitWithAddrSignal; - sigemptyset( &addrSignals.sa_mask ); - addrSignals.sa_flags = SA_SIGINFO; - - verify( sigaction(SIGSEGV, &addrSignals, 0) == 0 ); - verify( sigaction(SIGBUS, &addrSignals, 0) == 0 ); - verify( sigaction(SIGILL, &addrSignals, 0) == 0 ); - verify( sigaction(SIGFPE, &addrSignals, 0) == 0 ); - - verify( signal(SIGABRT, abruptQuit) != SIG_ERR ); - verify( signal(SIGQUIT, abruptQuit) != SIG_ERR ); - verify( signal(SIGPIPE, SIG_IGN) != SIG_ERR ); - - setupSIGTRAPforGDB(); - - // asyncSignals is a global variable listing the signals that should be handled by the - // interrupt thread, once it is started via startSignalProcessingThread(). - sigemptyset( &asyncSignals ); - sigaddset( &asyncSignals, SIGHUP ); - sigaddset( &asyncSignals, SIGINT ); - sigaddset( &asyncSignals, SIGTERM ); - sigaddset( &asyncSignals, SIGUSR1 ); - - set_terminate( myterminate ); - set_new_handler( my_new_handler ); - } - - void startSignalProcessingThread() { - verify( pthread_sigmask( SIG_SETMASK, &asyncSignals, 0 ) == 0 ); - boost::thread it( signalProcessingThread ); - } - -#else // WIN32 - void consoleTerminate( const char* controlCodeName ) { - Client::initThread( "consoleTerminate" ); - log() << "got " << controlCodeName << ", will terminate after current cmd ends" << endl; - exitCleanly( EXIT_KILL ); - } - - BOOL WINAPI CtrlHandler( DWORD fdwCtrlType ) { - - switch( fdwCtrlType ) { - - case CTRL_C_EVENT: - rawOut( "Ctrl-C signal" ); - consoleTerminate( "CTRL_C_EVENT" ); - return TRUE ; - - case CTRL_CLOSE_EVENT: - rawOut( "CTRL_CLOSE_EVENT signal" ); - consoleTerminate( "CTRL_CLOSE_EVENT" ); - return TRUE ; - - case CTRL_BREAK_EVENT: - rawOut( "CTRL_BREAK_EVENT signal" ); - consoleTerminate( "CTRL_BREAK_EVENT" ); - return TRUE; - - case CTRL_LOGOFF_EVENT: - // only sent to services, and only in pre-Vista Windows; FALSE means ignore - return FALSE; - - case CTRL_SHUTDOWN_EVENT: - rawOut( "CTRL_SHUTDOWN_EVENT signal" ); - consoleTerminate( "CTRL_SHUTDOWN_EVENT" ); - return TRUE; - - default: - return FALSE; - } - } - - // called by mongoAbort() - extern void (*reportEventToSystem)(const char *msg); - void reportEventToSystemImpl(const char *msg) { - static ::HANDLE hEventLog = RegisterEventSource( NULL, TEXT("mongod") ); - if( hEventLog ) { - std::wstring s = toNativeString(msg); - LPCTSTR txt = s.c_str(); - BOOL ok = ReportEvent( - hEventLog, EVENTLOG_ERROR_TYPE, - 0, 0, NULL, - 1, - 0, - &txt, - 0); - wassert(ok); - } - } - - void myPurecallHandler() { - printStackTrace(); - mongoAbort("pure virtual"); - } - - void setupSignalHandlers() { - reportEventToSystem = reportEventToSystemImpl; - setWindowsUnhandledExceptionFilter(); - massert(10297, - "Couldn't register Windows Ctrl-C handler", - SetConsoleCtrlHandler(static_cast(CtrlHandler), TRUE)); - _set_purecall_handler( myPurecallHandler ); - } - - void startSignalProcessingThread() {} - -#endif // if !defined(_WIN32) - -} // namespace mongo diff --git a/src/mongo/db/db.h b/src/mongo/db/db.h index 4b7c07d83a6..9ff1a1541a7 100644 --- a/src/mongo/db/db.h +++ b/src/mongo/db/db.h @@ -12,22 +12,41 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once #include "mongo/pch.h" -#include "../util/net/message.h" -#include "pdfile.h" -#include "curop.h" -#include "client.h" -#include "databaseholder.h" + +#include "mongo/db/client.h" +#include "mongo/db/curop.h" +#include "mongo/db/catalog/database_holder.h" +#include "mongo/db/pdfile.h" +#include "mongo/util/net/message.h" namespace mongo { // todo: relocked is being called when there was no unlock below. // that is weird. + /** + * Releases the current lock for the duration of its lifetime. + * + * WARNING: do not put in a smart pointer or any other class. If you absolutely must, you need + * to add the throw(DBException) annotation to it's destructor. + */ struct dbtemprelease { Client::Context * _context; scoped_ptr tr; @@ -36,7 +55,6 @@ namespace mongo { _context = c.getContext(); verify( Lock::isLocked() ); if( Lock::nested() ) { - Lock::nested(); massert(10298 , "can't temprelease nested lock", false); } if ( _context ) { @@ -46,34 +64,7 @@ namespace mongo { verify( c.curop() ); c.curop()->yielded(); } - ~dbtemprelease() { - tr.reset(); - if ( _context ) - _context->relocked(); - } - }; - - /** must be write locked - no verify(and no release) if nested write lock - a lot like dbtempreleasecond, eliminate? - */ - struct dbtempreleasewritelock { - Client::Context * _context; - int _locktype; - scoped_ptr tr; - dbtempreleasewritelock() { - const Client& c = cc(); - _context = c.getContext(); - verify( Lock::isW() ); - if( Lock::nested() ) - return; - if ( _context ) - _context->unlocked(); - tr.reset(new Lock::TempRelease); - verify( c.curop() ); - c.curop()->yielded(); - } - ~dbtempreleasewritelock() { + ~dbtemprelease() throw(DBException) { tr.reset(); if ( _context ) _context->relocked(); @@ -81,7 +72,10 @@ namespace mongo { }; /** - only does a temp release if we're not nested and have a lock + * only does a temp release if we're not nested and have a lock + * + * WARNING: do not put in a smart pointer or any other class. If you absolutely must, you need + * to add the throw(DBException) annotation to it's destructor. */ class dbtempreleasecond : boost::noncopyable { dbtemprelease * real; @@ -95,7 +89,7 @@ namespace mongo { } } } - ~dbtempreleasecond() { + ~dbtempreleasecond() throw(DBException) { if ( real ) { delete real; real = 0; @@ -104,4 +98,6 @@ namespace mongo { bool unlocked() const { return real != 0; } }; + extern void (*snmpInit)(); + } // namespace mongo diff --git a/src/mongo/db/db_10.sln b/src/mongo/db/db_10.sln index a711543d4bf..3c6e7abd4ce 100644 --- a/src/mongo/db/db_10.sln +++ b/src/mongo/db/db_10.sln @@ -21,19 +21,10 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "other", "other", "{12B11474-2D74-48C3-BB3D-F03249BEA88F}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongod", "mongod.vcxproj", "{215B2D68-0A70-4D10-8E75-B31010C62A91}" - ProjectSection(ProjectDependencies) = postProject - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} = {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} - EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongos", "..\s\mongos.vcxproj", "{E03717ED-69B4-4D21-BC55-DF6690B585C6}" - ProjectSection(ProjectDependencies) = postProject - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} = {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} - EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test", "..\dbtests\test.vcxproj", "{215B2D68-0A70-4D10-8E75-B33010C62A91}" - ProjectSection(ProjectDependencies) = postProject - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} = {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} - EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bsondemo", "..\bson\bsondemo\bsondemo.vcxproj", "{C9DB5EB7-81AA-4185-BAA1-DA035654402F}" EndProject @@ -42,14 +33,9 @@ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "simple_client_demo", "..\client\examples\simple_client_demo.vcxproj", "{89C30BC3-2874-4F2C-B4DA-EB04E9782236}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongo", "..\shell\mongo.vcxproj", "{FE959BD8-8EE2-4555-AE59-9FA14FFD410E}" - ProjectSection(ProjectDependencies) = postProject - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} = {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} - EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongoperf", "..\client\examples\mongoperf.vcxproj", "{79D4E297-BFB7-4FF2-9B13-08A146582E46}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SpiderMonkey pre-build step", "..\..\third_party\js-1.7\SpiderMonkey-prebuild.vcxproj", "{7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Mixed Platforms = Debug|Mixed Platforms @@ -230,30 +216,6 @@ Global {79D4E297-BFB7-4FF2-9B13-08A146582E46}.Win2008PlusRelease|Mixed Platforms.ActiveCfg = Win2008PlusRelease|Win32 {79D4E297-BFB7-4FF2-9B13-08A146582E46}.Win2008PlusRelease|Win32.ActiveCfg = Win2008PlusRelease|Win32 {79D4E297-BFB7-4FF2-9B13-08A146582E46}.Win2008PlusRelease|x64.ActiveCfg = Win2008PlusRelease|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|Mixed Platforms.ActiveCfg = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|Mixed Platforms.Build.0 = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|Win32.ActiveCfg = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|Win32.Build.0 = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|x64.ActiveCfg = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|x64.Build.0 = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|Mixed Platforms.ActiveCfg = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|Mixed Platforms.Build.0 = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|Win32.ActiveCfg = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|Win32.Build.0 = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|x64.ActiveCfg = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|x64.Build.0 = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|Mixed Platforms.ActiveCfg = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|Mixed Platforms.Build.0 = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|Win32.ActiveCfg = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|Win32.Build.0 = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|x64.ActiveCfg = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|x64.Build.0 = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|Mixed Platforms.ActiveCfg = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|Mixed Platforms.Build.0 = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|Win32.ActiveCfg = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|Win32.Build.0 = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|x64.ActiveCfg = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp index 33b77f455d6..25255a5b847 100644 --- a/src/mongo/db/dbcommands.cpp +++ b/src/mongo/db/dbcommands.cpp @@ -14,32 +14,45 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . -*/ - -/* SHARDING: - I believe this file is for mongod only. - See s/commands_public.cpp for mongos. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" #include -#include "mongo/base/counter.h" +#include "mongo/base/disallow_copying.h" #include "mongo/base/init.h" #include "mongo/base/status.h" #include "mongo/bson/util/builder.h" +#include "mongo/db/audit.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/user_management_commands_parser.h" +#include "mongo/db/auth/user_name.h" #include "mongo/db/background.h" -#include "mongo/db/btreecursor.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" #include "mongo/db/commands/server_status.h" +#include "mongo/db/commands/shutdown.h" #include "mongo/db/db.h" +#include "mongo/db/dbhelpers.h" #include "mongo/db/dur_stats.h" -#include "mongo/db/index_update.h" +#include "mongo/db/index_builder.h" #include "mongo/db/instance.h" #include "mongo/db/introspect.h" #include "mongo/db/jsobj.h" @@ -47,266 +60,26 @@ #include "mongo/db/kill_current_op.h" #include "mongo/db/lasterror.h" #include "mongo/db/ops/count.h" +#include "mongo/db/ops/insert.h" #include "mongo/db/pdfile.h" -#include "mongo/db/queryoptimizer.h" -#include "mongo/db/repl.h" -#include "mongo/db/repl_block.h" -#include "mongo/db/replutil.h" -#include "mongo/db/stats/timer_stats.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/repair_database.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/write_concern.h" +#include "mongo/s/d_logic.h" #include "mongo/s/d_writeback.h" #include "mongo/s/stale_exception.h" // for SendStaleConfigException #include "mongo/scripting/engine.h" #include "mongo/server.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/lruishmap.h" #include "mongo/util/md5.hpp" namespace mongo { - /* reset any errors so that getlasterror comes back clean. - - useful before performing a long series of operations where we want to - see if any of the operations triggered an error, but don't want to check - after each op as that woudl be a client/server turnaround. - */ - class CmdResetError : public Command { - public: - virtual LockType locktype() const { return NONE; } - virtual bool logTheOp() { - return false; - } - virtual bool slaveOk() const { - return true; - } - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} // No auth required - virtual void help( stringstream& help ) const { - help << "reset error state (used with getpreverror)"; - } - CmdResetError() : Command("resetError", false, "reseterror") {} - bool run(const string& db, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - LastError *le = lastError.get(); - verify( le ); - le->reset(); - return true; - } - } cmdResetError; - - /* set by replica sets if specified in the configuration. - a pointer is used to avoid any possible locking issues with lockless reading (see below locktype() is NONE - and would like to keep that) - (for now, it simply orphans any old copy as config changes should be extremely rare). - note: once non-null, never goes to null again. - */ - BSONObj *getLastErrorDefault = 0; - - static TimerStats gleWtimeStats; - static ServerStatusMetricField displayGleLatency( "getLastError.wtime", &gleWtimeStats ); - - static Counter64 gleWtimeouts; - static ServerStatusMetricField gleWtimeoutsDisplay( "getLastError.wtimeouts", &gleWtimeouts ); - - class CmdGetLastError : public Command { - public: - CmdGetLastError() : Command("getLastError", false, "getlasterror") { } - virtual LockType locktype() const { return NONE; } - virtual bool logTheOp() { return false; } - virtual bool slaveOk() const { return true; } - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} // No auth required - virtual void help( stringstream& help ) const { - help << "return error status of the last operation on this connection\n" - << "options:\n" - << " { fsync:true } - fsync before returning, or wait for journal commit if running with --journal\n" - << " { j:true } - wait for journal commit if running with --journal\n" - << " { w:n } - await replication to n servers (including self) before returning\n" - << " { wtimeout:m} - timeout for w in m milliseconds"; - } - bool run(const string& dbname, BSONObj& _cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - LastError *le = lastError.disableForCommand(); - - bool err = false; - - if ( le->nPrev != 1 ) { - err = LastError::noError.appendSelf( result , false ); - le->appendSelfStatus( result ); - } - else { - err = le->appendSelf( result , false ); - } - - Client& c = cc(); - c.appendLastOp( result ); - - result.appendNumber( "connectionId" , c.getConnectionId() ); // for sharding; also useful in general for debugging - - BSONObj cmdObj = _cmdObj; - { - BSONObj::iterator i(_cmdObj); - i.next(); - if( !i.more() ) { - /* empty, use default */ - BSONObj *def = getLastErrorDefault; - if( def ) - cmdObj = *def; - } - } - - if ( cmdObj["j"].trueValue() ) { - if( !getDur().awaitCommit() ) { - // --journal is off - result.append("jnote", "journaling not enabled on this server"); - } - if( cmdObj["fsync"].trueValue() ) { - errmsg = "fsync and j options are not used together"; - return false; - } - } - else if ( cmdObj["fsync"].trueValue() ) { - Timer t; - if( !getDur().awaitCommit() ) { - // if get here, not running with --journal - log() << "fsync from getlasterror" << endl; - result.append( "fsyncFiles" , MemoryMappedFile::flushAll( true ) ); - } - else { - // this perhaps is temp. how long we wait for the group commit to occur. - result.append( "waited", t.millis() ); - } - } - - if ( err ) { - // doesn't make sense to wait for replication - // if there was an error - return true; - } - - BSONElement e = cmdObj["w"]; - if ( e.ok() ) { - - if ( cmdLine.configsvr && (!e.isNumber() || e.numberInt() > 1) ) { - // w:1 on config servers should still work, but anything greater than that - // should not. - result.append( "wnote", "can't use w on config servers" ); - result.append( "err", "norepl" ); - return true; - } - - int timeout = cmdObj["wtimeout"].numberInt(); - TimerHolder timer( &gleWtimeStats ); - - long long passes = 0; - char buf[32]; - OpTime op(c.getLastOp()); - - if ( op.isNull() ) { - if ( anyReplEnabled() ) { - result.append( "wnote" , "no write has been done on this connection" ); - } - else if ( e.isNumber() && e.numberInt() <= 1 ) { - // don't do anything - // w=1 and no repl, so this is fine - } - else { - // w=2 and no repl - stringstream errmsg; - errmsg << "no replication has been enabled, so w=" << - e.toString(false) << " won't work"; - result.append( "wnote" , errmsg.str() ); - result.append( "err", "norepl" ); - return true; - } - - result.appendNull( "err" ); - return true; - } - - if ( !theReplSet && !e.isNumber() ) { - result.append( "wnote", "cannot use non integer w values for non-replica sets" ); - result.append( "err", "noreplset" ); - return true; - } - - while ( 1 ) { - - if ( !_isMaster() ) { - // this should be in the while loop in case we step down - errmsg = "not master"; - result.append( "wnote", "no longer primary" ); - result.append( "code" , 10990 ); - return false; - } - - // check this first for w=0 or w=1 - if ( opReplicatedEnough( op, e ) ) { - break; - } - - // if replication isn't enabled (e.g., config servers) - if ( ! anyReplEnabled() ) { - result.append( "err", "norepl" ); - return true; - } - - - if ( timeout > 0 && timer.millis() >= timeout ) { - gleWtimeouts.increment(); - result.append( "wtimeout" , true ); - errmsg = "timed out waiting for slaves"; - result.append( "waited" , timer.millis() ); - result.append("writtenTo", getHostsWrittenTo(op)); - result.append( "err" , "timeout" ); - return true; - } - - verify( sprintf( buf , "w block pass: %lld" , ++passes ) < 30 ); - c.curop()->setMessage( buf ); - sleepmillis(1); - killCurrentOp.checkForInterrupt(); - } - - result.append("writtenTo", getHostsWrittenTo(op)); - int myMillis = timer.recordMillis(); - result.appendNumber( "wtime" , myMillis ); - } - - result.appendNull( "err" ); - return true; - } - - } cmdGetLastError; - - class CmdGetPrevError : public Command { - public: - virtual LockType locktype() const { return NONE; } - virtual bool logTheOp() { - return false; - } - virtual void help( stringstream& help ) const { - help << "check for errors since last reseterror commandcal"; - } - virtual bool slaveOk() const { - return true; - } - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} // No auth required - CmdGetPrevError() : Command("getPrevError", false, "getpreverror") {} - bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - LastError *le = lastError.disableForCommand(); - le->appendSelf( result ); - if ( le->valid ) - result.append( "nPrev", le->nPrev ); - else - result.append( "nPrev", -1 ); - return true; - } - } cmdGetPrevError; - CmdShutdown cmdShutdown; void CmdShutdown::help( stringstream& help ) const { @@ -321,7 +94,10 @@ namespace mongo { bool CmdShutdown::run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { bool force = cmdObj.hasField("force") && cmdObj["force"].trueValue(); - if (!force && theReplSet && theReplSet->isPrimary()) { + if (!force && + theReplSet && + theReplSet->getConfig().members.size() > 1 && + theReplSet->isPrimary()) { long long timeout, now, start; timeout = now = start = curTimeMicros64()/1000000; if (cmdObj.hasField("timeoutSecs")) { @@ -385,7 +161,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::dropDatabase); - out->push_back(Privilege(dbname, actions)); + out->push_back(Privilege(ResourcePattern::forDatabaseName(dbname), actions)); } // this is suboptimal but syncDataAndTruncateJournal is called from dropDatabase, and that @@ -393,10 +169,34 @@ namespace mongo { virtual bool lockGlobally() const { return true; } virtual LockType locktype() const { return WRITE; } + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + invariant(db); + std::list collections; + db->namespaceIndex().getNamespaces(collections, true /* onlyCollections */); + + std::vector allKilledIndexes; + for (std::list::iterator it = collections.begin(); + it != collections.end(); + ++it) { + std::string ns = *it; + + IndexCatalog::IndexKillCriteria criteria; + criteria.ns = ns; + std::vector killedIndexes = + IndexBuilder::killMatchingIndexBuilds(db->getCollection(ns), criteria); + allKilledIndexes.insert(allKilledIndexes.end(), + killedIndexes.begin(), + killedIndexes.end()); + } + return allKilledIndexes; + } + CmdDropDatabase() : Command("dropDatabase") {} bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { // disallow dropping the config database - if ( cmdLine.configsvr && ( dbname == "config" ) ) { + if (serverGlobalParams.configsvr && (dbname == "config")) { errmsg = "Cannot drop 'config' database if mongod started with --configsvr"; return false; } @@ -405,6 +205,7 @@ namespace mongo { int p = (int) e.number(); if ( p != 1 ) return false; + stopIndexBuilds(cc().database(), cmdObj); dropDatabase(dbname); result.append( "dropped" , dbname ); log() << "dropDatabase " << dbname << " finished" << endl; @@ -432,22 +233,55 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::repairDatabase); - out->push_back(Privilege(dbname, actions)); + out->push_back(Privilege(ResourcePattern::forDatabaseName(dbname), actions)); } CmdRepairDatabase() : Command("repairDatabase") {} + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + invariant(db); + std::list collections; + db->namespaceIndex().getNamespaces(collections, true /* onlyCollections */); + + std::vector allKilledIndexes; + for (std::list::iterator it = collections.begin(); + it != collections.end(); + ++it) { + std::string ns = *it; + + IndexCatalog::IndexKillCriteria criteria; + criteria.ns = ns; + std::vector killedIndexes = + IndexBuilder::killMatchingIndexBuilds(db->getCollection(ns), criteria); + allKilledIndexes.insert(allKilledIndexes.end(), + killedIndexes.begin(), + killedIndexes.end()); + } + return allKilledIndexes; + } + bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { BSONElement e = cmdObj.firstElement(); log() << "repairDatabase " << dbname << endl; int p = (int) e.number(); + if ( p != 1 ) { errmsg = "bad option"; return false; } + + std::vector indexesInProg = stopIndexBuilds(cc().database(), cmdObj); + e = cmdObj.getField( "preserveClonedFilesOnFailure" ); bool preserveClonedFilesOnFailure = e.isBoolean() && e.boolean(); e = cmdObj.getField( "backupOriginalFiles" ); bool backupOriginalFiles = e.isBoolean() && e.boolean(); - return repairDatabase( dbname, errmsg, preserveClonedFilesOnFailure, backupOriginalFiles ); + Status status = + repairDatabase( dbname, preserveClonedFilesOnFailure, backupOriginalFiles ); + + IndexBuilder::restoreIndexes(indexesInProg); + + return appendCommandStatus( result, status ); } } cmdRepairDatabase; @@ -468,18 +302,34 @@ namespace mongo { help << "http://dochub.mongodb.org/core/databaseprofiler"; } virtual LockType locktype() const { return WRITE; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::profileEnable); - out->push_back(Privilege(dbname, actions)); + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + + if (cmdObj.firstElement().numberInt() == -1 && !cmdObj.hasField("slowms")) { + // If you just want to get the current profiling level you can do so with just + // read access to system.profile, even if you can't change the profiling level. + if (authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString(dbname, + "system.profile")), + ActionType::find)) { + return Status::OK(); + } + } + + if (authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forDatabaseName(dbname), ActionType::enableProfiler)) { + return Status::OK(); + } + + return Status(ErrorCodes::Unauthorized, "unauthorized"); } CmdProfile() : Command("profile") {} bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { BSONElement e = cmdObj.firstElement(); result.append("was", cc().database()->getProfilingLevel()); - result.append("slowms", cmdLine.slowMS ); + result.append("slowms", serverGlobalParams.slowMS); int p = (int) e.number(); bool ok = false; @@ -492,7 +342,7 @@ namespace mongo { BSONElement slow = cmdObj["slowms"]; if ( slow.isNumber() ) - cmdLine.slowMS = slow.numberInt(); + serverGlobalParams.slowMS = slow.numberInt(); return ok; } @@ -516,17 +366,6 @@ namespace mongo { } } cmdgetoptime; - /* - class Cmd : public Command { - public: - Cmd() : Command("") { } - bool adminOnly() const { return true; } - bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result) { - return true; - } - } cmd; - */ - class CmdDiagLogging : public Command { public: virtual bool slaveOk() const { @@ -543,13 +382,14 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::diagLogging); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { int was = _diaglog.setLevel( cmdObj.firstElement().numberInt() ); _diaglog.flush(); - if ( !cmdLine.quiet ) - tlog() << "CMD: diagLogging set to " << _diaglog.getLevel() << " from: " << was << endl; + if (!serverGlobalParams.quiet) { + MONGO_TLOG(0) << "CMD: diagLogging set to " << _diaglog.getLevel() << " from: " << was << endl; + } result.append( "was" , was ); return true; } @@ -574,22 +414,53 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::dropCollection); - out->push_back(Privilege(dbname, actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } virtual void help( stringstream& help ) const { help << "drop a collection\n{drop : }"; } virtual LockType locktype() const { return WRITE; } + + virtual std::vector stopIndexBuilds(Database* db, + const BSONObj& cmdObj) { + std::string nsToDrop = db->name() + '.' + cmdObj.firstElement().valuestr(); + + IndexCatalog::IndexKillCriteria criteria; + criteria.ns = nsToDrop; + return IndexBuilder::killMatchingIndexBuilds(db->getCollection(nsToDrop), criteria); + } + virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { string nsToDrop = dbname + '.' + cmdObj.firstElement().valuestr(); - NamespaceDetails *d = nsdetails(nsToDrop); - if ( !cmdLine.quiet ) - tlog() << "CMD: drop " << nsToDrop << endl; - if ( d == 0 ) { + if (!serverGlobalParams.quiet) { + MONGO_TLOG(0) << "CMD: drop " << nsToDrop << endl; + } + + if ( nsToDrop.find( '$' ) != string::npos ) { + errmsg = "can't drop collection with reserved $ character in name"; + return false; + } + + Collection* coll = cc().database()->getCollection( nsToDrop ); + // If collection does not exist, short circuit and return. + if ( !coll ) { errmsg = "ns not found"; return false; } - uassert( 10039 , "can't drop collection with reserved $ character in name", strchr(nsToDrop.c_str(), '$') == 0 ); - dropCollection( nsToDrop, errmsg, result ); - return true; + + int numIndexes = coll->getIndexCatalog()->numIndexesTotal(); + + stopIndexBuilds(cc().database(), cmdObj); + + result.append( "ns", nsToDrop ); + result.append( "nIndexesWas", numIndexes ); + + Status s = cc().database()->dropCollection( nsToDrop ); + + if ( s.isOK() ) + return true; + + appendCommandStatus( result, s ); + + return false; } } cmdDrop; @@ -612,7 +483,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { @@ -632,7 +503,7 @@ namespace mongo { string ns = parseNs(dbname, cmdObj); string err; int errCode; - long long n = runCount(ns.c_str(), cmdObj, err, errCode); + long long n = runCount(ns, cmdObj, err, errCode); long long nn = n; bool ok = true; if ( n == -1 ) { @@ -644,6 +515,7 @@ namespace mongo { ok = false; if ( !err.empty() ) { errmsg = err; + result.append("code", errCode); return false; } } @@ -670,141 +542,63 @@ namespace mongo { help << "create a collection explicitly\n" "{ create: [, capped: , size: , max: ] }"; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::createCollection); - out->push_back(Privilege(dbname, actions)); - } - virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { - uassert(15888, "must pass name of collection to create", cmdObj.firstElement().valuestrsafe()[0] != '\0'); - string ns = dbname + '.' + cmdObj.firstElement().valuestr(); - string err; - uassert(14832, "specify size: when capped is true", !cmdObj["capped"].trueValue() || cmdObj["size"].isNumber() || cmdObj.hasField("$nExtents")); - bool ok = userCreateNS(ns.c_str(), cmdObj, err, ! fromRepl ); - if ( !ok && !err.empty() ) - errmsg = err; - return ok; - } - } cmdCreate; - - /* "dropIndexes" is now the preferred form - "deleteIndexes" deprecated */ - class CmdDropIndexes : public Command { - public: - virtual bool logTheOp() { - return true; - } - virtual bool slaveOk() const { - return false; - } - virtual LockType locktype() const { return WRITE; } - virtual void help( stringstream& help ) const { - help << "drop indexes for a collection"; - } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::dropIndexes); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - CmdDropIndexes() : Command("dropIndexes", false, "deleteIndexes") { } - bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& anObjBuilder, bool /*fromRepl*/) { - BSONElement e = jsobj.firstElement(); - string toDeleteNs = dbname + '.' + e.valuestr(); - NamespaceDetails *d = nsdetails(toDeleteNs); - if ( !cmdLine.quiet ) - tlog() << "CMD: dropIndexes " << toDeleteNs << endl; - if ( d ) { - BSONElement f = jsobj.getField("index"); - if ( f.type() == String ) { - return dropIndexes( d, toDeleteNs.c_str(), f.valuestr(), errmsg, anObjBuilder, false ); - } - else if ( f.type() == Object ) { - int idxId = d->findIndexByKeyPattern( f.embeddedObject() ); - if ( idxId < 0 ) { - errmsg = "can't find index with key:"; - errmsg += f.embeddedObject().toString(); - return false; - } - else { - IndexDetails& ii = d->idx( idxId ); - string iName = ii.indexName(); - return dropIndexes( d, toDeleteNs.c_str(), iName.c_str() , errmsg, anObjBuilder, false ); - } - } - else { - errmsg = "invalid index name spec"; - return false; + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + AuthorizationSession* authzSession = client->getAuthorizationSession(); + if (cmdObj["capped"].trueValue()) { + if (!authzSession->isAuthorizedForActionsOnResource( + parseResourcePattern(dbname, cmdObj), ActionType::convertToCapped)) { + return Status(ErrorCodes::Unauthorized, "unauthorized"); } } - else { - errmsg = "ns not found"; - return false; + + // ActionType::createCollection or ActionType::insert are both acceptable + if (authzSession->isAuthorizedForActionsOnResource( + parseResourcePattern(dbname, cmdObj), ActionType::createCollection) || + authzSession->isAuthorizedForActionsOnResource( + parseResourcePattern(dbname, cmdObj), ActionType::insert)) { + return Status::OK(); } - } - } cmdDropIndexes; - class CmdReIndex : public Command { - public: - virtual bool logTheOp() { return false; } // only reindexes on the one node - virtual bool slaveOk() const { return true; } // can reindex on a secondary - virtual LockType locktype() const { return WRITE; } - virtual void help( stringstream& help ) const { - help << "re-index a collection"; + return Status(ErrorCodes::Unauthorized, "unauthorized"); } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::reIndex); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - CmdReIndex() : Command("reIndex") { } - bool run(const string& dbname , BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) { - static DBDirectClient db; - - BSONElement e = jsobj.firstElement(); - string toDeleteNs = dbname + '.' + e.valuestr(); - NamespaceDetails *d = nsdetails(toDeleteNs); - tlog() << "CMD: reIndex " << toDeleteNs << endl; - BackgroundOperation::assertNoBgOpInProgForNs(toDeleteNs.c_str()); + virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { + BSONObjIterator it(cmdObj); - if ( ! d ) { - errmsg = "ns not found"; - return false; - } + // Extract ns from first cmdObj element. + BSONElement firstElt = it.next(); + uassert(15888, + "must pass name of collection to create", + firstElt.valuestrsafe()[0] != '\0'); - list all; - auto_ptr i = db.query( dbname + ".system.indexes" , BSON( "ns" << toDeleteNs ) , 0 , 0 , 0 , QueryOption_SlaveOk ); - BSONObjBuilder b; - while ( i->more() ) { - BSONObj o = i->next().removeField("v").getOwned(); - b.append( BSONObjBuilder::numStr( all.size() ) , o ); - all.push_back( o ); + Status status = userAllowedCreateNS( dbname, firstElt.valuestr() ); + if ( !status.isOK() ) { + return appendCommandStatus( result, status ); } + string ns = dbname + '.' + firstElt.valuestr(); - bool ok = dropIndexes( d, toDeleteNs.c_str(), "*" , errmsg, result, true ); - if ( ! ok ) { - errmsg = "dropIndexes failed"; - return false; + // Build options object from remaining cmdObj elements. + BSONObjBuilder optionsBuilder; + while (it.more()) { + optionsBuilder.append(it.next()); } + BSONObj options = optionsBuilder.obj(); + uassert(14832, + "specify size: when capped is true", + !options["capped"].trueValue() || options["size"].isNumber() || + options.hasField("$nExtents")); - for ( list::iterator i=all.begin(); i!=all.end(); i++ ) { - BSONObj o = *i; - LOG(1) << "reIndex ns: " << toDeleteNs << " index: " << o << endl; - string systemIndexesNs = - Namespace( toDeleteNs.c_str() ).getSisterNS( "system.indexes" ); - theDataFileMgr.insertWithObjMod( systemIndexesNs.c_str(), o, false, true ); + // Create collection. + string err; + bool ok = userCreateNS(ns.c_str(), options, err, !fromRepl); + if (!ok && !err.empty()) { + errmsg = err; } - - result.append( "nIndexes" , (int)all.size() ); - result.appendArray( "indexes" , b.obj() ); - return true; + return ok; } - } cmdReIndex; + } cmdCreate; class CmdListDatabases : public Command { public: @@ -824,7 +618,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::listDatabases); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } CmdListDatabases() : Command("listDatabases" , true ) {} bool run(const string& dbname , BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) { @@ -833,12 +627,12 @@ namespace mongo { vector< BSONObj > dbInfos; set seen; - boost::intmax_t totalSize = 0; + intmax_t totalSize = 0; for ( vector< string >::iterator i = dbNames.begin(); i != dbNames.end(); ++i ) { BSONObjBuilder b; b.append( "name", *i ); - boost::intmax_t size = dbSize( i->c_str() ); + intmax_t size = dbSize( i->c_str() ); b.append( "sizeOnDisk", (double) size ); totalSize += size; @@ -852,7 +646,8 @@ namespace mongo { seen.insert( i->c_str() ); } - // TODO: erh 1/1/2010 I think this is broken where path != dbpath ?? + // TODO: erh 1/1/2010 I think this is broken where + // path != storageGlobalParams.dbpath ?? set allShortNames; { Lock::GlobalRead lk; @@ -898,13 +693,13 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::closeAllDatabases); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } CmdCloseAllDatabases() : Command( "closeAllDatabases" ) {} bool run(const string& dbname , BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) { bool ok; try { - ok = dbHolderW().closeAll( dbpath , result, false ); + ok = dbHolderW().closeAll(storageGlobalParams.dbpath, result, false); } catch(DBException&) { throw; @@ -928,23 +723,23 @@ namespace mongo { help << " example: { filemd5 : ObjectId(aaaaaaa) , root : \"fs\" }"; } virtual LockType locktype() const { return READ; } + + virtual std::string parseNs(const std::string& dbname, const BSONObj& cmdObj) const { + std::string collectionName = cmdObj.getStringField("root"); + if (collectionName.empty()) + collectionName = "fs"; + collectionName += ".chunks"; + return NamespaceString(dbname, collectionName).ns(); + } + virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), ActionType::find)); } + bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { - string ns = dbname; - ns += "."; - { - string root = jsobj.getStringField( "root" ); - if ( root.size() == 0 ) - root = "fs"; - ns += root; - } - ns += ".chunks"; // make this an option in jsobj + const std::string ns = parseNs(dbname, jsobj); // Check shard version at startup. // This will throw before we've done any work if shard version is outdated @@ -976,25 +771,29 @@ namespace mongo { BSONObj query = BSON( "files_id" << jsobj["filemd5"] << "n" << GTE << n ); BSONObj sort = BSON( "files_id" << 1 << "n" << 1 ); - shared_ptr cursor = NamespaceDetailsTransient::bestGuessCursor(ns.c_str(), - query, sort); - if ( ! cursor ) { - errmsg = "need an index on { files_id : 1 , n : 1 }"; - return false; + CanonicalQuery* cq; + if (!CanonicalQuery::canonicalize(ns, query, sort, BSONObj(), &cq).isOK()) { + uasserted(17240, "Can't canonicalize query " + query.toString()); + return 0; } - auto_ptr cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns.c_str())); - while ( cursor->ok() ) { - if ( ! cursor->matcher()->matchesCurrent( cursor.get() ) ) { - log() << "**** NOT MATCHING ****" << endl; - PRINT(cursor->current()); - cursor->advance(); - continue; - } + Runner* rawRunner; + if (!getRunner(cq, &rawRunner, QueryPlannerParams::NO_TABLE_SCAN).isOK()) { + uasserted(17241, "Can't get runner for query " + query.toString()); + return 0; + } + + auto_ptr runner(rawRunner); - BSONObj obj = cursor->current(); - cursor->advance(); + // The runner must be registered to be informed of DiskLoc deletions and NS dropping + // when we yield the lock below. + const ScopedRunnerRegistration safety(runner.get()); + const ChunkVersion shardVersionAtStart = shardingState.getVersion(ns); + + BSONObj obj; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { BSONElement ne = obj["n"]; verify(ne.isNumber()); int myn = ne.numberInt(); @@ -1012,36 +811,35 @@ namespace mongo { int len; const char * data = owned["data"].binDataClean( len ); - ClientCursor::YieldLock yield (cc.get()); + // Save state, yield, run the MD5, and reacquire lock. + runner->saveState(); + try { + dbtempreleasecond yield; + md5_append( &st , (const md5_byte_t*)(data) , len ); n++; } - catch (...) { - if ( ! yield.stillOk() ) // relocks - cc.release(); - throw; + catch (SendStaleConfigException&) { + log() << "metadata changed during filemd5" << endl; + break; } - try { // SERVER-5752 may make this try unnecessary - if ( ! yield.stillOk() ) { // relocks and checks shard version - cc.release(); - if (!partialOk) - uasserted(13281, "File deleted during filemd5 command"); + // Have the lock again. See if we were killed. + if (!runner->restoreState()) { + if (!partialOk) { + uasserted(13281, "File deleted during filemd5 command"); } } - catch(SendStaleConfigException& e){ - // return partial results. - // Mongos will get the error at the start of the next call if it doesn't update first. - log() << "Config changed during filemd5 - command will resume " << endl; - // useful for debugging but off by default to avoid looking like a scary error. - LOG(1) << "filemd5 stale config exception: " << e.what() << endl; + if (!shardingState.getVersion(ns).isWriteCompatibleWith(shardVersionAtStart)) { + // return partial results. Mongos will get the error at the start of the next + // call if it doesn't update first. + log() << "Config changed during filemd5 - command will resume " << endl; break; } } - if (partialOk) result.appendBinData("md5state", sizeof(st), BinDataGeneral, &st); @@ -1086,7 +884,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { Timer timer; @@ -1098,9 +896,9 @@ namespace mongo { bool estimate = jsobj["estimate"].trueValue(); Client::Context ctx( ns ); - NamespaceDetails *d = nsdetails(ns); + Collection* collection = ctx.db()->getCollection( ns ); - if ( ! d || d->stats.nrecords == 0 ) { + if ( !collection || collection->numRecords() == 0 ) { result.appendNumber( "size" , 0 ); result.appendNumber( "numObjects" , 0 ); result.append( "millis" , timer.millis() ); @@ -1109,15 +907,16 @@ namespace mongo { result.appendBool( "estimate" , estimate ); - shared_ptr c; + auto_ptr runner; if ( min.isEmpty() && max.isEmpty() ) { if ( estimate ) { - result.appendNumber( "size" , d->stats.datasize ); - result.appendNumber( "numObjects" , d->stats.nrecords ); + result.appendNumber( "size" , collection->details()->dataSize() ); + result.appendNumber( "numObjects", + static_cast( collection->numRecords() ) ); result.append( "millis" , timer.millis() ); return 1; } - c = theDataFileMgr.findAll( ns.c_str() ); + runner.reset(InternalPlanner::collectionScan(ns)); } else if ( min.isEmpty() || max.isEmpty() ) { errmsg = "only one of min or max specified"; @@ -1130,8 +929,9 @@ namespace mongo { keyPattern = Helpers::inferKeyPattern( min ); } - const IndexDetails *idx = d->findIndexByPrefix( keyPattern , - true ); /* require single key */ + IndexDescriptor *idx = + collection->getIndexCatalog()->findIndexByPrefix( keyPattern, true ); /* require single key */ + if ( idx == NULL ) { errmsg = "couldn't find valid index containing key pattern"; return false; @@ -1141,22 +941,24 @@ namespace mongo { min = Helpers::toKeyFormat( kp.extendRangeBound( min, false ) ); max = Helpers::toKeyFormat( kp.extendRangeBound( max, false ) ); - c.reset( BtreeCursor::make( d, *idx, min, max, false, 1 ) ); + runner.reset(InternalPlanner::indexScan(collection, idx, min, max, false)); } - long long avgObjSize = d->stats.datasize / d->stats.nrecords; + long long avgObjSize = collection->details()->dataSize() / collection->numRecords(); long long maxSize = jsobj["maxSize"].numberLong(); long long maxObjects = jsobj["maxObjects"].numberLong(); long long size = 0; long long numObjects = 0; - while( c->ok() ) { + DiskLoc loc; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &loc))) { if ( estimate ) size += avgObjSize; else - size += c->currLoc().rec()->netLength(); + size += loc.rec()->netLength(); numObjects++; @@ -1165,8 +967,10 @@ namespace mongo { result.appendBool( "maxReached" , true ); break; } + } - c->advance(); + if (Runner::RUNNER_EOF != state) { + warning() << "Internal error while reading " << ns << endl; } ostringstream os; @@ -1186,25 +990,31 @@ namespace mongo { namespace { long long getIndexSizeForCollection(string db, string ns, BSONObjBuilder* details=NULL, int scale = 1 ) { Lock::assertAtLeastReadLocked(ns); + Client::Context ctx( ns ); - NamespaceDetails * nsd = nsdetails( ns ); - if ( ! nsd ) + Collection* coll = ctx.db()->getCollection( ns ); + if ( !coll ) return 0; + IndexCatalog::IndexIterator ii = + coll->getIndexCatalog()->getIndexIterator( true /*includeUnfinishedIndexes*/ ); + long long totalSize = 0; - NamespaceDetails::IndexIterator ii = nsd->ii(); while ( ii.more() ) { - IndexDetails& d = ii.next(); - string collNS = d.indexNamespace(); - NamespaceDetails * mine = nsdetails( collNS ); - if ( ! mine ) { - log() << "error: have index [" << collNS << "] but no NamespaceDetails" << endl; + IndexDescriptor* d = ii.next(); + string indNS = d->indexNamespace(); + Collection* indColl = ctx.db()->getCollection( indNS ); + if ( ! indColl ) { + log() << "error: have index descriptor [" << indNS + << "] but no entry in the index collection." << endl; continue; } - totalSize += mine->stats.datasize; - if ( details ) - details->appendNumber( d.indexName() , mine->stats.datasize / scale ); + totalSize += indColl->dataSize(); + if ( details ) { + long long const indexSize = indColl->dataSize() / scale; + details->appendNumber( d->indexName() , indexSize ); + } } return totalSize; } @@ -1224,15 +1034,14 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::collStats); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { string ns = dbname + "." + jsobj.firstElement().valuestr(); Client::Context cx( ns ); - - NamespaceDetails * nsd = nsdetails( ns ); - if ( ! nsd ) { - errmsg = "ns not found"; + Collection* collection = cx.db()->getCollection( ns ); + if ( !collection ) { + errmsg = "Collection [" + ns + "] not found."; return false; } @@ -1253,19 +1062,22 @@ namespace mongo { bool verbose = jsobj["verbose"].trueValue(); - long long size = nsd->stats.datasize / scale; - result.appendNumber( "count" , nsd->stats.nrecords ); + long long size = collection->dataSize() / scale; + long long numRecords = collection->numRecords(); + result.appendNumber( "count" , numRecords ); result.appendNumber( "size" , size ); - if( nsd->stats.nrecords ) - result.append ( "avgObjSize" , double(size) / double(nsd->stats.nrecords) ); + if( numRecords ) + result.append( "avgObjSize" , collection->averageObjectSize() ); int numExtents; BSONArrayBuilder extents; - - result.appendNumber( "storageSize" , nsd->storageSize( &numExtents , verbose ? &extents : 0 ) / scale ); + result.appendNumber( "storageSize", + static_cast( collection->storageSize( &numExtents , verbose ? &extents : 0 ) / scale ) ); result.append( "numExtents" , numExtents ); - result.append( "nindexes" , nsd->nIndexes ); - result.append( "lastExtentSize" , nsd->lastExtentSize / scale ); + result.append( "nindexes" , collection->getIndexCatalog()->numIndexesReady() ); + + NamespaceDetails* nsd = collection->details(); + result.append( "lastExtentSize" , nsd->lastExtentSize() / scale ); result.append( "paddingFactor" , nsd->paddingFactor() ); result.append( "systemFlags" , nsd->systemFlags() ); result.append( "userFlags" , nsd->userFlags() ); @@ -1274,8 +1086,8 @@ namespace mongo { result.appendNumber( "totalIndexSize" , getIndexSizeForCollection(dbname, ns, &indexSizes, scale) / scale ); result.append("indexSizes", indexSizes.obj()); - if ( nsd->isCapped() ) { - result.append( "capped" , nsd->isCapped() ); + if ( collection->isCapped() ) { + result.append( "capped" , collection->isCapped() ); result.appendNumber( "max" , nsd->maxCappedDocs() ); } @@ -1303,23 +1115,32 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::collMod); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { string ns = dbname + "." + jsobj.firstElement().valuestr(); Client::Context ctx( ns ); - NamespaceDetails* nsd = nsdetails( ns ); - if ( ! nsd ) { + + Collection* coll = ctx.db()->getCollection( ns ); + if ( !coll ) { errmsg = "ns does not exist"; return false; } + NamespaceDetails* nsd = coll->details(); + bool ok = true; BSONForEach( e, jsobj ) { if ( str::equals( "collMod", e.fieldName() ) ) { // no-op } + else if ( str::startsWith( e.fieldName(), "$" ) ) { + // no-op: ignore top-level fields prefixed with $. They are for the command processor. + } + else if ( LiteParsedQuery::cmdOptionMaxTimeMS == e.fieldNameStringData() ) { + // no-op + } else if ( str::equals( "usePowerOf2Sizes", e.fieldName() ) ) { bool oldPowerOf2 = nsd->isUserFlagSet(NamespaceDetails::Flag_UsePowerOf2Sizes); bool newPowerOf2 = e.trueValue(); @@ -1357,16 +1178,15 @@ namespace mongo { continue; } - int idxNo = nsd->findIndexByKeyPattern( keyPattern ); - if( idxNo < 0 ){ + const IndexDescriptor* idx = coll->getIndexCatalog() + ->findIndexByKeyPattern( keyPattern ); + if ( idx == NULL ) { errmsg = str::stream() << "cannot find index " << keyPattern << " for ns " << ns; ok = false; continue; } - - IndexDetails idx = nsd->idx( idxNo ); - BSONElement oldExpireSecs = idx.info.obj().getField("expireAfterSeconds"); + BSONElement oldExpireSecs = idx->infoObj().getField("expireAfterSeconds"); if( oldExpireSecs.eoo() ){ errmsg = "no expireAfterSeconds field to update"; ok = false; @@ -1381,7 +1201,8 @@ namespace mongo { if ( oldExpireSecs != newExpireSecs ) { // change expireAfterSeconds result.appendAs( oldExpireSecs, "expireAfterSeconds_old" ); - nsd->updateTTLIndex( idxNo , newExpireSecs ); + idx = coll->getIndexCatalog() + ->updateTTLSetting( idx, newExpireSecs.numberLong() ); result.appendAs( newExpireSecs , "expireAfterSeconds_new" ); } } @@ -1410,7 +1231,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::dbStats); - out->push_back(Privilege(dbname, actions)); + out->push_back(Privilege(ResourcePattern::forDatabaseName(dbname), actions)); } bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { int scale = 1; @@ -1428,8 +1249,12 @@ namespace mongo { list collections; Database* d = cc().database(); + + if ( d && ( d->isEmpty() || d->getExtentManager().numFiles() == 0 ) ) + d = NULL; + if ( d ) - d->namespaceIndex.getNamespaces( collections ); + d->namespaceIndex().getNamespaces( collections ); long long ncollections = 0; long long objects = 0; @@ -1442,25 +1267,25 @@ namespace mongo { for (list::const_iterator it = collections.begin(); it != collections.end(); ++it) { const string ns = *it; - NamespaceDetails * nsd = nsdetails( ns ); - if ( ! nsd ) { + Collection* collection = d->getCollection( ns ); + if ( !collection ) { errmsg = "missing ns: "; errmsg += ns; return false; } ncollections += 1; - objects += nsd->stats.nrecords; - size += nsd->stats.datasize; + objects += collection->numRecords(); + size += collection->dataSize(); int temp; - storageSize += nsd->storageSize( &temp ); + storageSize += collection->storageSize( &temp, NULL ); numExtents += temp; - indexes += nsd->nIndexes; + indexes += collection->getIndexCatalog()->numIndexesTotal(); indexSize += getIndexSizeForCollection(dbname, ns); } - + result.append ( "db" , dbname ); result.appendNumber( "collections" , ncollections ); result.appendNumber( "objects" , objects ); @@ -1470,164 +1295,38 @@ namespace mongo { result.appendNumber( "numExtents" , numExtents ); result.appendNumber( "indexes" , indexes ); result.appendNumber( "indexSize" , indexSize / scale ); - result.appendNumber( "fileSize" , d->fileSize() / scale ); - if( d ) - result.appendNumber( "nsSizeMB", (int) d->namespaceIndex.fileLength() / 1024 / 1024 ); - - BSONObjBuilder dataFileVersion( result.subobjStart( "dataFileVersion" ) ); - if ( d && !d->isEmpty() ) { - DataFileHeader* header = d->getFile( 0 )->getHeader(); - dataFileVersion.append( "major", header->version ); - dataFileVersion.append( "minor", header->versionMinor ); - } - dataFileVersion.done(); - - return true; - } - } cmdDBStats; - - /* convertToCapped seems to use this */ - class CmdCloneCollectionAsCapped : public Command { - public: - CmdCloneCollectionAsCapped() : Command( "cloneCollectionAsCapped" ) {} - virtual bool slaveOk() const { return false; } - virtual LockType locktype() const { return WRITE; } - virtual void help( stringstream &help ) const { - help << "{ cloneCollectionAsCapped:, toCollection:, size: }"; - } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet sourceActions; - sourceActions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), sourceActions)); - - ActionSet targetActions; - targetActions.addAction(ActionType::insert); - targetActions.addAction(ActionType::ensureIndex); - std::string collection = cmdObj.getStringField("toCollection"); - uassert(16708, "bad 'toCollection' value", !collection.empty()); - - std::string targetNs = dbname + "." + collection; - - out->push_back(Privilege(targetNs, targetActions)); - } - bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { - string from = jsobj.getStringField( "cloneCollectionAsCapped" ); - string to = jsobj.getStringField( "toCollection" ); - long long size = (long long)jsobj.getField( "size" ).number(); - - if ( from.empty() || to.empty() || size == 0 ) { - errmsg = "invalid command spec"; - return false; - } - - string fromNs = dbname + "." + from; - string toNs = dbname + "." + to; - NamespaceDetails *nsd = nsdetails( fromNs ); - massert( 10301 , "source collection " + fromNs + " does not exist", nsd ); - long long excessSize = nsd->stats.datasize - size * 2; // datasize and extentSize can't be compared exactly, so add some padding to 'size' - DiskLoc extent = nsd->firstExtent; - for( ; excessSize > extent.ext()->length && extent != nsd->lastExtent; extent = extent.ext()->xnext ) { - excessSize -= extent.ext()->length; - LOG( 2 ) << "cloneCollectionAsCapped skipping extent of size " << extent.ext()->length << endl; - LOG( 6 ) << "excessSize: " << excessSize << endl; - } - DiskLoc startLoc = extent.ext()->firstRecord; - - CursorId id; - { - shared_ptr c = theDataFileMgr.findAll( fromNs.c_str(), startLoc ); - ClientCursor *cc = new ClientCursor(0, c, fromNs.c_str()); - id = cc->cursorid(); - } - - DBDirectClient client; - Client::Context ctx( toNs ); - BSONObjBuilder spec; - spec.appendBool( "capped", true ); - spec.append( "size", double( size ) ); - if (jsobj.hasField("temp")) - spec.append(jsobj["temp"]); - if ( !userCreateNS( toNs.c_str(), spec.done(), errmsg, true ) ) - return false; - - auto_ptr< DBClientCursor > c = client.getMore( fromNs, id ); - while( c->more() ) { - BSONObj obj = c->next(); - theDataFileMgr.insertAndLog( toNs.c_str(), obj, true ); - getDur().commitIfNeeded(); + if ( d ) { + result.appendNumber( "fileSize" , d->fileSize() / scale ); + result.appendNumber( "nsSizeMB", (int) d->namespaceIndex().fileLength() / 1024 / 1024 ); } - - return true; - } - } cmdCloneCollectionAsCapped; - - /* jan2010: - Converts the given collection to a capped collection w/ the specified size. - This command is not highly used, and is not currently supported with sharded - environments. - */ - class CmdConvertToCapped : public Command { - public: - CmdConvertToCapped() : Command( "convertToCapped" ) {} - virtual bool slaveOk() const { return false; } - virtual LockType locktype() const { return WRITE; } - // calls renamecollection which does a global lock, so we must too: - virtual bool lockGlobally() const { return true; } - virtual void help( stringstream &help ) const { - help << "{ convertToCapped:, size: }"; - } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::convertToCapped); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - bool run(const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { - BackgroundOperation::assertNoBgOpInProgForDb(dbname.c_str()); - - string from = jsobj.getStringField( "convertToCapped" ); - long long size = (long long)jsobj.getField( "size" ).number(); - - if ( from.empty() || size == 0 ) { - errmsg = "invalid command spec"; - return false; + else { + result.appendNumber( "fileSize" , 0 ); } - string shortTmpName = str::stream() << "tmp.convertToCapped." << from; - string longTmpName = str::stream() << dbname << "." << shortTmpName; - - DBDirectClient client; - client.dropCollection( longTmpName ); - - BSONObj info; - if ( !client.runCommand( dbname , - BSON( "cloneCollectionAsCapped" << from << "toCollection" << shortTmpName << "size" << double( size ) << "temp" << true ), - info ) ) { - errmsg = "cloneCollectionAsCapped failed: " + info.toString(); - return false; + BSONObjBuilder dataFileVersion( result.subobjStart( "dataFileVersion" ) ); + if ( d ) { + int major, minor; + d->getFileFormat( &major, &minor ); + dataFileVersion.append( "major", major ); + dataFileVersion.append( "minor", minor ); } + dataFileVersion.done(); - if ( !client.dropCollection( dbname + "." + from ) ) { - errmsg = "failed to drop original collection"; - return false; - } + if ( d ){ + int freeListSize = 0; + int64_t freeListSpace = 0; + d->getExtentManager().freeListStats( &freeListSize, &freeListSpace ); - if ( !client.runCommand( "admin", - BSON( "renameCollection" << longTmpName << - "to" << ( dbname + "." + from ) << - "stayTemp" << false // explicit - ), - info ) ) { - errmsg = "renameCollection failed: " + info.toString(); - return false; + BSONObjBuilder extentFreeList( result.subobjStart( "extentFreeList" ) ); + extentFreeList.append( "num", freeListSize ); + extentFreeList.appendNumber( "totalSize", + static_cast( freeListSpace / scale ) ); + extentFreeList.done(); } return true; } - } cmdConvertToCapped; + } cmdDBStats; /* Returns client's uri */ class CmdWhatsMyUri : public Command { @@ -1640,7 +1339,6 @@ namespace mongo { virtual void help( stringstream &help ) const { help << "{whatsmyuri:1}"; } - virtual bool requiresAuth() { return false; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) {} // No auth required @@ -1651,269 +1349,32 @@ namespace mongo { } } cmdWhatsMyUri; - /* For testing only, not for general use. Enabled via command-line */ - class GodInsert : public Command { + class AvailableQueryOptions: public Command { public: - GodInsert() : Command( "godinsert" ) { } - virtual bool adminOnly() const { return false; } - virtual bool logTheOp() { return false; } - virtual bool slaveOk() const { return true; } - virtual LockType locktype() const { return NONE; } - // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} - virtual void help( stringstream &help ) const { - help << "internal. for testing only."; - } - virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { - string coll = cmdObj[ "godinsert" ].valuestrsafe(); - log() << "test only command godinsert invoked coll:" << coll << endl; - uassert( 13049, "godinsert must specify a collection", !coll.empty() ); - string ns = dbname + "." + coll; - BSONObj obj = cmdObj[ "obj" ].embeddedObjectUserCheck(); - { - Lock::DBWrite lk(ns); - Client::Context ctx( ns ); - theDataFileMgr.insertWithObjMod( ns.c_str(), obj, false, true ); - } - return true; - } - }; - - MONGO_INITIALIZER(RegisterGodInsertCmd)(InitializerContext* context) { - if (Command::testCommandsEnabled) { - // Leaked intentionally: a Command registers itself when constructed. - new GodInsert(); - } - return Status::OK(); - } - - class DBHashCmd : public Command { - public: - DBHashCmd() : Command( "dbHash", false, "dbhash" ) {} - virtual bool slaveOk() const { return true; } - virtual LockType locktype() const { return READ; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::dbHash); - out->push_back(Privilege(dbname, actions)); - } - virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { - Timer timer; - - set desiredCollections; - if ( cmdObj["collections"].type() == Array ) { - BSONObjIterator i( cmdObj["collections"].Obj() ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( e.type() != String ) { - errmsg = "collections entries have to be strings"; - return false; - } - desiredCollections.insert( e.String() ); - } - } - - list colls; - Database* db = cc().database(); - if ( db ) - db->namespaceIndex.getNamespaces( colls ); - colls.sort(); - - result.appendNumber( "numCollections" , (long long)colls.size() ); - result.append( "host" , prettyHostName() ); - - md5_state_t globalState; - md5_init(&globalState); - - BSONObjBuilder bb( result.subobjStart( "collections" ) ); - for ( list::iterator i=colls.begin(); i != colls.end(); i++ ) { - string fullCollectionName = *i; - string shortCollectionName = fullCollectionName.substr( dbname.size() + 1 ); - - if ( shortCollectionName.find( "system." ) == 0 ) - continue; - - if ( desiredCollections.size() > 0 && - desiredCollections.count( shortCollectionName ) == 0 ) - continue; - - shared_ptr cursor; - - NamespaceDetails * nsd = nsdetails( fullCollectionName ); - - // debug SERVER-761 - NamespaceDetails::IndexIterator ii = nsd->ii(); - while( ii.more() ) { - const IndexDetails &idx = ii.next(); - if ( !idx.head.isValid() || !idx.info.isValid() ) { - log() << "invalid index for ns: " << fullCollectionName << " " << idx.head << " " << idx.info; - if ( idx.info.isValid() ) - log() << " " << idx.info.obj(); - log() << endl; - } - } - - int idNum = nsd->findIdIndex(); - if ( idNum >= 0 ) { - cursor.reset( BtreeCursor::make( nsd, - nsd->idx( idNum ), - BSONObj(), - BSONObj(), - false, - 1 ) ); - } - else if ( nsd->isCapped() ) { - cursor = findTableScan( fullCollectionName.c_str() , BSONObj() ); - } - else { - log() << "can't find _id index for: " << fullCollectionName << endl; - continue; - } - - md5_state_t st; - md5_init(&st); - - long long n = 0; - while ( cursor->ok() ) { - BSONObj c = cursor->current(); - md5_append( &st , (const md5_byte_t*)c.objdata() , c.objsize() ); - n++; - cursor->advance(); - } - md5digest d; - md5_finish(&st, d); - string hash = digestToString( d ); - - bb.append( shortCollectionName, hash ); - - md5_append( &globalState , (const md5_byte_t*)hash.c_str() , hash.size() ); - } - bb.done(); - - md5digest d; - md5_finish(&globalState, d); - string hash = digestToString( d ); - - result.append( "md5" , hash ); - result.appendNumber( "timeMillis", timer.millis() ); - return 1; + AvailableQueryOptions(): Command("availableQueryOptions", + false, + "availablequeryoptions") { } - } dbhashCmd; - - /* for diagnostic / testing purposes. Enabled via command line. */ - class CmdSleep : public Command { - public: virtual LockType locktype() const { return NONE; } - virtual bool adminOnly() const { return true; } - virtual bool logTheOp() { return false; } virtual bool slaveOk() const { return true; } - virtual void help( stringstream& help ) const { - help << "internal testing command. Makes db block (in a read lock) for 100 seconds\n"; - help << "w:true write lock. secs:"; - } - // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} - CmdSleep() : Command("sleep") { } - bool run(const string& ns, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - log() << "test only command sleep invoked" << endl; - int secs = 100; - if ( cmdObj["secs"].isNumber() ) - secs = cmdObj["secs"].numberInt(); - if( cmdObj.getBoolField("w") ) { - Lock::GlobalWrite lk; - sleepsecs(secs); - } - else { - Lock::GlobalRead lk; - sleepsecs(secs); - } - return true; - } - }; - MONGO_INITIALIZER(RegisterSleepCmd)(InitializerContext* context) { - if (Command::testCommandsEnabled) { - // Leaked intentionally: a Command registers itself when constructed. - new CmdSleep(); - } - return Status::OK(); - } - - // Testing only, enabled via command-line. - class CapTrunc : public Command { - public: - CapTrunc() : Command( "captrunc" ) {} - virtual bool slaveOk() const { return false; } - virtual LockType locktype() const { return WRITE; } - // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} - virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { - string coll = cmdObj[ "captrunc" ].valuestrsafe(); - uassert( 13416, "captrunc must specify a collection", !coll.empty() ); - string ns = dbname + "." + coll; - int n = cmdObj.getIntField( "n" ); - - // inclusive range? - bool inc = cmdObj.getBoolField( "inc" ); - NamespaceDetails *nsd = nsdetails( ns ); - ReverseCappedCursor c( nsd ); - massert( 13417, "captrunc collection not found or empty", c.ok() ); - for( int i = 0; i < n; ++i ) { - massert( 13418, "captrunc invalid n", c.advance() ); - } - DiskLoc end = c.currLoc(); - nsd->cappedTruncateAfter( ns.c_str(), end, inc ); + virtual bool isWriteCommandForConfigServer() const { return false; } + virtual Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) { + return Status::OK(); + } + + virtual bool run(const string& dbname, + BSONObj& cmdObj, + int, + string& errmsg, + BSONObjBuilder& result, + bool) { + result << "options" << QueryOption_AllSupported; return true; } - }; - MONGO_INITIALIZER(RegisterCapTruncCmd)(InitializerContext* context) { - if (Command::testCommandsEnabled) { - // Leaked intentionally: a Command registers itself when constructed. - new CapTrunc(); - } - return Status::OK(); - } - - // Testing-only, enabled via command line. - class EmptyCapped : public Command { - public: - EmptyCapped() : Command( "emptycapped" ) {} - virtual bool slaveOk() const { return false; } - virtual LockType locktype() const { return WRITE; } - virtual bool logTheOp() { return true; } - // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} - virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { - string coll = cmdObj[ "emptycapped" ].valuestrsafe(); - uassert( 13428, "emptycapped must specify a collection", !coll.empty() ); - string ns = dbname + "." + coll; - NamespaceDetails *nsd = nsdetails( ns ); - massert( 13429, "emptycapped no such collection", nsd ); - nsd->emptyCappedCollection( ns.c_str() ); - return true; - } - }; - MONGO_INITIALIZER(RegisterEmptyCappedCmd)(InitializerContext* context) { - if (Command::testCommandsEnabled) { - // Leaked intentionally: a Command registers itself when constructed. - new EmptyCapped(); - } - return Status::OK(); - } + } availableQueryOptionsCmd; bool _execCommand(Command *c, const string& dbname, @@ -1942,10 +1403,65 @@ namespace mongo { } } + /* Sometimes we cannot set maintenance mode, in which case the call to setMaintenanceMode will + return false. This class does not treat that case as an error which means that anybody + using it is assuming it is ok to continue execution without maintenance mode. This + assumption needs to be audited and documented. */ + class MaintenanceModeSetter { + public: + MaintenanceModeSetter() : maintenanceModeSet(theReplSet->setMaintenanceMode(true)) {} + ~MaintenanceModeSetter() { + if(maintenanceModeSet) + theReplSet->setMaintenanceMode(false); + } + private: + bool maintenanceModeSet; + }; + + + /** + * RAII class to optionally set an impersonated username list into the authorization session + * for the duration of the life of this object + */ + class ImpersonationSessionGuard { + MONGO_DISALLOW_COPYING(ImpersonationSessionGuard); + public: + ImpersonationSessionGuard(AuthorizationSession* authSession, + bool fieldIsPresent, + const std::vector &parsedUserNames, + const std::vector &parsedRoleNames): + _authSession(authSession), _impersonation(false) { + if (fieldIsPresent) { + massert(17317, "impersonation unexpectedly active", + !authSession->isImpersonating()); + authSession->setImpersonatedUserData(parsedUserNames, parsedRoleNames); + _impersonation = true; + } + } + ~ImpersonationSessionGuard() { + if (_impersonation) { + _authSession->clearImpersonatedUserData(); + } + } + private: + AuthorizationSession* _authSession; + bool _impersonation; + }; + + namespace { + void appendGLEHelperData(BSONObjBuilder& bob, const OpTime& opTime, const OID& oid) { + BSONObjBuilder subobj(bob.subobjStart(kGLEStatsFieldName)); + subobj.appendTimestamp(kGLEStatsLastOpTimeFieldName, opTime.asDate()); + subobj.appendOID(kGLEStatsElectionIdFieldName, const_cast(&oid)); + subobj.done(); + } + } + /** * this handles - auth - maintenance mode + - opcounters - locking - context then calls run() @@ -1957,35 +1473,8 @@ namespace mongo { BSONObj& cmdObj, BSONObjBuilder& result, bool fromRepl ) { - std::string dbname = nsToDatabase( cmdns ); - - if (c->adminOnly() && c->localHostOnlyIfNoAuth(cmdObj) && noauth && - !client.getIsLocalHostConnection()) { - log() << "command denied: " << cmdObj.toString() << endl; - appendCommandStatus(result, - false, - "unauthorized: this command must run from localhost when running " - "db without auth"); - return; - } - - if ( c->adminOnly() && ! fromRepl && dbname != "admin" ) { - log() << "command denied: " << cmdObj.toString() << endl; - appendCommandStatus(result, false, "access denied; use admin db"); - return; - } - - if (!noauth && c->requiresAuth()) { - std::vector privileges; - c->addRequiredPrivileges(dbname, cmdObj, &privileges); - Status status = client.getAuthorizationManager()->checkAuthForPrivileges(privileges); - if (!status.isOK()) { - log() << "command denied: " << cmdObj.toString() << endl; - appendCommandStatus(result, false, status.reason()); - return; - } - } + scoped_ptr mmSetter; if ( cmdObj["help"].trueValue() ) { client.curop()->ensureStarted(); @@ -1998,15 +1487,39 @@ namespace mongo { return; } - bool canRunHere = - isMaster( dbname.c_str() ) || - c->slaveOk() || - ( c->slaveOverrideOk() && ( queryOptions & QueryOption_SlaveOk ) ) || - fromRepl; + // Handle command option impersonatedUsers and impersonatedRoles. + // This must come before _checkAuthorization(), as there is some command parsing logic + // in that code path that must not see the impersonated user and roles array elements. + std::vector parsedUserNames; + std::vector parsedRoleNames; + AuthorizationSession* authSession = client.getAuthorizationSession(); + + bool rolesFieldIsPresent = false; + bool usersFieldIsPresent = false; + audit::parseAndRemoveImpersonatedRolesField(cmdObj, + authSession, + &parsedRoleNames, + &rolesFieldIsPresent); + audit::parseAndRemoveImpersonatedUsersField(cmdObj, + authSession, + &parsedUserNames, + &usersFieldIsPresent); + if (rolesFieldIsPresent != usersFieldIsPresent) { + // If there is a version mismatch between the mongos and the mongod, + // the mongos may fail to pass the role information, causing an error. + Status s(ErrorCodes::IncompatibleAuditMetadata, + "Audit metadata does not include both user and role information."); + appendCommandStatus(result, s); + return; + } + ImpersonationSessionGuard impersonationSession(authSession, + usersFieldIsPresent, + parsedUserNames, + parsedRoleNames); - if ( ! canRunHere ) { - result.append( "note" , "from execCommand" ); - appendCommandStatus(result, false, "not master"); + Status status = _checkAuthorization(c, &client, dbname, cmdObj, fromRepl); + if (!status.isOK()) { + appendCommandStatus(result, status); return; } @@ -2016,11 +1529,44 @@ namespace mongo { return; } - if ( c->adminOnly() ) + if ( c->adminOnly() ) { LOG( 2 ) << "command: " << cmdObj << endl; + } + + client.curop()->setCommand(c); - if (c->maintenanceMode() && theReplSet && theReplSet->isSecondary()) { - theReplSet->setMaintenanceMode(true); + if (c->maintenanceMode() && theReplSet) { + mmSetter.reset(new MaintenanceModeSetter()); + } + + if (c->shouldAffectCommandCounter()) { + // If !fromRepl, globalOpCounters need to be incremented. Otherwise, replOpCounters + // need to be incremented. + OpCounters* opCounters = fromRepl ? &replOpCounters : &globalOpCounters; + opCounters->gotCommand(); + } + + // Handle command option maxTimeMS. + StatusWith maxTimeMS = LiteParsedQuery::parseMaxTimeMSCommand(cmdObj); + if (!maxTimeMS.isOK()) { + appendCommandStatus(result, false, maxTimeMS.getStatus().reason()); + return; + } + if (cmdObj.hasField("$maxTimeMS")) { + appendCommandStatus(result, + false, + "no such command option $maxTimeMS; use maxTimeMS instead"); + return; + } + + client.curop()->setMaxTimeMicros(static_cast(maxTimeMS.getValue()) + * 1000); + try { + killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. + } + catch (UserException& e) { + appendCommandStatus(result, e.toStatus()); + return; } std::string errmsg; @@ -2028,6 +1574,19 @@ namespace mongo { if ( c->locktype() == Command::NONE ) { verify( !c->lockGlobally() ); + + bool canRunHere = + isMaster( dbname.c_str() ) || + c->slaveOk() || + ( c->slaveOverrideOk() && ( queryOptions & QueryOption_SlaveOk ) ) || + fromRepl; + + if ( ! canRunHere ) { + result.append( "note" , "from execCommand" ); + appendCommandStatus(result, false, "not master"); + return; + } + // we also trust that this won't crash retval = true; @@ -2043,7 +1602,20 @@ namespace mongo { scoped_ptr lk; if( c->lockGlobally() ) lk.reset( new Lock::GlobalRead() ); - Client::ReadContext ctx(ns , dbpath); // read locks + Client::ReadContext ctx(ns, storageGlobalParams.dbpath); // read locks + + bool canRunHere = + isMaster( dbname.c_str() ) || + c->slaveOk() || + ( c->slaveOverrideOk() && ( queryOptions & QueryOption_SlaveOk ) ) || + fromRepl; + + if ( ! canRunHere ) { + result.append( "note" , "from execCommand" ); + appendCommandStatus(result, false, "not master"); + return; + } + client.curop()->ensureStarted(); retval = _execCommand(c, dbname, cmdObj, queryOptions, errmsg, result, fromRepl); } @@ -2056,25 +1628,43 @@ namespace mongo { } if( global && Lock::isLocked() == 'w' ) { // can't go w->W - log() << "need glboal W lock but already have w on command : " << cmdObj.toString() << endl; + log() << "need global W lock but already have w on command : " << cmdObj.toString() << endl; } } scoped_ptr lk( global ? static_cast( new Lock::GlobalWrite() ) : static_cast( new Lock::DBWrite( dbname ) ) ); + + bool canRunHere = + isMaster( dbname.c_str() ) || + c->slaveOk() || + ( c->slaveOverrideOk() && ( queryOptions & QueryOption_SlaveOk ) ) || + fromRepl; + + if ( ! canRunHere ) { + result.append( "note" , "from execCommand" ); + appendCommandStatus(result, false, "not master"); + return; + } + client.curop()->ensureStarted(); - Client::Context ctx(dbname, dbpath); + Client::Context ctx(dbname, storageGlobalParams.dbpath); retval = _execCommand(c, dbname, cmdObj, queryOptions, errmsg, result, fromRepl); if ( retval && c->logTheOp() && ! fromRepl ) { logOp("c", cmdns, cmdObj); } } - if (c->maintenanceMode() && theReplSet) { - theReplSet->setMaintenanceMode(false); - } - appendCommandStatus(result, retval, errmsg); + + // For commands from mongos, append some info to help getLastError(w) work. + if (theReplSet && shardingState.enabled()) { + // Detect mongos connections by looking for setShardVersion to have been run previously + // on this connection. + if (shardingState.needCollectionMetadata(dbname)) { + appendGLEHelperData(result, client.getLastOp(), theReplSet->getElectionId()); + } + } return; } @@ -2089,8 +1679,7 @@ namespace mongo { bool _runCommands(const char *ns, BSONObj& _cmdobj, BufBuilder &b, BSONObjBuilder& anObjBuilder, bool fromRepl, int queryOptions) { string dbname = nsToDatabase( ns ); - if( logLevel >= 1 ) - log() << "run command " << ns << ' ' << _cmdobj << endl; + LOG(2) << "run command " << ns << ' ' << _cmdobj << endl; const char *p = strchr(ns, '.'); if ( !p ) return false; @@ -2104,6 +1693,16 @@ namespace mongo { : str::equals("query", e.fieldName()))) { jsobj = e.embeddedObject(); + if (_cmdobj.hasField("$maxTimeMS")) { + Command::appendCommandStatus(anObjBuilder, + false, + "cannot use $maxTimeMS query option with " + "commands; use maxTimeMS command option " + "instead"); + BSONObj x = anObjBuilder.done(); + b.appendBuf(x.objdata(), x.objsize()); + return true; + } } else { jsobj = _cmdobj; @@ -2129,11 +1728,12 @@ namespace mongo { Command::appendCommandStatus(anObjBuilder, false, str::stream() << "no such cmd: " << e.fieldName()); + anObjBuilder.append("code", ErrorCodes::CommandNotFound); anObjBuilder.append("bad cmd" , _cmdobj ); } BSONObj x = anObjBuilder.done(); - b.appendBuf((void*) x.objdata(), x.objsize()); + b.appendBuf(x.objdata(), x.objsize()); return true; } diff --git a/src/mongo/db/dbcommands_admin.cpp b/src/mongo/db/dbcommands_admin.cpp index 623a397201c..6fca2d5908a 100644 --- a/src/mongo/db/dbcommands_admin.cpp +++ b/src/mongo/db/dbcommands_admin.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ /** @@ -34,15 +46,16 @@ #include "mongo/base/status.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/privilege.h" -#include "mongo/db/cmdline.h" #include "mongo/db/commands.h" #include "mongo/db/curop-inl.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index/index_descriptor.h" #include "mongo/db/jsobj.h" #include "mongo/db/kill_current_op.h" -#include "mongo/db/namespace-inl.h" #include "mongo/db/pdfile.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/storage_options.h" #include "mongo/scripting/engine.h" #include "mongo/util/alignedbuilder.h" #include "mongo/util/background.h" @@ -66,7 +79,6 @@ namespace mongo { virtual bool adminOnly() const { return true; } virtual void help(stringstream& h) const { h << "test how long to write and fsync to a test file in the journal/ directory"; } // No auth needed because it only works when enabled via command line. - virtual bool requiresAuth() { return false; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) {} @@ -123,7 +135,8 @@ namespace mongo { catch(...) { } try { - result.append("onSamePartition", onSamePartition(dur::getJournalDir().string(), dbpath)); + result.append("onSamePartition", onSamePartition(dur::getJournalDir().string(), + storageGlobalParams.dbpath)); } catch(...) { } @@ -138,322 +151,6 @@ namespace mongo { return Status::OK(); } - class ValidateCmd : public Command { - public: - ValidateCmd() : Command( "validate" ) {} - - virtual bool slaveOk() const { - return true; - } - - virtual void help(stringstream& h) const { h << "Validate contents of a namespace by scanning its data structures for correctness. Slow.\n" - "Add full:true option to do a more thorough check"; } - - virtual LockType locktype() const { return READ; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::validate); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - //{ validate: "collectionnamewithoutthedbpart" [, scandata: ] [, full: } */ - - bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { - string ns = dbname + "." + cmdObj.firstElement().valuestrsafe(); - NamespaceDetails * d = nsdetails( ns ); - if ( !cmdLine.quiet ) - tlog() << "CMD: validate " << ns << endl; - - if ( ! d ) { - errmsg = "ns not found"; - return false; - } - - result.append( "ns", ns ); - validateNS( ns.c_str() , d, cmdObj, result); - return true; - } - - private: - void validateNS(const char *ns, - NamespaceDetails *d, - const BSONObj& cmdObj, - BSONObjBuilder& result) { - const bool full = cmdObj["full"].trueValue(); - const bool scanData = full || cmdObj["scandata"].trueValue(); - - bool valid = true; - BSONArrayBuilder errors; // explanation(s) for why valid = false - if ( d->isCapped() ){ - result.append("capped", d->isCapped()); - result.appendNumber("max", d->maxCappedDocs()); - } - - result.append("firstExtent", str::stream() << d->firstExtent.toString() - << " ns:" << d->firstExtent.ext()->nsDiagnostic.toString()); - result.append( "lastExtent", str::stream() << d->lastExtent.toString() - << " ns:" << d->lastExtent.ext()->nsDiagnostic.toString()); - - BSONArrayBuilder extentData; - int extentCount = 0; - try { - d->firstExtent.ext()->assertOk(); - d->lastExtent.ext()->assertOk(); - - DiskLoc extentDiskLoc = d->firstExtent; - while (!extentDiskLoc.isNull()) { - Extent* thisExtent = extentDiskLoc.ext(); - if (full) { - extentData << thisExtent->dump(); - } - if (!thisExtent->validates(extentDiskLoc, &errors)) { - valid = false; - } - DiskLoc nextDiskLoc = thisExtent->xnext; - if (extentCount > 0 && !nextDiskLoc.isNull() - && nextDiskLoc.ext()->xprev != extentDiskLoc) { - StringBuilder sb; - sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString() - << " in extent " << nextDiskLoc.toString() - << " does not point to extent " << extentDiskLoc.toString(); - errors << sb.str(); - valid = false; - } - if (nextDiskLoc.isNull() && extentDiskLoc != d->lastExtent) { - StringBuilder sb; - sb << "'lastExtent' pointer " << d->lastExtent.toString() - << " does not point to last extent in list " << extentDiskLoc.toString(); - errors << sb.str(); - valid = false; - } - extentDiskLoc = nextDiskLoc; - extentCount++; - killCurrentOp.checkForInterrupt(); - } - } - catch (const DBException& e) { - StringBuilder sb; - sb << "exception validating extent " << extentCount - << ": " << e.what(); - errors << sb.str(); - valid = false; - } - result.append("extentCount", extentCount); - - if ( full ) - result.appendArray( "extents" , extentData.arr() ); - - result.appendNumber("datasize", d->stats.datasize); - result.appendNumber("nrecords", d->stats.nrecords); - result.appendNumber("lastExtentSize", d->lastExtentSize); - result.appendNumber("padding", d->paddingFactor()); - - try { - - bool testingLastExtent = false; - try { - if (d->firstExtent.isNull()) { - errors << "'firstExtent' pointer is null"; - valid=false; - } - else { - result.append("firstExtentDetails", d->firstExtent.ext()->dump()); - if (!d->firstExtent.ext()->xprev.isNull()) { - StringBuilder sb; - sb << "'xprev' pointer in 'firstExtent' " << d->firstExtent.toString() - << " is " << d->firstExtent.ext()->xprev.toString() - << ", should be null"; - errors << sb.str(); - valid=false; - } - } - testingLastExtent = true; - if (d->lastExtent.isNull()) { - errors << "'lastExtent' pointer is null"; - valid=false; - } - else { - if (d->firstExtent != d->lastExtent) { - result.append("lastExtentDetails", d->lastExtent.ext()->dump()); - if (!d->lastExtent.ext()->xnext.isNull()) { - StringBuilder sb; - sb << "'xnext' pointer in 'lastExtent' " << d->lastExtent.toString() - << " is " << d->lastExtent.ext()->xnext.toString() - << ", should be null"; - errors << sb.str(); - valid = false; - } - } - } - } - catch (const DBException& e) { - StringBuilder sb; - sb << "exception processing '" - << (testingLastExtent ? "lastExtent" : "firstExtent") - << "': " << e.what(); - errors << sb.str(); - valid = false; - } - - set recs; - if( scanData ) { - shared_ptr c = theDataFileMgr.findAll(ns); - int n = 0; - int nInvalid = 0; - long long len = 0; - long long nlen = 0; - int outOfOrder = 0; - DiskLoc cl_last; - while ( c->ok() ) { - n++; - - DiskLoc cl = c->currLoc(); - if ( n < 1000000 ) - recs.insert(cl); - if ( d->isCapped() ) { - if ( cl < cl_last ) - outOfOrder++; - cl_last = cl; - } - - Record *r = c->_current(); - len += r->lengthWithHeaders(); - nlen += r->netLength(); - - if (full){ - BSONObj obj = BSONObj::make(r); - if (!obj.isValid() || !obj.valid()){ // both fast and deep checks - valid = false; - if (nInvalid == 0) // only log once; - errors << "invalid bson object detected (see logs for more info)"; - - nInvalid++; - if (strcmp("_id", obj.firstElementFieldName()) == 0){ - try { - obj.firstElement().validate(); // throws on error - log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl; - } - catch(...){ - log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl; - } - } - else { - log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl; - } - } - } - - c->advance(); - } - if ( d->isCapped() && !d->capLooped() ) { - result.append("cappedOutOfOrder", outOfOrder); - if ( outOfOrder > 1 ) { - valid = false; - errors << "too many out of order records"; - } - } - result.append("objectsFound", n); - - if (full) { - result.append("invalidObjects", nInvalid); - } - - result.appendNumber("bytesWithHeaders", len); - result.appendNumber("bytesWithoutHeaders", nlen); - } - - BSONArrayBuilder deletedListArray; - for ( int i = 0; i < Buckets; i++ ) { - deletedListArray << d->deletedList[i].isNull(); - } - - int ndel = 0; - long long delSize = 0; - int incorrect = 0; - for ( int i = 0; i < Buckets; i++ ) { - DiskLoc loc = d->deletedList[i]; - try { - int k = 0; - while ( !loc.isNull() ) { - if ( recs.count(loc) ) - incorrect++; - ndel++; - - if ( loc.questionable() ) { - if( d->isCapped() && !loc.isValid() && i == 1 ) { - /* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid - see comments in namespace.h - */ - break; - } - - string err( str::stream() << "bad pointer in deleted record list: " - << loc.toString() - << " bucket: " << i - << " k: " << k ); - errors << err; - valid = false; - break; - } - DeletedRecord *d = loc.drec(); - delSize += d->lengthWithHeaders(); - loc = d->nextDeleted(); - k++; - killCurrentOp.checkForInterrupt(); - } - } - catch (...) { - errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i)); - valid = false; - } - } - result.appendNumber("deletedCount", ndel); - result.appendNumber("deletedSize", delSize); - - if ( incorrect ) { - errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list"); - valid = false; - } - - int idxn = 0; - try { - result.append("nIndexes", d->nIndexes); - BSONObjBuilder indexes; // not using subObjStart to be exception safe - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - IndexDetails& id = i.next(); - log() << "validating index " << idxn << ": " << id.indexNamespace() << endl; - long long keys = id.idxInterface().fullValidate(id.head, id.keyPattern()); - indexes.appendNumber(id.indexNamespace(), keys); - idxn++; - } - result.append("keysPerIndex", indexes.done()); - } - catch (...) { - errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn)); - valid=false; - } - - } - catch (AssertionException) { - errors << "exception during validate"; - valid = false; - } - - result.appendBool("valid", valid); - result.append("errors", errors.arr()); - - if ( !full ){ - result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan."); - } - - if ( !valid ) { - result.append("advice", "ns corrupt, requires repair"); - } - - } - } validateCmd; } diff --git a/src/mongo/db/dbcommands_generic.cpp b/src/mongo/db/dbcommands_generic.cpp index e4528e605ae..22aa29fc16f 100644 --- a/src/mongo/db/dbcommands_generic.cpp +++ b/src/mongo/db/dbcommands_generic.cpp @@ -14,38 +14,54 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" + +#include +#include "mongo/bson/util/builder.h" +#include "mongo/client/dbclient_rs.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/privilege.h" -#include "pdfile.h" -#include "jsobj.h" -#include "../bson/util/builder.h" -#include -#include "introspect.h" -#include "../client/dbclient_rs.h" -#include "../util/lruishmap.h" -#include "../util/md5.hpp" -#include "../util/processinfo.h" -#include "json.h" -#include "repl.h" -#include "repl_block.h" -#include "replutil.h" -#include "commands.h" -#include "db.h" -#include "instance.h" -#include "lasterror.h" -#include "../scripting/engine.h" -#include "stats/counters.h" -#include "background.h" -#include "../util/version.h" -#include "../util/ramlog.h" -#include "repl/multicmd.h" -#include "server.h" +#include "mongo/db/background.h" +#include "mongo/db/commands.h" +#include "mongo/db/commands/shutdown.h" +#include "mongo/db/db.h" +#include "mongo/db/instance.h" +#include "mongo/db/introspect.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/lasterror.h" +#include "mongo/db/log_process_details.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/repl/multicmd.h" +#include "mongo/db/repl/write_concern.h" +#include "mongo/db/server_options.h" +#include "mongo/db/stats/counters.h" +#include "mongo/scripting/engine.h" +#include "mongo/server.h" +#include "mongo/util/fail_point.h" +#include "mongo/util/fail_point_service.h" +#include "mongo/util/lruishmap.h" +#include "mongo/util/md5.hpp" +#include "mongo/util/processinfo.h" +#include "mongo/util/ramlog.h" +#include "mongo/util/version_reporting.h" namespace mongo { @@ -107,7 +123,6 @@ namespace mongo { CmdBuildInfo() : Command( "buildInfo", true, "buildinfo" ) {} virtual bool slaveOk() const { return true; } virtual bool adminOnly() const { return false; } - virtual bool requiresAuth() { return false; } virtual LockType locktype() const { return NONE; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, @@ -137,7 +152,6 @@ namespace mongo { virtual bool slaveOk() const { return true; } virtual void help( stringstream &help ) const { help << "a way to check that the server is alive. responds immediately even if server is in a db lock."; } virtual LockType locktype() const { return NONE; } - virtual bool requiresAuth() { return false; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) {} // No auth required @@ -152,7 +166,6 @@ namespace mongo { FeaturesCmd() : Command( "features", true ) {} void help(stringstream& h) const { h << "return build level feature settings"; } virtual bool slaveOk() const { return true; } - virtual bool readOnly() { return true; } virtual LockType locktype() const { return NONE; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, @@ -190,7 +203,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::hostInfo); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { ProcessInfo p; @@ -227,10 +240,13 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::logRotate); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } virtual bool run(const string& ns, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - return rotateLogs(); + bool didRotate = rotateLogs(); + if (didRotate) + logProcessDetailsForLogRotate(); + return didRotate; } } logRotateCmd; @@ -261,7 +277,6 @@ namespace mongo { c->help( help ); temp.append( "help" , help.str() ); } - temp.append( "lockType" , c->locktype() ); temp.append( "slaveOk" , c->slaveOk() ); temp.append( "adminOnly" , c->adminOnly() ); //optionally indicates that the command can be forced to run on a slave/secondary @@ -275,7 +290,20 @@ namespace mongo { } listCommandsCmd; + namespace { + MONGO_FP_DECLARE(crashOnShutdown); + + int* volatile illegalAddress; + } // namespace + bool CmdShutdown::shutdownHelper() { + MONGO_FAIL_POINT_BLOCK(crashOnShutdown, crashBlock) { + const std::string crashHow = crashBlock.getData()["how"].str(); + if (crashHow == "fault") { + ++*illegalAddress; + } + ::abort(); + } Client * c = currentClient.get(); if ( c ) { c->shutdown(); @@ -311,21 +339,6 @@ namespace mongo { } } cmdForceError; - class AvailableQueryOptions : public Command { - public: - AvailableQueryOptions() : Command( "availableQueryOptions" , false , "availablequeryoptions" ) {} - virtual bool slaveOk() const { return true; } - virtual LockType locktype() const { return NONE; } - virtual bool requiresAuth() { return false; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} // No auth required - virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { - result << "options" << QueryOption_AllSupported; - return true; - } - } availableQueryOptionsCmd; - class GetLogCmd : public Command { public: GetLogCmd() : Command( "getLog" ){} @@ -338,7 +351,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::getLog); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } virtual void help( stringstream& help ) const { help << "{ getLog : '*' } OR { getLog : 'global' }"; @@ -358,20 +371,18 @@ namespace mongo { result.appendArray( "names" , arr.arr() ); } else { - RamLog* rl = RamLog::get( p ); - if ( ! rl ) { + RamLog* ramlog = RamLog::getIfExists(p); + if ( ! ramlog ) { errmsg = str::stream() << "no RamLog named: " << p; return false; } + RamLog::LineIterator rl(ramlog); - result.appendNumber( "totalLinesWritten", rl->getTotalLinesWritten() ); - - vector lines; - rl->get( lines ); + result.appendNumber( "totalLinesWritten", rl.getTotalLinesWritten() ); BSONArrayBuilder arr( result.subarrayStart( "log" ) ); - for ( unsigned i=0; i* out) { ActionSet actions; actions.addAction(ActionType::getCmdLineOpts); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } virtual bool run(const string&, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - result.append("argv", CmdLine::getArgvArray()); - result.append("parsed", CmdLine::getParsedOpts()); + result.append("argv", serverGlobalParams.argvArray); + result.append("parsed", serverGlobalParams.parsedOpts); return true; } diff --git a/src/mongo/db/dbeval.cpp b/src/mongo/db/dbeval.cpp index 5a6cc464c34..c25588c8bf1 100644 --- a/src/mongo/db/dbeval.cpp +++ b/src/mongo/db/dbeval.cpp @@ -14,21 +14,35 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "pdfile.h" -#include "jsobj.h" -#include "../bson/util/builder.h" +#include "mongo/pch.h" + #include -#include "introspect.h" -#include "../util/lruishmap.h" -#include "json.h" -#include "repl.h" -#include "commands.h" -#include "cmdline.h" -#include "../scripting/engine.h" +#include "mongo/bson/util/builder.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/commands.h" +#include "mongo/db/introspect.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/pdfile.h" +#include "mongo/scripting/engine.h" +#include "mongo/util/lruishmap.h" namespace mongo { @@ -57,7 +71,9 @@ namespace mongo { return false; } - auto_ptr s = globalScriptEngine->getPooledScope( dbName, "dbeval" ); + const string userToken = ClientBasic::getCurrent()->getAuthorizationSession() + ->getAuthenticatedUserNamesToken(); + auto_ptr s = globalScriptEngine->getPooledScope( dbName, "dbeval" + userToken ); ScriptingFunction f = s->createFunction(code); if ( f == 0 ) { errmsg = (string)"compile failed: " + s->getError(); @@ -83,9 +99,9 @@ namespace mongo { int res; { Timer t; - res = s->invoke(f, &args, 0, cmdLine.quota ? 10 * 60 * 1000 : 0 ); + res = s->invoke(f, &args, 0, storageGlobalParams.quota ? 10 * 60 * 1000 : 0); int m = t.millis(); - if ( m > cmdLine.slowMS ) { + if (m > serverGlobalParams.slowMS) { out() << "dbeval slow, time: " << dec << m << "ms " << dbName << endl; if ( m >= 1000 ) log() << code << endl; else OCCASIONALLY log() << code << endl; @@ -119,9 +135,8 @@ namespace mongo { virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { - // $eval can do pretty much anything, so require all privileges. - out->push_back(Privilege(PrivilegeSet::WILDCARD_RESOURCE, - AuthorizationManager::getAllUserActions())); + + RoleGraph::generateUniversalPrivileges(out); } CmdEval() : Command("eval", false, "$eval") { } bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index 262c2480a4f..6a6156b8723 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -14,60 +14,77 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "db.h" -#include "dbhelpers.h" -#include "json.h" -#include "mongo/db/btreecursor.h" -#include "pdfile.h" -#include "oplog.h" -#include "ops/update.h" -#include "ops/delete.h" -#include "queryoptimizercursor.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/pagefault.h" -#include "mongo/db/repl_block.h" -#include "mongo/s/d_logic.h" +#include "mongo/pch.h" -#include +#include "mongo/db/dbhelpers.h" #include #include +#include + +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/db.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/json.h" +#include "mongo/db/index/btree_access_method.h" +#include "mongo/db/ops/delete.h" +#include "mongo/db/ops/update.h" +#include "mongo/db/ops/update_lifecycle_impl.h" +#include "mongo/db/ops/update_request.h" +#include "mongo/db/ops/update_result.h" +#include "mongo/db/pagefault.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/write_concern.h" +#include "mongo/db/storage_options.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/s/d_logic.h" namespace mongo { const BSONObj reverseNaturalObj = BSON( "$natural" << -1 ); void Helpers::ensureIndex(const char *ns, BSONObj keyPattern, bool unique, const char *name) { - NamespaceDetails *d = nsdetails(ns); - if( d == 0 ) - return; + Database* db = cc().database(); + verify(db); - { - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - if( i.next().keyPattern().woCompare(keyPattern) == 0 ) - return; - } - } - - if( d->nIndexes >= NamespaceDetails::NIndexesMax ) { - problem() << "Helper::ensureIndex fails, MaxIndexes exceeded " << ns << '\n'; + Collection* collection = db->getCollection( ns ); + if ( !collection ) return; - } - string system_indexes = cc().database()->name + ".system.indexes"; + ensureIndex( collection, keyPattern, unique, name ); + } + void Helpers::ensureIndex(Collection* collection, + BSONObj keyPattern, bool unique, const char *name) { BSONObjBuilder b; b.append("name", name); - b.append("ns", ns); + b.append("ns", collection->ns()); b.append("key", keyPattern); b.appendBool("unique", unique); BSONObj o = b.done(); - theDataFileMgr.insert(system_indexes.c_str(), o.objdata(), o.objsize()); + Status status = collection->getIndexCatalog()->createIndex( o, false ); + if ( status.code() == ErrorCodes::IndexAlreadyExists ) + return; + uassertStatusOK( status ); } /* fetch a single object from collection ns that matches query @@ -85,108 +102,118 @@ namespace mongo { set your db SavedContext first */ DiskLoc Helpers::findOne(const StringData& ns, const BSONObj &query, bool requireIndex) { - shared_ptr c = - NamespaceDetailsTransient::getCursor( ns, query, BSONObj(), - requireIndex ? - QueryPlanSelectionPolicy::indexOnly() : - QueryPlanSelectionPolicy::any() ); - while( c->ok() ) { - if ( c->currentMatches() && !c->getsetdup( c->currLoc() ) ) { - return c->currLoc(); - } - c->advance(); + CanonicalQuery* cq; + massert(17244, "Could not canonicalize " + query.toString(), + CanonicalQuery::canonicalize(ns.toString(), query, &cq).isOK()); + + Runner* rawRunner; + size_t options = requireIndex ? QueryPlannerParams::NO_TABLE_SCAN : QueryPlannerParams::DEFAULT; + massert(17245, "Could not get runner for query " + query.toString(), + getRunner(cq, &rawRunner, options).isOK()); + + auto_ptr runner(rawRunner); + Runner::RunnerState state; + DiskLoc loc; + if (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &loc))) { + return loc; } return DiskLoc(); } bool Helpers::findById(Client& c, const char *ns, BSONObj query, BSONObj& result , - bool * nsFound , bool * indexFound ) { + bool* nsFound , bool* indexFound ) { Lock::assertAtLeastReadLocked(ns); Database *database = c.database(); verify( database ); - NamespaceDetails *d = database->namespaceIndex.details(ns); - if ( ! d ) + + Collection* collection = database->getCollection( ns ); + if ( !collection ) { return false; + } + if ( nsFound ) - *nsFound = 1; + *nsFound = true; + + IndexCatalog* catalog = collection->getIndexCatalog(); + const IndexDescriptor* desc = catalog->findIdIndex(); - int idxNo = d->findIdIndex(); - if ( idxNo < 0 ) + if ( !desc ) return false; + if ( indexFound ) *indexFound = 1; - IndexDetails& i = d->idx( idxNo ); + // See SERVER-12397. This may not always be true. + BtreeBasedAccessMethod* accessMethod = + static_cast(catalog->getIndex( desc )); - BSONObj key = i.getKeyFromQuery( query ); - - DiskLoc loc = i.idxInterface().findSingle(i , i.head , key); + DiskLoc loc = accessMethod->findSingle( query["_id"].wrap() ); if ( loc.isNull() ) return false; - result = loc.obj(); + result = collection->docFor( loc ); return true; } - DiskLoc Helpers::findById(NamespaceDetails *d, BSONObj idquery) { - verify(d); - int idxNo = d->findIdIndex(); - uassert(13430, "no _id index", idxNo>=0); - IndexDetails& i = d->idx( idxNo ); - BSONObj key = i.getKeyFromQuery( idquery ); - return i.idxInterface().findSingle(i , i.head , key); + DiskLoc Helpers::findById(Collection* collection, const BSONObj& idquery) { + verify(collection); + IndexCatalog* catalog = collection->getIndexCatalog(); + const IndexDescriptor* desc = catalog->findIdIndex(); + uassert(13430, "no _id index", desc); + // See SERVER-12397. This may not always be true. + BtreeBasedAccessMethod* accessMethod = + static_cast(catalog->getIndex( desc )); + return accessMethod->findSingle( idquery["_id"].wrap() ); } vector Helpers::findAll( const string& ns , const BSONObj& query ) { - Lock::assertAtLeastReadLocked( ns ); + Lock::assertAtLeastReadLocked(ns); + Client::Context ctx(ns); - vector all; + CanonicalQuery* cq; + uassert(17236, "Could not canonicalize " + query.toString(), + CanonicalQuery::canonicalize(ns, query, &cq).isOK()); - Client::Context tx( ns ); - - shared_ptr c = NamespaceDetailsTransient::getCursor( ns.c_str(), query ); + Runner* rawRunner; + uassert(17237, "Could not get runner for query " + query.toString(), + getRunner(cq, &rawRunner).isOK()); - while( c->ok() ) { - if ( c->currentMatches() && !c->getsetdup( c->currLoc() ) ) { - all.push_back( c->current() ); - } - c->advance(); + vector all; + + auto_ptr runner(rawRunner); + Runner::RunnerState state; + BSONObj obj; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { + all.push_back(obj); } return all; } bool Helpers::isEmpty(const char *ns) { - Client::Context context(ns, dbpath); - shared_ptr c = DataFileMgr::findAll(ns); - return !c->ok(); + Client::Context context(ns, storageGlobalParams.dbpath); + auto_ptr runner(InternalPlanner::collectionScan(ns)); + return Runner::RUNNER_EOF == runner->getNext(NULL, NULL); } /* Get the first object from a collection. Generally only useful if the collection - only ever has a single object -- which is a "singleton collection. + only ever has a single object -- which is a "singleton collection". Note that the + BSONObj returned is *not* owned and will become invalid if the database is closed. Returns: true if object exists. */ bool Helpers::getSingleton(const char *ns, BSONObj& result) { Client::Context context(ns); - - shared_ptr c = DataFileMgr::findAll(ns); - if ( !c->ok() ) { - context.getClient()->curop()->done(); - return false; - } - - result = c->current(); + auto_ptr runner(InternalPlanner::collectionScan(ns)); + Runner::RunnerState state = runner->getNext(&result, NULL); context.getClient()->curop()->done(); - return true; + return Runner::RUNNER_ADVANCED == state; } bool Helpers::getLast(const char *ns, BSONObj& result) { Client::Context ctx(ns); - shared_ptr c = findTableScan(ns, reverseNaturalObj); - if( !c->ok() ) - return false; - result = c->current(); - return true; + auto_ptr runner(InternalPlanner::collectionScan(ns, InternalPlanner::BACKWARD)); + Runner::RunnerState state = runner->getNext(&result, NULL); + return Runner::RUNNER_ADVANCED == state; } void Helpers::upsert( const string& ns , const BSONObj& o, bool fromMigrate ) { @@ -196,20 +223,53 @@ namespace mongo { OpDebug debug; Client::Context context(ns); - updateObjects(ns.c_str(), o, /*pattern=*/id, /*upsert=*/true, /*multi=*/false , /*logtheop=*/true , debug, fromMigrate ); + + const NamespaceString requestNs(ns); + UpdateRequest request(requestNs); + + request.setQuery(id); + request.setUpdates(o); + request.setUpsert(); + request.setUpdateOpLog(); + request.setFromMigration(fromMigrate); + UpdateLifecycleImpl updateLifecycle(true, requestNs); + request.setLifecycle(&updateLifecycle); + + update(request, &debug); } void Helpers::putSingleton(const char *ns, BSONObj obj) { OpDebug debug; Client::Context context(ns); - updateObjects(ns, obj, /*pattern=*/BSONObj(), /*upsert=*/true, /*multi=*/false , /*logtheop=*/true , debug ); + + const NamespaceString requestNs(ns); + UpdateRequest request(requestNs); + + request.setUpdates(obj); + request.setUpsert(); + request.setUpdateOpLog(); + UpdateLifecycleImpl updateLifecycle(true, requestNs); + request.setLifecycle(&updateLifecycle); + + update(request, &debug); + context.getClient()->curop()->done(); } void Helpers::putSingletonGod(const char *ns, BSONObj obj, bool logTheOp) { OpDebug debug; Client::Context context(ns); - _updateObjects(/*god=*/true, ns, obj, /*pattern=*/BSONObj(), /*upsert=*/true, /*multi=*/false , logTheOp , debug ); + + const NamespaceString requestNs(ns); + UpdateRequest request(requestNs); + + request.setGod(); + request.setUpdates(obj); + request.setUpsert(); + request.setUpdateOpLog(logTheOp); + + update(request, &debug); + context.getClient()->curop()->done(); } @@ -229,17 +289,68 @@ namespace mongo { return kpBuilder.obj(); } - long long Helpers::removeRange( const string& ns , - const BSONObj& min , - const BSONObj& max , - const BSONObj& keyPattern , - bool maxInclusive , - bool secondaryThrottle , - RemoveCallback * callback, - bool fromMigrate, - bool onlyRemoveOrphanedDocs ) { + bool findShardKeyIndexPattern_inlock( const string& ns, + const BSONObj& shardKeyPattern, + BSONObj* indexPattern ) { + verify( Lock::isLocked() ); + Collection* collection = cc().database()->getCollection( ns ); + if ( !collection ) + return false; + + // Allow multiKey based on the invariant that shard keys must be single-valued. + // Therefore, any multi-key index prefixed by shard key cannot be multikey over + // the shard key fields. + const IndexDescriptor* idx = + collection->getIndexCatalog()->findIndexByPrefix(shardKeyPattern, + false /* allow multi key */); + if ( idx == NULL ) + return false; + *indexPattern = idx->keyPattern().getOwned(); + return true; + } + + bool findShardKeyIndexPattern( const string& ns, + const BSONObj& shardKeyPattern, + BSONObj* indexPattern ) { + Client::ReadContext context( ns ); + return findShardKeyIndexPattern_inlock( ns, shardKeyPattern, indexPattern ); + } + + long long Helpers::removeRange( const KeyRange& range, + bool maxInclusive, + bool secondaryThrottle, + RemoveSaver* callback, + bool fromMigrate, + bool onlyRemoveOrphanedDocs ) + { Timer rangeRemoveTimer; + const string& ns = range.ns; + + // The IndexChunk has a keyPattern that may apply to more than one index - we need to + // select the index and get the full index keyPattern here. + BSONObj indexKeyPatternDoc; + if ( !findShardKeyIndexPattern( ns, + range.keyPattern, + &indexKeyPatternDoc ) ) + { + warning() << "no index found to clean data over range of type " + << range.keyPattern << " in " << ns << endl; + return -1; + } + + KeyPattern indexKeyPattern( indexKeyPatternDoc ); + + // Extend bounds to match the index we found + + // Extend min to get (min, MinKey, MinKey, ....) + const BSONObj& min = + Helpers::toKeyFormat(indexKeyPattern.extendRangeBound(range.minKey, + false)); + // If upper bound is included, extend max to get (max, MaxKey, MaxKey, ...) + // If not included, extend max to get (max, MinKey, MinKey, ....) + const BSONObj& max = + Helpers::toKeyFormat( indexKeyPattern.extendRangeBound(range.maxKey,maxInclusive)); LOG(1) << "begin removal of " << min << " to " << max << " in " << ns << (secondaryThrottle ? " (waiting for secondaries)" : "" ) << endl; @@ -247,50 +358,61 @@ namespace mongo { Client& c = cc(); long long numDeleted = 0; - PageFaultRetryableSection pgrs; long long millisWaitingForReplication = 0; while ( 1 ) { - try { - + // Scoping for write lock. + { Client::WriteContext ctx(ns); + Collection* collection = ctx.ctx().db()->getCollection( ns ); + if ( !collection ) break; - scoped_ptr c; - - { - NamespaceDetails* nsd = nsdetails( ns ); - if ( ! nsd ) - break; - - int ii = nsd->findIndexByKeyPattern( keyPattern ); - verify( ii >= 0 ); - - IndexDetails& i = nsd->idx( ii ); - - // Extend min to get (min, MinKey, MinKey, ....) - KeyPattern kp( keyPattern ); - BSONObj newMin = Helpers::toKeyFormat( kp.extendRangeBound( min, false ) ); - // If upper bound is included, extend max to get (max, MaxKey, MaxKey, ...) - // If not included, extend max to get (max, MinKey, MinKey, ....) - BSONObj newMax = Helpers::toKeyFormat( kp.extendRangeBound(max, maxInclusive) ); - - c.reset( BtreeCursor::make( nsd, i, newMin, newMax, maxInclusive, 1 ) ); + int oldYieldCount = c.curop()->numYields(); + + IndexDescriptor* desc = + collection->getIndexCatalog()->findIndexByKeyPattern( indexKeyPattern.toBSON() ); + + auto_ptr runner(InternalPlanner::indexScan(collection, desc, min, max, + maxInclusive, + InternalPlanner::FORWARD, + InternalPlanner::IXSCAN_FETCH)); + + runner->setYieldPolicy(Runner::YIELD_AUTO); + + DiskLoc rloc; + BSONObj obj; + Runner::RunnerState state; + // This may yield so we cannot touch nsd after this. + state = runner->getNext(&obj, &rloc); + runner.reset(); + if (Runner::RUNNER_EOF == state) { break; } + + if (Runner::RUNNER_DEAD == state) { + warning() << "cursor died: aborting deletion for " + << min << " to " << max << " in " << ns + << endl; + break; } - - if ( ! c->ok() ) { - // we're done + + if (Runner::RUNNER_ERROR == state) { + warning() << "cursor error while trying to delete " + << min << " to " << max + << " in " << ns << ": " + << WorkingSetCommon::toStatusString(obj) << endl; break; } - - DiskLoc rloc = c->currLoc(); - BSONObj obj = c->current(); - // this is so that we don't have to handle this cursor in the delete code - c.reset(0); + verify(Runner::RUNNER_ADVANCED == state); - if (fromMigrate && onlyRemoveOrphanedDocs) { + int newYieldCount = c.curop()->numYields(); + if (oldYieldCount != newYieldCount && !_isMaster()) { + warning() << "current node is not primary anymore, " + << "aborting removeRange" << endl; + return numDeleted; + } + if ( onlyRemoveOrphanedDocs ) { // Do a final check in the write lock to make absolutely sure that our // collection hasn't been modified in a way that invalidates our migration // cleanup. @@ -300,30 +422,34 @@ namespace mongo { verify(shardingState.enabled()); // In write lock, so will be the most up-to-date version - ShardChunkManagerPtr managerNow = shardingState.getShardChunkManager(ns); - - if (!managerNow || managerNow->belongsToMe(obj)) { + CollectionMetadataPtr metadataNow = shardingState.getCollectionMetadata( ns ); + + bool docIsOrphan; + if ( metadataNow ) { + KeyPattern kp( metadataNow->getKeyPattern() ); + BSONObj key = kp.extractSingleKey( obj ); + docIsOrphan = !metadataNow->keyBelongsToMe( key ) + && !metadataNow->keyIsPending( key ); + } + else { + docIsOrphan = false; + } - warning() << "aborting migration cleanup for chunk " - << min << " to " << max - << (managerNow ? (string)" at document " + obj.toString() : "") + if ( !docIsOrphan ) { + warning() << "aborting migration cleanup for chunk " << min << " to " << max + << ( metadataNow ? (string) " at document " + obj.toString() : "" ) << ", collection " << ns << " has changed " << endl; - break; } } - + if ( callback ) callback->goingToDelete( obj ); - - logOp( "d" , ns.c_str() , rloc.obj()["_id"].wrap() , 0 , 0 , fromMigrate ); - theDataFileMgr.deleteRecord(ns.c_str() , rloc.rec(), rloc); + + logOp("d", ns.c_str(), obj["_id"].wrap(), 0, 0, fromMigrate); + c.database()->getCollection( ns )->deleteDocument( rloc ); numDeleted++; } - catch( PageFaultException& e ) { - e.touch(); - continue; - } Timer secondaryThrottleTime; @@ -341,7 +467,6 @@ namespace mongo { sleepmicros( micros ); } } - } if ( secondaryThrottle ) @@ -354,15 +479,103 @@ namespace mongo { return numDeleted; } + const long long Helpers::kMaxDocsPerChunk( 250000 ); + + // Used by migration clone step + // TODO: Cannot hook up quite yet due to _trackerLocks in shared migration code. + Status Helpers::getLocsInRange( const KeyRange& range, + long long maxChunkSizeBytes, + set* locs, + long long* numDocs, + long long* estChunkSizeBytes ) + { + const string ns = range.ns; + *estChunkSizeBytes = 0; + *numDocs = 0; + + Client::ReadContext ctx( ns ); + Collection* collection = ctx.ctx().db()->getCollection( ns ); + if ( !collection ) return Status( ErrorCodes::NamespaceNotFound, ns ); + + // Require single key + + IndexDescriptor *idx = + collection->getIndexCatalog()->findIndexByPrefix( range.keyPattern, true ); + + if ( idx == NULL ) { + return Status( ErrorCodes::IndexNotFound, range.keyPattern.toString() ); + } + + // use the average object size to estimate how many objects a full chunk would carry + // do that while traversing the chunk's range using the sharding index, below + // there's a fair amount of slack before we determine a chunk is too large because object + // sizes will vary + long long avgDocsWhenFull; + long long avgDocSizeBytes; + const long long totalDocsInNS = collection->numRecords(); + if ( totalDocsInNS > 0 ) { + // TODO: Figure out what's up here + avgDocSizeBytes = collection->details()->dataSize() / totalDocsInNS; + avgDocsWhenFull = maxChunkSizeBytes / avgDocSizeBytes; + avgDocsWhenFull = std::min( kMaxDocsPerChunk + 1, + 130 * avgDocsWhenFull / 100 /* slack */); + } + else { + avgDocSizeBytes = 0; + avgDocsWhenFull = kMaxDocsPerChunk + 1; + } + + // Assume both min and max non-empty, append MinKey's to make them fit chosen index + KeyPattern idxKeyPattern( idx->keyPattern() ); + BSONObj min = Helpers::toKeyFormat( idxKeyPattern.extendRangeBound( range.minKey, false ) ); + BSONObj max = Helpers::toKeyFormat( idxKeyPattern.extendRangeBound( range.maxKey, false ) ); + + + // do a full traversal of the chunk and don't stop even if we think it is a large chunk + // we want the number of records to better report, in that case + bool isLargeChunk = false; + long long docCount = 0; + + auto_ptr runner(InternalPlanner::indexScan(collection, idx, min, max, false)); + // we can afford to yield here because any change to the base data that we might miss is + // already being queued and will be migrated in the 'transferMods' stage + runner->setYieldPolicy(Runner::YIELD_AUTO); + + DiskLoc loc; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &loc))) { + if ( !isLargeChunk ) { + locs->insert( loc ); + } + + if ( ++docCount > avgDocsWhenFull ) { + isLargeChunk = true; + } + } + + *numDocs = docCount; + *estChunkSizeBytes = docCount * avgDocSizeBytes; + + if ( isLargeChunk ) { + stringstream ss; + ss << estChunkSizeBytes; + return Status( ErrorCodes::InvalidLength, ss.str() ); + } + + return Status::OK(); + } + + void Helpers::emptyCollection(const char *ns) { Client::Context context(ns); deleteObjects(ns, BSONObj(), false); } - RemoveSaver::RemoveSaver( const string& a , const string& b , const string& why) : _out(0) { + Helpers::RemoveSaver::RemoveSaver( const string& a , const string& b , const string& why) + : _out(0) { static int NUM = 0; - _root = dbpath; + _root = storageGlobalParams.dbpath; if ( a.size() ) _root /= a; if ( b.size() ) @@ -376,7 +589,7 @@ namespace mongo { _file /= ss.str(); } - RemoveSaver::~RemoveSaver() { + Helpers::RemoveSaver::~RemoveSaver() { if ( _out ) { _out->close(); delete _out; @@ -384,7 +597,7 @@ namespace mongo { } } - void RemoveSaver::goingToDelete( const BSONObj& o ) { + void Helpers::RemoveSaver::goingToDelete( const BSONObj& o ) { if ( ! _out ) { boost::filesystem::create_directories( _root ); _out = new ofstream(); diff --git a/src/mongo/db/dbhelpers.h b/src/mongo/db/dbhelpers.h index 9ec14031d9d..a8aebc58c25 100644 --- a/src/mongo/db/dbhelpers.h +++ b/src/mongo/db/dbhelpers.h @@ -1,43 +1,58 @@ -/* @file dbhelpers.h - - db helpers are helper functions and classes that let us easily manipulate the local - database instance in-proc. -*/ - /** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ + * Copyright (C) 2008 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #pragma once +// TODO: Remove #include "mongo/pch.h" -#include "client.h" -#include "db.h" + +#include "mongo/db/client.h" +#include "mongo/db/db.h" +#include "mongo/db/keypattern.h" +#include "mongo/s/range_arithmetic.h" namespace mongo { extern const BSONObj reverseNaturalObj; // {"$natural": -1 } + class Collection; class Cursor; - class CoveredIndexMatcher; /** - all helpers assume locking is handled above them + * db helpers are helper functions and classes that let us easily manipulate the local + * database instance in-proc. + * + * all helpers assume locking is handled above them */ struct Helpers { + class RemoveSaver; + /* ensure the specified index exists. @param keyPattern key pattern, e.g., { ts : 1 } @@ -46,11 +61,14 @@ namespace mongo { This method can be a little (not much) cpu-slow, so you may wish to use OCCASIONALLY ensureIndex(...); - Note: use ensureHaveIdIndex() for the _id index: it is faster. Note: does nothing if collection does not yet exist. */ static void ensureIndex(const char *ns, BSONObj keyPattern, bool unique, const char *name); + // same as other ensureIndex + static void ensureIndex(Collection* collection, + BSONObj keyPattern, bool unique, const char *name); + /* fetch a single object from collection ns that matches query. set your db SavedContext first. @@ -77,9 +95,10 @@ namespace mongo { static bool findById(Client&, const char *ns, BSONObj query, BSONObj& result , bool * nsFound = 0 , bool * indexFound = 0 ); - /* uasserts if no _id index. - @return null loc if not found */ - static DiskLoc findById(NamespaceDetails *d, BSONObj query); + /* TODO: should this move into Collection? + * uasserts if no _id index. + * @return null loc if not found */ + static DiskLoc findById(Collection* collection, const BSONObj& query); /** Get/put the first (or last) object from a collection. Generally only useful if the collection only ever has a single object -- which is a "singleton collection". @@ -120,15 +139,9 @@ namespace mongo { */ static BSONObj inferKeyPattern( const BSONObj& o ); - class RemoveCallback { - public: - virtual ~RemoveCallback() {} - virtual void goingToDelete( const BSONObj& o ) = 0; - }; - /** - * Takes a range, specified by a min and max, and an index, specified by - * keyPattern, and removes all the documents in that range found by iterating + * Takes a namespace range, specified by a min and max and qualified by an index pattern, + * and removes all the documents in that range found by iterating * over the given index. Caller is responsible for insuring that min/max are * compatible with the given keyPattern (e.g min={a:100} is compatible with * keyPattern={a:1,b:1} since it can be extended to {a:100,b:minKey}, but @@ -136,19 +149,44 @@ namespace mongo { * * Caller must hold a write lock on 'ns' * + * Returns -1 when no usable index exists + * * Does oplog the individual document deletions. * // TODO: Refactor this mechanism, it is growing too large */ - static long long removeRange( const string& ns , - const BSONObj& min , - const BSONObj& max , - const BSONObj& keyPattern , - bool maxInclusive = false , - bool secondaryThrottle = false , - RemoveCallback * callback = 0, + static long long removeRange( const KeyRange& range, + bool maxInclusive = false, + bool secondaryThrottle = false, + RemoveSaver* callback = NULL, bool fromMigrate = false, bool onlyRemoveOrphanedDocs = false ); + + // TODO: This will supersede Chunk::MaxObjectsPerChunk + static const long long kMaxDocsPerChunk; + + /** + * Get sorted disklocs that belong to a range of a namespace defined over an index + * key pattern (KeyRange). + * + * @param chunk range of a namespace over an index key pattern. + * @param maxChunkSizeBytes max number of bytes that we will retrieve locs for, if the + * range is estimated larger (from avg doc stats) we will stop recording locs. + * @param locs set to record locs in + * @param estChunkSizeBytes chunk size estimated from doc count and avg doc size + * @param chunkTooBig whether the chunk was estimated larger than our maxChunkSizeBytes + * @param errmsg filled with textual description of error if this call return false + * + * @return NamespaceNotFound if the namespace doesn't exist + * @return IndexNotFound if the index pattern doesn't match any indexes + * @return InvalidLength if the estimated size exceeds maxChunkSizeBytes + */ + static Status getLocsInRange( const KeyRange& range, + long long maxChunkSizeBytes, + set* locs, + long long* numDocs, + long long* estChunkSizeBytes ); + /** * Remove all documents from a collection. * You do not need to set the database before calling. @@ -156,24 +194,22 @@ namespace mongo { */ static void emptyCollection(const char *ns); - }; - - /** - * user for saving deleted bson objects to a flat file - */ - class RemoveSaver : public Helpers::RemoveCallback , boost::noncopyable { - public: - RemoveSaver( const string& type , const string& ns , const string& why); - ~RemoveSaver(); + /** + * for saving deleted bson objects to a flat file + */ + class RemoveSaver : public boost::noncopyable { + public: + RemoveSaver(const string& type, const string& ns, const string& why); + ~RemoveSaver(); - void goingToDelete( const BSONObj& o ); + void goingToDelete( const BSONObj& o ); - private: - boost::filesystem::path _root; - boost::filesystem::path _file; - ofstream* _out; + private: + boost::filesystem::path _root; + boost::filesystem::path _file; + ofstream* _out; + }; }; - } // namespace mongo diff --git a/src/mongo/db/dbmessage.cpp b/src/mongo/db/dbmessage.cpp index 3f1e866291f..5fac84ebf24 100644 --- a/src/mongo/db/dbmessage.cpp +++ b/src/mongo/db/dbmessage.cpp @@ -15,8 +15,9 @@ * limitations under the License. */ -#include "pch.h" -#include "dbmessage.h" +#include "mongo/pch.h" + +#include "mongo/db/dbmessage.h" namespace mongo { @@ -52,6 +53,107 @@ namespace mongo { return ss.str(); } + DbMessage::DbMessage(const Message& msg) : _msg(msg), _nsStart(NULL), _mark(NULL), _nsLen(0) { + // for received messages, Message has only one buffer + _theEnd = _msg.singleData()->_data + _msg.singleData()->dataLen(); + _nextjsobj = _msg.singleData()->_data; + + _reserved = readAndAdvance(); + + // Read packet for NS + if (messageShouldHaveNs()) { + + // Limit = buffer size of message - + // (first int4 in message which is either flags or a zero constant) + size_t limit = _msg.singleData()->dataLen() - sizeof(int); + + _nsStart = _nextjsobj; + _nsLen = strnlen(_nsStart, limit); + + // Validate there is room for a null byte in the buffer + // Strings can be zero length + uassert(18633, "Failed to parse ns string", _nsLen <= (limit - 1)); + + _nextjsobj += _nsLen + 1; // skip namespace + null + } + } + + const char * DbMessage::getns() const { + verify(messageShouldHaveNs()); + return _nsStart; + } + + int DbMessage::getQueryNToReturn() const { + verify(messageShouldHaveNs()); + const char* p = _nsStart + _nsLen + 1; + checkRead(p, 2); + + return ((reinterpret_cast(p)))[1]; + } + + int DbMessage::pullInt() { + return readAndAdvance(); + } + + long long DbMessage::pullInt64() { + return readAndAdvance(); + } + + const long long* DbMessage::getArray(size_t count) const { + checkRead(_nextjsobj, count); + return reinterpret_cast(_nextjsobj); + } + + BSONObj DbMessage::nextJsObj() { + massert(10304, + "Client Error: Remaining data too small for BSON object", + _nextjsobj != NULL && _theEnd - _nextjsobj >= 5); + + if (serverGlobalParams.objcheck) { + Status status = validateBSON(_nextjsobj, _theEnd - _nextjsobj); + massert(10307, + str::stream() << "Client Error: bad object in message: " << status.reason(), + status.isOK()); + } + + BSONObj js(_nextjsobj); + verify(js.objsize() >= 5); + verify(js.objsize() <= (_theEnd - _nextjsobj)); + + _nextjsobj += js.objsize(); + if (_nextjsobj >= _theEnd) + _nextjsobj = NULL; + return js; + } + + void DbMessage::markReset(const char * toMark = NULL) { + if (toMark == NULL) { + toMark = _mark; + } + + verify(toMark); + _nextjsobj = toMark; + } + + template + void DbMessage::checkRead(const char* start, size_t count) const { + if ((_theEnd - start) < static_cast(sizeof(T) * count)) { + uassert(18634, "Not enough data to read", false); + } + } + + template + T DbMessage::read() const { + checkRead(_nextjsobj, 1); + + return *(reinterpret_cast(_nextjsobj)); + } + + template T DbMessage::readAndAdvance() { + T t = read(); + _nextjsobj += sizeof(T); + return t; + } void replyToQuery(int queryResultFlags, AbstractMessagingPort* p, Message& requestMsg, diff --git a/src/mongo/db/dbmessage.h b/src/mongo/db/dbmessage.h index aeb93134f51..2376e1f002c 100644 --- a/src/mongo/db/dbmessage.h +++ b/src/mongo/db/dbmessage.h @@ -14,16 +14,27 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "jsobj.h" -#include "namespace-inl.h" -#include "../util/net/message.h" -#include "../client/constants.h" -#include "instance.h" #include "mongo/bson/bson_validate.h" +#include "mongo/client/constants.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/net/message.h" +#include "mongo/util/net/message_port.h" namespace mongo { @@ -110,138 +121,74 @@ namespace mongo { See http://dochub.mongodb.org/core/mongowireprotocol */ class DbMessage { + // Assume sizeof(int) == 4 bytes + BOOST_STATIC_ASSERT(sizeof(int) == 4); + public: - DbMessage(const Message& _m) : m(_m) , mark(0) { - // for received messages, Message has only one buffer - theEnd = _m.singleData()->_data + _m.header()->dataLen(); - char *r = _m.singleData()->_data; - reserved = (int *) r; - data = r + 4; - nextjsobj = data; + // Note: DbMessage constructor reads the first 4 bytes and stores it in reserved + DbMessage(const Message& msg); + + // Indicates whether this message is expected to have a ns + // or in the case of dbMsg, a string in the same place as ns + bool messageShouldHaveNs() const { + return (_msg.operation() >= dbMsg) & (_msg.operation() <= dbDelete); } - /** the 32 bit field before the ns + /** the 32 bit field before the ns * track all bit usage here as its cross op * 0: InsertOption_ContinueOnError * 1: fromWriteback */ - int& reservedField() { return *reserved; } - - const char * getns() const { - return data; - } - void getns(Namespace& ns) const { - ns = data; - } + int reservedField() const { return _reserved; } - const char * afterNS() const { - return data + strlen( data ) + 1; - } + const char * getns() const; + int getQueryNToReturn() const; - int getInt( int num ) const { - const int * foo = (const int*)afterNS(); - return foo[num]; - } - - int getQueryNToReturn() const { - return getInt( 1 ); - } + int pullInt(); + long long pullInt64(); + const long long* getArray(size_t count) const; - /** - * get an int64 at specified offsetBytes after ns - */ - long long getInt64( int offsetBytes ) const { - const char * x = afterNS(); - x += offsetBytes; - const long long * ll = (const long long*)x; - return ll[0]; + /* for insert and update msgs */ + bool moreJSObjs() const { + return _nextjsobj != 0; } - void resetPull() { nextjsobj = data; } - int pullInt() const { return pullInt(); } - int& pullInt() { - if ( nextjsobj == data ) - nextjsobj += strlen(data) + 1; // skip namespace - int& i = *((int *)nextjsobj); - nextjsobj += 4; - return i; - } - long long pullInt64() const { - return pullInt64(); - } - long long &pullInt64() { - if ( nextjsobj == data ) - nextjsobj += strlen(data) + 1; // skip namespace - long long &i = *((long long *)nextjsobj); - nextjsobj += 8; - return i; - } + BSONObj nextJsObj(); - OID* getOID() const { - return (OID *) (data + strlen(data) + 1); // skip namespace - } + const Message& msg() const { return _msg; } - void getQueryStuff(const char *&query, int& ntoreturn) { - int *i = (int *) (data + strlen(data) + 1); - ntoreturn = *i; - i++; - query = (const char *) i; + const char * markGet() const { + return _nextjsobj; } - /* for insert and update msgs */ - bool moreJSObjs() const { - return nextjsobj != 0; + void markSet() { + _mark = _nextjsobj; } - BSONObj nextJsObj() { - if ( nextjsobj == data ) { - nextjsobj += strlen(data) + 1; // skip namespace - massert( 13066 , "Message contains no documents", theEnd > nextjsobj ); - } - massert( 10304, - "Client Error: Remaining data too small for BSON object", - theEnd - nextjsobj >= 5 ); - - if ( cmdLine.objcheck ) { - Status status = validateBSON( nextjsobj, theEnd - nextjsobj ); - massert( 10307, - str::stream() << "Client Error: bad object in message: " << status.reason(), - status.isOK() ); - } - BSONObj js(nextjsobj); - verify( js.objsize() >= 5 ); - verify( js.objsize() < ( theEnd - data ) ); + void markReset(const char * toMark); - nextjsobj += js.objsize(); - if ( nextjsobj >= theEnd ) - nextjsobj = 0; - return js; - } + private: + // Check if we have enough data to read + template + void checkRead(const char* start, size_t count = 0) const; - const Message& msg() const { return m; } + // Read some type without advancing our position + template + T read() const; - const char * markGet() { - return nextjsobj; - } + // Read some type, and advance our position + template T readAndAdvance(); - void markSet() { - mark = nextjsobj; - } + const Message& _msg; + int _reserved; // flags or zero depending on packet, starts the packet - void markReset( const char * toMark = 0) { - if( toMark == 0 ) toMark = mark; - verify( toMark ); - nextjsobj = toMark; - } + const char* _nsStart; // start of namespace string, +4 from message start + const char* _nextjsobj; // current position reading packet + const char* _theEnd; // end of packet - private: - const Message& m; - int* reserved; - const char *data; - const char *nextjsobj; - const char *theEnd; + const char* _mark; - const char * mark; + unsigned int _nsLen; }; @@ -255,7 +202,10 @@ namespace mongo { BSONObj query; BSONObj fields; - /* parses the message into the above fields */ + /** + * parses the message into the above fields + * Warning: constructor mutates DbMessage. + */ QueryMessage(DbMessage& d) { ns = d.getns(); ntoskip = d.pullInt(); @@ -268,6 +218,20 @@ namespace mongo { } }; + /** + * A response to a DbMessage. + */ + struct DbResponse { + Message *response; + MSGID responseTo; + string exhaustNS; /* points to ns if exhaust mode. 0=normal mode*/ + DbResponse(Message *r, MSGID rt) : response(r), responseTo(rt){ } + DbResponse() { + response = 0; + } + ~DbResponse() { delete response; } + }; + void replyToQuery(int queryResultFlags, AbstractMessagingPort* p, Message& requestMsg, void *data, int size, diff --git a/src/mongo/db/dbmessage_test.cpp b/src/mongo/db/dbmessage_test.cpp new file mode 100644 index 00000000000..867a52d9885 --- /dev/null +++ b/src/mongo/db/dbmessage_test.cpp @@ -0,0 +1,143 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include + +#include "mongo/bson/util/builder.h" +#include "mongo/db/dbmessage.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { + using std::string; + + // Test if the reserved field is short of 4 bytes + TEST(DBMessage1, ShortFlags) { + BufBuilder b; + string ns("test"); + + b.appendChar( 1 ); + + Message toSend; + toSend.setData( dbDelete , b.buf() , b.len() ); + + ASSERT_THROWS(DbMessage d1(toSend), UserException); + } + + // Test a short NS missing a trailing null + TEST(DBMessage1, BadNS) { + BufBuilder b; + + b.appendNum( static_cast(1) ); + b.appendChar( 'b' ); + b.appendChar( 'a' ); + b.appendChar( 'd' ); + // Forget to append \0 + + Message toSend; + toSend.setData( dbDelete , b.buf() , b.len() ); + + ASSERT_THROWS(DbMessage d1(toSend), UserException); + } + + // Test a valid kill message and try an extra pull + TEST(DBMessage1, GoodKill) { + BufBuilder b; + + b.appendNum( static_cast(1) ); + b.appendNum( static_cast(3) ); + + Message toSend; + toSend.setData( dbKillCursors , b.buf() , b.len() ); + + DbMessage d1(toSend); + ASSERT_EQUALS(3, d1.pullInt()); + + ASSERT_THROWS(d1.pullInt(), UserException); + } + + // Try a bad read of a type too large + TEST(DBMessage1, GoodKill2) { + BufBuilder b; + + b.appendNum( static_cast(1) ); + b.appendNum( static_cast(3) ); + + Message toSend; + toSend.setData( dbKillCursors , b.buf() , b.len() ); + + DbMessage d1(toSend); + ASSERT_THROWS(d1.pullInt64(), UserException); + } + + // Test a basic good insert, and an extra read + TEST(DBMessage1, GoodInsert) { + BufBuilder b; + string ns("test"); + + b.appendNum( static_cast(1) ); + b.appendStr(ns); + b.appendNum( static_cast(3) ); + b.appendNum( static_cast(39) ); + + Message toSend; + toSend.setData( dbInsert , b.buf() , b.len() ); + + DbMessage d1(toSend); + ASSERT_EQUALS(3, d1.pullInt()); + ASSERT_EQUALS(39, d1.pullInt()); + ASSERT_THROWS(d1.pullInt(), UserException); + } + + // Test a basic good insert, and an extra read + TEST(DBMessage1, GoodInsert2) { + BufBuilder b; + string ns("test"); + + b.appendNum( static_cast(1) ); + b.appendStr(ns); + b.appendNum( static_cast(3) ); + b.appendNum( static_cast(39) ); + + BSONObj bo = BSON( "ts" << 0 ); + bo.appendSelfToBufBuilder( b ); + + Message toSend; + toSend.setData( dbInsert , b.buf() , b.len() ); + + DbMessage d1(toSend); + ASSERT_EQUALS(3, d1.pullInt()); + + + ASSERT_EQUALS(39, d1.pullInt()); + BSONObj bo2 = d1.nextJsObj(); + ASSERT_THROWS(d1.nextJsObj(), MsgAssertionException); + } + + + +} // mongo namespace diff --git a/src/mongo/db/dbwebserver.cpp b/src/mongo/db/dbwebserver.cpp index d05cf596c7e..d66d7741ff9 100644 --- a/src/mongo/db/dbwebserver.cpp +++ b/src/mongo/db/dbwebserver.cpp @@ -17,370 +17,97 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" + +#include "mongo/db/dbwebserver.h" + +#include +#include +#include "mongo/base/init.h" #include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/principal.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" -#include "../util/net/miniwebserver.h" -#include "../util/mongoutils/html.h" -#include "../util/md5.hpp" -#include "db.h" -#include "instance.h" -#include "stats/snapshots.h" -#include "background.h" -#include "commands.h" -#include "../util/version.h" -#include "../util/ramlog.h" -#include "pcrecpp.h" -#include "../util/admin_access.h" -#include "dbwebserver.h" -#include +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/background.h" +#include "mongo/db/commands.h" +#include "mongo/db/db.h" +#include "mongo/db/instance.h" +#include "mongo/db/stats/snapshots.h" +#include "mongo/util/admin_access.h" +#include "mongo/util/md5.hpp" +#include "mongo/util/mongoutils/html.h" +#include "mongo/util/ramlog.h" +#include "mongo/util/version.h" +#include "mongo/util/version_reporting.h" + namespace mongo { using namespace mongoutils::html; using namespace bson; - time_t started = time(0); - - struct Timing { - Timing() { - start = timeLocked = 0; - } - unsigned long long start, timeLocked; - }; - - class DbWebServer : public MiniWebServer { - public: - DbWebServer(const string& ip, int port, const AdminAccess* webUsers) - : MiniWebServer("admin web console", ip, port), _webUsers(webUsers) { - WebStatusPlugin::initAll(); - } - - private: - const AdminAccess* _webUsers; // not owned here - - void doUnlockedStuff(stringstream& ss) { - /* this is in the header already ss << "port: " << port << '\n'; */ - ss << "
";
-            ss << mongodVersion() << '\n';
-            ss << "git hash: " << gitVersion() << '\n';
-            ss << "sys info: " << sysInfo() << '\n';
-            ss << "uptime: " << time(0)-started << " seconds\n";
-            ss << "
"; - } - - void _authorizePrincipal(const std::string& principalName, bool readOnly) { - Principal* principal = new Principal(PrincipalName(principalName, "local")); - ActionSet actions = AuthorizationManager::getActionsForOldStyleUser( - "admin", readOnly); - - AuthorizationManager* authorizationManager = cc().getAuthorizationManager(); - authorizationManager->addAuthorizedPrincipal(principal); - Status status = authorizationManager->acquirePrivilege( - Privilege(PrivilegeSet::WILDCARD_RESOURCE, actions), principal->getName()); - verify (status == Status::OK()); - } - - bool allowed( const char * rq , vector& headers, const SockAddr &from ) { - if ( from.isLocalHost() || !_webUsers->haveAdminUsers() ) { - _authorizePrincipal("RestUser", false); - return true; - } - - string auth = getHeader( rq , "Authorization" ); - - if ( auth.size() > 0 && auth.find( "Digest " ) == 0 ) { - auth = auth.substr( 7 ) + ", "; - - map parms; - pcrecpp::StringPiece input( auth ); - - string name, val; - pcrecpp::RE re("(\\w+)=\"?(.*?)\"?,\\s*"); - while ( re.Consume( &input, &name, &val) ) { - parms[name] = val; - } - - BSONObj user = _webUsers->getAdminUser( parms["username"] ); - if ( ! user.isEmpty() ) { - string ha1 = user["pwd"].str(); - string ha2 = md5simpledigest( (string)"GET" + ":" + parms["uri"] ); - - stringstream r; - r << ha1 << ':' << parms["nonce"]; - if ( parms["nc"].size() && parms["cnonce"].size() && parms["qop"].size() ) { - r << ':'; - r << parms["nc"]; - r << ':'; - r << parms["cnonce"]; - r << ':'; - r << parms["qop"]; - } - r << ':'; - r << ha2; - string r1 = md5simpledigest( r.str() ); - - if ( r1 == parms["response"] ) { - std::string principalName = user["user"].str(); - bool readOnly = user[ "readOnly" ].isBoolean() && - user[ "readOnly" ].boolean(); - - _authorizePrincipal(principalName, readOnly); - return true; - } - } - } - - stringstream authHeader; - authHeader - << "WWW-Authenticate: " - << "Digest realm=\"mongo\", " - << "nonce=\"abc\", " - << "algorithm=MD5, qop=\"auth\" " - ; - - headers.push_back( authHeader.str() ); - return 0; - } - - virtual void doRequest( - const char *rq, // the full request - string url, - // set these and return them: - string& responseMsg, - int& responseCode, - vector& headers, // if completely empty, content-type: text/html will be added - const SockAddr &from - ) { - if ( url.size() > 1 ) { - - if ( ! allowed( rq , headers, from ) ) { - responseCode = 401; - headers.push_back( "Content-Type: text/plain;charset=utf-8" ); - responseMsg = "not allowed\n"; - return; - } - - { - BSONObj params; - const size_t pos = url.find( "?" ); - if ( pos != string::npos ) { - MiniWebServer::parseParams( params , url.substr( pos + 1 ) ); - url = url.substr(0, pos); - } - - DbWebHandler * handler = DbWebHandler::findHandler( url ); - if ( handler ) { - if ( handler->requiresREST( url ) && ! cmdLine.rest ) { - _rejectREST( responseMsg , responseCode , headers ); - } - else { - string callback = params.getStringField("jsonp"); - uassert(13453, "server not started with --jsonp", callback.empty() || cmdLine.jsonp); - - handler->handle( rq , url , params , responseMsg , responseCode , headers , from ); - - if (responseCode == 200 && !callback.empty()) { - responseMsg = callback + '(' + responseMsg + ')'; - } - } - return; - } - } - - - if ( ! cmdLine.rest ) { - _rejectREST( responseMsg , responseCode , headers ); - return; - } - - responseCode = 404; - headers.push_back( "Content-Type: text/html;charset=utf-8" ); - responseMsg = "unknown url\n"; - return; - } - - // generate home page - - if ( ! allowed( rq , headers, from ) ) { - responseCode = 401; - headers.push_back( "Content-Type: text/plain;charset=utf-8" ); - responseMsg = "not allowed\n"; - return; - } - - responseCode = 200; - stringstream ss; - string dbname; - { - stringstream z; - z << cmdLine.binaryName << ' ' << prettyHostName(); - dbname = z.str(); - } - ss << start(dbname) << h2(dbname); - ss << "

List all commands | \n"; - ss << "Replica set status

\n"; - - //ss << "_status"; - { - const map *m = Command::webCommands(); - if( m ) { - ss << - a("", - "These read-only context-less commands can be executed from the web interface. " - "Results are json format, unless ?text=1 is appended in which case the result is output as text " - "for easier human viewing", - "Commands") - << ": "; - for( map::const_iterator i = m->begin(); i != m->end(); i++ ) { - stringstream h; - i->second->help(h); - string help = h.str(); - ss << "first << "?text=1\""; - if( help != "no help defined" ) - ss << " title=\"" << help << '"'; - ss << ">" << i->first << " "; - } - ss << '\n'; - } - } - ss << '\n'; - /* - ss << "HTTP admin port:" << _port << "

\n"; - */ - - doUnlockedStuff(ss); - - WebStatusPlugin::runAll( ss ); - - ss << "\n"; - responseMsg = ss.str(); - headers.push_back( "Content-Type: text/html;charset=utf-8" ); - } - - void _rejectREST( string& responseMsg , int& responseCode, vector& headers ) { - responseCode = 403; - stringstream ss; - ss << "REST is not enabled. use --rest to turn on.\n"; - ss << "check that port " << _port << " is secured for the network too.\n"; - responseMsg = ss.str(); - headers.push_back( "Content-Type: text/plain;charset=utf-8" ); - } - - }; - // --- - - bool prisort( const Prioritizable * a , const Prioritizable * b ) { - return a->priority() < b->priority(); +namespace { + + void doUnlockedStuff(stringstream& ss) { + /* this is in the header already ss << "port: " << port << '\n'; */ + ss << "

";
+        ss << mongodVersion() << '\n';
+        ss << "git hash: " << gitVersion() << '\n';
+        ss << openSSLVersion("OpenSSL version: ", "\n");
+        ss << "sys info: " << sysInfo() << '\n';
+        ss << "uptime: " << time(0) - serverGlobalParams.started << " seconds\n";
+        ss << "
"; } - // -- status framework --- - WebStatusPlugin::WebStatusPlugin( const string& secionName , double priority , const string& subheader ) - : Prioritizable(priority), _name( secionName ) , _subHeading( subheader ) { - if ( ! _plugins ) - _plugins = new vector(); - _plugins->push_back( this ); - } - void WebStatusPlugin::initAll() { - if ( ! _plugins ) - return; - - sort( _plugins->begin(), _plugins->end() , prisort ); - - for ( unsigned i=0; i<_plugins->size(); i++ ) - (*_plugins)[i]->init(); + bool prisort(const Prioritizable * a, const Prioritizable * b) { + return a->priority() < b->priority(); } - void WebStatusPlugin::runAll( stringstream& ss ) { - if ( ! _plugins ) - return; - - for ( unsigned i=0; i<_plugins->size(); i++ ) { - WebStatusPlugin * p = (*_plugins)[i]; - ss << "
\n" - << "" << p->_name << ""; - - ss << " " << p->_subHeading; - - ss << "
\n"; - p->run(ss); + struct Timing { + Timing() { + start = timeLocked = 0; } + unsigned long long start, timeLocked; + }; - } - - vector * WebStatusPlugin::_plugins = 0; - - // -- basic statuc plugins -- class LogPlugin : public WebStatusPlugin { public: - LogPlugin() : WebStatusPlugin( "Log" , 100 ), _log(0) { - } + LogPlugin() + : WebStatusPlugin("Log", 100), + _log(RamLog::get("global")) { - virtual void init() { - _log = RamLog::get( "global" ); - if ( ! _log ) { - _log = new RamLog("global"); - Logstream::get().addGlobalTee( _log ); - } } - virtual void run( stringstream& ss ) { - _log->toHTML( ss ); - } - RamLog * _log; - }; - - LogPlugin * logPlugin = new LogPlugin(); + virtual void init() {} - // -- handler framework --- - - DbWebHandler::DbWebHandler( const string& name , double priority , bool requiresREST ) - : Prioritizable(priority), _name(name) , _requiresREST(requiresREST) { - - { - // setup strings - _defaultUrl = "/"; - _defaultUrl += name; - - stringstream ss; - ss << name << " priority: " << priority << " rest: " << requiresREST; - _toString = ss.str(); - } - - { - // add to handler list - if ( ! _handlers ) - _handlers = new vector(); - _handlers->push_back( this ); - sort( _handlers->begin() , _handlers->end() , prisort ); + virtual void run(stringstream& ss) { + _log->toHTML(ss); } - } - - DbWebHandler * DbWebHandler::findHandler( const string& url ) { - if ( ! _handlers ) - return 0; - for ( unsigned i=0; i<_handlers->size(); i++ ) { - DbWebHandler * h = (*_handlers)[i]; - if ( h->handles( url ) ) - return h; - } - - return 0; - } - - vector * DbWebHandler::_handlers = 0; + private: + RamLog* const _log; + }; - // --- basic handlers --- class FavIconHandler : public DbWebHandler { public: @@ -396,6 +123,7 @@ namespace mongo { } faviconHandler; + class StatusHandler : public DbWebHandler { public: StatusHandler() : DbWebHandler( "_status" , 1 , false ) {} @@ -448,6 +176,7 @@ namespace mongo { } statusHandler; + class CommandListHandler : public DbWebHandler { public: CommandListHandler() : DbWebHandler( "_commands" , 1 , true ) {} @@ -474,6 +203,7 @@ namespace mongo { } } commandListHandler; + class CommandsHandler : public DbWebHandler { public: CommandsHandler() : DbWebHandler( "DUMMY COMMANDS" , 2 , true ) {} @@ -536,14 +266,301 @@ namespace mongo { } commandsHandler; - // --- external ---- - void webServerThread(const AdminAccess* adminAccess) { - boost::scoped_ptr adminAccessPtr(adminAccess); // adminAccess is owned here + MONGO_INITIALIZER(WebStatusLogPlugin)(InitializerContext*) { + if (serverGlobalParams.isHttpInterfaceEnabled) { + new LogPlugin(); + } + return Status::OK(); + } + +} // namespace + + + DbWebServer::DbWebServer(const string& ip, int port, const AdminAccess* webUsers) + : MiniWebServer("admin web console", ip, port), + _webUsers(webUsers) { + + WebStatusPlugin::initAll(); + } + + bool DbWebServer::_allowed(const char * rq, vector& headers, const SockAddr &from) { + if ( from.isLocalHost() || !_webUsers->haveAdminUsers() ) { + // TODO(spencer): should the above check use "&&" not "||"? Currently this is much + // more permissive than the server's localhost auth bypass. + cc().getAuthorizationSession()->grantInternalAuthorization(); + return true; + } + + string auth = getHeader( rq , "Authorization" ); + + if ( auth.size() > 0 && auth.find( "Digest " ) == 0 ) { + auth = auth.substr( 7 ) + ", "; + + map parms; + pcrecpp::StringPiece input( auth ); + + string name, val; + pcrecpp::RE re("(\\w+)=\"?(.*?)\"?,\\s*"); + while ( re.Consume( &input, &name, &val) ) { + parms[name] = val; + } + + // Only users in the admin DB are visible by the webserver + UserName userName(parms["username"], "admin"); + User* user; + AuthorizationManager& authzManager = + cc().getAuthorizationSession()->getAuthorizationManager(); + Status status = authzManager.acquireUser(userName, &user); + if (!status.isOK()) { + if (status.code() != ErrorCodes::UserNotFound) { + uasserted(17051, status.reason()); + } + } else { + uassert(17090, + "External users don't have a password", + !user->getCredentials().isExternal); + string ha1 = user->getCredentials().password; + authzManager.releaseUser(user); + string ha2 = md5simpledigest( (string)"GET" + ":" + parms["uri"] ); + + stringstream r; + r << ha1 << ':' << parms["nonce"]; + if ( parms["nc"].size() && parms["cnonce"].size() && parms["qop"].size() ) { + r << ':'; + r << parms["nc"]; + r << ':'; + r << parms["cnonce"]; + r << ':'; + r << parms["qop"]; + } + r << ':'; + r << ha2; + string r1 = md5simpledigest( r.str() ); + + if ( r1 == parms["response"] ) { + Status status = cc().getAuthorizationSession()->addAndAuthorizeUser(userName); + uassertStatusOK(status); + + return true; + } + } + } + + stringstream authHeader; + authHeader + << "WWW-Authenticate: " + << "Digest realm=\"mongo\", " + << "nonce=\"abc\", " + << "algorithm=MD5, qop=\"auth\" " + ; + + headers.push_back( authHeader.str() ); + return 0; + } + + void DbWebServer::doRequest(const char *rq, + string url, + string& responseMsg, + int& responseCode, + vector& headers, + const SockAddr &from) { + if ( url.size() > 1 ) { + + if (!_allowed(rq, headers, from)) { + responseCode = 401; + headers.push_back( "Content-Type: text/plain;charset=utf-8" ); + responseMsg = "not allowed\n"; + return; + } + + { + BSONObj params; + const size_t pos = url.find( "?" ); + if ( pos != string::npos ) { + MiniWebServer::parseParams( params , url.substr( pos + 1 ) ); + url = url.substr(0, pos); + } + + DbWebHandler * handler = DbWebHandler::findHandler( url ); + if ( handler ) { + if (handler->requiresREST(url) && !serverGlobalParams.rest) { + _rejectREST( responseMsg , responseCode , headers ); + } + else { + string callback = params.getStringField("jsonp"); + uassert(13453, "server not started with --jsonp", + callback.empty() || serverGlobalParams.jsonp); + + handler->handle( rq , url , params , responseMsg , responseCode , headers , from ); + + if (responseCode == 200 && !callback.empty()) { + responseMsg = callback + '(' + responseMsg + ')'; + } + } + return; + } + } + + + if (!serverGlobalParams.rest) { + _rejectREST( responseMsg , responseCode , headers ); + return; + } + + responseCode = 404; + headers.push_back( "Content-Type: text/html;charset=utf-8" ); + responseMsg = "unknown url\n"; + return; + } + + // generate home page + + if (!_allowed(rq, headers, from)) { + responseCode = 401; + headers.push_back( "Content-Type: text/plain;charset=utf-8" ); + responseMsg = "not allowed\n"; + return; + } + + responseCode = 200; + stringstream ss; + string dbname; + { + stringstream z; + z << serverGlobalParams.binaryName << ' ' << prettyHostName(); + dbname = z.str(); + } + ss << start(dbname) << h2(dbname); + ss << "

List all commands | \n"; + ss << "Replica set status

\n"; + + //ss << "_status"; + { + const map *m = Command::webCommands(); + if( m ) { + ss << + a("", + "These read-only context-less commands can be executed from the web interface. " + "Results are json format, unless ?text=1 is appended in which case the result is output as text " + "for easier human viewing", + "Commands") + << ": "; + for( map::const_iterator i = m->begin(); i != m->end(); i++ ) { + stringstream h; + i->second->help(h); + string help = h.str(); + ss << "first << "?text=1\""; + if( help != "no help defined" ) + ss << " title=\"" << help << '"'; + ss << ">" << i->first << " "; + } + ss << '\n'; + } + } + ss << '\n'; + + doUnlockedStuff(ss); + + WebStatusPlugin::runAll( ss ); + + ss << "\n"; + responseMsg = ss.str(); + headers.push_back( "Content-Type: text/html;charset=utf-8" ); + } + + void DbWebServer::_rejectREST(string& responseMsg, + int& responseCode, + vector& headers) { + responseCode = 403; + stringstream ss; + ss << "REST is not enabled. use --rest to turn on.\n"; + ss << "check that port " << _port << " is secured for the network too.\n"; + responseMsg = ss.str(); + headers.push_back("Content-Type: text/plain;charset=utf-8"); + } + + + WebStatusPlugin::WebStatusPlugin( const string& secionName , double priority , const string& subheader ) + : Prioritizable(priority), _name( secionName ) , _subHeading( subheader ) { + if ( ! _plugins ) + _plugins = new vector(); + _plugins->push_back( this ); + } + + void WebStatusPlugin::initAll() { + if ( ! _plugins ) + return; + + sort( _plugins->begin(), _plugins->end() , prisort ); + + for ( unsigned i=0; i<_plugins->size(); i++ ) + (*_plugins)[i]->init(); + } + + void WebStatusPlugin::runAll( stringstream& ss ) { + if ( ! _plugins ) + return; + + for ( unsigned i=0; i<_plugins->size(); i++ ) { + WebStatusPlugin * p = (*_plugins)[i]; + ss << "
\n" + << "" << p->_name << ""; + + ss << " " << p->_subHeading; + + ss << "
\n"; + + p->run(ss); + } + + } + + vector * WebStatusPlugin::_plugins = 0; + + DbWebHandler::DbWebHandler( const string& name , double priority , bool requiresREST ) + : Prioritizable(priority), _name(name) , _requiresREST(requiresREST) { + + { + // setup strings + _defaultUrl = "/"; + _defaultUrl += name; + + stringstream ss; + ss << name << " priority: " << priority << " rest: " << requiresREST; + _toString = ss.str(); + } + + { + // add to handler list + if ( ! _handlers ) + _handlers = new vector(); + _handlers->push_back( this ); + sort( _handlers->begin() , _handlers->end() , prisort ); + } + } + + DbWebHandler * DbWebHandler::findHandler( const string& url ) { + if ( ! _handlers ) + return 0; + + for ( unsigned i=0; i<_handlers->size(); i++ ) { + DbWebHandler * h = (*_handlers)[i]; + if ( h->handles( url ) ) + return h; + } + + return 0; + } + + vector * DbWebHandler::_handlers = 0; + + + void webServerListenThread(boost::shared_ptr dbWebServer) { Client::initThread("websvr"); - const int p = cmdLine.port + 1000; - DbWebServer mini(cmdLine.bind_ip, p, adminAccessPtr.get()); - mini.initAndListen(); + + dbWebServer->initAndListen(); + cc().shutdown(); } diff --git a/src/mongo/db/dbwebserver.h b/src/mongo/db/dbwebserver.h index 04715963249..1d867f7973c 100644 --- a/src/mongo/db/dbwebserver.h +++ b/src/mongo/db/dbwebserver.h @@ -15,9 +15,26 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "../util/admin_access.h" +#include +#include + +#include "mongo/util/admin_access.h" +#include "mongo/util/net/miniwebserver.h" +#include "mongo/util/net/sock.h" namespace mongo { @@ -79,7 +96,29 @@ namespace mongo { }; - void webServerThread( const AdminAccess* admins ); + class DbWebServer : public MiniWebServer { + public: + DbWebServer(const std::string& ip, int port, const AdminAccess* webUsers); + + private: + virtual void doRequest(const char *rq, + std::string url, + std::string& responseMsg, + int& responseCode, + std::vector& headers, + const SockAddr &from); + + bool _allowed(const char* rq, std::vector& headers, const SockAddr& from); + void _rejectREST(std::string& responseMsg, + int& responseCode, + std::vector& headers); + + + // not owned here + const AdminAccess* _webUsers; + }; + + void webServerListenThread(boost::shared_ptr dbWebServer); string prettyHostName(); }; diff --git a/src/mongo/db/diskloc.h b/src/mongo/db/diskloc.h index 70acc975d95..9fa401c6f60 100644 --- a/src/mongo/db/diskloc.h +++ b/src/mongo/db/diskloc.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* @file diskloc.h @@ -31,7 +43,7 @@ namespace mongo { class Record; class DeletedRecord; class Extent; - class MongoDataFile; + class DataFile; class DiskLoc; template< class Version > class BtreeBucket; @@ -75,7 +87,7 @@ namespace mongo { _a = -1; ofs = 0; /* note NullOfs is different. todo clean up. see refs to NullOfs in code - use is valid but outside DiskLoc context so confusing as-is. */ } - void assertOk() { verify(!isNull()); } + void assertOk() const { verify(!isNull()); } void setInvalid() { _a = -2; ofs = 0; @@ -152,19 +164,26 @@ namespace mongo { (think of this as an unchecked type cast) Note: set your Context first so that the database to which the diskloc applies is known. */ - BSONObj obj() const; - Record* rec() const; - DeletedRecord* drec() const; - Extent* ext() const; + BSONObj obj() const; // TODO(ERH): remove + Record* rec() const; // TODO(ERH): remove + DeletedRecord* drec() const; // TODO(ERH): remove + Extent* ext() const; // TODO(ERH): remove template< class V > - const BtreeBucket * btree() const; + const BtreeBucket * btree() const; // TODO(ERH): remove // Explicitly signals we are writing and casts away const template< class V > - BtreeBucket * btreemod() const; + BtreeBucket * btreemod() const; // TODO(ERH): remove - /*MongoDataFile& pdf() const;*/ + /// members for Sorter + struct SorterDeserializeSettings {}; // unused + void serializeForSorter(BufBuilder& buf) const { buf.appendStruct(*this); } + static DiskLoc deserializeForSorter(BufReader& buf, const SorterDeserializeSettings&) { + return buf.read(); + } + int memUsageForSorter() const { return sizeof(DiskLoc); } + DiskLoc getOwned() const { return *this; } }; #pragma pack() diff --git a/src/mongo/db/diskloc_test.cpp b/src/mongo/db/diskloc_test.cpp index db3e0c8a65b..cae5a62ef17 100644 --- a/src/mongo/db/diskloc_test.cpp +++ b/src/mongo/db/diskloc_test.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ /** Unit tests for DiskLoc. */ diff --git a/src/mongo/db/driverHelpers.cpp b/src/mongo/db/driverHelpers.cpp index 261ac966bdb..b827aef5451 100644 --- a/src/mongo/db/driverHelpers.cpp +++ b/src/mongo/db/driverHelpers.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /** @@ -22,23 +34,20 @@ */ -#include "pch.h" +#include "mongo/pch.h" #include #include #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/privilege.h" -#include "jsobj.h" -#include "pdfile.h" -#include "namespace-inl.h" -#include "commands.h" -#include "cmdline.h" -#include "curop-inl.h" -#include "../util/background.h" -#include "../scripting/engine.h" +#include "mongo/db/commands.h" +#include "mongo/db/curop-inl.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pdfile.h" +#include "mongo/scripting/engine.h" +#include "mongo/util/background.h" namespace mongo { diff --git a/src/mongo/db/dur.cpp b/src/mongo/db/dur.cpp index 9c336751446..99f24406f68 100644 --- a/src/mongo/db/dur.cpp +++ b/src/mongo/db/dur.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* @@ -28,7 +40,9 @@ have to handle falling behind which would use too much ram (going back into a read lock would suffice to stop that). for now (1.7.5/1.8.0) we are in read lock which is not ideal. WRITETODATAFILES - apply the writes back to the non-private MMF after they are for certain in redo log + actually write to the database data files in this phase. currently done by memcpy'ing the writes back to + the non-private MMF. alternatively one could write to the files the traditional way; however the way our + storage engine works that isn't any faster (actually measured a tiny bit slower). REMAPPRIVATEVIEW we could in a write lock quickly flip readers back to the main view, then stay in read lock and do our real remapping. with many files (e.g., 1000), remapping could be time consuming (several ms), so we don't want @@ -38,42 +52,42 @@ mutexes: - READLOCK dbMutex + READLOCK dbMutex (big 'R') LOCK groupCommitMutex PREPLOGBUFFER() READLOCK mmmutex commitJob.reset() - UNLOCK dbMutex // now other threads can write + UNLOCK dbMutex // now other threads can write WRITETOJOURNAL() WRITETODATAFILES() UNLOCK mmmutex UNLOCK groupCommitMutex - on the next write lock acquisition for dbMutex: // see MongoMutex::_acquiredWriteLock() - REMAPPRIVATEVIEW() + every Nth groupCommit, at the end, we REMAPPRIVATEVIEW() at the end of the work. because of + that we are in W lock for that groupCommit, which is nonideal of course. - @see https://docs.google.com/drawings/edit?id=1TklsmZzm7ohIZkwgeK6rMvsdaR13KjtJYMsfLr175Zc + @see https://docs.google.com/drawings/edit?id=1TklsmZzm7ohIZkwgeK6rMvsdaR13KjtJYMsfLr175Zc */ -#include "pch.h" +#include "mongo/pch.h" #include -#include "cmdline.h" -#include "client.h" -#include "dur.h" -#include "dur_journal.h" -#include "dur_commitjob.h" -#include "dur_recover.h" -#include "dur_stats.h" -#include "../util/concurrency/race.h" -#include "../util/mongoutils/hash.h" -#include "../util/mongoutils/str.h" -#include "../util/timer.h" -#include "mongo/util/stacktrace.h" -#include "../server.h" +#include "mongo/db/client.h" #include "mongo/db/commands/fsync.h" #include "mongo/db/commands/server_status.h" +#include "mongo/db/dur.h" +#include "mongo/db/dur_commitjob.h" +#include "mongo/db/dur_journal.h" +#include "mongo/db/dur_recover.h" +#include "mongo/db/dur_stats.h" +#include "mongo/db/storage_options.h" +#include "mongo/server.h" +#include "mongo/util/concurrency/race.h" +#include "mongo/util/mongoutils/hash.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/stacktrace.h" +#include "mongo/util/timer.h" using namespace mongoutils; @@ -150,8 +164,8 @@ namespace mongo { "writeToDataFiles" << (unsigned) (_writeToDataFilesMicros/1000) << "remapPrivateView" << (unsigned) (_remapPrivateViewMicros/1000) ); - if( cmdLine.journalCommitInterval != 0 ) - b << "journalCommitIntervalMs" << cmdLine.journalCommitInterval; + if (storageGlobalParams.journalCommitInterval != 0) + b << "journalCommitIntervalMs" << storageGlobalParams.journalCommitInterval; return b.obj(); } @@ -180,6 +194,17 @@ namespace mongo { cc().writeHappened(); } + bool NonDurableImpl::commitNow() { + cc().checkpointHappened(); + return false; + } + + bool NonDurableImpl::commitIfNeeded(bool) { + cc().checkpointHappened(); + return false; + } + + void assertLockedForCommitting(); static DurableImpl* durableImpl = new DurableImpl(); @@ -200,6 +225,7 @@ namespace mongo { bool DurableImpl::commitNow() { stats.curr->_earlyCommits++; groupCommit(0); + cc().checkpointHappened(); return true; } @@ -244,8 +270,9 @@ namespace mongo { return p; } - bool DurableImpl::aCommitIsNeeded() const { + bool DurableImpl::isCommitNeeded() const { DEV commitJob._nSinceCommitIfNeededCall = 0; + unspoolWriteIntents(); return commitJob.bytes() > UncommittedBytesLimit; } @@ -260,10 +287,16 @@ namespace mongo { bool NOINLINE_DECL DurableImpl::_aCommitIsNeeded() { switch (Lock::isLocked()) { case '\0': { - DEV log() << "commitIfNeeded but we are unlocked that is ok but why do we get here" << endl; + // lock_w() can call in this state at times if a commit is needed before attempting + // its lock. Lock::GlobalRead r; if( commitJob.bytes() < UncommittedBytesLimit ) { // someone else beat us to it + // + // note before of 'R' state, many threads can pile-in to this point and + // still fall through to below, and they will exit without doing work later + // once inside groupCommitMutex. this is all likely inefficient. maybe + // groupCommitMutex should be on top. return false; } commitNow(); @@ -271,15 +304,11 @@ namespace mongo { } case 'w': { if( Lock::atLeastReadLocked("local") ) { - error() << "can't commitNow from commitIfNeeded, as we are in local db lock" << endl; - printStackTrace(); - dassert(false); // this will make _DEBUG builds terminate. so we will notice in buildbot. + LOG(2) << "can't commitNow from commitIfNeeded, as we are in local db lock"; return false; } if( Lock::atLeastReadLocked("admin") ) { - error() << "can't commitNow from commitIfNeeded, as we are in admin db lock" << endl; - printStackTrace(); - dassert(false); + LOG(2) << "can't commitNow from commitIfNeeded, as we are in admin db lock"; return false; } @@ -317,6 +346,10 @@ namespace mongo { perf note: this function is called a lot, on every lock_w() ... and usually returns right away */ bool DurableImpl::commitIfNeeded(bool force) { + // this is safe since since conceptually if you call commitIfNeeded, we're at a valid + // spot in an operation to be terminated. + cc().checkpointHappened(); + unspoolWriteIntents(); DEV commitJob._nSinceCommitIfNeededCall = 0; if( likely( commitJob.bytes() < UncommittedBytesLimit && !force ) ) { @@ -338,12 +371,12 @@ namespace mongo { static int n; ++n; - verify(debug && cmdLine.dur); + verify(debug && storageGlobalParams.dur); if (commitJob.writes().empty()) return; const WriteIntent &i = commitJob.lastWrite(); size_t ofs; - MongoMMF *mmf = privateViews.find(i.start(), ofs); + DurableMappedFile *mmf = privateViews.find(i.start(), ofs); if( mmf == 0 ) return; size_t past = ofs + i.length(); @@ -378,8 +411,8 @@ namespace mongo { public: validateSingleMapMatches(unsigned long long& bytes) :_bytes(bytes) {} void operator () (MongoFile *mf) { - if( mf->isMongoMMF() ) { - MongoMMF *mmf = (MongoMMF*) mf; + if( mf->isDurableMappedFile() ) { + DurableMappedFile *mmf = (DurableMappedFile*) mf; const unsigned char *p = (const unsigned char *) mmf->getView(); const unsigned char *w = (const unsigned char *) mmf->view_write(); @@ -437,7 +470,7 @@ namespace mongo { /** (SLOW) diagnostic to check that the private view and the non-private view are in sync. */ void debugValidateAllMapsMatch() { - if( ! (cmdLine.durOptions & CmdLine::DurParanoid) ) + if (!(storageGlobalParams.durOptions & StorageGlobalParams::DurParanoid)) return; unsigned long long bytes = 0; @@ -466,7 +499,7 @@ namespace mongo { // remapping. unsigned long long now = curTimeMicros64(); double fraction = (now-lastRemap)/2000000.0; - if( cmdLine.durOptions & CmdLine::DurAlwaysRemap ) + if (storageGlobalParams.durOptions & StorageGlobalParams::DurAlwaysRemap) fraction = 1; lastRemap = now; @@ -515,8 +548,8 @@ namespace mongo { Timer t; for( unsigned x = 0; x < ntodo; x++ ) { dassert( i != e ); - if( (*i)->isMongoMMF() ) { - MongoMMF *mmf = (MongoMMF*) *i; + if( (*i)->isDurableMappedFile() ) { + DurableMappedFile *mmf = (DurableMappedFile*) *i; verify(mmf); if( mmf->willNeedRemap() ) { mmf->willNeedRemap() = false; @@ -566,7 +599,11 @@ namespace mongo { } JSectHeader h; - PREPLOGBUFFER(h,ab); // need to be in readlock (writes excluded) for this + // need to be in readlock (writes excluded) for this as write intent stuctures point into + // the private mmap for their actual data. i suppose we could lock individual databases + // and do them one at a time or in parallel (surely the latter would make sense if one went + // that route...) + PREPLOGBUFFER(h,ab); LockMongoFilesShared lk3; @@ -636,7 +673,8 @@ namespace mongo { AlignedBuilder &ab = __theBuilder; // we need to make sure two group commits aren't running at the same time - // (and we are only read locked in the dbMutex, so it could happen) + // (and we are only read locked in the dbMutex, so it could happen -- while + // there is only one dur thread, "early commits" can be done by other threads) SimpleMutex::scoped_lock lk(commitJob.groupCommitMutex); commitJob.commitingBegin(); @@ -673,6 +711,8 @@ namespace mongo { // DEV verify( !commitJob.hasWritten() ); if( !Lock::isW() ) { + // todo: note we end up here i believe if our lock state is X -- and that might not be what we want. + // REMAPPRIVATEVIEW needs done in a write lock (as there is a short window during remapping when each view // might not exist) thus we do it later. // @@ -692,19 +732,19 @@ namespace mongo { else { stats.curr->_commitsInWriteLock++; // however, if we are already write locked, we must do it now -- up the call tree someone - // may do a write without a new lock acquisition. this can happen when MongoMMF::close() calls + // may do a write without a new lock acquisition. this can happen when DurableMappedFile::close() calls // this method when a file (and its views) is about to go away. // REMAPPRIVATEVIEW(); } } - /** locking: in read lock when called - or, for early commits (commitIfNeeded), in write lock + /** locking: in at least 'R' when called + or, for early commits (commitIfNeeded), in W or X @param lwg set if the durcommitthread *only* -- then we will upgrade the lock - to W so we can remapprivateview. only durcommitthread as more than one - thread upgrading would potentially deadlock - @see MongoMMF::close() + to W so we can remapprivateview. only durcommitthread calls with + lgw != 0 as more than one thread upgrading would deadlock + @see DurableMappedFile::close() */ static void groupCommit(Lock::GlobalWrite *lgw) { try { @@ -734,7 +774,9 @@ namespace mongo { const int N = 10; static int n; - if( privateMapBytes < UncommittedBytesLimit && ++n % N && (cmdLine.durOptions&CmdLine::DurAlwaysRemap)==0 ) { + if (privateMapBytes < UncommittedBytesLimit && ++n % N && + (storageGlobalParams.durOptions & + StorageGlobalParams::DurAlwaysRemap) == 0) { // limited locks version doesn't do any remapprivateview at all, so only try this if privateMapBytes // is in an acceptable range. also every Nth commit, we do everything so we can do some remapping; // remapping a lot all at once could cause jitter from a large amount of copy-on-writes all at once. @@ -751,11 +793,11 @@ namespace mongo { groupCommit(&w); } - /** called when a MongoMMF is closing -- we need to go ahead and group commit in that case before its + /** called when a DurableMappedFile is closing -- we need to go ahead and group commit in that case before its views disappear */ void closingFileNotification() { - if (!cmdLine.dur) + if (!storageGlobalParams.dur) return; if( Lock::isLocked() ) { @@ -777,7 +819,8 @@ namespace mongo { bool samePartition = true; try { - const string dbpathDir = boost::filesystem::path(dbpath).string(); + const std::string dbpathDir = + boost::filesystem::path(storageGlobalParams.dbpath).string(); samePartition = onSamePartition(getJournalDir().string(), dbpathDir); } catch(...) { @@ -786,7 +829,7 @@ namespace mongo { while( !inShutdown() ) { RACECHECK - unsigned ms = cmdLine.journalCommitInterval; + unsigned ms = storageGlobalParams.journalCommitInterval; if( ms == 0 ) { // use default ms = samePartition ? 100 : 30; @@ -829,17 +872,17 @@ namespace mongo { /** at startup, recover, and then start the journal threads */ void startup() { - if( !cmdLine.dur ) + if (!storageGlobalParams.dur) return; #if defined(_DURABLEDEFAULTON) DEV { if( time(0) & 1 ) { - cmdLine.durOptions |= CmdLine::DurAlwaysCommit; + storageGlobalParams.durOptions |= StorageGlobalParams::DurAlwaysCommit; log() << "_DEBUG _DURABLEDEFAULTON : forcing DurAlwaysCommit mode for this run" << endl; } if( time(0) & 2 ) { - cmdLine.durOptions |= CmdLine::DurAlwaysRemap; + storageGlobalParams.durOptions |= StorageGlobalParams::DurAlwaysRemap; log() << "_DEBUG _DURABLEDEFAULTON : forcing DurAlwaysRemap mode for this run" << endl; } } @@ -892,7 +935,7 @@ namespace mongo { virtual bool includeByDefault() const { return true; } BSONObj generateSection(const BSONElement& configElement) const { - if ( ! cmdLine.dur ) + if (!storageGlobalParams.dur) return BSONObj(); return dur::stats.asObj(); } diff --git a/src/mongo/db/dur.h b/src/mongo/db/dur.h index 604ed2e23b1..f4982694184 100644 --- a/src/mongo/db/dur.h +++ b/src/mongo/db/dur.h @@ -14,12 +14,24 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "diskloc.h" -#include "mongommf.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/storage/durable_mapped_file.h" namespace mongo { @@ -37,7 +49,7 @@ namespace mongo { /** Call during startup so durability module can initialize Throws if fatal error - Does nothing if cmdLine.dur is false + Does nothing if storageGlobalParams.dur is false */ void startup(); @@ -120,7 +132,7 @@ namespace mongo { virtual bool commitIfNeeded(bool force=false) = 0; /** @return true if time to commit but does NOT do a commit */ - virtual bool aCommitIsNeeded() const = 0; + virtual bool isCommitNeeded() const = 0; /** Declare write intent for a DiskLoc. @see DiskLoc::writing() */ inline DiskLoc& writingDiskLoc(DiskLoc& d) { return *((DiskLoc*) writingPtr(&d, sizeof(d))); } @@ -135,7 +147,7 @@ namespace mongo { inline T* alreadyDeclared(T *x) { #if defined(_TESTINTENT) - return (T*) MongoMMF::switchToPrivateView(x); + return (T*) DurableMappedFile::switchToPrivateView(x); #else return x; #endif @@ -157,6 +169,8 @@ namespace mongo { */ virtual void syncDataAndTruncateJournal() = 0; + virtual bool isDurable() const = 0; + static DurableInterface& getDur() { return *_impl; } private: @@ -186,10 +200,11 @@ namespace mongo { void declareWriteIntent(void *, unsigned); void createdFile(const std::string& filename, unsigned long long len) { } bool awaitCommit() { return false; } - bool commitNow() { return false; } - bool commitIfNeeded(bool) { return false; } - bool aCommitIsNeeded() const { return false; } + bool commitNow(); + bool commitIfNeeded(bool); + bool isCommitNeeded() const { return false; } void syncDataAndTruncateJournal() {} + bool isDurable() const { return false; } }; class DurableImpl : public DurableInterface { @@ -201,9 +216,10 @@ namespace mongo { void createdFile(const std::string& filename, unsigned long long len); bool awaitCommit(); bool commitNow(); - bool aCommitIsNeeded() const; + bool isCommitNeeded() const; bool commitIfNeeded(bool); void syncDataAndTruncateJournal(); + bool isDurable() const { return true; } }; } // namespace dur diff --git a/src/mongo/db/dur_commitjob.cpp b/src/mongo/db/dur_commitjob.cpp index 96eb203e1bc..01ccee557d5 100644 --- a/src/mongo/db/dur_commitjob.cpp +++ b/src/mongo/db/dur_commitjob.cpp @@ -14,14 +14,28 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "dur_commitjob.h" -#include "dur_stats.h" -#include "taskqueue.h" -#include "client.h" -#include "../util/concurrency/threadlocal.h" +#include "mongo/pch.h" + +#include "mongo/db/dur_commitjob.h" + +#include "mongo/db/client.h" +#include "mongo/db/dur_stats.h" +#include "mongo/db/taskqueue.h" +#include "mongo/util/concurrency/threadlocal.h" #include "mongo/util/stacktrace.h" namespace mongo { @@ -122,6 +136,7 @@ namespace mongo { /** base declare write intent function that all the helpers call. */ /** we batch up our write intents so that we do not have to synchronize too often */ void DurableImpl::declareWriteIntent(void *p, unsigned len) { + dassert( Lock::somethingWriteLocked() ); cc().writeHappened(); MemoryMappedFile::makeWritable(p, len); ThreadLocalIntents *t = tlIntents.getMake(); @@ -165,7 +180,8 @@ namespace mongo { /** note an operation other than a "basic write" */ void CommitJob::noteOp(shared_ptr p) { - dassert( cmdLine.dur ); + dassert( Lock::somethingWriteLocked() ); + dassert(storageGlobalParams.dur); // DurOp's are rare so it is ok to have the lock cost here SimpleMutex::scoped_lock lk(groupCommitMutex); cc().writeHappened(); @@ -199,6 +215,7 @@ namespace mongo { } void CommitJob::note(void* p, int len) { + dassert( Lock::somethingWriteLocked() ); groupCommitMutex.dassertLocked(); dassert( _hasWritten ); @@ -220,7 +237,7 @@ namespace mongo { static int n; if( ++n < 10000 ) { size_t ofs; - MongoMMF *mmf = privateViews._find(w.p, ofs); + DurableMappedFile *mmf = privateViews._find(w.p, ofs); if( mmf ) { log() << "DEBUG note write intent " << w.p << ' ' << mmf->filename() << " ofs:" << hex << ofs << " len:" << w.len << endl; } diff --git a/src/mongo/db/dur_commitjob.h b/src/mongo/db/dur_commitjob.h index ed7fd9a46ff..6d5863e0c7d 100644 --- a/src/mongo/db/dur_commitjob.h +++ b/src/mongo/db/dur_commitjob.h @@ -14,18 +14,29 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "../util/alignedbuilder.h" -#include "../util/mongoutils/hash.h" -#include "../util/concurrency/synchronization.h" -#include "cmdline.h" -#include "durop.h" -#include "dur.h" -#include "taskqueue.h" -#include "d_concurrency.h" +#include "mongo/db/d_concurrency.h" +#include "mongo/db/dur.h" +#include "mongo/db/durop.h" +#include "mongo/db/taskqueue.h" +#include "mongo/util/alignedbuilder.h" +#include "mongo/util/concurrency/synchronization.h" +#include "mongo/util/mongoutils/hash.h" namespace mongo { namespace dur { diff --git a/src/mongo/db/dur_journal.cpp b/src/mongo/db/dur_journal.cpp index cd413eab0a6..bb466139f2e 100644 --- a/src/mongo/db/dur_journal.cpp +++ b/src/mongo/db/dur_journal.cpp @@ -14,35 +14,50 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "client.h" -#include "namespace.h" -#include "dur_journal.h" -#include "dur_journalformat.h" -#include "dur_stats.h" -#include "../util/logfile.h" -#include "../util/timer.h" -#include "../util/alignedbuilder.h" -#include "../util/net/listen.h" // getelapsedtimemillis +#include "mongo/pch.h" + +#include "mongo/db/dur_journal.h" + #include #include -#include "../util/mongoutils/str.h" -#include "dur_journalimpl.h" -#include "../util/file.h" -#include "../util/checksum.h" -#include "../util/concurrency/race.h" -#include "../util/compress.h" -#include "../util/progress_meter.h" -#include "../server.h" -#include "../util/mmap.h" +#include + +#include "mongo/base/init.h" +#include "mongo/db/client.h" +#include "mongo/db/dur_journalformat.h" +#include "mongo/db/dur_journalimpl.h" +#include "mongo/db/dur_stats.h" +#include "mongo/db/storage_options.h" #include "mongo/platform/random.h" +#include "mongo/server.h" +#include "mongo/util/alignedbuilder.h" +#include "mongo/util/checksum.h" +#include "mongo/util/compress.h" +#include "mongo/util/concurrency/race.h" +#include "mongo/util/file.h" +#include "mongo/util/logfile.h" +#include "mongo/util/mmap.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/net/listen.h" // getelapsedtimemillis +#include "mongo/util/progress_meter.h" +#include "mongo/util/timer.h" using namespace mongoutils; -#include - namespace mongo { class AlignedBuilder; @@ -64,6 +79,14 @@ namespace mongo { unsigned long long DataLimitPerJournalFile = (sizeof(void*)==4) ? 256 * 1024 * 1024 : 1 * 1024 * 1024 * 1024; #endif + MONGO_INITIALIZER(InitializeJournalingParams)(InitializerContext* context) { + if (storageGlobalParams.smallfiles == true) { + verify(dur::DataLimitPerJournalFile >= 128 * 1024 * 1024); + dur::DataLimitPerJournalFile = 128 * 1024 * 1024; + } + return Status::OK(); + } + BOOST_STATIC_ASSERT( sizeof(Checksum) == 16 ); BOOST_STATIC_ASSERT( sizeof(JHeader) == 8192 ); BOOST_STATIC_ASSERT( sizeof(JSectHeader) == 20 ); @@ -76,7 +99,7 @@ namespace mongo { void removeOldJournalFile(boost::filesystem::path p); boost::filesystem::path getJournalDir() { - boost::filesystem::path p(dbpath); + boost::filesystem::path p(storageGlobalParams.dbpath); p /= "journal"; return p; } @@ -392,12 +415,12 @@ namespace mongo { } void preallocateFiles() { - if (! (cmdLine.durOptions & CmdLine::DurNoCheckSpace)) + if (!(storageGlobalParams.durOptions & StorageGlobalParams::DurNoCheckSpace)) checkFreeSpace(); if( exists(preallocPath(0)) || // if enabled previously, keep using exists(preallocPath(1)) || - ( cmdLine.preallocj && preallocateIsFaster() ) ) { + (storageGlobalParams.preallocj && preallocateIsFaster()) ) { usingPreallocate = true; try { _preallocateFiles(); diff --git a/src/mongo/db/dur_journal.h b/src/mongo/db/dur_journal.h index 6decdc0a003..9e7fb5bf4b6 100644 --- a/src/mongo/db/dur_journal.h +++ b/src/mongo/db/dur_journal.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once diff --git a/src/mongo/db/dur_journalformat.h b/src/mongo/db/dur_journalformat.h index 10ed8487b71..9e35e10cfe2 100644 --- a/src/mongo/db/dur_journalformat.h +++ b/src/mongo/db/dur_journalformat.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once diff --git a/src/mongo/db/dur_journalimpl.h b/src/mongo/db/dur_journalimpl.h index ded9e4640a0..bafc0570f19 100644 --- a/src/mongo/db/dur_journalimpl.h +++ b/src/mongo/db/dur_journalimpl.h @@ -14,12 +14,24 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "dur_journalformat.h" -#include "../util/logfile.h" +#include "mongo/db/dur_journalformat.h" +#include "mongo/util/logfile.h" namespace mongo { namespace dur { diff --git a/src/mongo/db/dur_preplogbuffer.cpp b/src/mongo/db/dur_preplogbuffer.cpp index a6622c19e52..9686c21a526 100644 --- a/src/mongo/db/dur_preplogbuffer.cpp +++ b/src/mongo/db/dur_preplogbuffer.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* @@ -24,19 +36,19 @@ @see https://docs.google.com/drawings/edit?id=1TklsmZzm7ohIZkwgeK6rMvsdaR13KjtJYMsfLr175Zc */ -#include "pch.h" -#include "cmdline.h" -#include "dur.h" -#include "dur_journal.h" -#include "dur_journalimpl.h" -#include "dur_commitjob.h" -#include "../util/mongoutils/hash.h" -#include "../util/mongoutils/str.h" -#include "../util/alignedbuilder.h" +#include "mongo/pch.h" + +#include "mongo/db/dur.h" +#include "mongo/db/dur_commitjob.h" +#include "mongo/db/dur_journal.h" +#include "mongo/db/dur_journalimpl.h" +#include "mongo/db/dur_stats.h" +#include "mongo/server.h" +#include "mongo/util/alignedbuilder.h" +#include "mongo/util/mongoutils/hash.h" +#include "mongo/util/mongoutils/str.h" #include "mongo/util/stacktrace.h" -#include "../util/timer.h" -#include "dur_stats.h" -#include "../server.h" +#include "mongo/util/timer.h" using namespace mongoutils; @@ -47,8 +59,8 @@ namespace mongo { RelativePath local = RelativePath::fromRelativePath("local"); - static MongoMMF* findMMF_inlock(void *ptr, size_t &ofs) { - MongoMMF *f = privateViews.find_inlock(ptr, ofs); + static DurableMappedFile* findMMF_inlock(void *ptr, size_t &ofs) { + DurableMappedFile *f = privateViews.find_inlock(ptr, ofs); if( f == 0 ) { error() << "findMMF_inlock failed " << privateViews.numberOfViews_inlock() << endl; printStackTrace(); // we want a stack trace and the assert below didn't print a trace once in the real world - not sure why @@ -62,7 +74,7 @@ namespace mongo { /** put the basic write operation into the buffer (bb) to be journaled */ static void prepBasicWrite_inlock(AlignedBuilder&bb, const WriteIntent *i, RelativePath& lastDbPath) { size_t ofs = 1; - MongoMMF *mmf = findMMF_inlock(i->start(), /*out*/ofs); + DurableMappedFile *mmf = findMMF_inlock(i->start(), /*out*/ofs); if( unlikely(!mmf->willNeedRemap()) ) { // tag this mmf as needed a remap of its private view later. @@ -72,7 +84,7 @@ namespace mongo { } // since we have already looked up the mmf, we go ahead and remember the write view location - // so we don't have to find the MongoMMF again later in WRITETODATAFILES() + // so we don't have to find the DurableMappedFile again later in WRITETODATAFILES() // // this was for WRITETODATAFILES_Impl2 so commented out now // @@ -129,7 +141,12 @@ namespace mongo { assertNothingSpooled(); const vector& _intents = commitJob.getIntentsSorted(); - verify( !_intents.empty() ); + + // right now the durability code assumes there is at least one write intent + // this does not have to be true in theory as i could just add or delete a file + // callers have to ensure they do at least something for now even though its ugly + // until this can be addressed + fassert( 17388, !_intents.empty() ); WriteIntent last; for( vector::const_iterator i = _intents.begin(); i != _intents.end(); i++ ) { @@ -161,7 +178,7 @@ namespace mongo { @return partially populated sectheader and _ab set */ static void _PREPLOGBUFFER(JSectHeader& h, AlignedBuilder& bb) { - verify( cmdLine.dur ); + verify(storageGlobalParams.dur); assertLockedForCommitting(); resetLogBuffer(h, bb); // adds JSectHeader diff --git a/src/mongo/db/dur_recover.cpp b/src/mongo/db/dur_recover.cpp index 717bdcf0e5a..2e972b45d86 100644 --- a/src/mongo/db/dur_recover.cpp +++ b/src/mongo/db/dur_recover.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -24,9 +36,8 @@ #include #include -#include "mongo/db/cmdline.h" #include "mongo/db/curop.h" -#include "mongo/db/database.h" +#include "mongo/db/catalog/database.h" #include "mongo/db/db.h" #include "mongo/db/dur.h" #include "mongo/db/dur_commitjob.h" @@ -35,9 +46,9 @@ #include "mongo/db/dur_stats.h" #include "mongo/db/durop.h" #include "mongo/db/kill_current_op.h" -#include "mongo/db/mongommf.h" -#include "mongo/db/namespace.h" +#include "mongo/db/storage/durable_mapped_file.h" #include "mongo/db/pdfile.h" +#include "mongo/db/storage_options.h" #include "mongo/util/bufreader.h" #include "mongo/util/checksum.h" #include "mongo/util/compress.h" @@ -49,6 +60,15 @@ using namespace mongoutils; namespace mongo { + /** + * Thrown when a journal section is corrupt. This is considered OK as long as it occurs while + * processing the last file. Processing stops at the first corrupt section. + * + * Any logging about the nature of the corruption should happen before throwing as this class + * contains no data. + */ + class JournalSectionCorruptException {}; + namespace dur { struct ParsedJournalEntry { /*copyable*/ @@ -112,9 +132,10 @@ namespace mongo { verify( doDurOpsRecovering ); bool ok = uncompress((const char *)compressed, compressedLen, &_uncompressed); if( !ok ) { - // it should always be ok (i think?) as there is a previous check to see that the JSectFooter is ok + // We check the checksum before we uncompress, but this may still fail as the + // checksum isn't foolproof. log() << "couldn't uncompress journal section" << endl; - msgasserted(15874, "couldn't uncompress journal section"); + throw JournalSectionCorruptException(); } const char *p = _uncompressed.c_str(); verify( compressedLen == _h.sectionLen() - sizeof(JSectFooter) - sizeof(JSectHeader) ); @@ -160,9 +181,14 @@ namespace mongo { case JEntry::OpCode_DbContext: { _lastDbName = (const char*) _entries->pos(); - const unsigned limit = std::min((unsigned)Namespace::MaxNsLen, _entries->remaining()); + const unsigned limit = std::min((unsigned)Namespace::MaxNsLenWithNUL, + _entries->remaining()); const unsigned len = strnlen(_lastDbName, limit); - massert(13533, "problem processing journal file during recovery", _lastDbName[len] == '\0'); + if (_lastDbName[len] != '\0') { + log() << "problem processing journal file during recovery"; + throw JournalSectionCorruptException(); + } + _entries->skip(len+1); // skip '\0' too _entries->read(lenOrOpCode); // read this for the fall through } @@ -195,7 +221,7 @@ namespace mongo { ss << fileNo; // relative name -> full path name - boost::filesystem::path full(dbpath); + boost::filesystem::path full(storageGlobalParams.dbpath); full /= ss.str(); return full.string(); } @@ -222,7 +248,7 @@ namespace mongo { LockMongoFilesShared::assertAtLeastReadLocked(); } - MongoMMF* RecoveryJob::Last::newEntry(const dur::ParsedJournalEntry& entry, RecoveryJob& rj) { + DurableMappedFile* RecoveryJob::Last::newEntry(const dur::ParsedJournalEntry& entry, RecoveryJob& rj) { int num = entry.e->getFileNo(); if( num == fileNo && entry.dbName == dbName ) return mmf; @@ -230,20 +256,20 @@ namespace mongo { string fn = fileName(entry.dbName, num); MongoFile *file; { - MongoFileFinder finder; // must release lock before creating new MongoMMF + MongoFileFinder finder; // must release lock before creating new DurableMappedFile file = finder.findByPath(fn); } if (file) { - verify(file->isMongoMMF()); - mmf = (MongoMMF*)file; + verify(file->isDurableMappedFile()); + mmf = (DurableMappedFile*)file; } else { if( !rj._recovering ) { log() << "journal error applying writes, file " << fn << " is not open" << endl; verify(false); } - boost::shared_ptr sp (new MongoMMF); + boost::shared_ptr sp (new DurableMappedFile); verify(sp->open(fn, false)); rj._mmfs.push_back(sp); mmf = sp.get(); @@ -261,7 +287,7 @@ namespace mongo { verify(entry.dbName); verify((size_t)strnlen(entry.dbName, MaxDatabaseNameLen) < MaxDatabaseNameLen); - MongoMMF *mmf = last.newEntry(entry, *this); + DurableMappedFile *mmf = last.newEntry(entry, *this); if ((entry.e->ofs + entry.e->len) <= mmf->length()) { verify(mmf->view_write()); @@ -307,28 +333,28 @@ namespace mongo { } } - MongoMMF* RecoveryJob::getMongoMMF(const ParsedJournalEntry& entry) { + DurableMappedFile* RecoveryJob::getDurableMappedFile(const ParsedJournalEntry& entry) { verify(entry.dbName); verify((size_t)strnlen(entry.dbName, MaxDatabaseNameLen) < MaxDatabaseNameLen); const string fn = fileName(entry.dbName, entry.e->getFileNo()); MongoFile* file; { - MongoFileFinder finder; // must release lock before creating new MongoMMF + MongoFileFinder finder; // must release lock before creating new DurableMappedFile file = finder.findByPath(fn); } - MongoMMF* mmf; + DurableMappedFile* mmf; if (file) { - verify(file->isMongoMMF()); - mmf = (MongoMMF*)file; + verify(file->isDurableMappedFile()); + mmf = (DurableMappedFile*)file; } else { if( !_recovering ) { log() << "journal error applying writes, file " << fn << " is not open" << endl; verify(false); } - boost::shared_ptr sp (new MongoMMF); + boost::shared_ptr sp (new DurableMappedFile); verify(sp->open(fn, false)); _mmfs.push_back(sp); mmf = sp.get(); @@ -338,8 +364,10 @@ namespace mongo { } void RecoveryJob::applyEntries(const vector &entries) { - bool apply = (cmdLine.durOptions & CmdLine::DurScanOnly) == 0; - bool dump = cmdLine.durOptions & CmdLine::DurDumpJournal; + bool apply = (storageGlobalParams.durOptions & + StorageGlobalParams::DurScanOnly) == 0; + bool dump = storageGlobalParams.durOptions & + StorageGlobalParams::DurDumpJournal; if( dump ) log() << "BEGIN section" << endl; @@ -357,10 +385,15 @@ namespace mongo { scoped_lock lk(_mx); RACECHECK - /** todo: we should really verify the checksum to see that seqNumber is ok? - that is expensive maybe there is some sort of checksum of just the header - within the header itself - */ + // Check the footer checksum before doing anything else. + if (_recovering) { + verify( ((const char *)h) + sizeof(JSectHeader) == p ); + if (!f->checkHash(h, len + sizeof(JSectHeader))) { + log() << "journal section checksum doesn't match"; + throw JournalSectionCorruptException(); + } + } + if( _recovering && _lastDataSyncedFromLastRun > h->seqNumber + ExtraKeepTimeMs ) { if( h->seqNumber != _lastSeqMentionedInConsoleLog ) { static int n; @@ -403,14 +436,6 @@ namespace mongo { entries.push_back(e); } - // after the entries check the footer checksum - if( _recovering ) { - verify( ((const char *)h) + sizeof(JSectHeader) == p ); - if( !f->checkHash(h, len + sizeof(JSectHeader)) ) { - msgasserted(13594, "journal checksum doesn't match"); - } - } - // got all the entries for one group commit. apply them: applyEntries(entries); } @@ -429,24 +454,25 @@ namespace mongo { JHeader h; br.read(h); - /* [dm] not automatically handled. we should eventually handle this automatically. i think: - (1) if this is the final journal file - (2) and the file size is just the file header in length (or less) -- this is a bit tricky to determine if prealloced - then can just assume recovery ended cleanly and not error out (still should log). - */ - uassert(13537, - "journal file header invalid. This could indicate corruption in a journal file, or perhaps a crash where sectors in file header were in flight written out of order at time of crash (unlikely but possible).", - h.valid()); + if (!h.valid()) { + log() << "Journal file header invalid. This could indicate corruption, or " + << "an unclean shutdown while writing the first section in a journal " + << "file."; + throw JournalSectionCorruptException(); + } if( !h.versionOk() ) { log() << "journal file version number mismatch got:" << hex << h._version << " expected:" << hex << (unsigned) JHeader::CurrentVersion << ". if you have just upgraded, recover with old version of mongod, terminate cleanly, then upgrade." << endl; + // Not using JournalSectionCurruptException as we don't want to ignore + // journal files on upgrade. uasserted(13536, str::stream() << "journal version number mismatch " << h._version); } fileId = h.fileId; - if(cmdLine.durOptions & CmdLine::DurDumpJournal) { + if (storageGlobalParams.durOptions & + StorageGlobalParams::DurDumpJournal) { log() << "JHeader::fileId=" << fileId << endl; } } @@ -456,7 +482,8 @@ namespace mongo { JSectHeader h; br.peek(h); if( h.fileId != fileId ) { - if( debug || (cmdLine.durOptions & CmdLine::DurDumpJournal) ) { + if (debug || (storageGlobalParams.durOptions & + StorageGlobalParams::DurDumpJournal)) { log() << "Ending processFileBuffer at differing fileId want:" << fileId << " got:" << h.fileId << endl; log() << " sect len:" << h.sectionLen() << " seqnum:" << h.seqNumber << endl; } @@ -473,8 +500,13 @@ namespace mongo { killCurrentOp.checkForInterrupt(false); } } - catch( BufReader::eof& ) { - if( cmdLine.durOptions & CmdLine::DurDumpJournal ) + catch (const BufReader::eof&) { + if (storageGlobalParams.durOptions & StorageGlobalParams::DurDumpJournal) + log() << "ABRUPT END" << endl; + return true; // abrupt end + } + catch (const JournalSectionCorruptException&) { + if (storageGlobalParams.durOptions & StorageGlobalParams::DurDumpJournal) log() << "ABRUPT END" << endl; return true; // abrupt end } @@ -523,8 +555,10 @@ namespace mongo { close(); - if( cmdLine.durOptions & CmdLine::DurScanOnly ) { - uasserted(13545, str::stream() << "--durOptions " << (int) CmdLine::DurScanOnly << " (scan only) specified"); + if (storageGlobalParams.durOptions & StorageGlobalParams::DurScanOnly) { + uasserted(13545, str::stream() << "--durOptions " + << (int) StorageGlobalParams::DurScanOnly + << " (scan only) specified"); } log() << "recover cleaning up" << endl; @@ -535,7 +569,7 @@ namespace mongo { } void _recover() { - verify( cmdLine.dur ); + verify(storageGlobalParams.dur); boost::filesystem::path p = getJournalDir(); if( !exists(p) ) { @@ -569,7 +603,7 @@ namespace mongo { // at this point in the program so it wouldn't have been a true problem (I think) // can't lock groupCommitMutex here as - // MongoMMF::close()->closingFileNotication()->groupCommit() will lock it + // DurableMappedFile::close()->closingFileNotication()->groupCommit() will lock it // and that would be recursive. // // SimpleMutex::scoped_lock lk2(commitJob.groupCommitMutex); diff --git a/src/mongo/db/dur_recover.h b/src/mongo/db/dur_recover.h index 4cb36eec9aa..b36c7ea6562 100644 --- a/src/mongo/db/dur_recover.h +++ b/src/mongo/db/dur_recover.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -26,7 +38,7 @@ #include "mongo/util/file.h" namespace mongo { - class MongoMMF; + class DurableMappedFile; namespace dur { struct ParsedJournalEntry; @@ -37,9 +49,9 @@ namespace mongo { static class Last { public: Last(); - MongoMMF* newEntry(const ParsedJournalEntry&, RecoveryJob&); + DurableMappedFile* newEntry(const ParsedJournalEntry&, RecoveryJob&); private: - MongoMMF *mmf; + DurableMappedFile *mmf; string dbName; int fileNo; } last; @@ -62,9 +74,9 @@ namespace mongo { bool processFileBuffer(const void *, unsigned len); bool processFile(boost::filesystem::path journalfile); void _close(); // doesn't lock - MongoMMF* getMongoMMF(const ParsedJournalEntry& entry); + DurableMappedFile* getDurableMappedFile(const ParsedJournalEntry& entry); - list > _mmfs; + list > _mmfs; unsigned long long _lastDataSyncedFromLastRun; unsigned long long _lastSeqMentionedInConsoleLog; diff --git a/src/mongo/db/dur_stats.h b/src/mongo/db/dur_stats.h index a3b00952a97..e2c0b29353e 100644 --- a/src/mongo/db/dur_stats.h +++ b/src/mongo/db/dur_stats.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ namespace mongo { diff --git a/src/mongo/db/dur_writetodatafiles.cpp b/src/mongo/db/dur_writetodatafiles.cpp index f6978920e14..d9a2e345115 100644 --- a/src/mongo/db/dur_writetodatafiles.cpp +++ b/src/mongo/db/dur_writetodatafiles.cpp @@ -14,9 +14,21 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/db/dur_commitjob.h" #include "mongo/db/dur_recover.h" @@ -25,9 +37,6 @@ #include "mongo/util/timer.h" namespace mongo { -#ifdef _WIN32 - extern SimpleMutex globalFlushMutex; // defined in mongo/util/mmap_win.cpp -#endif namespace dur { void debugValidateAllMapsMatch(); @@ -79,7 +88,7 @@ namespace mongo { - we couldn't do it from the private views then as they may be changing. would have to then be from the journal alignedbuffer. - we need to be careful the file isn't unmapped on us -- perhaps a mutex or something - with MongoMMF on closes or something to coordinate that. + with DurableMappedFile on closes or something to coordinate that. concurrency: in mmmutex, not necessarily in dbMutex @@ -87,9 +96,6 @@ namespace mongo { */ void WRITETODATAFILES(const JSectHeader& h, AlignedBuilder& uncompressed) { -#ifdef _WIN32 - SimpleMutex::scoped_lock _globalFlushMutex(globalFlushMutex); -#endif Timer t; WRITETODATAFILES_Impl1(h, uncompressed); unsigned long long m = t.micros(); diff --git a/src/mongo/db/durop.cpp b/src/mongo/db/durop.cpp index 400884bfb50..3a7fadf753e 100644 --- a/src/mongo/db/durop.cpp +++ b/src/mongo/db/durop.cpp @@ -14,16 +14,31 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "d_concurrency.h" -#include "../util/alignedbuilder.h" -#include "../util/mongoutils/str.h" -#include "../util/file.h" -#include "mongommf.h" -#include "durop.h" -#include "../util/file_allocator.h" +#include "mongo/pch.h" + +#include "mongo/db/durop.h" + +#include "mongo/db/d_concurrency.h" +#include "mongo/db/repair_database.h" +#include "mongo/db/storage/durable_mapped_file.h" +#include "mongo/util/alignedbuilder.h" +#include "mongo/util/file.h" +#include "mongo/util/file_allocator.h" +#include "mongo/util/mongoutils/str.h" using namespace mongoutils; @@ -31,10 +46,6 @@ using namespace mongoutils; namespace mongo { - extern string dbpath; // --dbpath parm - - void _deleteDataFiles(const char *); - namespace dur { /** read a durop from journal file referenced by br. @@ -79,7 +90,7 @@ namespace mongo { /** throws */ void DropDbOp::replay() { log() << "recover replay drop db " << _db << endl; - _deleteDataFiles(_db.c_str()); + _deleteDataFiles(_db); } FileCreatedOp::FileCreatedOp(const std::string& f, unsigned long long l) : diff --git a/src/mongo/db/durop.h b/src/mongo/db/durop.h index 77b2a8a34f8..6b0f00ea9c0 100644 --- a/src/mongo/db/durop.h +++ b/src/mongo/db/durop.h @@ -14,13 +14,25 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "dur_journalformat.h" -#include "../util/bufreader.h" -#include "../util/paths.h" +#include "mongo/db/dur_journalformat.h" +#include "mongo/util/bufreader.h" +#include "mongo/util/paths.h" namespace mongo { diff --git a/src/mongo/db/exec/2d.cpp b/src/mongo/db/exec/2d.cpp new file mode 100644 index 00000000000..218d320e88b --- /dev/null +++ b/src/mongo/db/exec/2d.cpp @@ -0,0 +1,287 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/2d.h" + +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + TwoD::TwoD(const TwoDParams& params, WorkingSet* ws) + : _params(params), _workingSet(ws), _descriptor(NULL), _am(NULL) { + + init(); + } + + TwoD::~TwoD() { } + + bool TwoD::isEOF() { + return NULL == _browse.get(); + } + + void TwoD::init() { + Database* database = cc().database(); + if ( !database ) return; + + Collection* collection = database->getCollection( _params.ns ); + if ( !collection ) return; + + _descriptor = collection->getIndexCatalog()->findIndexByKeyPattern(_params.indexKeyPattern); + if ( _descriptor == NULL ) return; + + _am = static_cast( collection->getIndexCatalog()->getIndex( _descriptor ) ); + verify( _am ); + + if (NULL != _params.gq.getGeometry()._cap.get()) { + _browse.reset(new twod_exec::GeoCircleBrowse(_params, _am)); + } + else if (NULL != _params.gq.getGeometry()._polygon.get()) { + _browse.reset(new twod_exec::GeoPolygonBrowse(_params, _am)); + } + else { + verify(NULL != _params.gq.getGeometry()._box.get()); + _browse.reset(new twod_exec::GeoBoxBrowse(_params, _am)); + } + + // Fill out static portion of plan stats. + // We will retrieve the geo hashes used by the geo browser + // when the search is complete. + _specificStats.type = _browse->_type; + _specificStats.field = _params.gq.getField(); + _specificStats.converterParams = _browse->_converter->getParams(); + } + + PlanStage::StageState TwoD::work(WorkingSetID* out) { + if (isEOF()) { return PlanStage::IS_EOF; } + + verify(NULL != _browse.get()); + + if (!_browse->ok()) { + // Grab geo hashes before disposing geo browser. + _specificStats.expPrefixes.swap(_browse->_expPrefixes); + _browse.reset(); + return PlanStage::IS_EOF; + } + + WorkingSetID id = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(id); + member->loc = _browse->currLoc(); + member->obj = member->loc.obj(); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + + _browse->advance(); + + *out = id; + _commonStats.advanced++; + _commonStats.works++; + return PlanStage::ADVANCED; + } + + void TwoD::prepareToYield() { + if (NULL != _browse) { + _browse->noteLocation(); + } + } + + void TwoD::recoverFromYield() { + if (NULL != _browse) { + _browse->checkLocation(); + } + } + + void TwoD::invalidate(const DiskLoc& dl, InvalidationType type) { + if (NULL != _browse) { + // If the invalidation actually tossed out a result... + if (_browse->invalidate(dl)) { + // Create a new WSM + WorkingSetID id = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(id); + member->loc = dl; + member->obj = member->loc.obj(); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + + // And flag it for later. + WorkingSetCommon::fetchAndInvalidateLoc(member); + _workingSet->flagForReview(id); + } + } + } + + PlanStageStats* TwoD::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_GEO_2D)); + ret->specific.reset(new TwoDStats(_specificStats)); + return ret.release(); + } +} + +namespace mongo { +namespace twod_exec { + + + // + // Impls of browse below + // + + // + // GeoCircleBrowse + // + + GeoCircleBrowse::GeoCircleBrowse(const TwoDParams& params, TwoDAccessMethod* accessMethod) + : GeoBrowse(accessMethod, "circle", params.filter) { + + _converter = accessMethod->getParams().geoHashConverter; + + const CapWithCRS& cap = *params.gq.getGeometry()._cap; + + _startPt = cap.circle.center; + _start = _converter->hash(_startPt); + _maxDistance = cap.circle.radius; + + if (FLAT == cap.crs) { + _type = GEO_PLANE; + xScanDistance = _maxDistance + _converter->getError(); + yScanDistance = _maxDistance + _converter->getError(); + } else { + _type = GEO_SPHERE; + yScanDistance = rad2deg(_maxDistance) + _converter->getError(); + xScanDistance = computeXScanDistance(_startPt.y, yScanDistance); + } + + // Bounding box includes fudge factor. + // TODO: Is this correct, since fudge factor may be spherically transformed? + _bBox._min = Point(_startPt.x - xScanDistance, _startPt.y - yScanDistance); + _bBox._max = Point(_startPt.x + xScanDistance, _startPt.y + yScanDistance); + + ok(); + } + + GeoAccumulator::KeyResult GeoCircleBrowse::approxKeyCheck(const Point& p, double& d) { + // Inexact hash distance checks. + double error = 0; + switch (_type) { + case GEO_PLANE: + d = distance(_startPt, p); + error = _converter->getError(); + break; + case GEO_SPHERE: { + checkEarthBounds(p); + d = spheredist_deg(_startPt, p); + error = _converter->getErrorSphere(); + break; + } + default: verify(false); + } + + // If our distance is in the error bounds... + if(d >= _maxDistance - error && d <= _maxDistance + error) return BORDER; + return d > _maxDistance ? BAD : GOOD; + } + + bool GeoCircleBrowse::exactDocCheck(const Point& p, double& d){ + switch (_type) { + case GEO_PLANE: { + if(distanceWithin(_startPt, p, _maxDistance)) return true; + break; + } + case GEO_SPHERE: + checkEarthBounds(p); + if(spheredist_deg(_startPt, p) <= _maxDistance) return true; + break; + default: verify(false); + } + + return false; + } + + // + // GeoBoxBrowse + // + + GeoBoxBrowse::GeoBoxBrowse(const TwoDParams& params, TwoDAccessMethod* accessMethod) + : GeoBrowse(accessMethod, "box", params.filter) { + + _converter = accessMethod->getParams().geoHashConverter; + + _want = params.gq.getGeometry()._box->box; + _wantRegion = _want; + // Need to make sure we're checking regions within error bounds of where we want + _wantRegion.fudge(_converter->getError()); + fixBox(_wantRegion); + fixBox(_want); + + Point center = _want.center(); + _start = _converter->hash(center.x, center.y); + + _fudge = _converter->getError(); + _wantLen = _fudge + + std::max((_want._max.x - _want._min.x), + (_want._max.y - _want._min.y)) / 2; + + ok(); + } + + void GeoBoxBrowse::fixBox(Box& box) { + if(box._min.x > box._max.x) + std::swap(box._min.x, box._max.x); + if(box._min.y > box._max.y) + std::swap(box._min.y, box._max.y); + + double gMin = _converter->getMin(); + double gMax = _converter->getMax(); + + if(box._min.x < gMin) box._min.x = gMin; + if(box._min.y < gMin) box._min.y = gMin; + if(box._max.x > gMax) box._max.x = gMax; + if(box._max.y > gMax) box._max.y = gMax; + } + + // + // GeoPolygonBrowse + // + + GeoPolygonBrowse::GeoPolygonBrowse(const TwoDParams& params, TwoDAccessMethod* accessMethod) + : GeoBrowse(accessMethod, "polygon", params.filter) { + + _converter = accessMethod->getParams().geoHashConverter; + + _poly = params.gq.getGeometry()._polygon->oldPolygon; + _bounds = _poly.bounds(); + // We need to check regions within the error bounds of these bounds + _bounds.fudge(_converter->getError()); + // We don't need to look anywhere outside the space + _bounds.truncate(_converter->getMin(), _converter->getMax()); + _maxDim = _converter->getError() + _bounds.maxDim() / 2; + + ok(); + } + +} // namespace twod_exec +} // namespace mongo diff --git a/src/mongo/db/exec/2d.h b/src/mongo/db/exec/2d.h new file mode 100644 index 00000000000..eaaf0bdbdb9 --- /dev/null +++ b/src/mongo/db/exec/2d.h @@ -0,0 +1,181 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/2dcommon.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/geo/geoquery.h" + +#pragma once + +namespace mongo { + + struct TwoDParams { + TwoDParams() : filter(NULL) { } + GeoQuery gq; + MatchExpression* filter; + BSONObj indexKeyPattern; + string ns; + }; + + class TwoD : public PlanStage { + public: + TwoD(const TwoDParams& params, WorkingSet* ws); + virtual ~TwoD(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + void init(); + + scoped_ptr _browse; + TwoDParams _params; + WorkingSet* _workingSet; + IndexDescriptor* _descriptor; + TwoDAccessMethod* _am; + CommonStats _commonStats; + TwoDStats _specificStats; + }; +} + +namespace mongo { +namespace twod_exec { + + // + // Impls of browse below + // + + class GeoCircleBrowse : public GeoBrowse { + public: + GeoCircleBrowse(const TwoDParams& params, TwoDAccessMethod* accessMethod); + + virtual GeoHash expandStartHash() { return _start; } + + virtual bool fitsInBox(double width) { + return width >= std::max(xScanDistance, yScanDistance); + } + + virtual double intersectsBox(Box& cur) { + return cur.intersects(_bBox); + } + + virtual KeyResult approxKeyCheck(const Point& p, double& d); + + virtual bool exactDocCheck(const Point& p, double& d); + + GeoDistType _type; + GeoHash _start; + Point _startPt; + double _maxDistance; // user input + double xScanDistance; // effected by GeoDistType + double yScanDistance; // effected by GeoDistType + Box _bBox; + + shared_ptr _converter; + }; + + class GeoBoxBrowse : public GeoBrowse { + public: + GeoBoxBrowse(const TwoDParams& params, TwoDAccessMethod* accessMethod); + + void fixBox(Box& box); + + virtual GeoHash expandStartHash() { + return _start; + } + + virtual bool fitsInBox(double width) { + return width >= _wantLen; + } + + virtual double intersectsBox(Box& cur) { + return cur.intersects(_wantRegion); + } + + virtual KeyResult approxKeyCheck(const Point& p, double& d) { + if(_want.onBoundary(p, _fudge)) return BORDER; + else return _want.inside(p, _fudge) ? GOOD : BAD; + + } + + virtual bool exactDocCheck(const Point& p, double& d){ + return _want.inside(p); + } + + Box _want; + Box _wantRegion; + double _wantLen; + double _fudge; + GeoHash _start; + shared_ptr _converter; + }; + + class GeoPolygonBrowse : public GeoBrowse { + public: + GeoPolygonBrowse(const TwoDParams& params, TwoDAccessMethod* accessMethod); + + // The initial geo hash box for our first expansion + virtual GeoHash expandStartHash() { + return _converter->hash(_bounds.center()); + } + + // Whether the current box width is big enough for our search area + virtual bool fitsInBox(double width) { + return _maxDim <= width; + } + + // Whether the current box overlaps our search area + virtual double intersectsBox(Box& cur) { + return cur.intersects(_bounds); + } + + virtual KeyResult approxKeyCheck(const Point& p, double& d) { + int in = _poly.contains(p, _converter->getError()); + if(in == 0) return BORDER; + else return in > 0 ? GOOD : BAD; + } + + virtual bool exactDocCheck(const Point& p, double& d){ + return _poly.contains(p); + } + + private: + Polygon _poly; + Box _bounds; + double _maxDim; + GeoHash _start; + shared_ptr _converter; + }; +} // namespace twod_exec +} // namespace mongo diff --git a/src/mongo/db/exec/2dcommon.cpp b/src/mongo/db/exec/2dcommon.cpp new file mode 100644 index 00000000000..bf8f72b57f8 --- /dev/null +++ b/src/mongo/db/exec/2dcommon.cpp @@ -0,0 +1,680 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/2dcommon.h" + +#include "mongo/db/matcher/matchable.h" +#include "mongo/db/query/index_bounds_builder.h" + +namespace mongo { +namespace twod_exec { + + // + // A MatchableDocument that will load the doc if need be but records if it does. + // + + class GeoMatchableDocument : public MatchableDocument { + public: + GeoMatchableDocument(const BSONObj& keyPattern, const BSONObj& key, DiskLoc loc, bool *fetched) + : _keyPattern(keyPattern), + _key(key), + _loc(loc), + _fetched(fetched) { } + + BSONObj toBSON() const { + *_fetched = true; + return _loc.obj(); + } + + virtual ElementIterator* allocateIterator(const ElementPath* path) const { + BSONObjIterator keyPatternIt(_keyPattern); + BSONObjIterator keyDataIt(_key); + + // Skip the "2d"-indexed stuff. We might have a diff. predicate over that field + // and those can't be covered. + keyPatternIt.next(); + keyDataIt.next(); + + // Look in the key. + while (keyPatternIt.more()) { + BSONElement keyPatternElt = keyPatternIt.next(); + verify(keyDataIt.more()); + BSONElement keyDataElt = keyDataIt.next(); + + if (path->fieldRef().equalsDottedField(keyPatternElt.fieldName())) { + if (Array == keyDataElt.type()) { + return new SimpleArrayElementIterator(keyDataElt, true); + } + else { + return new SingleElementElementIterator(keyDataElt); + } + } + } + + // All else fails, fetch. + *_fetched = true; + return new BSONElementIterator(path, _loc.obj()); + } + + virtual void releaseIterator( ElementIterator* iterator ) const { + delete iterator; + } + + private: + BSONObj _keyPattern; + BSONObj _key; + DiskLoc _loc; + bool* _fetched; + }; + + + // + // GeoAccumulator + // + + GeoAccumulator::GeoAccumulator(TwoDAccessMethod* accessMethod, MatchExpression* filter) + : _accessMethod(accessMethod), _converter(accessMethod->getParams().geoHashConverter), + _filter(filter), + _lookedAt(0), _matchesPerfd(0), _objectsLoaded(0), _pointsLoaded(0), _found(0) { } + + GeoAccumulator::~GeoAccumulator() { } + + void GeoAccumulator::add(const GeoIndexEntry& node) { + _lookedAt++; + + // Approximate distance check using key data + double keyD = 0; + Point keyP(_converter->unhashToPoint(node._key.firstElement())); + KeyResult keyOk = approxKeyCheck(keyP, keyD); + if (keyOk == BAD) { + return; + } + + // Check for match using other key (and potentially doc) criteria + // Remember match results for each object + map::iterator match = _matched.find(node.recordLoc); + bool newDoc = match == _matched.end(); + + //cout << "newDoc: " << newDoc << endl; + if(newDoc) { + bool fetched = false; + + if (NULL != _filter) { + GeoMatchableDocument md(_accessMethod->getDescriptor()->keyPattern(), + node._key, + node.recordLoc, + &fetched); + bool good = _filter->matches(&md); + + _matchesPerfd++; + + if (fetched) { + _objectsLoaded++; + } + + if (! good) { + _matched[ node.recordLoc ] = false; + return; + } + } + // Don't double-count. + if (!fetched) { + _objectsLoaded++; + } + } else if(!((*match).second)) { + return; + } + + // Exact check with particular data fields + // Can add multiple points + int diff = addSpecific(node, keyP, keyOk == BORDER, keyD, newDoc); + if(diff > 0) _found += diff; + else _found -= -diff; + } + + void GeoAccumulator::getPointsFor(const BSONObj& key, const BSONObj& obj, + vector &locsForNode, bool allPoints) { + // Find all the location objects from the keys + vector locs; + _accessMethod->getKeys(obj, allPoints ? locsForNode : locs); + ++_pointsLoaded; + + if (allPoints) return; + if (locs.size() == 1){ + locsForNode.push_back(locs[0]); + return; + } + + // Find the particular location we want + GeoHash keyHash(key.firstElement(), _converter->getBits()); + + for(vector< BSONObj >::iterator i = locs.begin(); i != locs.end(); ++i) { + // Ignore all locations not hashed to the key's hash, since we may see + // those later + if(_converter->hash(*i) != keyHash) continue; + locsForNode.push_back(*i); + } + } + + // + // BtreeLocation + // + + // static + bool BtreeLocation::hasPrefix(const BSONObj& key, const GeoHash& hash) { + BSONElement e = key.firstElement(); + if (e.eoo()) { return false; } + return GeoHash(e).hasPrefix(hash); + } + + void BtreeLocation::advance() { + WorkingSetID id = WorkingSet::INVALID_ID; + for (;;) { + PlanStage::StageState state = _scan->work(&id); + if (PlanStage::ADVANCED == state) { + break; + } + else if (PlanStage::NEED_TIME == state) { + continue; + } + else { + // Error or EOF. Either way, stop. + _eof = true; + return; + } + } + verify(WorkingSet::INVALID_ID != id); + WorkingSetMember* wsm = _ws->get(id); + verify(WorkingSetMember::LOC_AND_IDX == wsm->state); + _key = wsm->keyData[0].keyData; + _loc = wsm->loc; + _ws->free(id); + } + + // Returns the min and max keys which bound a particular location. + // The only time these may be equal is when we actually equal the location + // itself, otherwise our expanding algorithm will fail. + // static + bool BtreeLocation::initial(const IndexDescriptor* descriptor, const TwoDIndexingParams& params, + BtreeLocation& min, BtreeLocation& max, GeoHash start) { + verify(descriptor); + + min._eof = false; + max._eof = false; + + // Add the range for the 2d indexed field to the keys used. + + // Two scans: one for min one for max. + IndexScanParams minParams; + minParams.direction = -1; + minParams.descriptor = descriptor; + minParams.bounds.fields.resize(descriptor->keyPattern().nFields()); + minParams.doNotDedup = true; + // First field of start key goes (MINKEY, start] (in reverse) + BSONObjBuilder firstBob; + firstBob.appendMinKey(""); + start.appendToBuilder(&firstBob, ""); + minParams.bounds.fields[0].intervals.push_back(Interval(firstBob.obj(), false, true)); + + IndexScanParams maxParams; + maxParams.direction = 1; + maxParams.descriptor = descriptor; + maxParams.bounds.fields.resize(descriptor->keyPattern().nFields()); + // Don't have the ixscan dedup since we want dup DiskLocs because of multi-point docs. + maxParams.doNotDedup = true; + // First field of end key goes (start, MAXKEY) + BSONObjBuilder secondBob; + start.appendToBuilder(&secondBob, ""); + secondBob.appendMaxKey(""); + maxParams.bounds.fields[0].intervals.push_back(Interval(secondBob.obj(), false, false)); + + BSONObjIterator it(descriptor->keyPattern()); + BSONElement kpElt = it.next(); + maxParams.bounds.fields[0].name = kpElt.fieldName(); + minParams.bounds.fields[0].name = kpElt.fieldName(); + // Fill out the non-2d indexed fields with the "all values" interval, aligned properly. + size_t idx = 1; + while (it.more()) { + kpElt = it.next(); + maxParams.bounds.fields[idx].intervals.push_back(IndexBoundsBuilder::allValues()); + minParams.bounds.fields[idx].intervals.push_back(IndexBoundsBuilder::allValues()); + maxParams.bounds.fields[idx].name = kpElt.fieldName(); + minParams.bounds.fields[idx].name = kpElt.fieldName(); + if (kpElt.number() == -1) { + IndexBoundsBuilder::reverseInterval(&minParams.bounds.fields[idx].intervals[0]); + IndexBoundsBuilder::reverseInterval(&maxParams.bounds.fields[idx].intervals[0]); + } + ++idx; + } + + for (size_t i = 0; i < minParams.bounds.fields.size(); ++i) { + IndexBoundsBuilder::reverseInterval(&minParams.bounds.fields[i].intervals[0]); + } + + //cout << "keyPattern " << descriptor->keyPattern().toString() << endl; + //cout << "minBounds " << minParams.bounds.toString() << endl; + //cout << "maxBounds " << maxParams.bounds.toString() << endl; + verify(minParams.bounds.isValidFor(descriptor->keyPattern(), -1)); + verify(maxParams.bounds.isValidFor(descriptor->keyPattern(), 1)); + + min._ws.reset(new WorkingSet()); + min._scan.reset(new IndexScan(minParams, min._ws.get(), NULL)); + + max._ws.reset(new WorkingSet()); + max._scan.reset(new IndexScan(maxParams, max._ws.get(), NULL)); + + min.advance(); + max.advance(); + + return !max._eof || !min._eof; + } + + // + // GeoBrowse + // + + GeoBrowse::GeoBrowse(TwoDAccessMethod* accessMethod, string type, MatchExpression* filter) + : GeoAccumulator(accessMethod, filter), + _type(type), _firstCall(true), _nscanned(), + _centerPrefix(0, 0, 0), + _descriptor(accessMethod->getDescriptor()), + _converter(accessMethod->getParams().geoHashConverter), + _params(accessMethod->getParams()) { + + // Set up the initial expand state + _state = START; + _neighbor = -1; + _foundInExp = 0; + + } + + bool GeoBrowse::ok() { + /* + cout << "Checking cursor, in state " << (int) _state << ", first call " + << _firstCall << ", empty : " << _cur.isEmpty() + << ", stack : " << _stack.size() << endl; + */ + + bool first = _firstCall; + + if (_firstCall) { + fillStack(maxPointsHeuristic); + _firstCall = false; + } + + if (_stack.size()) { + if (first) { ++_nscanned; } + return true; + } + + while (moreToDo()) { + fillStack(maxPointsHeuristic); + if (! _cur.isEmpty()) { + if (first) { ++_nscanned; } + return true; + } + } + + return !_cur.isEmpty(); + } + + bool GeoBrowse::advance() { + _cur._o = BSONObj(); + + if (_stack.size()) { + _cur = _stack.front(); + _stack.pop_front(); + ++_nscanned; + return true; + } + + if (! moreToDo()) return false; + + while (_cur.isEmpty() && moreToDo()){ + fillStack(maxPointsHeuristic); + } + + return ! _cur.isEmpty() && ++_nscanned; + } + + void GeoBrowse::noteLocation() { + // Remember where our _max, _min are + _min.prepareToYield(); + _max.prepareToYield(); + } + + /* called before query getmore block is iterated */ + void GeoBrowse::checkLocation() { + // We can assume an error was thrown earlier if this database somehow disappears + // Recall our _max, _min + _min.recoverFromYield(); + _max.recoverFromYield(); + } + + Record* GeoBrowse::_current() { verify(ok()); return _cur._loc.rec(); } + BSONObj GeoBrowse::current() { verify(ok()); return _cur._o; } + DiskLoc GeoBrowse::currLoc() { verify(ok()); return _cur._loc; } + BSONObj GeoBrowse::currKey() const { return _cur._key; } + + // Are we finished getting points? + bool GeoBrowse::moreToDo() { return _state != DONE; } + + bool GeoBrowse::checkAndAdvance(BtreeLocation* bl, const GeoHash& hash, int& totalFound) { + if (bl->eof()) { return false; } + + //cout << "looking at " << bl->_loc.obj().toString() << " dl " << bl->_loc.toString() << endl; + + if (!BtreeLocation::hasPrefix(bl->_key, hash)) { return false; } + + totalFound++; + GeoIndexEntry n(bl->_loc, bl->_key); + add(n); + //cout << "adding\n"; + + bl->advance(); + + return true; + } + + + // Fills the stack, but only checks a maximum number of maxToCheck points at a time. + // Further calls to this function will continue the expand/check neighbors algorithm. + void GeoBrowse::fillStack(int maxToCheck, int maxToAdd, bool onlyExpand) { + if(maxToAdd < 0) maxToAdd = maxToCheck; + int maxFound = _foundInExp + maxToCheck; + verify(maxToCheck > 0); + verify(maxFound > 0); + verify(_found <= 0x7fffffff); // conversion to int + int maxAdded = static_cast(_found) + maxToAdd; + verify(maxAdded >= 0); // overflow check + + bool isNeighbor = _centerPrefix.constrains(); + + // Starting a box expansion + if (_state == START) { + // Get the very first hash point, if required + if(! isNeighbor) + _prefix = expandStartHash(); + + if (!BtreeLocation::initial(_descriptor, _params, _min, _max, _prefix)) { + _state = isNeighbor ? DONE_NEIGHBOR : DONE; + } else { + _state = DOING_EXPAND; + _lastPrefix.reset(); + } + } + + // Doing the actual box expansion + if (_state == DOING_EXPAND) { + while (true) { + // Record the prefix we're actively exploring... + _expPrefix.reset(new GeoHash(_prefix)); + + // Find points inside this prefix + while (checkAndAdvance(&_min, _prefix, _foundInExp) + && _foundInExp < maxFound && _found < maxAdded) {} + while (checkAndAdvance(&_max, _prefix, _foundInExp) + && _foundInExp < maxFound && _found < maxAdded) {} + + if(_foundInExp >= maxFound || _found >= maxAdded) return; + + // We've searched this prefix fully, remember + _lastPrefix.reset(new GeoHash(_prefix)); + + // If we've searched the entire space, we're finished. + if (! _prefix.constrains()) { + _state = DONE; + notePrefix(); + return; + } + + // If we won't fit in the box, and we're not doing a sub-scan, increase the size + if (! fitsInBox(_converter->sizeEdge(_prefix)) && _fringe.size() == 0) { + // If we're still not expanded bigger than the box size, expand again + _prefix = _prefix.up(); + continue; + } + + // We're done and our size is large enough + _state = DONE_NEIGHBOR; + + // Go to the next sub-box, if applicable + if(_fringe.size() > 0) _fringe.pop_back(); + // Go to the next neighbor if this was the last sub-search + if(_fringe.size() == 0) _neighbor++; + break; + } + notePrefix(); + } + + // If we doeighbors + if(onlyExpand) return; + + // If we're done expanding the current box... + if(_state == DONE_NEIGHBOR) { + // Iterate to the next neighbor + // Loop is useful for cases where we want to skip over boxes entirely, + // otherwise recursion increments the neighbors. + for (; _neighbor < 9; _neighbor++) { + // If we have no fringe for the neighbor, make sure we have the default fringe + if(_fringe.size() == 0) _fringe.push_back(""); + + if(! isNeighbor) { + _centerPrefix = _prefix; + _centerBox = _converter->unhashToBox(_centerPrefix); + isNeighbor = true; + } + + int i = (_neighbor / 3) - 1; + int j = (_neighbor % 3) - 1; + + if ((i == 0 && j == 0) || + (i < 0 && _centerPrefix.atMinX()) || + (i > 0 && _centerPrefix.atMaxX()) || + (j < 0 && _centerPrefix.atMinY()) || + (j > 0 && _centerPrefix.atMaxY())) { + + continue; // main box or wrapped edge + // TODO: We may want to enable wrapping in future, probably best as layer + // on top of this search. + } + + // Make sure we've got a reasonable center + verify(_centerPrefix.constrains()); + + GeoHash _neighborPrefix = _centerPrefix; + _neighborPrefix.move(i, j); + + while(_fringe.size() > 0) { + _prefix = _neighborPrefix + _fringe.back(); + Box cur(_converter->unhashToBox(_prefix)); + + double intAmt = intersectsBox(cur); + + // No intersection + if(intAmt <= 0) { + _fringe.pop_back(); + continue; + } else if(intAmt < 0.5 && _prefix.canRefine() + && _fringe.back().size() < 4 /* two bits */) { + // Small intersection, refine search + string lastSuffix = _fringe.back(); + _fringe.pop_back(); + _fringe.push_back(lastSuffix + "00"); + _fringe.push_back(lastSuffix + "01"); + _fringe.push_back(lastSuffix + "11"); + _fringe.push_back(lastSuffix + "10"); + continue; + } + + // Restart our search from a diff box. + _state = START; + verify(! onlyExpand); + verify(_found <= 0x7fffffff); + fillStack(maxFound - _foundInExp, maxAdded - static_cast(_found)); + // When we return from the recursive fillStack call, we'll either have + // checked enough points or be entirely done. Max recurse depth is < 8 * + // 16. + // If we're maxed out on points, return + if(_foundInExp >= maxFound || _found >= maxAdded) { + // Make sure we'll come back to add more points + verify(_state == DOING_EXPAND); + return; + } + + // Otherwise we must be finished to return + verify(_state == DONE); + return; + } + } + // Finished with neighbors + _state = DONE; + } + } + + bool GeoBrowse::remembered(BSONObj o){ + BSONObj seenId = o["_id"].wrap("").getOwned(); + if(_seenIds.find(seenId) != _seenIds.end()){ + return true; + } else{ + _seenIds.insert(seenId); + return false; + } + } + + int GeoBrowse::addSpecific(const GeoIndexEntry& node, const Point& keyP, bool onBounds, + double keyD, bool potentiallyNewDoc) { + int found = 0; + // We need to handle every possible point in this method, even those not in the key + // value, to avoid us tracking which hashes we've already seen. + if(! potentiallyNewDoc){ return 0; } + + // Final check for new doc + // OK to touch, since we're probably returning this object now + if(remembered(node.recordLoc.obj())) { + //cout << "remembered\n"; + return 0; + } + + if(! onBounds) { + //log() << "Added ind to " << _type << endl; + _stack.push_front(GeoPoint(node)); + found++; + } else { + // We now handle every possible point in the document, even those not in the key + // value, since we're iterating through them anyway - prevents us from having to + // save the hashes we've seen per-doc + // If we're filtering by hash, get the original + + vector< BSONObj > locs; + getPointsFor(node._key, node.recordLoc.obj(), locs, true); + for(vector< BSONObj >::iterator i = locs.begin(); i != locs.end(); ++i){ + double d = -1; + Point p(*i); + // We can avoid exact document checks by redoing approx checks, + // if the exact checks are more expensive. + bool needExact = true; + + if(! needExact || exactDocCheck(p, d)){ + //log() << "Added mult to " << _type << endl; + _stack.push_front(GeoPoint(node)); + found++; + // IExit after first point is added + break; + } + } + } + + while(_cur.isEmpty() && _stack.size() > 0){ + _cur = _stack.front(); + _stack.pop_front(); + } + + return found; + } + + long long GeoBrowse::nscanned() { + if (_firstCall) { ok(); } + return _nscanned; + } + + void GeoBrowse::explainDetails(BSONObjBuilder& b){ + b << "lookedAt" << _lookedAt; + b << "matchesPerfd" << _matchesPerfd; + b << "objectsLoaded" << _objectsLoaded; + b << "pointsLoaded" << _pointsLoaded; + // b << "pointsSavedForYield" << _nDirtied; + // b << "pointsChangedOnYield" << _nChangedOnYield; + // b << "pointsRemovedOnYield" << _nRemovedOnYield; + } + + bool GeoBrowse::invalidate(const DiskLoc& dl) { + if (_firstCall) { return false; } + + // Are we tossing out a result that we (probably) would have returned? + bool found = false; + + if (_cur._loc == dl) { + advance(); + found = true; + } + + list::iterator it = _stack.begin(); + while (it != _stack.end()) { + if (it->_loc == dl) { + list::iterator old = it; + it++; + _stack.erase(old); + found = true; + } + else { + it++; + } + } + + if (!_min.eof() && _min._loc == dl) { + _min.recoverFromYield(); + while (_min._loc == dl && !_min.eof()) { + _min.advance(); + } + _min.prepareToYield(); + } + + if (!_max.eof() && _max._loc == dl) { + _max.recoverFromYield(); + while (_max._loc == dl && !_max.eof()) { + _max.advance(); + } + _max.prepareToYield(); + } + + return found; + } + +} // namespace twod_exec +} // namespace mongo diff --git a/src/mongo/db/exec/2dcommon.h b/src/mongo/db/exec/2dcommon.h new file mode 100644 index 00000000000..9a4bfab9c85 --- /dev/null +++ b/src/mongo/db/exec/2dcommon.h @@ -0,0 +1,273 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/index_scan.h" +#include "mongo/db/geo/core.h" +#include "mongo/db/geo/hash.h" +#include "mongo/db/geo/shapes.h" +#include "mongo/db/pdfile.h" + +#include "mongo/db/index/2d_access_method.h" + +#pragma once + +namespace mongo { +namespace twod_exec { + + // + // Data structures + // + + enum GeoDistType { + GEO_PLANE, + GEO_SPHERE + }; + + class GeoIndexEntry { + public: + GeoIndexEntry(DiskLoc r, BSONObj k) : recordLoc(r), _key(k) { } + const DiskLoc recordLoc; + const BSONObj _key; + private: + GeoIndexEntry(); + }; + + class GeoPoint { + public: + GeoPoint() : _distance(-1), _exact(false) { } + + //// Distance not used //// + + GeoPoint(const GeoIndexEntry& node) + : _key(node._key), _loc(node.recordLoc), _o(node.recordLoc.obj()), + _distance(-1), _exact(false) { } + + //// Immediate initialization of distance //// + + GeoPoint(const GeoIndexEntry& node, double distance, bool exact) + : _key(node._key), _loc(node.recordLoc), _o(node.recordLoc.obj()), + _distance(distance), _exact(exact) { } + + GeoPoint(const GeoPoint& pt, double distance, bool exact) + : _key(pt.key()), _loc(pt.loc()), _o(pt.obj()), _distance(distance), _exact(exact) { } + + bool operator<(const GeoPoint& other) const { + if (_distance != other._distance) return _distance < other._distance; + if (_exact != other._exact) return _exact < other._exact; + return _loc < other._loc; + } + + double distance() const { return _distance; } + bool isExact() const { return _exact; } + BSONObj key() const { return _key; } + bool hasLoc() const { return _loc.isNull(); } + BSONObj obj() const { return _o; } + BSONObj pt() const { return _pt; } + bool isEmpty() { return _o.isEmpty(); } + + DiskLoc loc() const { + return _loc; + } + + string toString() const { + return str::stream() << "Point from " << _key << " - " << _o + << " dist : " << _distance << (_exact ? " (ex)" : " (app)"); + } + + BSONObj _key; + DiskLoc _loc; + BSONObj _o; + BSONObj _pt; + + double _distance; + bool _exact; + + BSONObj _id; + }; + + struct BtreeLocation { + BtreeLocation() : _eof(false) { } + + scoped_ptr _scan; + scoped_ptr _ws; + DiskLoc _loc; + BSONObj _key; + bool _eof; + + bool eof() { return _eof; } + + static bool hasPrefix(const BSONObj& key, const GeoHash& hash); + + void advance(); + + void prepareToYield() { _scan->prepareToYield(); } + void recoverFromYield() { _scan->recoverFromYield(); } + + // Returns the min and max keys which bound a particular location. + // The only time these may be equal is when we actually equal the location + // itself, otherwise our expanding algorithm will fail. + static bool initial(const IndexDescriptor* descriptor, const TwoDIndexingParams& params, + BtreeLocation& min, BtreeLocation& max, GeoHash start); + }; + + // + // Execution + // + + class GeoAccumulator { + public: + GeoAccumulator(TwoDAccessMethod* accessMethod, MatchExpression* filter); + + virtual ~GeoAccumulator(); + + enum KeyResult { BAD, BORDER, GOOD }; + + virtual void add(const GeoIndexEntry& node); + + long long found() const { return _found; } + + virtual void getPointsFor(const BSONObj& key, const BSONObj& obj, + vector &locsForNode, bool allPoints = false); + + virtual int addSpecific(const GeoIndexEntry& node, const Point& p, bool inBounds, double d, + bool newDoc) = 0; + + virtual KeyResult approxKeyCheck(const Point& p, double& keyD) = 0; + + TwoDAccessMethod* _accessMethod; + shared_ptr _converter; + map _matched; + + MatchExpression* _filter; + + long long _lookedAt; + long long _matchesPerfd; + long long _objectsLoaded; + long long _pointsLoaded; + long long _found; + }; + + class GeoBrowse : public GeoAccumulator { + public: + // The max points which should be added to an expanding box at one time + static const int maxPointsHeuristic = 50; + + // Expand states + enum State { + START, + DOING_EXPAND, + DONE_NEIGHBOR, + DONE + } _state; + + GeoBrowse(TwoDAccessMethod* accessMethod, string type, MatchExpression* filter); + + virtual bool ok(); + virtual bool advance(); + virtual void noteLocation(); + + /* called before query getmore block is iterated */ + virtual void checkLocation(); + + virtual Record* _current(); + virtual BSONObj current(); + virtual DiskLoc currLoc(); + virtual BSONObj currKey() const; + + // Are we finished getting points? + virtual bool moreToDo(); + + // Fills the stack, but only checks a maximum number of maxToCheck points at a time. + // Further calls to this function will continue the expand/check neighbors algorithm. + virtual void fillStack(int maxToCheck, int maxToAdd = -1, bool onlyExpand = false); + + bool checkAndAdvance(BtreeLocation* bl, const GeoHash& hash, int& totalFound); + + // The initial geo hash box for our first expansion + virtual GeoHash expandStartHash() = 0; + + // Whether the current box width is big enough for our search area + virtual bool fitsInBox(double width) = 0; + + // The amount the current box overlaps our search area + virtual double intersectsBox(Box& cur) = 0; + + virtual bool exactDocCheck(const Point& p, double& d) = 0; + + bool remembered(BSONObj o); + + virtual int addSpecific(const GeoIndexEntry& node, const Point& keyP, bool onBounds, + double keyD, bool potentiallyNewDoc); + + virtual long long nscanned(); + + virtual void explainDetails(BSONObjBuilder& b); + + void notePrefix() { _expPrefixes.push_back(_prefix); } + + /** + * Returns true if the result was actually invalidated, false otherwise. + */ + bool invalidate(const DiskLoc& dl); + + string _type; + list _stack; + set _seenIds; + + GeoPoint _cur; + bool _firstCall; + + long long _nscanned; + + // The current box we're expanding (-1 is first/center box) + int _neighbor; + + // The points we've found so far + int _foundInExp; + + // The current hash prefix we're expanding and the center-box hash prefix + GeoHash _prefix; + shared_ptr _lastPrefix; + GeoHash _centerPrefix; + list _fringe; + int recurseDepth; + Box _centerBox; + + // Start and end of our search range in the current box + BtreeLocation _min; + BtreeLocation _max; + + shared_ptr _expPrefix; + mutable vector _expPrefixes; + const IndexDescriptor* _descriptor; + shared_ptr _converter; + TwoDIndexingParams _params; + }; + +} // namespace twod_exec +} // namespace mongo diff --git a/src/mongo/db/exec/2dnear.cpp b/src/mongo/db/exec/2dnear.cpp new file mode 100644 index 00000000000..5f27fab1581 --- /dev/null +++ b/src/mongo/db/exec/2dnear.cpp @@ -0,0 +1,530 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/2dnear.h" + +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + TwoDNear::TwoDNear(const TwoDNearParams& params, WorkingSet* ws) { + _params = params; + _workingSet = ws; + _initted = false; + } + + TwoDNear::~TwoDNear() { } + + bool TwoDNear::isEOF() { + return _initted && _results.empty(); + } + + PlanStage::StageState TwoDNear::work(WorkingSetID* out) { + ++_commonStats.works; + if (!_initted) { + _initted = true; + + Database* db = cc().database(); + if ( !db ) + return PlanStage::IS_EOF; + Collection* collection = db->getCollection( _params.ns ); + if ( !collection ) + return PlanStage::IS_EOF; + + IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern(_params.indexKeyPattern); + if ( desc == NULL ) + return PlanStage::IS_EOF; + TwoDAccessMethod* am = static_cast( collection->getIndexCatalog()->getIndex( desc ) ); + + auto_ptr search; + search.reset(new twod_exec::GeoSearch(am, + _params.nearQuery.centroid.oldPoint, + _params.numWanted, + _params.filter, + _params.nearQuery.maxDistance, + _params.nearQuery.isNearSphere ? twod_exec::GEO_SPHERE + : twod_exec::GEO_PLANE)); + + // This is where all the work is done. :( + search->exec(); + _specificStats.objectsLoaded = search->_objectsLoaded; + _specificStats.nscanned = search->_lookedAt; + + for (twod_exec::GeoHopper::Holder::iterator it = search->_points.begin(); + it != search->_points.end(); it++) { + + WorkingSetID id = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(id); + member->loc = it->_loc; + member->obj = member->loc.obj(); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + if (_params.addDistMeta) { + member->addComputed(new GeoDistanceComputedData(it->_distance)); + } + if (_params.addPointMeta) { + member->addComputed(new GeoNearPointComputedData(it->_pt)); + } + _results.push(Result(id, it->_distance)); + _invalidationMap.insert(pair(it->_loc, id)); + } + } + + if (isEOF()) { return PlanStage::IS_EOF; } + + Result result = _results.top(); + _results.pop(); + *out = result.id; + + // Remove from invalidation map. + WorkingSetMember* member = _workingSet->get(*out); + + // The WSM may have been mutated or deleted so it may not have a loc. + if (member->hasLoc()) { + typedef multimap::iterator MMIT; + pair range = _invalidationMap.equal_range(member->loc); + for (MMIT it = range.first; it != range.second; ++it) { + if (it->second == *out) { + _invalidationMap.erase(it); + break; + } + } + } + + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + + void TwoDNear::prepareToYield() { + // Nothing to do here. + } + + void TwoDNear::recoverFromYield() { + // Also nothing to do here. + } + + void TwoDNear::invalidate(const DiskLoc& dl, InvalidationType type) { + // We do the same thing for mutation or deletion: fetch the doc and forget about the + // DiskLoc. 2d's near search computes all its results in one go so we know that we'll still + // return valid data. + typedef multimap::iterator MMIT; + pair range = _invalidationMap.equal_range(dl); + for (MMIT it = range.first; it != range.second; ++it) { + WorkingSetMember* member = _workingSet->get(it->second); + // If it's in the invalidation map it must have a DiskLoc. + verify(member->hasLoc()); + WorkingSetCommon::fetchAndInvalidateLoc(member); + verify(!member->hasLoc()); + } + _invalidationMap.erase(range.first, range.second); + } + + PlanStageStats* TwoDNear::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_GEO_NEAR_2D)); + ret->specific.reset(new TwoDNearStats(_specificStats)); + return ret.release(); + } + +} // namespace mongo + +namespace mongo { +namespace twod_exec { + + // + // GeoHopper + // + + GeoHopper::GeoHopper(TwoDAccessMethod* accessMethod, + unsigned max, + const Point& n, + MatchExpression* filter, + double maxDistance, + GeoDistType type) + : GeoBrowse(accessMethod, "search", filter), + _max(max), + _near(n), + _maxDistance(maxDistance), + _type(type), + _distError(type == GEO_PLANE + ? accessMethod->getParams().geoHashConverter->getError() + : accessMethod->getParams().geoHashConverter->getErrorSphere()), + _farthest(0) { } + + GeoAccumulator:: KeyResult GeoHopper::approxKeyCheck(const Point& p, double& d) { + // Always check approximate distance, since it lets us avoid doing + // checks of the rest of the object if it succeeds + switch (_type) { + case GEO_PLANE: + d = distance(_near, p); + break; + case GEO_SPHERE: + checkEarthBounds(p); + d = spheredist_deg(_near, p); + break; + default: verify(false); + } + verify(d >= 0); + + // If we need more points + double borderDist = (_points.size() < _max ? _maxDistance : farthest()); + + if (d >= borderDist - 2 * _distError && d <= borderDist + 2 * _distError) return BORDER; + else return d < borderDist ? GOOD : BAD; + } + + bool GeoHopper::exactDocCheck(const Point& p, double& d){ + bool within = false; + + // Get the appropriate distance for the type + switch (_type) { + case GEO_PLANE: + d = distance(_near, p); + within = distanceWithin(_near, p, _maxDistance); + break; + case GEO_SPHERE: + checkEarthBounds(p); + d = spheredist_deg(_near, p); + within = (d <= _maxDistance); + break; + default: verify(false); + } + + return within; + } + + + int GeoHopper::addSpecific(const GeoIndexEntry& node, const Point& keyP, bool onBounds, + double keyD, bool potentiallyNewDoc) { + // Unique documents + GeoPoint newPoint(node, keyD, false); + int prevSize = _points.size(); + + // STEP 1 : Remove old duplicate points from the set if needed + + // Lookup old point with same doc + map::iterator oldPointIt = _seenPts.find(newPoint.loc()); + + if(oldPointIt != _seenPts.end()){ + const GeoPoint& oldPoint = *(oldPointIt->second); + // We don't need to care if we've already seen this same approx pt or better, + // or we've already gone to disk once for the point + if(oldPoint < newPoint){ + return 0; + } + _points.erase(oldPointIt->second); + } + + //cout << "inserting point\n"; + Holder::iterator newIt = _points.insert(newPoint); + _seenPts[ newPoint.loc() ] = newIt; + + verify(_max > 0); + + Holder::iterator lastPtIt = _points.end(); + lastPtIt--; + _farthest = lastPtIt->distance() + 2 * _distError; + return _points.size() - prevSize; + } + + // Removes extra points from end of _points set. + // Check can be a bit costly if we have lots of exact points near borders, + // so we'll do this every once and awhile. + void GeoHopper::processExtraPoints(){ + if(_points.size() == 0) return; + int prevSize = _points.size(); + + // Erase all points from the set with a position >= _max *and* + // whose distance isn't close to the _max - 1 position distance + int numToErase = _points.size() - _max; + if(numToErase < 0) numToErase = 0; + + // Get the first point definitely in the _points array + Holder::iterator startErase = _points.end(); + for(int i = 0; i < numToErase + 1; i++) startErase--; + _farthest = startErase->distance() + 2 * _distError; + + startErase++; + while(numToErase > 0 && startErase->distance() <= _farthest){ + numToErase--; + startErase++; + verify(startErase != _points.end() || numToErase == 0); + } + + for(Holder::iterator i = startErase; i != _points.end(); ++i) { + _seenPts.erase(i->loc()); + } + + _points.erase(startErase, _points.end()); + + int diff = _points.size() - prevSize; + if(diff > 0) _found += diff; + else _found -= -diff; + } + + // + // GeoSearch + // + + GeoSearch::GeoSearch(TwoDAccessMethod* accessMethod, + const Point& startPt, + int numWanted, + MatchExpression* filter, + double maxDistance, + GeoDistType type) + : GeoHopper(accessMethod, numWanted, startPt, filter, maxDistance, type), + _start(accessMethod->getParams().geoHashConverter->hash(startPt.x, startPt.y)), + _numWanted(numWanted), + _type(type), + _params(accessMethod->getParams()) { + + _nscanned = 0; + _found = 0; + + if(_maxDistance < 0){ + _scanDistance = numeric_limits::max(); + } else if (type == GEO_PLANE) { + _scanDistance = maxDistance + _params.geoHashConverter->getError(); + } else if (type == GEO_SPHERE) { + checkEarthBounds(startPt); + // TODO: consider splitting into x and y scan distances + _scanDistance = computeXScanDistance(startPt.y, + rad2deg(_maxDistance) + _params.geoHashConverter->getError()); + } + + verify(_scanDistance > 0); + } + + void GeoSearch::exec() { + if(_numWanted == 0) return; + + /* + * Search algorithm + * 1) use geohash prefix to find X items + * 2) compute max distance from want to an item + * 3) find optimal set of boxes that complete circle + * 4) use regular btree cursors to scan those boxes + */ + + // Part 1 + { + do { + long long f = found(); + verify(f <= 0x7fffffff); + fillStack(maxPointsHeuristic, _numWanted - static_cast(f), true); + processExtraPoints(); + } while(_state != DONE && _state != DONE_NEIGHBOR && + found() < _numWanted && + (!_prefix.constrains() || + _params.geoHashConverter->sizeEdge(_prefix) <= _scanDistance)); + + // If we couldn't scan or scanned everything, we're done + if(_state == DONE){ + expandEndPoints(); + return; + } + } + + // Part 2 + { + // Find farthest distance for completion scan + double farDist = farthest(); + if(found() < _numWanted) { + // Not enough found in Phase 1 + farDist = _scanDistance; + } + else if (_type == GEO_PLANE) { + // Enough found, but need to search neighbor boxes + farDist += _params.geoHashConverter->getError(); + } + else if (_type == GEO_SPHERE) { + // Enough found, but need to search neighbor boxes + farDist = std::min(_scanDistance, + computeXScanDistance(_near.y, + rad2deg(farDist)) + + 2 * _params.geoHashConverter->getError()); + } + verify(farDist >= 0); + + // Find the box that includes all the points we need to return + _want = Box(_near.x - farDist, _near.y - farDist, farDist * 2); + + // Remember the far distance for further scans + _scanDistance = farDist; + + // Reset the search, our distances have probably changed + if(_state == DONE_NEIGHBOR){ + _state = DOING_EXPAND; + _neighbor = -1; + } + + // Do regular search in the full region + do { + fillStack(maxPointsHeuristic); + processExtraPoints(); + } + while(_state != DONE); + } + + expandEndPoints(); + } + + void GeoSearch::addExactPoints(const GeoPoint& pt, Holder& points, bool force){ + int before, after; + addExactPoints(pt, points, before, after, force); + } + + void GeoSearch::addExactPoints(const GeoPoint& pt, Holder& points, int& before, int& after, + bool force){ + before = 0; + after = 0; + + if(pt.isExact()){ + if(force) points.insert(pt); + return; + } + + vector locs; + // last argument is uniqueDocs + getPointsFor(pt.key(), pt.obj(), locs, true); + + GeoPoint nearestPt(pt, -1, true); + + for(vector::iterator i = locs.begin(); i != locs.end(); i++){ + Point loc(*i); + double d; + if(! exactDocCheck(loc, d)) continue; + + if(nearestPt.distance() < 0 || d < nearestPt.distance()){ + nearestPt._distance = d; + nearestPt._pt = *i; + continue; + } + } + + if(nearestPt.distance() >= 0){ + points.insert(nearestPt); + if(nearestPt < pt) before++; + else after++; + } + } + + // TODO: Refactor this back into holder class, allow to run periodically when we are seeing + // a lot of pts + void GeoSearch::expandEndPoints(bool finish) { + processExtraPoints(); + // All points in array *could* be in maxDistance + + // Step 1 : Trim points to max size TODO: This check will do little for now, but is + // skeleton for future work in incremental $near + // searches + if(_max > 0){ + int numToErase = _points.size() - _max; + if(numToErase > 0){ + Holder tested; + // Work backward through all points we're not sure belong in the set + Holder::iterator maybePointIt = _points.end(); + maybePointIt--; + double approxMin = maybePointIt->distance() - 2 * _distError; + + // Insert all + int erased = 0; + while(_points.size() > 0 + && (maybePointIt->distance() >= approxMin || erased < numToErase)){ + + Holder::iterator current = maybePointIt; + if (current != _points.begin()) + --maybePointIt; + + addExactPoints(*current, tested, true); + _points.erase(current); + erased++; + + if(tested.size()) + approxMin = tested.begin()->distance() - 2 * _distError; + } + + int numToAddBack = erased - numToErase; + verify(numToAddBack >= 0); + + Holder::iterator testedIt = tested.begin(); + for(int i = 0; i < numToAddBack && testedIt != tested.end(); i++){ + _points.insert(*testedIt); + testedIt++; + } + } + } + + // We've now trimmed first set of unneeded points + + // Step 2: iterate through all points and add as needed + unsigned expandedPoints = 0; + Holder::iterator it = _points.begin(); + double expandWindowEnd = -1; + + while(it != _points.end()){ + const GeoPoint& currPt = *it; + // TODO: If one point is exact, maybe not 2 * _distError + + // See if we're in an expand window + bool inWindow = currPt.distance() <= expandWindowEnd; + // If we're not, and we're done with points, break + if(! inWindow && expandedPoints >= _max) break; + + bool expandApprox = !currPt.isExact() && (finish || inWindow); + + if (expandApprox) { + // Add new point(s). These will only be added in a radius of 2 * _distError + // around the current point, so should not affect previously valid points. + int before, after; + addExactPoints(currPt, _points, before, after, false); + expandedPoints += before; + + if(_max > 0 && expandedPoints < _max) + expandWindowEnd = currPt.distance() + 2 * _distError; + + // Iterate to the next point + Holder::iterator current = it++; + // Erase the current point + _points.erase(current); + } else{ + expandedPoints++; + it++; + } + } + + // Finish + // TODO: Don't really need to trim? + for(; expandedPoints > _max; expandedPoints--) it--; + _points.erase(it, _points.end()); + } + +} // namespace twod_exec +} // namespace mongo diff --git a/src/mongo/db/exec/2dnear.h b/src/mongo/db/exec/2dnear.h new file mode 100644 index 00000000000..7b8a8b66fa4 --- /dev/null +++ b/src/mongo/db/exec/2dnear.h @@ -0,0 +1,184 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/2dcommon.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" + +namespace mongo { + + struct TwoDNearParams { + NearQuery nearQuery; + string ns; + BSONObj indexKeyPattern; + MatchExpression* filter; + int numWanted; + bool addPointMeta; + bool addDistMeta; + }; + + struct Result { + Result(WorkingSetID wsid, double dist) : id(wsid), distance(dist) { } + + bool operator<(const Result& other) const { + // We want increasing distance, not decreasing, so we reverse the <. + return distance > other.distance; + } + + WorkingSetID id; + + double distance; + }; + + class TwoDNear : public PlanStage { + public: + TwoDNear(const TwoDNearParams& params, WorkingSet* ws); + virtual ~TwoDNear(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + WorkingSet* _workingSet; + + // Stats + CommonStats _commonStats; + TwoDNearStats _specificStats; + + // We compute an annulus of results and cache it here. + priority_queue _results; + + // For fast invalidation. Perhaps not worth it. + // + // Multi-location docs mean that this is not one diskloc -> one WSID but one DiskLoc -> many + // WSIDs. + multimap _invalidationMap; + + TwoDNearParams _params; + + bool _initted; + }; + +} // namespace mongo + +namespace mongo { +namespace twod_exec { + + class GeoHopper : public GeoBrowse { + public: + typedef multiset Holder; + + GeoHopper(TwoDAccessMethod* accessMethod, + unsigned max, + const Point& n, + MatchExpression* filter, + double maxDistance = numeric_limits::max(), + GeoDistType type = GEO_PLANE); + + virtual KeyResult approxKeyCheck(const Point& p, double& d); + + virtual bool exactDocCheck(const Point& p, double& d); + + // Always in distance units, whether radians or normal + double farthest() const { return _farthest; } + + virtual int addSpecific(const GeoIndexEntry& node, const Point& keyP, bool onBounds, + double keyD, bool potentiallyNewDoc); + + // Removes extra points from end of _points set. + // Check can be a bit costly if we have lots of exact points near borders, + // so we'll do this every once and awhile. + void processExtraPoints(); + + unsigned _max; + Point _near; + Holder _points; + double _maxDistance; + GeoDistType _type; + double _distError; + double _farthest; + + // Safe to use currently since we don't yield in $near searches. If we do start to yield, + // we may need to replace dirtied disklocs in our holder / ensure our logic is correct. + map _seenPts; + }; + + class GeoSearch : public GeoHopper { + public: + GeoSearch(TwoDAccessMethod* accessMethod, + const Point& startPt, + int numWanted = 100, + MatchExpression* filter = NULL, + double maxDistance = numeric_limits::max(), + GeoDistType type = GEO_PLANE); + + void exec(); + + void addExactPoints(const GeoPoint& pt, Holder& points, bool force); + + void addExactPoints(const GeoPoint& pt, Holder& points, int& before, int& after, + bool force); + + // TODO: Refactor this back into holder class, allow to run periodically when we are seeing + // a lot of pts + void expandEndPoints(bool finish = true); + + virtual GeoHash expandStartHash() { return _start; } + + // Whether the current box width is big enough for our search area + virtual bool fitsInBox(double width) { return width >= _scanDistance; } + + // Whether the current box overlaps our search area + virtual double intersectsBox(Box& cur) { return cur.intersects(_want); } + + set< pair > _seen; + GeoHash _start; + int _numWanted; + double _scanDistance; + long long _nscanned; + int _found; + GeoDistType _type; + Box _want; + TwoDIndexingParams& _params; + }; + +} // namespace twod_exec +} // namespace mongo diff --git a/src/mongo/db/exec/SConscript b/src/mongo/db/exec/SConscript new file mode 100644 index 00000000000..d1e3015ae85 --- /dev/null +++ b/src/mongo/db/exec/SConscript @@ -0,0 +1,97 @@ +# -*- mode: python -*- + +Import("env") + +env.Library( + target = "working_set", + source = [ + "working_set.cpp", + ], + LIBDEPS = [ + "$BUILD_DIR/mongo/bson", + ], +) + +env.CppUnitTest( + target = "working_set_test", + source = [ + "working_set_test.cpp" + ], + LIBDEPS = [ + "working_set", + ], +) + +env.Library( + target = "mock_stage", + source = [ + "mock_stage.cpp", + ], + LIBDEPS = [ + "working_set", + ], +) + +env.Library( + target = 'exec', + source = [ + "2d.cpp", + "2dcommon.cpp", + "2dnear.cpp", + "and_hash.cpp", + "and_sorted.cpp", + "collection_scan.cpp", + "count.cpp", + "distinct_scan.cpp", + "fetch.cpp", + "index_scan.cpp", + "keep_mutations.cpp", + "limit.cpp", + "merge_sort.cpp", + "oplogstart.cpp", + "or.cpp", + "projection.cpp", + "projection_exec.cpp", + "s2near.cpp", + "shard_filter.cpp", + "skip.cpp", + "sort.cpp", + "stagedebug_cmd.cpp", + "text.cpp", + "working_set_common.cpp", + ], + LIBDEPS = [ + "$BUILD_DIR/mongo/bson", + ], +) + +env.CppUnitTest( + target = "sort_test", + source = [ + "sort_test.cpp", + ], + LIBDEPS = [ + "exec", + "mock_stage", + "$BUILD_DIR/mongo/serveronly", + "$BUILD_DIR/mongo/coreserver", + "$BUILD_DIR/mongo/coredb", + "$BUILD_DIR/mongo/mocklib", + ], + NO_CRUTCH = True, +) + +env.CppUnitTest( + target = "projection_exec_test", + source = [ + "projection_exec_test.cpp", + ], + LIBDEPS = [ + "exec", + "mock_stage", + "$BUILD_DIR/mongo/serveronly", + "$BUILD_DIR/mongo/coreserver", + "$BUILD_DIR/mongo/coredb", + ], + NO_CRUTCH = True, +) diff --git a/src/mongo/db/exec/and_common-inl.h b/src/mongo/db/exec/and_common-inl.h new file mode 100644 index 00000000000..7f133301a22 --- /dev/null +++ b/src/mongo/db/exec/and_common-inl.h @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +namespace mongo { + + class AndCommon { + public: + /** + * If src has any data dest doesn't, add that data to dest. + */ + static void mergeFrom(WorkingSetMember* dest, const WorkingSetMember& src) { + verify(dest->hasLoc()); + verify(src.hasLoc()); + verify(dest->loc == src.loc); + + // Merge computed data. + typedef WorkingSetComputedDataType WSCD; + for (WSCD i = WSCD(0); i < WSM_COMPUTED_NUM_TYPES; i = WSCD(i + 1)) { + if (!dest->hasComputed(i) && src.hasComputed(i)) { + dest->addComputed(src.getComputed(i)->clone()); + } + } + + if (dest->hasObj()) { + // The merged WSM that we're creating already has the full document, so there's + // nothing left to do. + return; + } + + if (src.hasObj()) { + // 'src' has the full document but 'dest' doesn't so we need to copy it over. + // + // The source diskloc must be in the "diskloc and unowned object" state rather than + // the "owned object" state. This is because we've just intersected according to + // diskloc. Since we merge based on finding working set members with matching + // disklocs, we shouldn't have a WSM that is missing the diskloc. + invariant(WorkingSetMember::LOC_AND_UNOWNED_OBJ == src.state); + + // Copy the object to 'dest'. + dest->obj = src.obj; + + // We have an object so we don't need key data. + dest->keyData.clear(); + + // 'dest' should be LOC_AND_UNOWNED_OBJ + dest->state = src.state; + + // Now 'dest' has the full object. No more work to do. + return; + } + + // If we're here, then both WSMs getting merged contain index keys. We need + // to merge the key data. + // + // This is N^2 but N is probably pretty small. Easy enough to revisit. + for (size_t i = 0; i < src.keyData.size(); ++i) { + bool found = false; + for (size_t j = 0; j < dest->keyData.size(); ++j) { + if (dest->keyData[j].indexKeyPattern == src.keyData[i].indexKeyPattern) { + found = true; + break; + } + } + if (!found) { dest->keyData.push_back(src.keyData[i]); } + } + } + }; + +} // namespace mongo + diff --git a/src/mongo/db/exec/and_hash.cpp b/src/mongo/db/exec/and_hash.cpp new file mode 100644 index 00000000000..28a6aadd779 --- /dev/null +++ b/src/mongo/db/exec/and_hash.cpp @@ -0,0 +1,515 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/and_hash.h" + +#include "mongo/db/exec/and_common-inl.h" +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/util/mongoutils/str.h" + +namespace { + + // Upper limit for buffered data. + // Stage execution will fail once size of all buffered data exceeds this threshold. + const size_t kDefaultMaxMemUsageBytes = 32 * 1024 * 1024; + +} // namespace + +namespace mongo { + + using std::auto_ptr; + + const size_t AndHashStage::kLookAheadWorks = 10; + + AndHashStage::AndHashStage(WorkingSet* ws, const MatchExpression* filter) + : _ws(ws), + _filter(filter), + _hashingChildren(true), + _currentChild(0), + _memUsage(0), + _maxMemUsage(kDefaultMaxMemUsageBytes) {} + + AndHashStage::AndHashStage(WorkingSet* ws, const MatchExpression* filter, size_t maxMemUsage) + : _ws(ws), + _filter(filter), + _hashingChildren(true), + _currentChild(0), + _memUsage(0), + _maxMemUsage(maxMemUsage) {} + + AndHashStage::~AndHashStage() { + for (size_t i = 0; i < _children.size(); ++i) { delete _children[i]; } + } + + void AndHashStage::addChild(PlanStage* child) { _children.push_back(child); } + + size_t AndHashStage::getMemUsage() const { + return _memUsage; + } + + bool AndHashStage::isEOF() { + // This is empty before calling work() and not-empty after. + if (_lookAheadResults.empty()) { return false; } + + // Either we're busy hashing children, in which case we're not done yet. + if (_hashingChildren) { return false; } + + // Or we're streaming in results from the last child. + + // If there's nothing to probe against, we're EOF. + if (_dataMap.empty()) { return true; } + + // Otherwise, we're done when the last child is done. + invariant(_children.size() >= 2); + return (WorkingSet::INVALID_ID == _lookAheadResults[_children.size() - 1]) + && _children[_children.size() - 1]->isEOF(); + } + + PlanStage::StageState AndHashStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (isEOF()) { return PlanStage::IS_EOF; } + + // Fast-path for one of our children being EOF immediately. We work each child a few times. + // If it hits EOF, the AND cannot output anything. If it produces a result, we stash that + // result in _lookAheadResults. + if (_lookAheadResults.empty()) { + // INVALID_ID means that the child didn't produce a valid result. + + // We specifically are not using .resize(size, value) here because C++11 builds don't + // seem to resolve WorkingSet::INVALID_ID during linking. + _lookAheadResults.resize(_children.size()); + for (size_t i = 0; i < _children.size(); ++i) { + _lookAheadResults[i] = WorkingSet::INVALID_ID; + } + + // Work each child some number of times until it's either EOF or produces + // a result. If it's EOF this whole stage will be EOF. If it produces a + // result we cache it for later. + for (size_t i = 0; i < _children.size(); ++i) { + PlanStage* child = _children[i]; + for (size_t j = 0; j < kLookAheadWorks; ++j) { + StageState childStatus = child->work(&_lookAheadResults[i]); + + if (PlanStage::IS_EOF == childStatus || PlanStage::DEAD == childStatus) { + + // A child went right to EOF. Bail out. + _hashingChildren = false; + _dataMap.clear(); + return PlanStage::IS_EOF; + } + else if (PlanStage::ADVANCED == childStatus) { + // We have a result cached in _lookAheadResults[i]. Stop looking at this + // child. + break; + } + else if (PlanStage::FAILURE == childStatus) { + // Propage error to parent. + *out = _lookAheadResults[i]; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == *out) { + mongoutils::str::stream ss; + ss << "hashed AND stage failed to read in look ahead results " + << "from child " << i; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + + _hashingChildren = false; + _dataMap.clear(); + return PlanStage::FAILURE; + } + // We ignore NEED_TIME. TODO: What do we want to do if the child provides + // NEED_FETCH? + } + } + + // We did a bunch of work above, return NEED_TIME to be fair. + return PlanStage::NEED_TIME; + } + + // An AND is either reading the first child into the hash table, probing against the hash + // table with subsequent children, or checking the last child's results to see if they're + // in the hash table. + + // We read the first child into our hash table. + if (_hashingChildren) { + // Check memory usage of previously hashed results. + if (_memUsage > _maxMemUsage) { + mongoutils::str::stream ss; + ss << "hashed AND stage buffered data usage of " << _memUsage + << " bytes exceeds internal limit of " << kDefaultMaxMemUsageBytes << " bytes"; + Status status(ErrorCodes::Overflow, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + return PlanStage::FAILURE; + } + + if (0 == _currentChild) { + return readFirstChild(out); + } + else if (_currentChild < _children.size() - 1) { + return hashOtherChildren(out); + } + else { + _hashingChildren = false; + // We don't hash our last child. Instead, we probe the table created from the + // previous children, returning results in the order of the last child. + // Fall through to below. + } + } + + // Returning results. We read from the last child and return the results that are in our + // hash map. + + // We should be EOF if we're not hashing results and the dataMap is empty. + verify(!_dataMap.empty()); + + // We probe _dataMap with the last child. + verify(_currentChild == _children.size() - 1); + + // Get the next result for the (_children.size() - 1)-th child. + StageState childStatus = workChild(_children.size() - 1, out); + if (PlanStage::ADVANCED != childStatus) { + return childStatus; + } + + // We know that we've ADVANCED. See if the WSM is in our table. + WorkingSetMember* member = _ws->get(*out); + + // Maybe the child had an invalidation. We intersect DiskLoc(s) so we can't do anything + // with this WSM. + if (!member->hasLoc()) { + _ws->flagForReview(*out); + return PlanStage::NEED_TIME; + } + + DataMap::iterator it = _dataMap.find(member->loc); + if (_dataMap.end() == it) { + // Child's output wasn't in every previous child. Throw it out. + _ws->free(*out); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else { + // Child's output was in every previous child. Merge any key data in + // the child's output and free the child's just-outputted WSM. + WorkingSetID hashID = it->second; + _dataMap.erase(it); + + WorkingSetMember* olderMember = _ws->get(hashID); + AndCommon::mergeFrom(olderMember, *member); + _ws->free(*out); + + // We should check for matching at the end so the matcher can use information in the + // indices of all our children. + if (Filter::passes(olderMember, _filter)) { + *out = hashID; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else { + _ws->free(hashID); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + } + + PlanStage::StageState AndHashStage::workChild(size_t childNo, WorkingSetID* out) { + if (WorkingSet::INVALID_ID != _lookAheadResults[childNo]) { + *out = _lookAheadResults[childNo]; + _lookAheadResults[childNo] = WorkingSet::INVALID_ID; + return PlanStage::ADVANCED; + } + else { + return _children[childNo]->work(out); + } + } + + PlanStage::StageState AndHashStage::readFirstChild(WorkingSetID* out) { + verify(_currentChild == 0); + + WorkingSetID id = WorkingSet::INVALID_ID; + StageState childStatus = workChild(0, &id); + + if (PlanStage::ADVANCED == childStatus) { + WorkingSetMember* member = _ws->get(id); + + // Maybe the child had an invalidation. We intersect DiskLoc(s) so we can't do anything + // with this WSM. + if (!member->hasLoc()) { + _ws->flagForReview(id); + return PlanStage::NEED_TIME; + } + + verify(member->hasLoc()); + verify(_dataMap.end() == _dataMap.find(member->loc)); + + _dataMap[member->loc] = id; + + // Update memory stats. + _memUsage += member->getMemUsage(); + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::IS_EOF == childStatus) { + // Done reading child 0. + _currentChild = 1; + + // If our first child was empty, don't scan any others, no possible results. + if (_dataMap.empty()) { + _hashingChildren = false; + return PlanStage::IS_EOF; + } + + ++_commonStats.needTime; + _specificStats.mapAfterChild.push_back(_dataMap.size()); + + return PlanStage::NEED_TIME; + } + else if (PlanStage::FAILURE == childStatus) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "hashed AND stage failed to read in results to from first child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return childStatus; + } + else { + if (PlanStage::NEED_FETCH == childStatus) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == childStatus) { + ++_commonStats.needTime; + } + + return childStatus; + } + } + + PlanStage::StageState AndHashStage::hashOtherChildren(WorkingSetID* out) { + verify(_currentChild > 0); + + WorkingSetID id = WorkingSet::INVALID_ID; + StageState childStatus = workChild(_currentChild, &id); + + if (PlanStage::ADVANCED == childStatus) { + WorkingSetMember* member = _ws->get(id); + + // Maybe the child had an invalidation. We intersect DiskLoc(s) so we can't do anything + // with this WSM. + if (!member->hasLoc()) { + _ws->flagForReview(id); + return PlanStage::NEED_TIME; + } + + verify(member->hasLoc()); + if (_dataMap.end() == _dataMap.find(member->loc)) { + // Ignore. It's not in any previous child. + } + else { + // We have a hit. Copy data into the WSM we already have. + _seenMap.insert(member->loc); + WorkingSetMember* olderMember = _ws->get(_dataMap[member->loc]); + size_t memUsageBefore = olderMember->getMemUsage(); + + AndCommon::mergeFrom(olderMember, *member); + + // Update memory stats. + _memUsage += olderMember->getMemUsage() - memUsageBefore; + } + _ws->free(id); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::IS_EOF == childStatus) { + // Finished with a child. + ++_currentChild; + + // Keep elements of _dataMap that are in _seenMap. + DataMap::iterator it = _dataMap.begin(); + while (it != _dataMap.end()) { + if (_seenMap.end() == _seenMap.find(it->first)) { + DataMap::iterator toErase = it; + ++it; + + // Update memory stats. + WorkingSetMember* member = _ws->get(toErase->second); + _memUsage -= member->getMemUsage(); + + _ws->free(toErase->second); + _dataMap.erase(toErase); + } + else { ++it; } + } + + _specificStats.mapAfterChild.push_back(_dataMap.size()); + + _seenMap.clear(); + + // _dataMap is now the intersection of the first _currentChild nodes. + + // If we have nothing to AND with after finishing any child, stop. + if (_dataMap.empty()) { + _hashingChildren = false; + return PlanStage::IS_EOF; + } + + // We've finished scanning all children. Return results with the next call to work(). + if (_currentChild == _children.size()) { + _hashingChildren = false; + } + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::FAILURE == childStatus) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "hashed AND stage failed to read in results from other child " + << _currentChild; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return childStatus; + } + else { + if (PlanStage::NEED_FETCH == childStatus) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == childStatus) { + ++_commonStats.needTime; + } + + return childStatus; + } + } + + void AndHashStage::prepareToYield() { + ++_commonStats.yields; + + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->prepareToYield(); + } + } + + void AndHashStage::recoverFromYield() { + ++_commonStats.unyields; + + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->recoverFromYield(); + } + } + + void AndHashStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + + if (isEOF()) { return; } + + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->invalidate(dl, type); + } + + // Invalidation can happen to our warmup results. If that occurs just + // flag it and forget about it. + for (size_t i = 0; i < _lookAheadResults.size(); ++i) { + if (WorkingSet::INVALID_ID != _lookAheadResults[i]) { + WorkingSetMember* member = _ws->get(_lookAheadResults[i]); + if (member->hasLoc() && member->loc == dl) { + WorkingSetCommon::fetchAndInvalidateLoc(member); + _ws->flagForReview(_lookAheadResults[i]); + _lookAheadResults[i] = WorkingSet::INVALID_ID; + } + } + } + + // If it's a deletion, we have to forget about the DiskLoc, and since the AND-ing is by + // DiskLoc we can't continue processing it even with the object. + // + // If it's a mutation the predicates implied by the AND-ing may no longer be true. + // + // So, we flag and try to pick it up later. + DataMap::iterator it = _dataMap.find(dl); + if (_dataMap.end() != it) { + WorkingSetID id = it->second; + WorkingSetMember* member = _ws->get(id); + verify(member->loc == dl); + + if (_hashingChildren) { + ++_specificStats.flaggedInProgress; + } + else { + ++_specificStats.flaggedButPassed; + } + + // Update memory stats. + _memUsage -= member->getMemUsage(); + + // The loc is about to be invalidated. Fetch it and clear the loc. + WorkingSetCommon::fetchAndInvalidateLoc(member); + + // Add the WSID to the to-be-reviewed list in the WS. + _ws->flagForReview(id); + + // And don't return it from this stage. + _dataMap.erase(it); + } + } + + PlanStageStats* AndHashStage::getStats() { + _commonStats.isEOF = isEOF(); + + _specificStats.memLimit = _maxMemUsage; + _specificStats.memUsage = _memUsage; + + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_AND_HASH)); + ret->specific.reset(new AndHashStats(_specificStats)); + for (size_t i = 0; i < _children.size(); ++i) { + ret->children.push_back(_children[i]->getStats()); + } + + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/and_hash.h b/src/mongo/db/exec/and_hash.h new file mode 100644 index 00000000000..a546057ffe2 --- /dev/null +++ b/src/mongo/db/exec/and_hash.h @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + /** + * Reads from N children, each of which must have a valid DiskLoc. Uses a hash table to + * intersect the outputs of the N children, and outputs the intersection. + * + * Preconditions: Valid DiskLoc. More than one child. + * + * Any DiskLoc that we keep a reference to that is invalidated before we are able to return it + * is fetched and added to the WorkingSet as "flagged for further review." Because this stage + * operates with DiskLocs, we are unable to evaluate the AND for the invalidated DiskLoc, and it + * must be fully matched later. + */ + class AndHashStage : public PlanStage { + public: + AndHashStage(WorkingSet* ws, const MatchExpression* filter); + + /** + * For testing only. Allows tests to set memory usage threshold. + */ + AndHashStage(WorkingSet* ws, const MatchExpression* filter, size_t maxMemUsage); + + virtual ~AndHashStage(); + + void addChild(PlanStage* child); + + /** + * Returns memory usage. + * For testing only. + */ + size_t getMemUsage() const; + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + static const size_t kLookAheadWorks; + + StageState readFirstChild(WorkingSetID* out); + StageState hashOtherChildren(WorkingSetID* out); + StageState workChild(size_t childNo, WorkingSetID* out); + + // Not owned by us. + WorkingSet* _ws; + + // Not owned by us. + const MatchExpression* _filter; + + // The stages we read from. Owned by us. + std::vector _children; + + // We want to see if any of our children are EOF immediately. This requires working them a + // few times to see if they hit EOF or if they produce a result. If they produce a result, + // we place that result here. + std::vector _lookAheadResults; + + // _dataMap is filled out by the first child and probed by subsequent children. This is the + // hash table that we create by intersecting _children and probe with the last child. + typedef unordered_map DataMap; + DataMap _dataMap; + + // Keeps track of what elements from _dataMap subsequent children have seen. + // Only used while _hashingChildren. + typedef unordered_set SeenMap; + SeenMap _seenMap; + + // True if we're still intersecting _children[0..._children.size()-1]. + bool _hashingChildren; + + // Which child are we currently working on? + size_t _currentChild; + + // Stats + CommonStats _commonStats; + AndHashStats _specificStats; + + // The usage in bytes of all buffered data that we're holding. + // Memory usage is calculated from keys held in _dataMap only. + // For simplicity, results in _lookAheadResults do not count towards the limit. + size_t _memUsage; + + // Upper limit for buffered data memory usage. + // Defaults to 32 MB (See kMaxBytes in and_hash.cpp). + size_t _maxMemUsage; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/and_sorted.cpp b/src/mongo/db/exec/and_sorted.cpp new file mode 100644 index 00000000000..51741e4f974 --- /dev/null +++ b/src/mongo/db/exec/and_sorted.cpp @@ -0,0 +1,311 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/and_sorted.h" + +#include "mongo/db/exec/and_common-inl.h" +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + AndSortedStage::AndSortedStage(WorkingSet* ws, const MatchExpression* filter) + : _ws(ws), _filter(filter), _targetNode(numeric_limits::max()), + _targetId(WorkingSet::INVALID_ID), _isEOF(false) { } + + AndSortedStage::~AndSortedStage() { + for (size_t i = 0; i < _children.size(); ++i) { delete _children[i]; } + } + + void AndSortedStage::addChild(PlanStage* child) { + _children.push_back(child); + } + + bool AndSortedStage::isEOF() { return _isEOF; } + + PlanStage::StageState AndSortedStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (isEOF()) { return PlanStage::IS_EOF; } + + if (0 == _specificStats.failedAnd.size()) { + _specificStats.failedAnd.resize(_children.size()); + } + + // If we don't have any nodes that we're work()-ing until they hit a certain DiskLoc... + if (0 == _workingTowardRep.size()) { + // Get a target DiskLoc. + return getTargetLoc(out); + } + + // Move nodes toward the target DiskLoc. + // If all nodes reach the target DiskLoc, return it. The next call to work() will set a new + // target. + return moveTowardTargetLoc(out); + } + + PlanStage::StageState AndSortedStage::getTargetLoc(WorkingSetID* out) { + verify(numeric_limits::max() == _targetNode); + verify(WorkingSet::INVALID_ID == _targetId); + verify(DiskLoc() == _targetLoc); + + // Pick one, and get a loc to work toward. + WorkingSetID id = WorkingSet::INVALID_ID; + StageState state = _children[0]->work(&id); + + if (PlanStage::ADVANCED == state) { + WorkingSetMember* member = _ws->get(id); + + // Maybe the child had an invalidation. We intersect DiskLoc(s) so we can't do anything + // with this WSM. + if (!member->hasLoc()) { + _ws->flagForReview(id); + return PlanStage::NEED_TIME; + } + + verify(member->hasLoc()); + + // We have a value from one child to AND with. + _targetNode = 0; + _targetId = id; + _targetLoc = member->loc; + + // We have to AND with all other children. + for (size_t i = 1; i < _children.size(); ++i) { + _workingTowardRep.push(i); + } + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::IS_EOF == state) { + _isEOF = true; + return state; + } + else if (PlanStage::FAILURE == state) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "sorted AND stage failed to read in results from first child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + _isEOF = true; + return state; + } + else { + if (PlanStage::NEED_FETCH == state) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == state) { + ++_commonStats.needTime; + } + + // NEED_TIME, NEED_YIELD. + return state; + } + } + + PlanStage::StageState AndSortedStage::moveTowardTargetLoc(WorkingSetID* out) { + verify(numeric_limits::max() != _targetNode); + verify(WorkingSet::INVALID_ID != _targetId); + + // We have nodes that haven't hit _targetLoc yet. + size_t workingChildNumber = _workingTowardRep.front(); + PlanStage* next = _children[workingChildNumber]; + WorkingSetID id = WorkingSet::INVALID_ID; + StageState state = next->work(&id); + + if (PlanStage::ADVANCED == state) { + WorkingSetMember* member = _ws->get(id); + + // Maybe the child had an invalidation. We intersect DiskLoc(s) so we can't do anything + // with this WSM. + if (!member->hasLoc()) { + _ws->flagForReview(id); + return PlanStage::NEED_TIME; + } + + verify(member->hasLoc()); + + if (member->loc == _targetLoc) { + // The front element has hit _targetLoc. Don't move it forward anymore/work on + // another element. + _workingTowardRep.pop(); + AndCommon::mergeFrom(_ws->get(_targetId), *member); + _ws->free(id); + + if (0 == _workingTowardRep.size()) { + WorkingSetID toReturn = _targetId; + WorkingSetMember* toMatchTest = _ws->get(toReturn); + + _targetNode = numeric_limits::max(); + _targetId = WorkingSet::INVALID_ID; + _targetLoc = DiskLoc(); + + // Everyone hit it, hooray. Return it, if it matches. + if (Filter::passes(toMatchTest, _filter)) { + if (NULL != _filter) { + ++_specificStats.matchTested; + } + + *out = toReturn; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else { + _ws->free(toReturn); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + // More children need to be advanced to _targetLoc. + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (member->loc < _targetLoc) { + // The front element of _workingTowardRep hasn't hit the thing we're AND-ing with + // yet. Try again later. + _ws->free(id); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else { + // member->loc > _targetLoc. + // _targetLoc wasn't successfully AND-ed with the other sub-plans. We toss it and + // try AND-ing with the next value. + _specificStats.failedAnd[_targetNode]++; + + _ws->free(_targetId); + _targetNode = workingChildNumber; + _targetLoc = member->loc; + _targetId = id; + _workingTowardRep = queue(); + for (size_t i = 0; i < _children.size(); ++i) { + if (workingChildNumber != i) { + _workingTowardRep.push(i); + } + } + // Need time to chase after the new _targetLoc. + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + else if (PlanStage::IS_EOF == state) { + _isEOF = true; + _ws->free(_targetId); + return state; + } + else if (PlanStage::FAILURE == state) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "sorted AND stage failed to read in results from child " << workingChildNumber; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + _isEOF = true; + _ws->free(_targetId); + return state; + } + else { + if (PlanStage::NEED_FETCH == state) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == state) { + ++_commonStats.needTime; + } + return state; + } + } + + void AndSortedStage::prepareToYield() { + ++_commonStats.yields; + + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->prepareToYield(); + } + } + + void AndSortedStage::recoverFromYield() { + ++_commonStats.unyields; + + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->recoverFromYield(); + } + } + + void AndSortedStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + + if (isEOF()) { return; } + + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->invalidate(dl, type); + } + + if (dl == _targetLoc) { + // We're in the middle of moving children forward until they hit _targetLoc, which is no + // longer a valid target. If it's a deletion we can't AND it with anything, if it's a + // mutation the predicates implied by the AND may no longer be true. So no matter what, + // fetch it, flag for review, and find another _targetLoc. + ++_specificStats.flagged; + + // The DiskLoc could still be a valid result so flag it and save it for later. + WorkingSetCommon::fetchAndInvalidateLoc(_ws->get(_targetId)); + _ws->flagForReview(_targetId); + + _targetId = WorkingSet::INVALID_ID; + _targetNode = numeric_limits::max(); + _targetLoc = DiskLoc(); + _workingTowardRep = queue(); + } + } + + PlanStageStats* AndSortedStage::getStats() { + _commonStats.isEOF = isEOF(); + + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_AND_SORTED)); + ret->specific.reset(new AndSortedStats(_specificStats)); + for (size_t i = 0; i < _children.size(); ++i) { + ret->children.push_back(_children[i]->getStats()); + } + + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/and_sorted.h b/src/mongo/db/exec/and_sorted.h new file mode 100644 index 00000000000..cf755228433 --- /dev/null +++ b/src/mongo/db/exec/and_sorted.h @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + /** + * Reads from N children, each of which must have a valid DiskLoc. Assumes each child produces + * DiskLocs in sorted order. Outputs the intersection of the DiskLocs outputted by the + * children. + * + * Preconditions: Valid DiskLoc. More than one child. + * + * Any DiskLoc that we keep a reference to that is invalidated before we are able to return it + * is fetched and added to the WorkingSet as "flagged for further review." Because this stage + * operates with DiskLocs, we are unable to evaluate the AND for the invalidated DiskLoc, and it + * must be fully matched later. + */ + class AndSortedStage : public PlanStage { + public: + AndSortedStage(WorkingSet* ws, const MatchExpression* filter); + virtual ~AndSortedStage(); + + void addChild(PlanStage* child); + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + // Find a node to AND against. + PlanStage::StageState getTargetLoc(WorkingSetID* out); + + // Move a child which hasn't advanced to the target node forward. + // Returns the target node in 'out' if all children successfully advance to it. + PlanStage::StageState moveTowardTargetLoc(WorkingSetID* out); + + // Not owned by us. + WorkingSet* _ws; + + // Not owned by us. + const MatchExpression* _filter; + + // Owned by us. + vector _children; + + // The current node we're AND-ing against. + size_t _targetNode; + DiskLoc _targetLoc; + WorkingSetID _targetId; + + // Nodes we're moving forward until they hit the element we're AND-ing. + // Everything in here has not advanced to _targetLoc yet. + // These are indices into _children. + std::queue _workingTowardRep; + + // If any child hits EOF or if we have any errors, we're EOF. + bool _isEOF; + + // Stats + CommonStats _commonStats; + AndSortedStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/collection_scan.cpp b/src/mongo/db/exec/collection_scan.cpp new file mode 100644 index 00000000000..1a29d8b76fb --- /dev/null +++ b/src/mongo/db/exec/collection_scan.cpp @@ -0,0 +1,206 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/collection_scan.h" + +#include "mongo/db/catalog/database.h" +#include "mongo/db/exec/collection_scan_common.h" +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/structure/collection_iterator.h" +#include "mongo/util/fail_point_service.h" + +#include "mongo/db/client.h" // XXX-ERH +#include "mongo/db/pdfile.h" // XXX-ERH/ACM + +namespace mongo { + + // Some fail points for testing. + MONGO_FP_DECLARE(collscanInMemoryFail); + MONGO_FP_DECLARE(collscanInMemorySucceed); + + // static + bool CollectionScan::diskLocInMemory(DiskLoc loc) { + if (MONGO_FAIL_POINT(collscanInMemoryFail)) { + return false; + } + + if (MONGO_FAIL_POINT(collscanInMemorySucceed)) { + return true; + } + + return loc.rec()->likelyInPhysicalMemory(); + } + + CollectionScan::CollectionScan(const CollectionScanParams& params, + WorkingSet* workingSet, + const MatchExpression* filter) + : _workingSet(workingSet), + _filter(filter), + _params(params), + _nsDropped(false) { + + // We pre-allocate a WSID and use it to pass up fetch requests. It is only + // used to pass up fetch requests and we should never use it for anything else. + _wsidForFetch = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(_wsidForFetch); + // Kind of a lie since the obj isn't pointing to the data at loc. but the obj + // won't be used. + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + } + + PlanStage::StageState CollectionScan::work(WorkingSetID* out) { + ++_commonStats.works; + if (_nsDropped) { return PlanStage::DEAD; } + + // Do some init if we haven't already. + if (NULL == _iter) { + Collection* collection = cc().database()->getCollection( _params.ns ); + if ( collection == NULL ) { + _nsDropped = true; + return PlanStage::DEAD; + } + + _iter.reset( collection->getIterator( _params.start, + _params.tailable, + _params.direction ) ); + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + + // See if the record we're about to access is in memory. If it's not, pass a fetch + // request up. + if (!isEOF()) { + DiskLoc curr = _iter->curr(); + if (!curr.isNull() && !diskLocInMemory(curr)) { + WorkingSetMember* member = _workingSet->get(_wsidForFetch); + member->loc = curr; + *out = _wsidForFetch; + return PlanStage::NEED_FETCH; + } + } + + // What we'll return to the user. + DiskLoc nextLoc; + + // Should we try getNext() on the underlying _iter if we're EOF? Yes, if we're tailable. + if (isEOF()) { + if (!_params.tailable) { + return PlanStage::IS_EOF; + } + else { + // See if _iter gives us anything new. + nextLoc = _iter->getNext(); + if (nextLoc.isNull()) { + // Nope, still EOF. + return PlanStage::IS_EOF; + } + } + } + else { + nextLoc = _iter->getNext(); + } + + WorkingSetID id = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(id); + member->loc = nextLoc; + member->obj = member->loc.obj(); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + + ++_specificStats.docsTested; + + if (Filter::passes(member, _filter)) { + *out = id; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else { + _workingSet->free(id); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + + bool CollectionScan::isEOF() { + if ((0 != _params.maxScan) && (_specificStats.docsTested >= _params.maxScan)) { + return true; + } + if (_nsDropped) { return true; } + if (NULL == _iter) { return false; } + return _iter->isEOF(); + } + + void CollectionScan::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + + // We don't care about mutations since we apply any filters to the result when we (possibly) + // return it. + if (INVALIDATION_DELETION != type) { + return; + } + + // If we're here, 'dl' is being deleted. + + // Deletions can harm the underlying CollectionIterator so we must pass them down. + if (NULL != _iter) { + _iter->invalidate(dl); + } + + // We might have 'dl' inside of the WSM that _wsidForFetch references. This is OK because + // the runner who handles the fetch request does so before releasing any locks (and allowing + // the DiskLoc to be deleted). We also don't use any data in the WSM referenced by + // _wsidForFetch so it's OK to leave the DiskLoc there. + } + + void CollectionScan::prepareToYield() { + ++_commonStats.yields; + if (NULL != _iter) { + _iter->prepareToYield(); + } + } + + void CollectionScan::recoverFromYield() { + ++_commonStats.unyields; + if (NULL != _iter) { + if (!_iter->recoverFromYield()) { + warning() << "Collection dropped or state deleted during yield of CollectionScan"; + _nsDropped = true; + } + } + } + + PlanStageStats* CollectionScan::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_COLLSCAN)); + ret->specific.reset(new CollectionScanStats(_specificStats)); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/collection_scan.h b/src/mongo/db/exec/collection_scan.h new file mode 100644 index 00000000000..e0d4eb945f8 --- /dev/null +++ b/src/mongo/db/exec/collection_scan.h @@ -0,0 +1,91 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/collection_scan_common.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/structure/collection_iterator.h" + +namespace mongo { + + class WorkingSet; + + /** + * Scans over a collection, starting at the DiskLoc provided in params and continuing until + * there are no more records in the collection. + * + * Preconditions: Valid DiskLoc. + */ + class CollectionScan : public PlanStage { + public: + CollectionScan(const CollectionScanParams& params, + WorkingSet* workingSet, + const MatchExpression* filter); + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + virtual void prepareToYield(); + virtual void recoverFromYield(); + + virtual PlanStageStats* getStats(); + + private: + /** + * Returns true if the record 'loc' references is in memory, false otherwise. + */ + static bool diskLocInMemory(DiskLoc loc); + + // WorkingSet is not owned by us. + WorkingSet* _workingSet; + + // The filter is not owned by us. + const MatchExpression* _filter; + + scoped_ptr _iter; + + CollectionScanParams _params; + + // True if nsdetails(_ns) == NULL on our first call to work. + bool _nsDropped; + + // If we want to return a DiskLoc and it points at something that's not in memory, we return + // a a "please page this in" result. We allocate one WSM for this purpose at construction + // and reuse it for any future fetch requests, changing the DiskLoc as appropriate. + WorkingSetID _wsidForFetch; + + // Stats + CommonStats _commonStats; + CollectionScanStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/collection_scan_common.h b/src/mongo/db/exec/collection_scan_common.h new file mode 100644 index 00000000000..9c88e7653f6 --- /dev/null +++ b/src/mongo/db/exec/collection_scan_common.h @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" + +namespace mongo { + + struct CollectionScanParams { + enum Direction { + FORWARD = 1, + BACKWARD = -1, + }; + + CollectionScanParams() : start(DiskLoc()), + direction(FORWARD), + tailable(false), + maxScan(0) { } + + // What collection? + string ns; + + // isNull by default. If you specify any value for this, you're responsible for the DiskLoc + // not being invalidated before the first call to work(...). + DiskLoc start; + + Direction direction; + + // Do we want the scan to be 'tailable'? Only meaningful if the collection is capped. + bool tailable; + + // If non-zero, how many documents will we look at? + size_t maxScan; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/count.cpp b/src/mongo/db/exec/count.cpp new file mode 100644 index 00000000000..29abe83d7b2 --- /dev/null +++ b/src/mongo/db/exec/count.cpp @@ -0,0 +1,204 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/count.h" + +#include "mongo/db/index/index_cursor.h" +#include "mongo/db/index/index_descriptor.h" + +namespace mongo { + + Count::Count(const CountParams& params, WorkingSet* workingSet) + : _workingSet(workingSet), + _descriptor(params.descriptor), + _iam(params.descriptor->getIndexCatalog()->getIndex(params.descriptor)), + _btreeCursor(NULL), + _params(params), + _hitEnd(false), + _shouldDedup(params.descriptor->isMultikey()) { } + + void Count::initIndexCursor() { + CursorOptions cursorOptions; + cursorOptions.direction = CursorOptions::INCREASING; + + IndexCursor *cursor; + Status s = _iam->newCursor(&cursor); + verify(s.isOK()); + verify(cursor); + + // Is this assumption always valid? See SERVER-12397 + _btreeCursor.reset(static_cast(cursor)); + _btreeCursor->setOptions(cursorOptions); + + // _btreeCursor points at our start position. We move it forward until it hits a cursor + // that points at the end. + _btreeCursor->seek(_params.startKey, !_params.startKeyInclusive); + + // Create the cursor that points at our end position. + IndexCursor* endCursor; + verify(_iam->newCursor(&endCursor).isOK()); + verify(endCursor); + + // Is this assumption always valid? See SERVER-12397 + _endCursor.reset(static_cast(endCursor)); + _endCursor->setOptions(cursorOptions); + + // If the end key is inclusive we want to point *past* it since that's the end. + _endCursor->seek(_params.endKey, _params.endKeyInclusive); + + // See if we've hit the end already. + checkEnd(); + } + + void Count::checkEnd() { + if (isEOF()) { return; } + + if (_endCursor->isEOF()) { + // If the endCursor is EOF we're only done when our 'current count position' hits EOF. + _hitEnd = _btreeCursor->isEOF(); + } + else { + // If not, we're only done when we hit the end cursor's (valid) position. + _hitEnd = _btreeCursor->pointsAt(*_endCursor.get()); + } + } + + PlanStage::StageState Count::work(WorkingSetID* out) { + if (NULL == _btreeCursor.get()) { + // First call to work(). Perform cursor init. + initIndexCursor(); + checkEnd(); + return PlanStage::NEED_TIME; + } + + if (isEOF()) { return PlanStage::IS_EOF; } + + DiskLoc loc = _btreeCursor->getValue(); + _btreeCursor->next(); + checkEnd(); + + if (_shouldDedup) { + if (_returned.end() != _returned.find(loc)) { + return PlanStage::NEED_TIME; + } + else { + _returned.insert(loc); + } + } + + *out = WorkingSet::INVALID_ID; + return PlanStage::ADVANCED; + } + + bool Count::isEOF() { + if (NULL == _btreeCursor.get()) { + // Have to call work() at least once. + return false; + } + + return _hitEnd || _btreeCursor->isEOF(); + } + + void Count::prepareToYield() { + if (isEOF() || (NULL == _btreeCursor.get())) { return; } + + verify(!_btreeCursor->isEOF()); + _btreeCursor->savePosition(); + if (!_endCursor->isEOF()) { + _endCursor->savePosition(); + } + } + + void Count::recoverFromYield() { + if (isEOF() || (NULL == _btreeCursor.get())) { return; } + + if (!_btreeCursor->restorePosition().isOK()) { + _hitEnd = true; + } + + if (_btreeCursor->isEOF()) { + _hitEnd = true; + return; + } + + // See if we're somehow already past our end key (maybe the thing we were pointing at got + // deleted...) + int cmp = _btreeCursor->getKey().woCompare(_params.endKey, _descriptor->keyPattern(), false); + if (cmp > 0 || (cmp == 0 && !_params.endKeyInclusive)) { + _hitEnd = true; + return; + } + + if (!_endCursor->isEOF()) { + if (!_endCursor->restorePosition().isOK()) { + _hitEnd = true; + return; + } + } + + // If we were EOF when we yielded we don't always want to have _btreeCursor run until + // EOF. New documents may have been inserted after our endKey and our end marker + // may be before them. + // + // As an example, say we're counting from 5 to 10 and the index only has keys + // for 6, 7, 8, and 9. btreeCursor will point at a 6 key at the start and the + // endCursor will be EOF. If we insert documents with keys 11 during a yield we + // need to relocate the endCursor to point at them as the "end key" of our count. + // + // If we weren't EOF our end position might have moved around. Relocate it. + _endCursor->seek(_params.endKey, _params.endKeyInclusive); + + // This can change during yielding. + _shouldDedup = _descriptor->isMultikey(); + + checkEnd(); + } + + void Count::invalidate(const DiskLoc& dl, InvalidationType type) { + // The only state we're responsible for holding is what DiskLocs to drop. If a document + // mutates the underlying index cursor will deal with it. + if (INVALIDATION_MUTATION == type) { + return; + } + + // If we see this DiskLoc again, it may not be the same document it was before, so we want + // to return it if we see it again. + unordered_set::iterator it = _returned.find(dl); + if (it != _returned.end()) { + _returned.erase(it); + } + } + + PlanStageStats* Count::getStats() { + // We don't collect stats since this stage is only used by the count command. + // If count ever collects stats we must implement this. + invariant(0); + return NULL; + } + +} // namespace mongo diff --git a/src/mongo/db/exec/count.h b/src/mongo/db/exec/count.h new file mode 100644 index 00000000000..31e2ad8ba96 --- /dev/null +++ b/src/mongo/db/exec/count.h @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/index/btree_index_cursor.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + class IndexAccessMethod; + class IndexDescriptor; + class WorkingSet; + + struct CountParams { + CountParams() : descriptor(NULL) { } + + // What index are we traversing? + const IndexDescriptor* descriptor; + + BSONObj startKey; + bool startKeyInclusive; + + BSONObj endKey; + bool endKeyInclusive; + }; + + /** + * Used by the count command. Scans an index from a start key to an end key. Does not create + * any WorkingSetMember(s) for any of the data, instead returning ADVANCED to indicate to the + * caller that another result should be counted. + * + * Only created through the getRunnerCount path, as count is the only operation that doesn't + * care about its data. + */ + class Count : public PlanStage { + public: + Count(const CountParams& params, WorkingSet* workingSet); + virtual ~Count() { } + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + /** + * Initialize the underlying IndexCursor + */ + void initIndexCursor(); + + /** + * See if we've hit the end yet. + */ + void checkEnd(); + + // The WorkingSet we annotate with results. Not owned by us. + WorkingSet* _workingSet; + + // Index access. Both pointers below are owned by Collection -> IndexCatalog. + const IndexDescriptor* _descriptor; + const IndexAccessMethod* _iam; + + // Our start cursor is _btreeCursor. + boost::scoped_ptr _btreeCursor; + + // Our end marker. + boost::scoped_ptr _endCursor; + + // Could our index have duplicates? If so, we use _returned to dedup. + unordered_set _returned; + + CountParams _params; + + bool _hitEnd; + + bool _shouldDedup; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/distinct_scan.cpp b/src/mongo/db/exec/distinct_scan.cpp new file mode 100644 index 00000000000..5c58ff40e08 --- /dev/null +++ b/src/mongo/db/exec/distinct_scan.cpp @@ -0,0 +1,239 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/distinct_scan.h" + +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index/index_cursor.h" +#include "mongo/db/index/index_descriptor.h" + +namespace mongo { + + DistinctScan::DistinctScan(const DistinctParams& params, WorkingSet* workingSet) + : _workingSet(workingSet), + _descriptor(params.descriptor), + _iam(params.descriptor->getIndexCatalog()->getIndex(params.descriptor)), + _btreeCursor(NULL), + _scanState(INITIALIZING), + _params(params) { } + + void DistinctScan::initIndexCursor() { + // This function transitions from the initializing state to CHECKING_END. If + // the initialization fails, however, then the state transitions to HIT_END. + invariant(INITIALIZING == _scanState); + + // Create an IndexCursor over the btree we're distinct-ing over. + CursorOptions cursorOptions; + + if (1 == _params.direction) { + cursorOptions.direction = CursorOptions::INCREASING; + } + else { + cursorOptions.direction = CursorOptions::DECREASING; + } + + IndexCursor *cursor; + Status s = _iam->newCursor(&cursor); + verify(s.isOK()); + verify(cursor); + // Is this assumption always valid? See SERVER-12397 + _btreeCursor.reset(static_cast(cursor)); + _btreeCursor->setOptions(cursorOptions); + + // Create a new bounds checker. The bounds checker gets our start key and assists in + // executing the scan and staying within the required bounds. + _checker.reset(new IndexBoundsChecker(&_params.bounds, + _descriptor->keyPattern(), + _params.direction)); + + int nFields = _descriptor->keyPattern().nFields(); + // The start key is dumped into these two. + vector key; + vector inc; + key.resize(nFields); + inc.resize(nFields); + if (_checker->getStartKey(&key, &inc)) { + _btreeCursor->seek(key, inc); + _keyElts.resize(nFields); + _keyEltsInc.resize(nFields); + } + else { + _scanState = HIT_END; + } + + // This method may throw an exception while it's doing initialization. If we've gotten + // here, then we've done all the initialization without an exception being thrown. This + // means it is safe to transition to the CHECKING_END state. In error cases, we transition + // to HIT_END, so we should not change state again here. + if (HIT_END != _scanState) { + _scanState = CHECKING_END; + } + } + + PlanStage::StageState DistinctScan::work(WorkingSetID* out) { + ++_commonStats.works; + + if (INITIALIZING == _scanState) { + invariant(NULL == _btreeCursor.get()); + initIndexCursor(); + } + + if (CHECKING_END == _scanState) { + checkEnd(); + } + + if (isEOF()) { + _commonStats.isEOF = true; + return PlanStage::IS_EOF; + } + + if (GETTING_NEXT == _scanState) { + // Grab the next (key, value) from the index. + BSONObj ownedKeyObj = _btreeCursor->getKey().getOwned(); + DiskLoc loc = _btreeCursor->getValue(); + + // The underlying IndexCursor points at the *next* thing we want to return. We do this + // so that if we're scanning an index looking for docs to delete we don't continually + // clobber the thing we're pointing at. + + // We skip to the next value of the _params.fieldNo-th field in the index key pattern. + // This is the field we're distinct-ing over. + _btreeCursor->skip(_btreeCursor->getKey(), + _params.fieldNo + 1, + true, + _keyElts, + _keyEltsInc); + + // On the next call to work, make sure that the cursor is still within the bounds. + _scanState = CHECKING_END; + + // Package up the result for the caller. + WorkingSetID id = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(id); + member->loc = loc; + member->keyData.push_back(IndexKeyDatum(_descriptor->keyPattern(), ownedKeyObj)); + member->state = WorkingSetMember::LOC_AND_IDX; + + *out = id; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + + bool DistinctScan::isEOF() { + if (INITIALIZING == _scanState) { + // Have to call work() at least once. + return false; + } + + return HIT_END == _scanState || _btreeCursor->isEOF(); + } + + void DistinctScan::prepareToYield() { + ++_commonStats.yields; + + if (isEOF() || INITIALIZING == _scanState) { return; } + // We save these so that we know if the cursor moves during the yield. If it moves, we have + // to make sure its ending position is valid w.r.t. our bounds. + _savedKey = _btreeCursor->getKey().getOwned(); + _savedLoc = _btreeCursor->getValue(); + _btreeCursor->savePosition(); + } + + void DistinctScan::recoverFromYield() { + ++_commonStats.unyields; + + if (isEOF() || INITIALIZING == _scanState) { return; } + + // We can have a valid position before we check isEOF(), restore the position, and then be + // EOF upon restore. + if (!_btreeCursor->restorePosition().isOK() || _btreeCursor->isEOF()) { + _scanState = HIT_END; + return; + } + + if (!_savedKey.binaryEqual(_btreeCursor->getKey()) || _savedLoc != _btreeCursor->getValue()) { + // Our restored position might be past endKey, see if we've hit the end. + _scanState = CHECKING_END; + } + } + + void DistinctScan::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + } + + void DistinctScan::checkEnd() { + if (isEOF()) { + _commonStats.isEOF = true; + return; + } + + // Use _checker to see how things are. + IndexBoundsChecker::KeyState keyState; + keyState = _checker->checkKey(_btreeCursor->getKey(), + &_keyEltsToUse, + &_movePastKeyElts, + &_keyElts, + &_keyEltsInc); + + if (IndexBoundsChecker::DONE == keyState) { + _scanState = HIT_END; + return; + } + + // This seems weird but it's the old definition of nscanned. + ++_specificStats.keysExamined; + + if (IndexBoundsChecker::VALID == keyState) { + _scanState = GETTING_NEXT; + return; + } + + verify(IndexBoundsChecker::MUST_ADVANCE == keyState); + _btreeCursor->skip(_btreeCursor->getKey(), _keyEltsToUse, _movePastKeyElts, + _keyElts, _keyEltsInc); + + // Must check underlying cursor EOF after every cursor movement. + if (_btreeCursor->isEOF()) { + _scanState = HIT_END; + } + } + + PlanStageStats* DistinctScan::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_DISTINCT)); + ret->specific.reset(new DistinctScanStats(_specificStats)); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/distinct_scan.h b/src/mongo/db/exec/distinct_scan.h new file mode 100644 index 00000000000..116762b4b2e --- /dev/null +++ b/src/mongo/db/exec/distinct_scan.h @@ -0,0 +1,151 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/index/btree_index_cursor.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/query/index_bounds.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + class IndexAccessMethod; + class IndexCursor; + class IndexDescriptor; + class WorkingSet; + + struct DistinctParams { + DistinctParams() : descriptor(NULL), + direction(1), + fieldNo(0) { } + + // What index are we traversing? + const IndexDescriptor* descriptor; + + // And in what direction? + int direction; + + // What are the bounds? + IndexBounds bounds; + + // What field in the index's key pattern is the one we're distinct-ing over? + // For example: + // If we have an index {a:1, b:1} we could use it to distinct over either 'a' or 'b'. + // If we distinct over 'a' the position is 0. + // If we distinct over 'b' the position is 1. + int fieldNo; + }; + + /** + * Used by the distinct command. Executes a mutated index scan over the provided bounds. + * However, rather than looking at every key in the bounds, it skips to the next value of the + * _params.fieldNo-th indexed field. This is because distinct only cares about distinct values + * for that field, so there is no point in examining all keys with the same value for that + * field. + * + * Only created through the getDistinctRunner path. See db/query/get_runner.cpp + */ + class DistinctScan : public PlanStage { + public: + /** + * Keeps track of what this distinct scan is currently doing so that it + * can do the right thing on the next call to work(). + */ + enum ScanState { + // Need to initialize the underlying index traversal machinery. + INITIALIZING, + + // Skipping keys in order to check whether we have reached the end. + CHECKING_END, + + // Retrieving the next key, and applying the filter if necessary. + GETTING_NEXT, + + // The index scan is finished. + HIT_END + }; + + DistinctScan(const DistinctParams& params, WorkingSet* workingSet); + virtual ~DistinctScan() { } + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + /** + * Initialize the underlying IndexCursor + */ + void initIndexCursor(); + + /** See if the cursor is pointing at or past _endKey, if _endKey is non-empty. */ + void checkEnd(); + + // The WorkingSet we annotate with results. Not owned by us. + WorkingSet* _workingSet; + + // Index access. + const IndexDescriptor* _descriptor; // owned by Collection -> IndexCatalog + const IndexAccessMethod* _iam; // owned by Collection -> IndexCatalog + + // The cursor we use to navigate the tree. + boost::scoped_ptr _btreeCursor; + + // Keeps track of what work we need to do next. + ScanState _scanState; + + // For yielding. + BSONObj _savedKey; + DiskLoc _savedLoc; + + DistinctParams _params; + + // _checker gives us our start key and ensures we stay in bounds. + boost::scoped_ptr _checker; + int _keyEltsToUse; + bool _movePastKeyElts; + vector _keyElts; + vector _keyEltsInc; + + // Stats + CommonStats _commonStats; + DistinctScanStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/fetch.cpp b/src/mongo/db/exec/fetch.cpp new file mode 100644 index 00000000000..d255050fffa --- /dev/null +++ b/src/mongo/db/exec/fetch.cpp @@ -0,0 +1,230 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/fetch.h" + +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/pdfile.h" +#include "mongo/util/fail_point_service.h" + +namespace mongo { + + // Some fail points for testing. + MONGO_FP_DECLARE(fetchInMemoryFail); + MONGO_FP_DECLARE(fetchInMemorySucceed); + + FetchStage::FetchStage(WorkingSet* ws, PlanStage* child, const MatchExpression* filter) + : _ws(ws), _child(child), _filter(filter), _idBeingPagedIn(WorkingSet::INVALID_ID) { } + + FetchStage::~FetchStage() { } + + bool FetchStage::isEOF() { + if (WorkingSet::INVALID_ID != _idBeingPagedIn) { + // We asked our parent for a page-in but he didn't get back to us. We still need to + // return the result that _idBeingPagedIn refers to. + return false; + } + + return _child->isEOF(); + } + + bool recordInMemory(const char* data) { + if (MONGO_FAIL_POINT(fetchInMemoryFail)) { + return false; + } + + if (MONGO_FAIL_POINT(fetchInMemorySucceed)) { + return true; + } + + return Record::likelyInPhysicalMemory(data); + } + + PlanStage::StageState FetchStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (isEOF()) { return PlanStage::IS_EOF; } + + // If we asked our parent for a page-in last time work(...) was called, finish the fetch. + if (WorkingSet::INVALID_ID != _idBeingPagedIn) { + return fetchCompleted(out); + } + + // If we're here, we're not waiting for a DiskLoc to be fetched. Get another to-be-fetched + // result from our child. + WorkingSetID id = WorkingSet::INVALID_ID; + StageState status = _child->work(&id); + + if (PlanStage::ADVANCED == status) { + WorkingSetMember* member = _ws->get(id); + + // If there's an obj there, there is no fetching to perform. + if (member->hasObj()) { + ++_specificStats.alreadyHasObj; + return returnIfMatches(member, id, out); + } + + // We need a valid loc to fetch from and this is the only state that has one. + verify(WorkingSetMember::LOC_AND_IDX == member->state); + verify(member->hasLoc()); + + Record* record = member->loc.rec(); + const char* data = record->dataNoThrowing(); + + if (!recordInMemory(data)) { + // member->loc points to a record that's NOT in memory. Pass a fetch request up. + verify(WorkingSet::INVALID_ID == _idBeingPagedIn); + _idBeingPagedIn = id; + *out = id; + ++_commonStats.needFetch; + return PlanStage::NEED_FETCH; + } + else { + // Don't need index data anymore as we have an obj. + member->keyData.clear(); + member->obj = BSONObj(data); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + return returnIfMatches(member, id, out); + } + } + else if (PlanStage::FAILURE == status) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "fetch stage failed to read in results from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return status; + } + else { + if (PlanStage::NEED_FETCH == status) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == status) { + ++_commonStats.needTime; + } + return status; + } + } + + void FetchStage::prepareToYield() { + ++_commonStats.yields; + _child->prepareToYield(); + } + + void FetchStage::recoverFromYield() { + ++_commonStats.unyields; + _child->recoverFromYield(); + } + + void FetchStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + + _child->invalidate(dl, type); + + // If we're holding on to an object that we're waiting for the runner to page in... + if (WorkingSet::INVALID_ID != _idBeingPagedIn) { + // And we haven't already invalidated it... + WorkingSetMember* member = _ws->get(_idBeingPagedIn); + if (member->hasLoc() && (member->loc == dl)) { + // Just fetch it now and kill the DiskLoc. + WorkingSetCommon::fetchAndInvalidateLoc(member); + } + } + } + + PlanStage::StageState FetchStage::fetchCompleted(WorkingSetID* out) { + WorkingSetMember* member = _ws->get(_idBeingPagedIn); + + // The DiskLoc we're waiting to page in was invalidated (forced fetch). Test for + // matching and maybe pass it up. + if (member->state == WorkingSetMember::OWNED_OBJ) { + WorkingSetID memberID = _idBeingPagedIn; + _idBeingPagedIn = WorkingSet::INVALID_ID; + return returnIfMatches(member, memberID, out); + } + + // Assume that the caller has fetched appropriately. + // TODO: Do we want to double-check the runner? Not sure how reliable likelyInMemory is + // on all platforms. + verify(member->hasLoc()); + verify(!member->hasObj()); + + // Make the (unowned) object. + Record* record = member->loc.rec(); + const char* data = record->dataNoThrowing(); + member->obj = BSONObj(data); + + // Don't need index data anymore as we have an obj. + member->keyData.clear(); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + verify(!member->obj.isOwned()); + + // Return the obj if it passes our filter. + WorkingSetID memberID = _idBeingPagedIn; + _idBeingPagedIn = WorkingSet::INVALID_ID; + return returnIfMatches(member, memberID, out); + } + + PlanStage::StageState FetchStage::returnIfMatches(WorkingSetMember* member, + WorkingSetID memberID, + WorkingSetID* out) { + if (Filter::passes(member, _filter)) { + if (NULL != _filter) { + ++_specificStats.matchTested; + } + + *out = memberID; + + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else { + _ws->free(memberID); + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + + PlanStageStats* FetchStage::getStats() { + _commonStats.isEOF = isEOF(); + + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_FETCH)); + ret->specific.reset(new FetchStats(_specificStats)); + ret->children.push_back(_child->getStats()); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/fetch.h b/src/mongo/db/exec/fetch.h new file mode 100644 index 00000000000..0b05dc389b5 --- /dev/null +++ b/src/mongo/db/exec/fetch.h @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" + +namespace mongo { + + /** + * This stage turns a DiskLoc into a BSONObj. + * + * In WorkingSetMember terms, it transitions from LOC_AND_IDX to LOC_AND_UNOWNED_OBJ by reading + * the record at the provided loc. Returns verbatim any data that already has an object. + * + * Preconditions: Valid DiskLoc. + */ + class FetchStage : public PlanStage { + public: + FetchStage(WorkingSet* ws, PlanStage* child, const MatchExpression* filter); + virtual ~FetchStage(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + PlanStageStats* getStats(); + + private: + /** + * If the member (with id memberID) passes our filter, set *out to memberID and return that + * ADVANCED. Otherwise, free memberID and return NEED_TIME. + */ + StageState returnIfMatches(WorkingSetMember* member, WorkingSetID memberID, + WorkingSetID* out); + + /** + * work(...) delegates to this when we're called after requesting a fetch. + */ + StageState fetchCompleted(WorkingSetID* out); + + // _ws is not owned by us. + WorkingSet* _ws; + scoped_ptr _child; + + // The filter is not owned by us. + const MatchExpression* _filter; + + // If we're fetching a DiskLoc and it points at something that's not in memory, we return a + // a "please page this in" result and hold on to the WSID until the next call to work(...). + WorkingSetID _idBeingPagedIn; + + // Stats + CommonStats _commonStats; + FetchStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/filter.h b/src/mongo/db/exec/filter.h new file mode 100644 index 00000000000..8fdd3b8be80 --- /dev/null +++ b/src/mongo/db/exec/filter.h @@ -0,0 +1,171 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/exec/working_set.h" +#include "mongo/db/matcher/matchable.h" + +namespace mongo { + + /** + * The MatchExpression uses the MatchableDocument interface to see if a document satisfies the + * expression. This wraps a WorkingSetMember in the MatchableDocument interface so that any of + * the WorkingSetMember's various types can be tested to see if they satisfy an expression. + */ + class WorkingSetMatchableDocument : public MatchableDocument { + public: + WorkingSetMatchableDocument(WorkingSetMember* wsm) : _wsm(wsm) { } + virtual ~WorkingSetMatchableDocument() { } + + // This is only called by a $where query. The query system must be smart enough to realize + // that it should do a fetch beforehand. + BSONObj toBSON() const { + invariant(_wsm->hasObj()); + return _wsm->obj; + } + + virtual ElementIterator* allocateIterator(const ElementPath* path) const { + // BSONElementIterator does some interesting things with arrays that I don't think + // SimpleArrayElementIterator does. + if (_wsm->hasObj()) { + return new BSONElementIterator(path, _wsm->obj); + } + + // NOTE: This (kind of) duplicates code in WorkingSetMember::getFieldDotted. + // Keep in sync w/that. + // Find the first field in the index key data described by path and return an iterator + // over it. + for (size_t i = 0; i < _wsm->keyData.size(); ++i) { + BSONObjIterator keyPatternIt(_wsm->keyData[i].indexKeyPattern); + BSONObjIterator keyDataIt(_wsm->keyData[i].keyData); + + while (keyPatternIt.more()) { + BSONElement keyPatternElt = keyPatternIt.next(); + invariant(keyDataIt.more()); + BSONElement keyDataElt = keyDataIt.next(); + + if (path->fieldRef().equalsDottedField(keyPatternElt.fieldName())) { + if (Array == keyDataElt.type()) { + return new SimpleArrayElementIterator(keyDataElt, true); + } + else { + return new SingleElementElementIterator(keyDataElt); + } + } + } + } + + // This should not happen. + massert(16920, "trying to match on unknown field: " + path->fieldRef().dottedField().toString(), + 0); + + return new SingleElementElementIterator(BSONElement()); + } + + virtual void releaseIterator( ElementIterator* iterator ) const { + delete iterator; + } + + private: + WorkingSetMember* _wsm; + }; + + class IndexKeyMatchableDocument : public MatchableDocument { + public: + IndexKeyMatchableDocument(const BSONObj& key, + const BSONObj& keyPattern) + : _keyPattern(keyPattern), _key(key) { } + + BSONObj toBSON() const { + // Planning shouldn't let this happen. + invariant(0); + } + + virtual ElementIterator* allocateIterator(const ElementPath* path) const { + BSONObjIterator keyPatternIt(_keyPattern); + BSONObjIterator keyDataIt(_key); + + while (keyPatternIt.more()) { + BSONElement keyPatternElt = keyPatternIt.next(); + invariant(keyDataIt.more()); + BSONElement keyDataElt = keyDataIt.next(); + + if (path->fieldRef().equalsDottedField(keyPatternElt.fieldName())) { + if (Array == keyDataElt.type()) { + return new SimpleArrayElementIterator(keyDataElt, true); + } + else { + return new SingleElementElementIterator(keyDataElt); + } + } + } + + // Planning should not let this happen. + massert(17409, + "trying to match on unknown field: " + path->fieldRef().dottedField().toString(), + 0); + + return new SingleElementElementIterator(BSONElement()); + } + + virtual void releaseIterator(ElementIterator* iterator) const { + delete iterator; + } + + private: + BSONObj _keyPattern; + BSONObj _key; + }; + + /** + * Used by every stage with a filter. + */ + class Filter { + public: + /** + * Returns true if filter is NULL or if 'wsm' satisfies the filter. + * Returns false if 'wsm' does not satisfy the filter. + */ + static bool passes(WorkingSetMember* wsm, const MatchExpression* filter) { + if (NULL == filter) { return true; } + WorkingSetMatchableDocument doc(wsm); + return filter->matches(&doc, NULL); + } + + static bool passes(const BSONObj& keyData, + const BSONObj& keyPattern, + const MatchExpression* filter) { + + if (NULL == filter) { return true; } + IndexKeyMatchableDocument doc(keyData, keyPattern); + return filter->matches(&doc, NULL); + } + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/index_scan.cpp b/src/mongo/db/exec/index_scan.cpp new file mode 100644 index 00000000000..2323b11102b --- /dev/null +++ b/src/mongo/db/exec/index_scan.cpp @@ -0,0 +1,368 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/index_scan.h" + +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index/index_cursor.h" +#include "mongo/db/index/index_descriptor.h" + +namespace { + + // Return a value in the set {-1, 0, 1} to represent the sign of parameter i. + int sgn(int i) { + if (i == 0) + return 0; + return i > 0 ? 1 : -1; + } + +} // namespace + +namespace mongo { + + IndexScan::IndexScan(const IndexScanParams& params, WorkingSet* workingSet, + const MatchExpression* filter) + : _workingSet(workingSet), + _scanState(INITIALIZING), + _filter(filter), + _shouldDedup(true), + _params(params), + _btreeCursor(NULL) { + _iam = _params.descriptor->getIndexCatalog()->getIndex(_params.descriptor); + _keyPattern = _params.descriptor->keyPattern().getOwned(); + + // We can't always access the descriptor in the call to getStats() so we pull + // any info we need for stats reporting out here. + _specificStats.indexName = _params.descriptor->indexName(); + _specificStats.isMultiKey = _params.descriptor->isMultikey(); + } + + void IndexScan::initIndexScan() { + // This function transitions from the initializing state to CHECKING_END. If + // the initialization fails, however, then the state transitions to HIT_END. + invariant(INITIALIZING == _scanState); + + // Perform the possibly heavy-duty initialization of the underlying index cursor. + if (_params.doNotDedup) { + _shouldDedup = false; + } + else { + _shouldDedup = _params.descriptor->isMultikey(); + } + + // Set up the index cursor. + CursorOptions cursorOptions; + + if (1 == _params.direction) { + cursorOptions.direction = CursorOptions::INCREASING; + } + else { + cursorOptions.direction = CursorOptions::DECREASING; + } + + IndexCursor *cursor; + Status s = _iam->newCursor(&cursor); + verify(s.isOK()); + _indexCursor.reset(cursor); + _indexCursor->setOptions(cursorOptions); + + if (_params.bounds.isSimpleRange) { + // Start at one key, end at another. + Status status = _indexCursor->seek(_params.bounds.startKey); + if (!status.isOK()) { + warning() << "IndexCursor seek failed: " << status.toString(); + _scanState = HIT_END; + } + if (!isEOF()) { + _specificStats.keysExamined = 1; + } + } + else { + // "Fast" Btree-specific navigation. + _btreeCursor = static_cast(_indexCursor.get()); + _checker.reset(new IndexBoundsChecker(&_params.bounds, + _keyPattern, + _params.direction)); + + int nFields = _keyPattern.nFields(); + vector key; + vector inc; + key.resize(nFields); + inc.resize(nFields); + if (_checker->getStartKey(&key, &inc)) { + _btreeCursor->seek(key, inc); + _keyElts.resize(nFields); + _keyEltsInc.resize(nFields); + } + else { + _scanState = HIT_END; + } + } + + // This method may throw an exception while it's doing initialization. If we've gotten + // here, then we've done all the initialization without an exception being thrown. This + // means it is safe to transition to the CHECKING_END state. In error cases, we transition + // to HIT_END, so we should not change state again here. + if (HIT_END != _scanState) { + _scanState = CHECKING_END; + } + } + + PlanStage::StageState IndexScan::work(WorkingSetID* out) { + ++_commonStats.works; + + if (INITIALIZING == _scanState) { + invariant(NULL == _indexCursor.get()); + initIndexScan(); + } + + if (CHECKING_END == _scanState) { + checkEnd(); + } + + if (isEOF()) { + _commonStats.isEOF = true; + return PlanStage::IS_EOF; + } + + if (GETTING_NEXT == _scanState) { + // Grab the next (key, value) from the index. + BSONObj keyObj = _indexCursor->getKey(); + DiskLoc loc = _indexCursor->getValue(); + + // Move to the next result. + // The underlying IndexCursor points at the *next* thing we want to return. We do this + // so that if we're scanning an index looking for docs to delete we don't continually + // clobber the thing we're pointing at. + _indexCursor->next(); + _scanState = CHECKING_END; + + if (_shouldDedup) { + ++_specificStats.dupsTested; + if (_returned.end() != _returned.find(loc)) { + ++_specificStats.dupsDropped; + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else { + _returned.insert(loc); + } + } + + if (Filter::passes(keyObj, _keyPattern, _filter)) { + if (NULL != _filter) { + ++_specificStats.matchTested; + } + + // We must make a copy of the on-disk data since it can mutate during the execution + // of this query. + BSONObj ownedKeyObj = keyObj.getOwned(); + + // Fill out the WSM. + WorkingSetID id = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(id); + member->loc = loc; + member->keyData.push_back(IndexKeyDatum(_keyPattern, ownedKeyObj)); + member->state = WorkingSetMember::LOC_AND_IDX; + + if (_params.addKeyMetadata) { + BSONObjBuilder bob; + bob.appendKeys(_keyPattern, ownedKeyObj); + member->addComputed(new IndexKeyComputedData(bob.obj())); + } + + *out = id; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + } + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + + bool IndexScan::isEOF() { + if (INITIALIZING == _scanState) { + // Have to call work() at least once. + return false; + } + + // If there's a limit on how many keys we can scan, we may be EOF when we hit that. + if (0 != _params.maxScan) { + if (_specificStats.keysExamined >= _params.maxScan) { + return true; + } + } + + return HIT_END == _scanState || _indexCursor->isEOF(); + } + + void IndexScan::prepareToYield() { + ++_commonStats.yields; + + if (isEOF() || INITIALIZING == _scanState) { return; } + + _savedKey = _indexCursor->getKey().getOwned(); + _savedLoc = _indexCursor->getValue(); + _indexCursor->savePosition(); + } + + void IndexScan::recoverFromYield() { + ++_commonStats.unyields; + + if (isEOF() || INITIALIZING == _scanState) { return; } + + // We can have a valid position before we check isEOF(), restore the position, and then be + // EOF upon restore. + if (!_indexCursor->restorePosition().isOK() || _indexCursor->isEOF()) { + _scanState = HIT_END; + return; + } + + if (!_savedKey.binaryEqual(_indexCursor->getKey()) + || _savedLoc != _indexCursor->getValue()) { + // Our restored position isn't the same as the saved position. When we call work() + // again we want to return where we currently point, not past it. + ++_specificStats.yieldMovedCursor; + + // Our restored position might be past endKey, see if we've hit the end. + _scanState = CHECKING_END; + } + } + + void IndexScan::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + + // The only state we're responsible for holding is what DiskLocs to drop. If a document + // mutates the underlying index cursor will deal with it. + if (INVALIDATION_MUTATION == type) { + return; + } + + // If we see this DiskLoc again, it may not be the same document it was before, so we want + // to return it if we see it again. + unordered_set::iterator it = _returned.find(dl); + if (it != _returned.end()) { + ++_specificStats.seenInvalidated; + _returned.erase(it); + } + } + + void IndexScan::checkEnd() { + if (isEOF()) { + _commonStats.isEOF = true; + return; + } + + if (_params.bounds.isSimpleRange) { + _scanState = GETTING_NEXT; + + // "Normal" start -> end scanning. + verify(NULL == _btreeCursor); + verify(NULL == _checker.get()); + + // If there is an empty endKey we will scan until we run out of index to scan over. + if (_params.bounds.endKey.isEmpty()) { return; } + + int cmp = sgn(_params.bounds.endKey.woCompare(_indexCursor->getKey(), _keyPattern)); + + if ((cmp != 0 && cmp != _params.direction) + || (cmp == 0 && !_params.bounds.endKeyInclusive)) { + _scanState = HIT_END; + } + else { + ++_specificStats.keysExamined; + } + } + else { + verify(NULL != _btreeCursor); + verify(NULL != _checker.get()); + + // Use _checker to see how things are. + IndexBoundsChecker::KeyState keyState; + keyState = _checker->checkKey(_indexCursor->getKey(), + &_keyEltsToUse, + &_movePastKeyElts, + &_keyElts, + &_keyEltsInc); + + if (IndexBoundsChecker::DONE == keyState) { + _scanState = HIT_END; + return; + } + + // This seems weird but it's the old definition of nscanned. + ++_specificStats.keysExamined; + + if (IndexBoundsChecker::VALID == keyState) { + _scanState = GETTING_NEXT; + return; + } + + verify(IndexBoundsChecker::MUST_ADVANCE == keyState); + _btreeCursor->skip(_indexCursor->getKey(), _keyEltsToUse, _movePastKeyElts, + _keyElts, _keyEltsInc); + + // Must check underlying cursor EOF after every cursor movement. + if (_btreeCursor->isEOF()) { + _scanState = HIT_END; + return; + } + } + } + + CommonStats* IndexScan::getCommonStats() { + return &_commonStats; + } + + IndexScanStats* IndexScan::getSpecificStats() { + return &_specificStats; + } + + PlanStageStats* IndexScan::getStats() { + // WARNING: this could be called even if the collection was dropped. Do not access any + // catalog information here. + _commonStats.isEOF = isEOF(); + + // These specific stats fields never change. + if (_specificStats.indexType.empty()) { + _specificStats.indexType = "BtreeCursor"; // TODO amName; + _specificStats.indexBounds = _params.bounds.toBSON(); + _specificStats.indexBoundsVerbose = _params.bounds.toString(); + _specificStats.direction = _params.direction; + _specificStats.keyPattern = _keyPattern; + } + + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_IXSCAN)); + ret->specific.reset(new IndexScanStats(_specificStats)); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/index_scan.h b/src/mongo/db/exec/index_scan.h new file mode 100644 index 00000000000..d5bffbf7439 --- /dev/null +++ b/src/mongo/db/exec/index_scan.h @@ -0,0 +1,173 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/index/btree_index_cursor.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/query/index_bounds.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + class IndexAccessMethod; + class IndexCursor; + class IndexDescriptor; + class WorkingSet; + + struct IndexScanParams { + IndexScanParams() : descriptor(NULL), + direction(1), + doNotDedup(false), + maxScan(0), + addKeyMetadata(false) { } + + const IndexDescriptor* descriptor; + + IndexBounds bounds; + + int direction; + + bool doNotDedup; + + // How many keys will we look at? + size_t maxScan; + + // Do we want to add the key as metadata? + bool addKeyMetadata; + }; + + /** + * Stage scans over an index from startKey to endKey, returning results that pass the provided + * filter. Internally dedups on DiskLoc. + * + * TODO: we probably should split this into 2 stages: one btree-only "fast" ixscan and one that + * strictly talks through the index API. Need to figure out what we really want to ship down + * through that API predicate-wise though, currently the language is a BSONObj but that's + * clearly not enough (or we need different index scan exec nodes per index type?). See + * SERVER-12397 for tracking. + * + * Sub-stage preconditions: None. Is a leaf and consumes no stage data. + */ + class IndexScan : public PlanStage { + public: + + /** + * Keeps track of what this index scan is currently doing so that it + * can do the right thing on the next call to work(). + */ + enum ScanState { + // Need to initialize the underlying index traversal machinery. + INITIALIZING, + + // Skipping keys in order to check whether we have reached the end. + CHECKING_END, + + // Retrieving the next key, and applying the filter if necessary. + GETTING_NEXT, + + // The index scan is finished. + HIT_END + }; + + IndexScan(const IndexScanParams& params, WorkingSet* workingSet, + const MatchExpression* filter); + + virtual ~IndexScan() { } + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + /** + * Get an unowned pointer to this stage's common stats. + */ + CommonStats* getCommonStats(); + + /** + * Get an unowned pointer to this stage's specific stats. + */ + IndexScanStats* getSpecificStats(); + + private: + /** + * Initialize the underlying IndexCursor, grab information from the catalog for stats. + */ + void initIndexScan(); + + /** See if the cursor is pointing at or past _endKey, if _endKey is non-empty. */ + void checkEnd(); + + // The WorkingSet we annotate with results. Not owned by us. + WorkingSet* _workingSet; + + // Index access. + const IndexAccessMethod* _iam; // owned by Collection -> IndexCatalog + scoped_ptr _indexCursor; + BSONObj _keyPattern; + + // Keeps track of what work we need to do next. + ScanState _scanState; + + // Contains expressions only over fields in the index key. We assume this is built + // correctly by whomever creates this class. + // The filter is not owned by us. + const MatchExpression* _filter; + + // Could our index have duplicates? If so, we use _returned to dedup. + bool _shouldDedup; + unordered_set _returned; + + // For yielding. + BSONObj _savedKey; + DiskLoc _savedLoc; + + IndexScanParams _params; + + // For our "fast" Btree-only navigation AKA the index bounds optimization. + scoped_ptr _checker; + BtreeIndexCursor* _btreeCursor; + int _keyEltsToUse; + bool _movePastKeyElts; + vector _keyElts; + vector _keyEltsInc; + + // Stats + CommonStats _commonStats; + IndexScanStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/keep_mutations.cpp b/src/mongo/db/exec/keep_mutations.cpp new file mode 100644 index 00000000000..a16d5f7e462 --- /dev/null +++ b/src/mongo/db/exec/keep_mutations.cpp @@ -0,0 +1,131 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/keep_mutations.h" +#include "mongo/db/exec/filter.h" + +namespace mongo { + + KeepMutationsStage::KeepMutationsStage(const MatchExpression* filter, + WorkingSet* ws, + PlanStage* child) + : _workingSet(ws), + _child(child), + _filter(filter), + _doneReadingChild(false), + _doneReturningFlagged(false) { } + + KeepMutationsStage::~KeepMutationsStage() { } + + bool KeepMutationsStage::isEOF() { + return _doneReadingChild && _doneReturningFlagged; + } + + PlanStage::StageState KeepMutationsStage::work(WorkingSetID* out) { + ++_commonStats.works; + + // If we've returned as many results as we're limited to, isEOF will be true. + if (isEOF()) { return PlanStage::IS_EOF; } + + // Stream child results until the child is all done. + if (!_doneReadingChild) { + StageState status = _child->work(out); + + // Child is still returning results. Pass them through. + if (PlanStage::IS_EOF != status) { + if (PlanStage::ADVANCED == status) { + ++_commonStats.advanced; + } + else if (PlanStage::NEED_TIME == status) { + ++_commonStats.needTime; + } + else if (PlanStage::NEED_FETCH == status) { + ++_commonStats.needFetch; + } + + return status; + } + + // Child is EOF. We want to stream flagged results if there are any. + _doneReadingChild = true; + + // Read out all of the flagged results from the working set. We can't iterate through + // the working set's flagged result set directly, since it may be modified later if + // further documents are invalidated during a yield. + std::copy(_workingSet->getFlagged().begin(), _workingSet->getFlagged().end(), + std::back_inserter(_flagged)); + _flaggedIterator = _flagged.begin(); + } + + // We're streaming flagged results. + invariant(!_doneReturningFlagged); + if (_flaggedIterator == _flagged.end()) { + _doneReturningFlagged = true; + return PlanStage::IS_EOF; + } + + WorkingSetID idToTest = *_flaggedIterator; + _flaggedIterator++; + + WorkingSetMember* member = _workingSet->get(idToTest); + if (Filter::passes(member, _filter)) { + *out = idToTest; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else { + _workingSet->free(idToTest); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + + void KeepMutationsStage::prepareToYield() { + ++_commonStats.yields; + _child->prepareToYield(); + } + + void KeepMutationsStage::recoverFromYield() { + ++_commonStats.unyields; + _child->recoverFromYield(); + } + + void KeepMutationsStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + _child->invalidate(dl, type); + } + + PlanStageStats* KeepMutationsStage::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_KEEP_MUTATIONS)); + // Takes ownership of the object returned from _child->getStats(). + ret->children.push_back(_child->getStats()); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/keep_mutations.h b/src/mongo/db/exec/keep_mutations.h new file mode 100644 index 00000000000..33f436a68b9 --- /dev/null +++ b/src/mongo/db/exec/keep_mutations.h @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/matcher/expression.h" + +namespace mongo { + + /** + * KeepMutationsStage passes all of its child's data through until the child is EOF. + * It then returns all flagged elements in the WorkingSet that pass the stage's filter. + * + * This stage is used to merge results that are invalidated mid-query back into the query + * results when possible. The query planner is responsible for determining when it's valid to + * merge these results. + */ + class KeepMutationsStage : public PlanStage { + public: + KeepMutationsStage(const MatchExpression* filter, WorkingSet* ws, PlanStage* child); + virtual ~KeepMutationsStage(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + // Not owned here. + WorkingSet* _workingSet; + + scoped_ptr _child; + + // Not owned here. Should be the full query expression tree. + const MatchExpression* _filter; + + // We read from our child... + bool _doneReadingChild; + + // ...until it's out of results, at which point we put any flagged results back in the query + // stream. + bool _doneReturningFlagged; + + // Stats. + CommonStats _commonStats; + + // Our copy of the working set's flagged results. + std::vector _flagged; + + // Iterator pointing into _flagged. + std::vector::const_iterator _flaggedIterator; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/limit.cpp b/src/mongo/db/exec/limit.cpp new file mode 100644 index 00000000000..e084781dbaf --- /dev/null +++ b/src/mongo/db/exec/limit.cpp @@ -0,0 +1,106 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/limit.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + LimitStage::LimitStage(int limit, WorkingSet* ws, PlanStage* child) + : _ws(ws), _child(child), _numToReturn(limit) { } + + LimitStage::~LimitStage() { } + + bool LimitStage::isEOF() { return (0 == _numToReturn) || _child->isEOF(); } + + PlanStage::StageState LimitStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (0 == _numToReturn) { + // We've returned as many results as we're limited to. + return PlanStage::IS_EOF; + } + + WorkingSetID id = WorkingSet::INVALID_ID; + StageState status = _child->work(&id); + + if (PlanStage::ADVANCED == status) { + *out = id; + --_numToReturn; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else if (PlanStage::FAILURE == status) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "limit stage failed to read in results from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return status; + } + else { + if (PlanStage::NEED_FETCH == status) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == status) { + ++_commonStats.needTime; + } + return status; + } + } + + void LimitStage::prepareToYield() { + ++_commonStats.yields; + _child->prepareToYield(); + } + + void LimitStage::recoverFromYield() { + ++_commonStats.unyields; + _child->recoverFromYield(); + } + + void LimitStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + _child->invalidate(dl, type); + } + + PlanStageStats* LimitStage::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_LIMIT)); + ret->children.push_back(_child->getStats()); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/limit.h b/src/mongo/db/exec/limit.h new file mode 100644 index 00000000000..8742dcbb0bd --- /dev/null +++ b/src/mongo/db/exec/limit.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" + +namespace mongo { + + /** + * This stage implements limit functionality. It only returns 'limit' results before EOF. + * + * Sort has a baked-in limit, as it can optimize the sort if it has a limit. + * + * Preconditions: None. + */ + class LimitStage : public PlanStage { + public: + LimitStage(int limit, WorkingSet* ws, PlanStage* child); + virtual ~LimitStage(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + WorkingSet* _ws; + scoped_ptr _child; + + // We only return this many results. + int _numToReturn; + + // Stats + CommonStats _commonStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/merge_sort.cpp b/src/mongo/db/exec/merge_sort.cpp new file mode 100644 index 00000000000..8e8c760c336 --- /dev/null +++ b/src/mongo/db/exec/merge_sort.cpp @@ -0,0 +1,255 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/merge_sort.h" + +#include "mongo/db/exec/working_set.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + MergeSortStage::MergeSortStage(const MergeSortStageParams& params, WorkingSet* ws) + : _ws(ws), _pattern(params.pattern), _dedup(params.dedup), + _merging(StageWithValueComparison(ws, params.pattern)) { } + + MergeSortStage::~MergeSortStage() { + for (size_t i = 0; i < _children.size(); ++i) { delete _children[i]; } + } + + void MergeSortStage::addChild(PlanStage* child) { + _children.push_back(child); + + // We have to call work(...) on every child before we can pick a min. + _noResultToMerge.push(child); + } + + bool MergeSortStage::isEOF() { + // If we have no more results to return, and we have no more children that we can call + // work(...) on to get results, we're done. + return _merging.empty() && _noResultToMerge.empty(); + } + + PlanStage::StageState MergeSortStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (isEOF()) { return PlanStage::IS_EOF; } + + if (!_noResultToMerge.empty()) { + // We have some child that we don't have a result from. Each child must have a result + // in order to pick the minimum result among all our children. Work a child. + PlanStage* child = _noResultToMerge.front(); + WorkingSetID id = WorkingSet::INVALID_ID; + StageState code = child->work(&id); + + if (PlanStage::ADVANCED == code) { + // If we're deduping... + if (_dedup) { + WorkingSetMember* member = _ws->get(id); + + if (!member->hasLoc()) { + // Can't dedup data unless there's a DiskLoc. We go ahead and use its + // result. + _noResultToMerge.pop(); + } + else { + ++_specificStats.dupsTested; + // ...and there's a diskloc and and we've seen the DiskLoc before + if (_seen.end() != _seen.find(member->loc)) { + // ...drop it. + _ws->free(id); + ++_commonStats.needTime; + ++_specificStats.dupsDropped; + return PlanStage::NEED_TIME; + } + else { + // Otherwise, note that we've seen it. + _seen.insert(member->loc); + // We're going to use the result from the child, so we remove it from + // the queue of children without a result. + _noResultToMerge.pop(); + } + } + } + else { + // Not deduping. We use any result we get from the child. Remove the child + // from the queue of things without a result. + _noResultToMerge.pop(); + } + + // Store the result in our list. + StageWithValue value; + value.id = id; + value.stage = child; + _mergingData.push_front(value); + + // Insert the result (indirectly) into our priority queue. + _merging.push(_mergingData.begin()); + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::IS_EOF == code) { + // There are no more results possible from this child. Don't bother with it + // anymore. + _noResultToMerge.pop(); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::FAILURE == code) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "merge sort stage failed to read in results from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return code; + } + else { + if (PlanStage::NEED_FETCH == code) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == code) { + ++_commonStats.needTime; + } + return code; + } + } + + // If we're here, for each non-EOF child, we have a valid WSID. + verify(!_merging.empty()); + + // Get the 'min' WSID. _merging is a priority queue so its top is the smallest. + MergingRef top = _merging.top(); + _merging.pop(); + + // Since we're returning the WSID that came from top->stage, we need to work(...) it again + // to get a new result. + _noResultToMerge.push(top->stage); + + // Save the ID that we're returning and remove the returned result from our data. + WorkingSetID idToTest = top->id; + _mergingData.erase(top); + + // Return the min. + *out = idToTest; + ++_commonStats.advanced; + + // But don't return it if it's flagged. + if (_ws->isFlagged(*out)) { + _ws->free(*out); + return PlanStage::NEED_TIME; + } + + return PlanStage::ADVANCED; + } + + void MergeSortStage::prepareToYield() { + ++_commonStats.yields; + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->prepareToYield(); + } + } + + void MergeSortStage::recoverFromYield() { + ++_commonStats.unyields; + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->recoverFromYield(); + } + } + + void MergeSortStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->invalidate(dl, type); + } + + // Go through our data and see if we're holding on to the invalidated loc. + for (list::iterator valueIt = _mergingData.begin(); valueIt != _mergingData.end(); valueIt++) { + WorkingSetMember* member = _ws->get(valueIt->id); + if (member->hasLoc() && (dl == member->loc)) { + // Force a fetch and flag. We could possibly merge this result back in later. + WorkingSetCommon::fetchAndInvalidateLoc(member); + _ws->flagForReview(valueIt->id); + ++_specificStats.forcedFetches; + } + } + + // If we see DL again it is not the same record as it once was so we still want to + // return it. + if (_dedup) { _seen.erase(dl); } + } + + // Is lhs less than rhs? Note that priority_queue is a max heap by default so we invert + // the return from the expected value. + bool MergeSortStage::StageWithValueComparison::operator()( + const MergingRef& lhs, const MergingRef& rhs) { + + WorkingSetMember* lhsMember = _ws->get(lhs->id); + WorkingSetMember* rhsMember = _ws->get(rhs->id); + + BSONObjIterator it(_pattern); + while (it.more()) { + BSONElement patternElt = it.next(); + string fn = patternElt.fieldName(); + + BSONElement lhsElt; + verify(lhsMember->getFieldDotted(fn, &lhsElt)); + + BSONElement rhsElt; + verify(rhsMember->getFieldDotted(fn, &rhsElt)); + + // false means don't compare field name. + int x = lhsElt.woCompare(rhsElt, false); + if (-1 == patternElt.number()) { x = -x; } + if (x != 0) { return x > 0; } + } + + // A comparator for use with sort is required to model a strict weak ordering, so + // to satisfy irreflexivity we must return 'false' for elements that we consider + // equivalent under the pattern. + return false; + } + + PlanStageStats* MergeSortStage::getStats() { + _commonStats.isEOF = isEOF(); + + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_SORT_MERGE)); + ret->specific.reset(new MergeSortStats(_specificStats)); + for (size_t i = 0; i < _children.size(); ++i) { + ret->children.push_back(_children[i]->getStats()); + } + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/merge_sort.h b/src/mongo/db/exec/merge_sort.h new file mode 100644 index 00000000000..82540a467ab --- /dev/null +++ b/src/mongo/db/exec/merge_sort.h @@ -0,0 +1,152 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + // External params for the merge sort stage. Declared below. + class MergeSortStageParams; + + /** + * Merges the outputs of N children, each of which is sorted in the order specified by + * 'pattern'. The output is sorted by 'pattern'. Practically speaking, all of this stage's + * children are indices. + * + * AKA the SERVER-1205 stage. Allows very efficient handling of the following query: + * find($or[{a:1}, {b:1}]).sort({c:1}) with indices {a:1, c:1} and {b:1, c:1}. + * + * Preconditions: For each field in 'pattern' all inputs in the child must handle a + * getFieldDotted for that field. + */ + class MergeSortStage : public PlanStage { + public: + MergeSortStage(const MergeSortStageParams& params, WorkingSet* ws); + virtual ~MergeSortStage(); + + void addChild(PlanStage* child); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + PlanStageStats* getStats(); + + private: + // Not owned by us. + WorkingSet* _ws; + + // The pattern that we're sorting by. + BSONObj _pattern; + + // Are we deduplicating on DiskLoc? + bool _dedup; + + // Which DiskLocs have we seen? + unordered_set _seen; + + // Owned by us. All the children we're reading from. + vector _children; + + // In order to pick the next smallest value, we need each child work(...) until it produces + // a result. This is the queue of children that haven't given us a result yet. + std::queue _noResultToMerge; + + // There is some confusing STL wrangling going on below. Here's a guide: + // + // We want to keep a priority_queue of results so we can quickly return the min result. + // + // If we receive an invalidate, we need to iterate over any cached state to see if the + // invalidate is relevant. + // + // We can't iterate over a priority_queue, so we keep the actual cached state in a list and + // have a priority_queue of iterators into that list. + // + // Why an iterator instead of a pointer? We need to be able to use the information in the + // priority_queue to remove the item from the list and quickly. + + struct StageWithValue { + StageWithValue() : id(WorkingSet::INVALID_ID), stage(NULL) { } + WorkingSetID id; + PlanStage* stage; + }; + + // We have a priority queue of these. + typedef list::iterator MergingRef; + + // The comparison function used in our priority queue. + class StageWithValueComparison { + public: + StageWithValueComparison(WorkingSet* ws, BSONObj pattern) + : _ws(ws), _pattern(pattern) {} + + // Is lhs less than rhs? Note that priority_queue is a max heap by default so we invert + // the return from the expected value. + bool operator()(const MergingRef& lhs, const MergingRef& rhs); + + private: + WorkingSet* _ws; + BSONObj _pattern; + }; + + // The min heap of the results we're returning. + std::priority_queue, StageWithValueComparison> _merging; + + // The data referred to by the _merging queue above. + list _mergingData; + + // Stats + CommonStats _commonStats; + MergeSortStats _specificStats; + }; + + // Parameters that must be provided to a MergeSortStage + class MergeSortStageParams { + public: + MergeSortStageParams() : dedup(true) { } + + // How we're sorting. + BSONObj pattern; + + // Do we deduplicate on DiskLoc? + bool dedup; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/mock_stage.cpp b/src/mongo/db/exec/mock_stage.cpp new file mode 100644 index 00000000000..1fbc01b06ae --- /dev/null +++ b/src/mongo/db/exec/mock_stage.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/mock_stage.h" +#include "mongo/db/exec/working_set_common.h" + +namespace mongo { + + MockStage::MockStage(WorkingSet* ws) : _ws(ws) { } + + PlanStage::StageState MockStage::work(WorkingSetID* out) { + if (isEOF()) { return PlanStage::IS_EOF; } + + StageState state = _results.front(); + _results.pop(); + + if (PlanStage::ADVANCED == state) { + *out = _members.front(); + _members.pop(); + } + + return state; + } + + bool MockStage::isEOF() { return _results.empty(); } + + void MockStage::pushBack(const PlanStage::StageState state) { + _results.push(state); + } + + void MockStage::pushBack(const WorkingSetMember& member) { + _results.push(PlanStage::ADVANCED); + + WorkingSetID id = _ws->allocate(); + WorkingSetMember* ourMember = _ws->get(id); + WorkingSetCommon::initFrom(ourMember, member); + + // member lives in _ws. We'll return it when _results hits ADVANCED. + _members.push(id); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/mock_stage.h b/src/mongo/db/exec/mock_stage.h new file mode 100644 index 00000000000..9476038a16a --- /dev/null +++ b/src/mongo/db/exec/mock_stage.h @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" + +namespace mongo { + + class DiskLoc; + + /** + * MockStage is a data-producing stage that is used for testing. Unlike the other two leaf + * stages (CollectionScan and IndexScan) MockStage does not require any underlying storage + * layer. + * + * A MockStage is "programmed" by pushing return values from work() onto its internal queue. + * Calls to MockStage::work() pop values off that queue and return them in FIFO order, + * annotating the working set with data when appropriate. + */ + class MockStage : public PlanStage { + public: + MockStage(WorkingSet* ws); + virtual ~MockStage() { } + + virtual StageState work(WorkingSetID* out); + + virtual bool isEOF(); + + // These don't really mean anything here. + // Some day we could count the # of calls to the yield functions to check that other stages + // have correct yielding behavior. + virtual void prepareToYield() { } + virtual void recoverFromYield() { } + virtual void invalidate(const DiskLoc& dl, InvalidationType type) { } + virtual PlanStageStats* getStats() { return NULL; } + + /** + * Add a result to the back of the queue. work() goes through the queue. + * Either no data is returned (just a state), or... + */ + void pushBack(const PlanStage::StageState state); + + /** + * ...data is returned (and we ADVANCED) + * + * Allocates a new member and copies 'member' into it. + * Does not take ownership of anything in 'member'. + */ + void pushBack(const WorkingSetMember& member); + + private: + // We don't own this. + WorkingSet* _ws; + + // The data we return. + std::queue _results; + std::queue _members; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/oplogstart.cpp b/src/mongo/db/exec/oplogstart.cpp new file mode 100644 index 00000000000..5e94e058d7f --- /dev/null +++ b/src/mongo/db/exec/oplogstart.cpp @@ -0,0 +1,222 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/oplogstart.h" + +#include "mongo/db/pdfile.h" +#include "mongo/db/storage/extent.h" + +namespace mongo { + + // Does not take ownership. + OplogStart::OplogStart(const string& ns, MatchExpression* filter, WorkingSet* ws) + : _needInit(true), + _backwardsScanning(false), + _extentHopping(false), + _done(false), + _workingSet(ws), + _ns(ns), + _filter(filter) { } + + OplogStart::~OplogStart() { } + + PlanStage::StageState OplogStart::work(WorkingSetID* out) { + // We do our (heavy) init in a work(), where work is expected. + if (_needInit) { + CollectionScanParams params; + params.ns = _ns; + params.direction = CollectionScanParams::BACKWARD; + _cs.reset(new CollectionScan(params, _workingSet, NULL)); + _nsd = nsdetails(_ns.c_str()); + _needInit = false; + _backwardsScanning = true; + _timer.reset(); + } + + // If we don't have a _curloc yet, then we'll have + // to backwards scan this time in order to initialize it. + if (_curloc.isNull() && !_extentHopping) { + return workBackwardsScan(out); + } + + // If we're reading backwards, try again. + if (_backwardsScanning) { + // Still have time to succeed with reading backwards. + if (_timer.seconds() < _backwardsScanTime) { + return workBackwardsScan(out); + } + switchToExtentHopping(); + } + + // Don't find it in time? Swing from extent to extent like tarzan.com. + verify(_extentHopping); + return workExtentHopping(out); + } + + PlanStage::StageState OplogStart::workExtentHopping(WorkingSetID* out) { + if (_curloc.isNull()) { + _done = true; + return PlanStage::IS_EOF; + } + + if (!_filter->matchesBSON(_curloc.obj())) { + _done = true; + WorkingSetID id = _workingSet->allocate(); + WorkingSetMember* member = _workingSet->get(id); + member->loc = _curloc; + member->obj = member->loc.obj(); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + *out = id; + return PlanStage::ADVANCED; + } + + _curloc = prevExtentFirstLoc(_nsd, _curloc); + return PlanStage::NEED_TIME; + } + + void OplogStart::switchToExtentHopping() { + // Transition from backwards scanning to extent hopping. + _backwardsScanning = false; + _extentHopping = true; + + // Toss the collection scan we were using. + _cs.reset(); + + // Set up our extent hopping state. Get the start of the extent that we were collection + // scanning. + Extent* e = _curloc.rec()->myExtent(_curloc); + if (!_nsd->capLooped() || (e->myLoc != _nsd->capExtent())) { + _curloc = e->firstRecord; + } + else { + // Direct quote: + // Likely we are on the fresh side of capExtent, so return first fresh + // record. If we are on the stale side of capExtent, then the collection is + // small and it doesn't matter if we start the extent scan with + // capFirstNewRecord. + _curloc = _nsd->capFirstNewRecord(); + } + } + + PlanStage::StageState OplogStart::workBackwardsScan(WorkingSetID* out) { + PlanStage::StageState state = _cs->work(out); + + // EOF. Just start from the beginning, which is where we've hit. + if (PlanStage::IS_EOF == state) { + _done = true; + return state; + } + + if (PlanStage::ADVANCED != state) { return state; } + + WorkingSetMember* member = _workingSet->get(*out); + verify(member->hasObj()); + verify(member->hasLoc()); + + if (!_filter->matchesBSON(member->obj)) { + _done = true; + // DiskLoc is returned in *out. + return PlanStage::ADVANCED; + } + else { + _curloc = member->loc; + _workingSet->free(*out); + return PlanStage::NEED_TIME; + } + } + + bool OplogStart::isEOF() { return _done; } + + void OplogStart::invalidate(const DiskLoc& dl, InvalidationType type) { + if (_needInit) { return; } + + if (INVALIDATION_DELETION != type) { return; } + + if (_backwardsScanning) { + _cs->invalidate(dl, type); + } + else { + verify(_extentHopping); + if (dl == _curloc) { + _curloc = DiskLoc(); + } + } + } + + void OplogStart::prepareToYield() { + if (_backwardsScanning) { + _cs->prepareToYield(); + } + } + + void OplogStart::recoverFromYield() { + if (_backwardsScanning) { + _cs->recoverFromYield(); + } + } + + // static + DiskLoc OplogStart::prevExtentFirstLoc(NamespaceDetails* nsd, const DiskLoc& rec ) { + Extent *e = rec.rec()->myExtent( rec ); + if (nsd->capLooped() ) { + while( true ) { + // Advance e to preceding extent (looping to lastExtent if necessary). + if ( e->xprev.isNull() ) { + e = nsd->lastExtent().ext(); + } + else { + e = e->xprev.ext(); + } + if ( e->myLoc == nsd->capExtent() ) { + // Reached the extent containing the oldest data in the collection. + return DiskLoc(); + } + if ( !e->firstRecord.isNull() ) { + // Return the first record of the first non empty extent encountered. + return e->firstRecord; + } + } + } + else { + while( true ) { + if ( e->xprev.isNull() ) { + // Reached the beginning of the collection. + return DiskLoc(); + } + e = e->xprev.ext(); + if ( !e->firstRecord.isNull() ) { + // Return the first record of the first non empty extent encountered. + return e->firstRecord; + } + } + } + } + + int OplogStart::_backwardsScanTime = 5; + +} // namespace mongo diff --git a/src/mongo/db/exec/oplogstart.h b/src/mongo/db/exec/oplogstart.h new file mode 100644 index 00000000000..159e8c82a2a --- /dev/null +++ b/src/mongo/db/exec/oplogstart.h @@ -0,0 +1,125 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/collection_scan.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/util/timer.h" + +namespace mongo { + + class NamespaceDetails; + + /** + * OplogStart walks a collection backwards to find the first object in the collection that + * matches the query. It's used by replication to efficiently find where the oplog should be + * replayed from. + * + * The oplog is always a capped collection. In capped collections, documents are oriented on + * disk according to insertion order. The oplog inserts documents with increasing timestamps. + * Queries on the oplog look for entries that are after a certain time. Therefore if we + * navigate backwards, the last document we encounter that satisfies our query (over the + * timestamp) is the first document we must scan from to answer the query. + * + * Why isn't this a normal reverse table scan, you may ask? We could be correct if we used a + * normal reverse collection scan. However, that's not fast enough. Since we know all + * documents are oriented on disk in insertion order, we know all documents in one extent were + * inserted before documents in a subsequent extent. As such we can skip through entire extents + * looking only at the first document. + * + * Why is this a stage? Because we want to yield, and we want to be notified of DiskLoc + * invalidations. :( + */ + class OplogStart : public PlanStage { + public: + // Does not take ownership. + OplogStart(const string& ns, MatchExpression* filter, WorkingSet* ws); + virtual ~OplogStart(); + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + virtual void prepareToYield(); + virtual void recoverFromYield(); + + // PS. don't call this. + virtual PlanStageStats* getStats() { return NULL; } + + // For testing only. + void setBackwardsScanTime(int newTime) { _backwardsScanTime = newTime; } + bool isExtentHopping() { return _extentHopping; } + bool isBackwardsScanning() { return _backwardsScanning; } + private: + // Copied verbatim. + static DiskLoc prevExtentFirstLoc(NamespaceDetails* nsd, const DiskLoc& rec); + + StageState workBackwardsScan(WorkingSetID* out); + + void switchToExtentHopping(); + + StageState workExtentHopping(WorkingSetID* out); + + // If we're backwards scanning we just punt to a collscan. + scoped_ptr _cs; + + // What's our current DiskLoc? Set by both collscan and extent hopping. + // Only written by collscan, read and written by extent hopping. + DiskLoc _curloc; + + // Have we done our heavy init yet? + bool _needInit; + + // Our first state: going backwards via a collscan. + bool _backwardsScanning; + + // Our second state: hopping backwards extent by extent. + bool _extentHopping; + + // Our final state: done. + bool _done; + + NamespaceDetails* _nsd; + + // We only go backwards via a collscan for a few seconds. + Timer _timer; + + // WorkingSet is not owned by us. + WorkingSet* _workingSet; + + string _ns; + + MatchExpression* _filter; + + static int _backwardsScanTime; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/or.cpp b/src/mongo/db/exec/or.cpp new file mode 100644 index 00000000000..83a441d3896 --- /dev/null +++ b/src/mongo/db/exec/or.cpp @@ -0,0 +1,184 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/or.h" +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + OrStage::OrStage(WorkingSet* ws, bool dedup, const MatchExpression* filter) + : _ws(ws), _filter(filter), _currentChild(0), _dedup(dedup) { } + + OrStage::~OrStage() { + for (size_t i = 0; i < _children.size(); ++i) { + delete _children[i]; + } + } + + void OrStage::addChild(PlanStage* child) { _children.push_back(child); } + + bool OrStage::isEOF() { return _currentChild >= _children.size(); } + + PlanStage::StageState OrStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (isEOF()) { return PlanStage::IS_EOF; } + + if (0 == _specificStats.matchTested.size()) { + _specificStats.matchTested = vector(_children.size(), 0); + } + + WorkingSetID id = WorkingSet::INVALID_ID; + StageState childStatus = _children[_currentChild]->work(&id); + + if (PlanStage::ADVANCED == childStatus) { + WorkingSetMember* member = _ws->get(id); + + // If we're deduping (and there's something to dedup by) + if (_dedup && member->hasLoc()) { + ++_specificStats.dupsTested; + + // ...and we've seen the DiskLoc before + if (_seen.end() != _seen.find(member->loc)) { + // ...drop it. + ++_specificStats.dupsDropped; + _ws->free(id); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else { + // Otherwise, note that we've seen it. + _seen.insert(member->loc); + } + } + + if (Filter::passes(member, _filter)) { + if (NULL != _filter) { + ++_specificStats.matchTested[_currentChild]; + } + // Match! return it. + *out = id; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else { + // Does not match, try again. + _ws->free(id); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + else if (PlanStage::IS_EOF == childStatus) { + // Done with _currentChild, move to the next one. + ++_currentChild; + + // Maybe we're out of children. + if (isEOF()) { + return PlanStage::IS_EOF; + } + else { + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + } + else if (PlanStage::FAILURE == childStatus) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "OR stage failed to read in results from child " << _currentChild; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return childStatus; + } + else { + if (PlanStage::NEED_FETCH == childStatus) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == childStatus) { + ++_commonStats.needTime; + } + + // NEED_TIME, ERROR, NEED_YIELD, pass them up. + return childStatus; + } + } + + void OrStage::prepareToYield() { + ++_commonStats.yields; + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->prepareToYield(); + } + } + + void OrStage::recoverFromYield() { + ++_commonStats.unyields; + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->recoverFromYield(); + } + } + + void OrStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + + if (isEOF()) { return; } + + for (size_t i = 0; i < _children.size(); ++i) { + _children[i]->invalidate(dl, type); + } + + // If we see DL again it is not the same record as it once was so we still want to + // return it. + if (_dedup && INVALIDATION_DELETION == type) { + unordered_set::iterator it = _seen.find(dl); + if (_seen.end() != it) { + ++_specificStats.locsForgotten; + _seen.erase(dl); + } + } + } + + PlanStageStats* OrStage::getStats() { + _commonStats.isEOF = isEOF(); + + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_OR)); + ret->specific.reset(new OrStats(_specificStats)); + for (size_t i = 0; i < _children.size(); ++i) { + ret->children.push_back(_children[i]->getStats()); + } + + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/or.h b/src/mongo/db/exec/or.h new file mode 100644 index 00000000000..fa765f67ea6 --- /dev/null +++ b/src/mongo/db/exec/or.h @@ -0,0 +1,87 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + /** + * This stage outputs the union of its children. It optionally deduplicates on DiskLoc. + * + * Preconditions: Valid DiskLoc. + * + * If we're deduping, we may fail to dedup any invalidated DiskLoc properly. + */ + class OrStage : public PlanStage { + public: + OrStage(WorkingSet* ws, bool dedup, const MatchExpression* filter); + virtual ~OrStage(); + + void addChild(PlanStage* child); + + virtual bool isEOF(); + + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + // Not owned by us. + WorkingSet* _ws; + + // The filter is not owned by us. + const MatchExpression* _filter; + + // Owned by us. + vector _children; + + // Which of _children are we calling work(...) on now? + size_t _currentChild; + + // True if we dedup on DiskLoc, false otherwise. + bool _dedup; + + // Which DiskLocs have we returned? + unordered_set _seen; + + // Stats + CommonStats _commonStats; + OrStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/plan_stage.h b/src/mongo/db/exec/plan_stage.h new file mode 100644 index 00000000000..6797841971e --- /dev/null +++ b/src/mongo/db/exec/plan_stage.h @@ -0,0 +1,225 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/exec/plan_stats.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/invalidation_type.h" + +namespace mongo { + + class DiskLoc; + + /** + * A PlanStage ("stage") is the basic building block of a "Query Execution Plan." A stage is + * the smallest piece of machinery used in executing a compiled query. Stages either access + * data (from a collection or an index) to create a stream of results, or transform a stream of + * results (e.g. AND, OR, SORT) to create a stream of results. + * + * Stages have zero or more input streams but only one output stream. Data-accessing stages are + * leaves and data-transforming stages have children. Stages can be connected together to form + * a tree which is then executed (see plan_runner.h) to solve a query. + * + * A stage's input and output are each typed. Only stages with compatible types can be + * connected. + * + * All of the stages of a QEP share a WorkingSet (see working_set.h). Data source stages + * allocate a slot in the WorkingSet, fill the slot with data, and return the ID of that slot. + * Subsequent stages fetch a WorkingSetElement by its ID and operate on the enclosed data. + * + * Stages do nothing unless work() is called. work() is a request to the stage to consume one + * unit of input. Some stages (e.g. AND, SORT) require many calls to work() before generating + * output as they must consume many units of input. These stages will inform the caller that + * they need more time, and work() must be called again in order to produce an output. + * + * Every stage of a query implements the PlanStage interface. Queries perform a unit of work + * and report on their subsequent status; see StatusCode for possible states. Query results are + * passed through the WorkingSet interface; see working_set.h for details. + * + * All synchronization is the responsibility of the caller. Queries must be told to yield with + * prepareToYield() if any underlying database state changes. If prepareToYield() is called, + * recoverFromYield() must be called again before any work() is done. + * + * Here is a very simple usage example: + * + * WorkingSet workingSet; + * PlanStage* rootStage = makeQueryPlan(&workingSet, ...); + * while (!rootStage->isEOF()) { + * WorkingSetID result; + * switch(rootStage->work(&result)) { + * case PlanStage::ADVANCED: + * // do something with result + * WorkingSetMember* member = workingSet.get(result); + * cout << "Result: " << member->obj << endl; + * break; + * case PlanStage::IS_EOF: + * // All done. Will fall out of while loop. + * break; + * case PlanStage::NEED_TIME: + * // Need more time. + * break; + * case PlanStage::FAILURE: + * // Throw exception or return error + * break; + * case PlanStage::NEED_FETCH: + * // Go to disk and fetch stuff. + * break; + * } + * + * if (shouldYield) { + * // Occasionally yield. + * stage->prepareToYield(); + * // Do work that requires a yield here (execute other plans, insert, delete, etc.). + * stage->recoverFromYield(); + * } + * } + */ + class PlanStage { + public: + virtual ~PlanStage() { } + + /** + * All possible return values of work(...) + */ + enum StageState { + // work(...) has returned a new result in its out parameter. The caller must free it + // from the working set when done with it. + ADVANCED, + + // work(...) won't do anything more. isEOF() will also be true. There is nothing + // output in the out parameter. + IS_EOF, + + // work(...) needs more time to product a result. Call work(...) again. There is + // nothing output in the out parameter. + NEED_TIME, + + // Something went wrong but it's not an internal error. Perhaps our collection was + // dropped or state deleted. + DEAD, + + // Something has gone unrecoverably wrong. Stop running this query. + // If the out parameter does not refer to an invalid working set member, + // call WorkingSetCommon::getStatusMemberObject() to get details on the failure. + // Any class implementing this interface must set the WSID out parameter to + // INVALID_ID or a valid WSM ID if FAILURE is returned. + FAILURE, + + // Something isn't in memory. Fetch it. + // + // Full fetch semantics: + // The fetch-requesting stage populates the out parameter of work(...) with a WSID that + // refers to a WSM with a valid loc. Each stage that receives a NEED_FETCH from a child + // must propagate the NEED_FETCH up and perform no work. The plan runner is responsible + // for paging in the data upon receipt of a NEED_FETCH. The plan runner does NOT free + // the WSID of the requested fetch. The stage that requested the fetch holds the WSID + // of the loc it wants fetched. On the next call to work() that stage can assume a + // fetch was performed on the WSM that the held WSID refers to. + NEED_FETCH, + }; + + static string stateStr(const StageState& state) { + if (ADVANCED == state) { + return "ADVANCED"; + } + else if (IS_EOF == state) { + return "IS_EOF"; + } + else if (NEED_TIME == state) { + return "NEED_TIME"; + } + else if (NEED_FETCH == state) { + return "NEED_FETCH"; + } + else { + verify(FAILURE == state); + return "FAILURE"; + } + } + + + /** + * Perform a unit of work on the query. Ask the stage to produce the next unit of output. + * Stage returns StageState::ADVANCED if *out is set to the next unit of output. Otherwise, + * returns another value of StageState to indicate the stage's status. + */ + virtual StageState work(WorkingSetID* out) = 0; + + /** + * Returns true if no more work can be done on the query / out of results. + */ + virtual bool isEOF() = 0; + + // + // Yielding and isolation semantics: + // + // Any data that is not inserted, deleted, or modified during a yield will be faithfully + // returned by a query that should return that data. + // + // Any data inserted, deleted, or modified during a yield that should be returned by a query + // may or may not be returned by that query. The query could return: nothing; the data + // before; the data after; or both the data before and the data after. + // + // In short, there is no isolation between a query and an insert/delete/update. AKA, + // READ_UNCOMMITTED. + // + + /** + * Notifies the stage that all locks are about to be released. The stage must save any + * state required to resume where it was before prepareToYield was called. + */ + virtual void prepareToYield() = 0; + + /** + * Notifies the stage that any required locks have been reacquired. The stage must restore + * any saved state and be ready to handle calls to work(). + * + * Can only be called after prepareToYield. + */ + virtual void recoverFromYield() = 0; + + /** + * Notifies a stage that a DiskLoc is going to be deleted (or in-place updated) so that the + * stage can invalidate or modify any state required to continue processing without this + * DiskLoc. + * + * Can only be called after a prepareToYield but before a recoverFromYield. + */ + virtual void invalidate(const DiskLoc& dl, InvalidationType type) = 0; + + /** + * Returns a tree of stats. See plan_stats.h for the details of this structure. If the + * stage has any children it must propagate the request for stats to them. + * + * Caller owns returned pointer. + */ + virtual PlanStageStats* getStats() = 0; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/plan_stats.cpp b/src/mongo/db/exec/plan_stats.cpp new file mode 100644 index 00000000000..bf079a7113a --- /dev/null +++ b/src/mongo/db/exec/plan_stats.cpp @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/plan_stats.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + void CommonStats::writeExplainTo(BSONObjBuilder* bob) const { + if (NULL == bob) { + return; + } + // potential overflow because original counters are unsigned 64-bit values + bob->append("works", static_cast(works)); + bob->append("advanced", static_cast(advanced)); + } + + // forward to CommonStats for now + // TODO: fill in specific stats + void PlanStageStats::writeExplainTo(BSONObjBuilder* bob) const { + common.writeExplainTo(bob); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/plan_stats.h b/src/mongo/db/exec/plan_stats.h new file mode 100644 index 00000000000..5266bf99491 --- /dev/null +++ b/src/mongo/db/exec/plan_stats.h @@ -0,0 +1,431 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/geo/hash.h" +#include "mongo/db/query/stage_types.h" +#include "mongo/platform/cstdint.h" + +namespace mongo { + + /** + * The interface all specific-to-stage stats provide. + */ + struct SpecificStats { + virtual ~SpecificStats() { } + + /** + * Make a deep copy. + */ + virtual SpecificStats* clone() const = 0; + }; + + // Every stage has CommonStats. + struct CommonStats { + CommonStats() : works(0), + yields(0), + unyields(0), + invalidates(0), + advanced(0), + needTime(0), + needFetch(0), + isEOF(false) { } + + // Count calls into the stage. + size_t works; + size_t yields; + size_t unyields; + size_t invalidates; + + // How many times was this state the return value of work(...)? + size_t advanced; + size_t needTime; + size_t needFetch; + + // TODO: have some way of tracking WSM sizes (or really any series of #s). We can measure + // the size of our inputs and the size of our outputs. We can do a lot with the WS here. + + // TODO: once we've picked a plan, collect different (or additional) stats for display to + // the user, eg. time_t totalTimeSpent; + + // TODO: keep track of total yield time / fetch time for a plan (done by runner) + + bool isEOF; + }; + + // The universal container for a stage's stats. + struct PlanStageStats { + PlanStageStats(const CommonStats& c, StageType t) : stageType(t), common(c) { } + + ~PlanStageStats() { + for (size_t i = 0; i < children.size(); ++i) { + delete children[i]; + } + } + + /** + * Make a deep copy. + */ + PlanStageStats* clone() const { + PlanStageStats* stats = new PlanStageStats(common, stageType); + if (specific.get()) { + stats->specific.reset(specific->clone()); + } + for (size_t i = 0; i < children.size(); ++i) { + invariant(children[i]); + stats->children.push_back(children[i]->clone()); + } + return stats; + } + + // See query/stage_type.h + StageType stageType; + + // Stats exported by implementing the PlanStage interface. + CommonStats common; + + // Per-stage place to stash additional information + boost::scoped_ptr specific; + + // The stats of the node's children. + std::vector children; + + private: + MONGO_DISALLOW_COPYING(PlanStageStats); + }; + + struct AndHashStats : public SpecificStats { + AndHashStats() : flaggedButPassed(0), + flaggedInProgress(0), + memUsage(0), + memLimit(0) { } + + virtual ~AndHashStats() { } + + virtual SpecificStats* clone() const { + AndHashStats* specific = new AndHashStats(*this); + return specific; + } + + // Invalidation counters. + // How many results had the AND fully evaluated but were invalidated? + size_t flaggedButPassed; + + // How many results were mid-AND but got flagged? + size_t flaggedInProgress; + + // How many entries are in the map after each child? + // child 'i' produced children[i].common.advanced DiskLocs, of which mapAfterChild[i] were + // intersections. + std::vector mapAfterChild; + + // mapAfterChild[mapAfterChild.size() - 1] WSMswere match tested. + // commonstats.advanced is how many passed. + + // What's our current memory usage? + size_t memUsage; + + // What's our memory limit? + size_t memLimit; + }; + + struct AndSortedStats : public SpecificStats { + AndSortedStats() : flagged(0), + matchTested(0) { } + + virtual ~AndSortedStats() { } + + virtual SpecificStats* clone() const { + AndSortedStats* specific = new AndSortedStats(*this); + return specific; + } + + // How many results from each child did not pass the AND? + std::vector failedAnd; + + // How many results were flagged via invalidation? + size_t flagged; + + // Fails == common.advanced - matchTested + size_t matchTested; + }; + + struct CollectionScanStats : public SpecificStats { + CollectionScanStats() : docsTested(0) { } + + virtual SpecificStats* clone() const { + CollectionScanStats* specific = new CollectionScanStats(*this); + return specific; + } + + // How many documents did we check against our filter? + size_t docsTested; + }; + + struct DistinctScanStats : public SpecificStats { + DistinctScanStats() : keysExamined(0) { } + + virtual SpecificStats* clone() const { + return new DistinctScanStats(*this); + } + + // How many keys did we look at while distinct-ing? + size_t keysExamined; + }; + + struct FetchStats : public SpecificStats { + FetchStats() : alreadyHasObj(0), + forcedFetches(0), + matchTested(0) { } + + virtual ~FetchStats() { } + + virtual SpecificStats* clone() const { + FetchStats* specific = new FetchStats(*this); + return specific; + } + + // Have we seen anything that already had an object? + size_t alreadyHasObj; + + // How many fetches weren't in memory? it's common.needFetch. + // How many total fetches did we do? it's common.advanced. + // So the number of fetches that were in memory are common.advanced - common.needFetch. + + // How many records were we forced to fetch as the result of an invalidation? + size_t forcedFetches; + + // We know how many passed (it's the # of advanced) and therefore how many failed. + size_t matchTested; + }; + + struct IndexScanStats : public SpecificStats { + IndexScanStats() : isMultiKey(false), + yieldMovedCursor(0), + dupsTested(0), + dupsDropped(0), + seenInvalidated(0), + matchTested(0), + keysExamined(0) { } + + virtual ~IndexScanStats() { } + + virtual SpecificStats* clone() const { + IndexScanStats* specific = new IndexScanStats(*this); + // BSON objects have to be explicitly copied. + specific->keyPattern = keyPattern.getOwned(); + specific->indexBounds = indexBounds.getOwned(); + return specific; + } + + // Index type being used. + std::string indexType; + + // name of the index being used + std::string indexName; + + BSONObj keyPattern; + + // A BSON (opaque, ie. hands off other than toString() it) representation of the bounds + // used. + BSONObj indexBounds; + + // Contains same information as indexBounds with the addition of inclusivity of bounds. + std::string indexBoundsVerbose; + + // >1 if we're traversing the index along with its order. <1 if we're traversing it + // against the order. + int direction; + + // Whether this index is over a field that contain array values. + bool isMultiKey; + + size_t yieldMovedCursor; + size_t dupsTested; + size_t dupsDropped; + + size_t seenInvalidated; + // TODO: we could track key sizes here. + + // We know how many passed (it's the # of advanced) and therefore how many failed. + size_t matchTested; + + // Number of entries retrieved from the index during the scan. + size_t keysExamined; + + }; + + struct OrStats : public SpecificStats { + OrStats() : dupsTested(0), + dupsDropped(0), + locsForgotten(0) { } + + virtual ~OrStats() { } + + virtual SpecificStats* clone() const { + OrStats* specific = new OrStats(*this); + return specific; + } + + size_t dupsTested; + size_t dupsDropped; + + // How many calls to invalidate(...) actually removed a DiskLoc from our deduping map? + size_t locsForgotten; + + // We know how many passed (it's the # of advanced) and therefore how many failed. + std::vector matchTested; + }; + + struct SortStats : public SpecificStats { + SortStats() : forcedFetches(0), memUsage(0), memLimit(0) { } + + virtual ~SortStats() { } + + virtual SpecificStats* clone() const { + SortStats* specific = new SortStats(*this); + return specific; + } + + // How many records were we forced to fetch as the result of an invalidation? + size_t forcedFetches; + + // What's our current memory usage? + size_t memUsage; + + // What's our memory limit? + size_t memLimit; + }; + + struct MergeSortStats : public SpecificStats { + MergeSortStats() : dupsTested(0), + dupsDropped(0), + forcedFetches(0) { } + + virtual ~MergeSortStats() { } + + virtual SpecificStats* clone() const { + MergeSortStats* specific = new MergeSortStats(*this); + return specific; + } + + size_t dupsTested; + size_t dupsDropped; + + // How many records were we forced to fetch as the result of an invalidation? + size_t forcedFetches; + }; + + struct S2NearStats : public SpecificStats { + S2NearStats() : nscanned(0), nscannedObjects(0), isMultiKey(false) { } + + virtual SpecificStats* clone() const { + S2NearStats* specific = new S2NearStats(*this); + return specific; + } + + size_t nscanned; + size_t nscannedObjects; + bool isMultiKey; + }; + + struct ShardingFilterStats : public SpecificStats { + ShardingFilterStats() : chunkSkips(0) { } + + virtual SpecificStats* clone() const { + ShardingFilterStats* specific = new ShardingFilterStats(*this); + return specific; + } + + size_t chunkSkips; + }; + + struct TwoDStats : public SpecificStats { + TwoDStats() { } + + virtual SpecificStats* clone() const { + TwoDStats* specific = new TwoDStats(*this); + return specific; + } + + // Type of GeoBrowse (box, circle, ...) + std::string type; + + // Field name in 2d index. + std::string field; + + // Geo hash converter parameters. + // Used to construct a geo hash converter to generate + // explain-style index bounds from geo hashes. + GeoHashConverter::Parameters converterParams; + + // Geo hashes generated by GeoBrowse::fillStack. + // Raw data for explain index bounds. + std::vector expPrefixes; + }; + + struct TwoDNearStats : public SpecificStats { + TwoDNearStats() : objectsLoaded(0), nscanned(0) { } + + virtual SpecificStats* clone() const { + TwoDNearStats* specific = new TwoDNearStats(*this); + return specific; + } + + size_t objectsLoaded; + + // Since 2d's near does all its work in one go we can't divine the real nscanned from + // anything else. + size_t nscanned; + }; + + struct TextStats : public SpecificStats { + TextStats() : keysExamined(0), fetches(0), parsedTextQuery() { } + + virtual SpecificStats* clone() const { + TextStats* specific = new TextStats(*this); + return specific; + } + + size_t keysExamined; + + size_t fetches; + + // Human-readable form of the FTSQuery associated with the text stage. + BSONObj parsedTextQuery; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/plan_stats_test.cpp b/src/mongo/db/exec/plan_stats_test.cpp new file mode 100644 index 00000000000..55ff41f9cad --- /dev/null +++ b/src/mongo/db/exec/plan_stats_test.cpp @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/exec/plan_stats.h + */ + +#include "mongo/db/exec/plan_stats.h" +#include "mongo/db/jsobj.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + /** + * Basic test on field initializers + */ + TEST(CommonStatsTest, defaultValues) { + CommonStats stats; + ASSERT_EQUALS(stats.works, static_cast(0)); + ASSERT_EQUALS(stats.yields, static_cast(0)); + ASSERT_EQUALS(stats.invalidates, static_cast(0)); + ASSERT_EQUALS(stats.advanced, static_cast(0)); + ASSERT_EQUALS(stats.needTime, static_cast(0)); + ASSERT_EQUALS(stats.needFetch, static_cast(0)); + ASSERT_FALSE(stats.isEOF); + } + + /** + * Verifies null argument check in CommonStats::writeExplainTo + */ + TEST(CommonStatsTest, writeExplainToNullBuilder) { + CommonStats stats; + stats.writeExplainTo(NULL); + } + + /** + * Verifies null argument check in PlanStageStats::writeExplainTo + */ + TEST(PlanStageStatsTest, writeExplainToNullBuilder) { + CommonStats stats; + PlanStageStats pss(stats); + pss.writeExplainTo(NULL); + } + + /** + * Checks BSON output of CommonStats::writeExplainTo to ensure it contains + * correct values for CommonStats fields + */ + TEST(CommonStatsTest, writeExplainTo) { + CommonStats stats; + stats.works = static_cast(2); + stats.advanced = static_cast(3); + BSONObjBuilder bob; + stats.writeExplainTo(&bob); + BSONObj obj = bob.done(); + ASSERT_TRUE(obj.hasField("works")); + ASSERT_EQUALS(obj.getIntField("works"), 2); + ASSERT_TRUE(obj.hasField("advanced")); + ASSERT_EQUALS(obj.getIntField("advanced"), 3); + } + + /** + * Checks BSON output of PlanStageStats::writeExplainTo to ensure it contains + * correct values for CommonStats fields + */ + TEST(PlanStageStatsTest, writeExplainTo) { + CommonStats stats; + stats.works = static_cast(2); + stats.advanced = static_cast(3); + BSONObjBuilder bob; + PlanStageStats pss(stats); + pss.writeExplainTo(&bob); + BSONObj obj = bob.done(); + ASSERT_TRUE(obj.hasField("works")); + ASSERT_EQUALS(obj.getIntField("works"), 2); + ASSERT_TRUE(obj.hasField("advanced")); + ASSERT_EQUALS(obj.getIntField("advanced"), 3); + } + +} // namespace diff --git a/src/mongo/db/exec/projection.cpp b/src/mongo/db/exec/projection.cpp new file mode 100644 index 00000000000..689d828b4b7 --- /dev/null +++ b/src/mongo/db/exec/projection.cpp @@ -0,0 +1,255 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/projection.h" + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + static const char* kIdField = "_id"; + + ProjectionStage::ProjectionStage(const ProjectionStageParams& params, + WorkingSet* ws, + PlanStage* child) + : _ws(ws), + _child(child), + _projImpl(params.projImpl) { + + if (ProjectionStageParams::NO_FAST_PATH == _projImpl) { + _exec.reset(new ProjectionExec(params.projObj, params.fullExpression)); + } + else { + // We shouldn't need the full expression if we're fast-pathing. + invariant(NULL == params.fullExpression); + + _projObj = params.projObj; + + // Sanity-check the input. + invariant(_projObj.isOwned()); + invariant(!_projObj.isEmpty()); + + // Figure out what fields are in the projection. + getSimpleInclusionFields(_projObj, &_includedFields); + + // If we're pulling data out of one index we can pre-compute the indices of the fields + // in the key that we pull data from and avoid looking up the field name each time. + if (ProjectionStageParams::COVERED_ONE_INDEX == params.projImpl) { + // Sanity-check. + _coveredKeyObj = params.coveredKeyObj; + invariant(_coveredKeyObj.isOwned()); + + BSONObjIterator kpIt(_coveredKeyObj); + while (kpIt.more()) { + BSONElement elt = kpIt.next(); + unordered_set::iterator fieldIt; + fieldIt = _includedFields.find(elt.fieldNameStringData()); + + if (_includedFields.end() == fieldIt) { + // Push an unused value on the back to keep _includeKey and _keyFieldNames + // in sync. + _keyFieldNames.push_back(StringData()); + _includeKey.push_back(false); + } + else { + // If we are including this key field store its field name. + _keyFieldNames.push_back(*fieldIt); + _includeKey.push_back(true); + } + } + } + else { + invariant(ProjectionStageParams::SIMPLE_DOC == params.projImpl); + } + } + } + + // static + void ProjectionStage::getSimpleInclusionFields(const BSONObj& projObj, + FieldSet* includedFields) { + // The _id is included by default. + bool includeId = true; + + // Figure out what fields are in the projection. TODO: we can get this from the + // ParsedProjection...modify that to have this type instead of a vector. + BSONObjIterator projObjIt(projObj); + while (projObjIt.more()) { + BSONElement elt = projObjIt.next(); + // Must deal with the _id case separately as there is an implicit _id: 1 in the + // projection. + if (mongoutils::str::equals(elt.fieldName(), kIdField) + && !elt.trueValue()) { + includeId = false; + continue; + } + includedFields->insert(elt.fieldNameStringData()); + } + + if (includeId) { + includedFields->insert(kIdField); + } + } + + // static + void ProjectionStage::transformSimpleInclusion(const BSONObj& in, + const FieldSet& includedFields, + BSONObjBuilder& bob) { + // Look at every field in the source document and see if we're including it. + BSONObjIterator inputIt(in); + while (inputIt.more()) { + BSONElement elt = inputIt.next(); + unordered_set::const_iterator fieldIt; + fieldIt = includedFields.find(elt.fieldNameStringData()); + if (includedFields.end() != fieldIt) { + // If so, add it to the builder. + bob.append(elt); + } + } + } + + Status ProjectionStage::transform(WorkingSetMember* member) { + // The default no-fast-path case. + if (ProjectionStageParams::NO_FAST_PATH == _projImpl) { + return _exec->transform(member); + } + + BSONObjBuilder bob; + + // Note that even if our fast path analysis is bug-free something that is + // covered might be invalidated and just be an obj. In this case we just go + // through the SIMPLE_DOC path which is still correct if the covered data + // is not available. + // + // SIMPLE_DOC implies that we expect an object so it's kind of redundant. + if ((ProjectionStageParams::SIMPLE_DOC == _projImpl) || member->hasObj()) { + // If we got here because of SIMPLE_DOC the planner shouldn't have messed up. + invariant(member->hasObj()); + + // Apply the SIMPLE_DOC projection. + transformSimpleInclusion(member->obj, _includedFields, bob); + } + else { + invariant(ProjectionStageParams::COVERED_ONE_INDEX == _projImpl); + // We're pulling data out of the key. + invariant(1 == member->keyData.size()); + size_t keyIndex = 0; + + // Look at every key element... + BSONObjIterator keyIterator(member->keyData[0].keyData); + while (keyIterator.more()) { + BSONElement elt = keyIterator.next(); + // If we're supposed to include it... + if (_includeKey[keyIndex]) { + // Do so. + bob.appendAs(elt, _keyFieldNames[keyIndex]); + } + ++keyIndex; + } + } + + member->state = WorkingSetMember::OWNED_OBJ; + member->keyData.clear(); + member->loc = DiskLoc(); + member->obj = bob.obj(); + return Status::OK(); + } + + ProjectionStage::~ProjectionStage() { } + + bool ProjectionStage::isEOF() { return _child->isEOF(); } + + PlanStage::StageState ProjectionStage::work(WorkingSetID* out) { + ++_commonStats.works; + + WorkingSetID id = WorkingSet::INVALID_ID; + StageState status = _child->work(&id); + + // Note that we don't do the normal if isEOF() return EOF thing here. Our child might be a + // tailable cursor and isEOF() would be true even if it had more data... + if (PlanStage::ADVANCED == status) { + WorkingSetMember* member = _ws->get(id); + // Punt to our specific projection impl. + Status projStatus = transform(member); + if (!projStatus.isOK()) { + warning() << "Couldn't execute projection, status = " + << projStatus.toString() << endl; + *out = WorkingSetCommon::allocateStatusMember(_ws, projStatus); + return PlanStage::FAILURE; + } + + *out = id; + ++_commonStats.advanced; + } + else if (PlanStage::FAILURE == status) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "projection stage failed to read in results from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + } + else if (PlanStage::NEED_FETCH == status) { + *out = id; + ++_commonStats.needFetch; + } + + return status; + } + + void ProjectionStage::prepareToYield() { + ++_commonStats.yields; + _child->prepareToYield(); + } + + void ProjectionStage::recoverFromYield() { + ++_commonStats.unyields; + _child->recoverFromYield(); + } + + void ProjectionStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + _child->invalidate(dl, type); + } + + PlanStageStats* ProjectionStage::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_PROJECTION)); + ret->children.push_back(_child->getStats()); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/projection.h b/src/mongo/db/exec/projection.h new file mode 100644 index 00000000000..e93ee682170 --- /dev/null +++ b/src/mongo/db/exec/projection.h @@ -0,0 +1,143 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/projection_exec.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" + +namespace mongo { + + struct ProjectionStageParams { + enum ProjectionImplementation { + // The default case. Will handle every projection. + NO_FAST_PATH, + + // The projection is simple inclusion and is totally covered by one index. + COVERED_ONE_INDEX, + + // The projection is simple inclusion and we expect an object. + SIMPLE_DOC + }; + + ProjectionStageParams() : projImpl(NO_FAST_PATH), fullExpression(NULL) { } + + ProjectionImplementation projImpl; + + // The projection object. We lack a ProjectionExpression or similar so we use a BSONObj. + BSONObj projObj; + + // If we have a positional or elemMatch projection we need a MatchExpression to pull out the + // right data. + // Not owned here, we do not take ownership. + const MatchExpression* fullExpression; + + // If (COVERED_ONE_INDEX == projObj) this is the key pattern we're extracting covered data + // from. Otherwise, this field is ignored. + BSONObj coveredKeyObj; + }; + + /** + * This stage computes a projection. + */ + class ProjectionStage : public PlanStage { + public: + ProjectionStage(const ProjectionStageParams& params, + WorkingSet* ws, + PlanStage* child); + + virtual ~ProjectionStage(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + PlanStageStats* getStats(); + + typedef unordered_set FieldSet; + + /** + * Given the projection spec for a simple inclusion projection, + * 'projObj', populates 'includedFields' with the set of field + * names to be included. + */ + static void getSimpleInclusionFields(const BSONObj& projObj, + FieldSet* includedFields); + + /** + * Applies a simple inclusion projection to 'in', including + * only the fields specified by 'includedFields'. + * + * The resulting document is constructed using 'bob'. + */ + static void transformSimpleInclusion(const BSONObj& in, + const FieldSet& includedFields, + BSONObjBuilder& bob); + + private: + Status transform(WorkingSetMember* member); + + scoped_ptr _exec; + + // _ws is not owned by us. + WorkingSet* _ws; + scoped_ptr _child; + + // Stats + CommonStats _commonStats; + + // Fast paths: + ProjectionStageParams::ProjectionImplementation _projImpl; + + // Used by all projection implementations. + BSONObj _projObj; + + // Data used for both SIMPLE_DOC and COVERED_ONE_INDEX paths. + // Has the field names present in the simple projection. + unordered_set _includedFields; + + // + // Used for the COVERED_ONE_INDEX path. + // + BSONObj _coveredKeyObj; + + // Field names can be empty in 2.4 and before so we can't use them as a sentinel value. + // If the i-th entry is true we include the i-th field in the key. + vector _includeKey; + + // If the i-th entry of _includeKey is true this is the field name for the i-th key field. + vector _keyFieldNames; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/projection_exec.cpp b/src/mongo/db/exec/projection_exec.cpp new file mode 100644 index 00000000000..27eed3f90d4 --- /dev/null +++ b/src/mongo/db/exec/projection_exec.cpp @@ -0,0 +1,551 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/projection_exec.h" + +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/query/lite_parsed_query.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + ProjectionExec::ProjectionExec() + : _include(true), + _special(false), + _includeID(true), + _skip(0), + _limit(-1), + _arrayOpType(ARRAY_OP_NORMAL), + _hasNonSimple(false), + _hasDottedField(false), + _queryExpression(NULL), + _hasReturnKey(false) { } + + + ProjectionExec::ProjectionExec(const BSONObj& spec, const MatchExpression* queryExpression) + : _include(true), + _special(false), + _source(spec), + _includeID(true), + _skip(0), + _limit(-1), + _arrayOpType(ARRAY_OP_NORMAL), + _hasNonSimple(false), + _hasDottedField(false), + _queryExpression(queryExpression), + _hasReturnKey(false) { + + // Are we including or excluding fields? + // -1 when we haven't initialized it. + // 1 when we're including + // 0 when we're excluding. + int include_exclude = -1; + + BSONObjIterator it(_source); + while (it.more()) { + BSONElement e = it.next(); + + if (!e.isNumber() && !e.isBoolean()) { + _hasNonSimple = true; + } + + if (Object == e.type()) { + BSONObj obj = e.embeddedObject(); + verify(1 == obj.nFields()); + + BSONElement e2 = obj.firstElement(); + if (mongoutils::str::equals(e2.fieldName(), "$slice")) { + if (e2.isNumber()) { + int i = e2.numberInt(); + if (i < 0) { + add(e.fieldName(), i, -i); // limit is now positive + } + else { + add(e.fieldName(), 0, i); + } + } + else { + verify(e2.type() == Array); + BSONObj arr = e2.embeddedObject(); + verify(2 == arr.nFields()); + + BSONObjIterator it(arr); + int skip = it.next().numberInt(); + int limit = it.next().numberInt(); + + verify(limit > 0); + + add(e.fieldName(), skip, limit); + } + } + else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) { + _arrayOpType = ARRAY_OP_ELEM_MATCH; + + // Create a MatchExpression for the elemMatch. + BSONObj elemMatchObj = e.wrap(); + verify(elemMatchObj.isOwned()); + _elemMatchObjs.push_back(elemMatchObj); + StatusWithMatchExpression swme = MatchExpressionParser::parse(elemMatchObj); + verify(swme.isOK()); + // And store it in _matchers. + _matchers[mongoutils::str::before(e.fieldName(), '.').c_str()] + = swme.getValue(); + + add(e.fieldName(), true); + } + else if (mongoutils::str::equals(e2.fieldName(), "$meta")) { + verify(String == e2.type()); + if (e2.valuestr() == LiteParsedQuery::metaTextScore) { + _meta[e.fieldName()] = META_TEXT_SCORE; + } + else if (e2.valuestr() == LiteParsedQuery::metaDiskLoc) { + _meta[e.fieldName()] = META_DISKLOC; + } + else if (e2.valuestr() == LiteParsedQuery::metaGeoNearPoint) { + _meta[e.fieldName()] = META_GEONEAR_POINT; + } + else if (e2.valuestr() == LiteParsedQuery::metaGeoNearDistance) { + _meta[e.fieldName()] = META_GEONEAR_DIST; + } + else if (e2.valuestr() == LiteParsedQuery::metaIndexKey) { + _hasReturnKey = true; + // The index key clobbers everything so just stop parsing here. + return; + } + else { + // This shouldn't happen, should be caught by parsing. + verify(0); + } + } + else { + verify(0); + } + } + else if (mongoutils::str::equals(e.fieldName(), "_id") && !e.trueValue()) { + _includeID = false; + } + else { + add(e.fieldName(), e.trueValue()); + + // Projections of dotted fields aren't covered. + if (mongoutils::str::contains(e.fieldName(), '.')) { + _hasDottedField = true; + } + + // Validate input. + if (include_exclude == -1) { + // If we haven't specified an include/exclude, initialize include_exclude. + // We expect further include/excludes to match it. + include_exclude = e.trueValue(); + _include = !e.trueValue(); + } + } + + if (mongoutils::str::contains(e.fieldName(), ".$")) { + _arrayOpType = ARRAY_OP_POSITIONAL; + } + } + } + + ProjectionExec::~ProjectionExec() { + for (FieldMap::const_iterator it = _fields.begin(); it != _fields.end(); ++it) { + delete it->second; + } + + for (Matchers::const_iterator it = _matchers.begin(); it != _matchers.end(); ++it) { + delete it->second; + } + } + + void ProjectionExec::add(const string& field, bool include) { + if (field.empty()) { // this is the field the user referred to + _include = include; + } + else { + _include = !include; + + const size_t dot = field.find('.'); + const string subfield = field.substr(0,dot); + const string rest = (dot == string::npos ? "" : field.substr(dot + 1, string::npos)); + + ProjectionExec*& fm = _fields[subfield.c_str()]; + + if (NULL == fm) { + fm = new ProjectionExec(); + } + + fm->add(rest, include); + } + } + + void ProjectionExec::add(const string& field, int skip, int limit) { + _special = true; // can't include or exclude whole object + + if (field.empty()) { // this is the field the user referred to + _skip = skip; + _limit = limit; + } + else { + const size_t dot = field.find('.'); + const string subfield = field.substr(0,dot); + const string rest = (dot == string::npos ? "" : field.substr(dot + 1, string::npos)); + + ProjectionExec*& fm = _fields[subfield.c_str()]; + + if (NULL == fm) { + fm = new ProjectionExec(); + } + + fm->add(rest, skip, limit); + } + } + + // + // Execution + // + + Status ProjectionExec::transform(WorkingSetMember* member) const { + if (_hasReturnKey) { + BSONObj keyObj; + + if (member->hasComputed(WSM_INDEX_KEY)) { + const IndexKeyComputedData* key + = static_cast(member->getComputed(WSM_INDEX_KEY)); + keyObj = key->getKey(); + } + + member->state = WorkingSetMember::OWNED_OBJ; + member->obj = keyObj; + member->keyData.clear(); + member->loc = DiskLoc(); + return Status::OK(); + } + + BSONObjBuilder bob; + if (member->hasObj()) { + MatchDetails matchDetails; + + // If it's a positional projection we need a MatchDetails. + if (transformRequiresDetails()) { + matchDetails.requestElemMatchKey(); + verify(NULL != _queryExpression); + verify(_queryExpression->matchesBSON(member->obj, &matchDetails)); + } + + Status projStatus = transform(member->obj, &bob, &matchDetails); + if (!projStatus.isOK()) { + return projStatus; + } + } + else { + verify(!requiresDocument()); + // Go field by field. + if (_includeID) { + BSONElement elt; + // Sometimes the _id field doesn't exist... + if (member->getFieldDotted("_id", &elt) && !elt.eoo()) { + bob.appendAs(elt, "_id"); + } + } + + BSONObjIterator it(_source); + while (it.more()) { + BSONElement specElt = it.next(); + if (mongoutils::str::equals("_id", specElt.fieldName())) { + continue; + } + + BSONElement keyElt; + // We can project a field that doesn't exist. We just ignore it. + if (member->getFieldDotted(specElt.fieldName(), &keyElt) && !keyElt.eoo()) { + bob.appendAs(keyElt, specElt.fieldName()); + } + } + } + + for (MetaMap::const_iterator it = _meta.begin(); it != _meta.end(); ++it) { + if (META_GEONEAR_DIST == it->second) { + if (member->hasComputed(WSM_COMPUTED_GEO_DISTANCE)) { + const GeoDistanceComputedData* dist + = static_cast( + member->getComputed(WSM_COMPUTED_GEO_DISTANCE)); + bob.append(it->first, dist->getDist()); + } + else { + return Status(ErrorCodes::InternalError, + "near loc dist requested but no data available"); + } + } + else if (META_GEONEAR_POINT == it->second) { + if (member->hasComputed(WSM_GEO_NEAR_POINT)) { + const GeoNearPointComputedData* point + = static_cast( + member->getComputed(WSM_GEO_NEAR_POINT)); + BSONObj ptObj = point->getPoint(); + if (ptObj.couldBeArray()) { + bob.appendArray(it->first, ptObj); + } + else { + bob.append(it->first, ptObj); + } + } + else { + return Status(ErrorCodes::InternalError, + "near loc proj requested but no data available"); + } + } + else if (META_TEXT_SCORE == it->second) { + if (member->hasComputed(WSM_COMPUTED_TEXT_SCORE)) { + const TextScoreComputedData* score + = static_cast( + member->getComputed(WSM_COMPUTED_TEXT_SCORE)); + bob.append(it->first, score->getScore()); + } + else { + bob.append(it->first, 0.0); + } + } + else if (META_DISKLOC == it->second) { + bob.append(it->first, member->loc.toBSONObj()); + } + } + + BSONObj newObj = bob.obj(); + member->state = WorkingSetMember::OWNED_OBJ; + member->obj = newObj; + member->keyData.clear(); + member->loc = DiskLoc(); + + return Status::OK(); + } + + Status ProjectionExec::transform(const BSONObj& in, BSONObj* out) const { + // If it's a positional projection we need a MatchDetails. + MatchDetails matchDetails; + if (transformRequiresDetails()) { + matchDetails.requestElemMatchKey(); + verify(NULL != _queryExpression); + verify(_queryExpression->matchesBSON(in, &matchDetails)); + } + + BSONObjBuilder bob; + Status s = transform(in, &bob, &matchDetails); + if (!s.isOK()) { + return s; + } + *out = bob.obj(); + return Status::OK(); + } + + Status ProjectionExec::transform(const BSONObj& in, + BSONObjBuilder* bob, + const MatchDetails* details) const { + + const ArrayOpType& arrayOpType = _arrayOpType; + + BSONObjIterator it(in); + while (it.more()) { + BSONElement elt = it.next(); + + // Case 1: _id + if (mongoutils::str::equals("_id", elt.fieldName())) { + if (_includeID) { + bob->append(elt); + } + continue; + } + + // Case 2: no array projection for this field. + Matchers::const_iterator matcher = _matchers.find(elt.fieldName()); + if (_matchers.end() == matcher) { + Status s = append(bob, elt, details, arrayOpType); + if (!s.isOK()) { + return s; + } + continue; + } + + // Case 3: field has array projection with $elemMatch specified. + if (ARRAY_OP_ELEM_MATCH != arrayOpType) { + return Status(ErrorCodes::BadValue, + "Matchers are only supported for $elemMatch"); + } + + MatchDetails arrayDetails; + arrayDetails.requestElemMatchKey(); + + if (matcher->second->matchesBSON(in, &arrayDetails)) { + FieldMap::const_iterator fieldIt = _fields.find(elt.fieldName()); + if (_fields.end() == fieldIt) { + return Status(ErrorCodes::BadValue, + "$elemMatch specified, but projection field not found."); + } + + BSONArrayBuilder arrBuilder; + BSONObjBuilder subBob; + + if (in.getField(elt.fieldName()).eoo()) { + return Status(ErrorCodes::InternalError, + "$elemMatch called on document element with eoo"); + } + + if (in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey()).eoo()) { + return Status(ErrorCodes::InternalError, + "$elemMatch called on array element with eoo"); + } + + arrBuilder.append( + in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey())); + subBob.appendArray(matcher->first, arrBuilder.arr()); + Status status = append(bob, subBob.done().firstElement(), details, arrayOpType); + if (!status.isOK()) { + return status; + } + } + } + + return Status::OK(); + } + + void ProjectionExec::appendArray(BSONObjBuilder* bob, const BSONObj& array, bool nested) const { + int skip = nested ? 0 : _skip; + int limit = nested ? -1 : _limit; + + if (skip < 0) { + skip = max(0, skip + array.nFields()); + } + + int index = 0; + BSONObjIterator it(array); + while (it.more()) { + BSONElement elt = it.next(); + + if (skip) { + skip--; + continue; + } + + if (limit != -1 && (limit-- == 0)) { + break; + } + + switch(elt.type()) { + case Array: { + BSONObjBuilder subBob; + appendArray(&subBob, elt.embeddedObject(), true); + bob->appendArray(bob->numStr(index++), subBob.obj()); + break; + } + case Object: { + BSONObjBuilder subBob; + BSONObjIterator jt(elt.embeddedObject()); + while (jt.more()) { + append(&subBob, jt.next()); + } + bob->append(bob->numStr(index++), subBob.obj()); + break; + } + default: + if (_include) { + bob->appendAs(elt, bob->numStr(index++)); + } + } + } + } + + Status ProjectionExec::append(BSONObjBuilder* bob, + const BSONElement& elt, + const MatchDetails* details, + const ArrayOpType arrayOpType) const { + + + // Skip if the field name matches a computed $meta field. + // $meta projection fields can exist at the top level of + // the result document and the field names cannot be dotted. + if (_meta.find(elt.fieldName()) != _meta.end()) { + return Status::OK(); + } + + FieldMap::const_iterator field = _fields.find(elt.fieldName()); + if (field == _fields.end()) { + if (_include) { + bob->append(elt); + } + return Status::OK(); + } + + ProjectionExec& subfm = *field->second; + if ((subfm._fields.empty() && !subfm._special) + || !(elt.type() == Object || elt.type() == Array)) { + // field map empty, or element is not an array/object + if (subfm._include) { + bob->append(elt); + } + } + else if (elt.type() == Object) { + BSONObjBuilder subBob; + BSONObjIterator it(elt.embeddedObject()); + while (it.more()) { + subfm.append(&subBob, it.next(), details, arrayOpType); + } + bob->append(elt.fieldName(), subBob.obj()); + } + else { + // Array + BSONObjBuilder matchedBuilder; + if (details && arrayOpType == ARRAY_OP_POSITIONAL) { + // $ positional operator specified + if (!details->hasElemMatchKey()) { + mongoutils::str::stream error; + error << "positional operator (" << elt.fieldName() + << ".$) requires corresponding field" + << " in query specifier"; + return Status(ErrorCodes::BadValue, error); + } + + if (elt.embeddedObject()[details->elemMatchKey()].eoo()) { + return Status(ErrorCodes::BadValue, + "positional operator element mismatch"); + } + + // append as the first and only element in the projected array + matchedBuilder.appendAs( elt.embeddedObject()[details->elemMatchKey()], "0" ); + } + else { + // append exact array; no subarray matcher specified + subfm.appendArray(&matchedBuilder, elt.embeddedObject()); + } + bob->appendArray(elt.fieldName(), matchedBuilder.obj()); + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/projection_exec.h b/src/mongo/db/exec/projection_exec.h new file mode 100644 index 00000000000..e358c76f686 --- /dev/null +++ b/src/mongo/db/exec/projection_exec.h @@ -0,0 +1,197 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/exec/working_set.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/util/string_map.h" + +namespace mongo { + + class ProjectionExec { + public: + /** + * A .find() projection can have an array operation, either an elemMatch or positional (or + * neither). + */ + enum ArrayOpType { + ARRAY_OP_NORMAL = 0, + ARRAY_OP_ELEM_MATCH, + ARRAY_OP_POSITIONAL + }; + + /** + * Projections based on data computed while answering a query, or other metadata about a + * document / query. + */ + enum MetaProjection { + META_TEXT_SCORE, + META_GEONEAR_DIST, + META_GEONEAR_POINT, + META_DISKLOC, + META_IX_KEY, + }; + + /** + * TODO: document why we like StringMap so much here + */ + typedef StringMap FieldMap; + typedef StringMap Matchers; + typedef StringMap MetaMap; + + ProjectionExec(const BSONObj& spec, const MatchExpression* queryExpression); + ~ProjectionExec(); + + /** + * Apply this projection to the 'member'. Changes the type to OWNED_OBJ. + */ + Status transform(WorkingSetMember* member) const; + + /** + * Apply this projection to the object 'in'. + * + * Upon success, 'out' is set to the new object and Status::OK() is returned. + * Otherwise, returns an error Status and *out is not mutated. + */ + Status transform(const BSONObj& in, BSONObj* out) const; + + private: + // + // Initialization + // + + ProjectionExec(); + + /** + * Add 'field' as a field name that is included or excluded as part of the projection. + */ + void add(const string& field, bool include); + + /** + * Add 'field' as a field name that is sliced as part of the projection. + */ + void add(const string& field, int skip, int limit); + + // + // Execution + // + + /** + * Apply the projection that 'this' represents to the object 'in'. 'details' is the result + * of a match evaluation of the full query on the object 'in'. This is only required + * if the projection is positional. + * + * If the projection is successfully computed, returns Status::OK() and stuff the result in + * 'bob'. + * Otherwise, returns error. + */ + Status transform(const BSONObj& in, + BSONObjBuilder* bob, + const MatchDetails* details = NULL) const; + + /** + * See transform(...) above. + */ + bool transformRequiresDetails() const { + return ARRAY_OP_POSITIONAL == _arrayOpType; + } + + /** + * Is the full document required to compute this projection? + */ + bool requiresDocument() const { + return _include || _hasNonSimple || _hasDottedField; + } + + /** + * Appends the element 'e' to the builder 'bob', possibly descending into sub-fields of 'e' + * if needed. + */ + Status append(BSONObjBuilder* bob, + const BSONElement& elt, + const MatchDetails* details = NULL, + const ArrayOpType arrayOpType = ARRAY_OP_NORMAL) const; + + /** + * Like append, but for arrays. + * Deals with slice and calls appendArray to preserve the array-ness. + */ + void appendArray(BSONObjBuilder* bob, const BSONObj& array, bool nested = false) const; + + // True if default at this level is to include. + bool _include; + + // True if this level can't be skipped or included without recursing. + bool _special; + + // We must group projections with common prefixes together. + // TODO: benchmark vector vs map + // + // Projection is a rooted tree. If we have {a.b: 1, a.c: 1} we don't want to + // double-traverse the document when we're projecting it. Instead, we have an entry in + // _fields for 'a' with two sub projections: b:1 and c:1. + FieldMap _fields; + + // The raw projection spec. that is passed into init(...) + BSONObj _source; + + // Should we include the _id field? + bool _includeID; + + // Arguments from the $slice operator. + int _skip; + int _limit; + + // Used for $elemMatch and positional operator ($) + Matchers _matchers; + + // The matchers above point into BSONObjs and this is where those objs live. + vector _elemMatchObjs; + + ArrayOpType _arrayOpType; + + // Is there an slice, elemMatch or meta operator? + bool _hasNonSimple; + + // Is there a projection over a dotted field or a $ positional operator? + bool _hasDottedField; + + // The full query expression. Used when we need MatchDetails. + const MatchExpression* _queryExpression; + + // Projections that aren't sourced from the document or index keys. + MetaMap _meta; + + // Do we have a returnKey projection? If so we *only* output the index key metadata. If + // it's not found we output nothing. + bool _hasReturnKey; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/projection_exec_test.cpp b/src/mongo/db/exec/projection_exec_test.cpp new file mode 100644 index 00000000000..59af37dd6d4 --- /dev/null +++ b/src/mongo/db/exec/projection_exec_test.cpp @@ -0,0 +1,219 @@ +/** + * Copyright (C) 2013 mongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/exec/projection_exec.cpp + */ + +#include "mongo/db/exec/projection_exec.h" + +#include +#include "mongo/db/json.h" +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + using std::auto_ptr; + + /** + * Utility function to create MatchExpression + */ + MatchExpression* parseMatchExpression(const BSONObj& obj) { + StatusWithMatchExpression status = MatchExpressionParser::parse(obj); + ASSERT_TRUE(status.isOK()); + MatchExpression* expr(status.getValue()); + return expr; + } + + // + // transform tests + // + + /** + * test function to verify results of transform() + * on a working set member. + * + * specStr - projection specification + * queryStr - query + * objStr - object to run projection on + * data - computed data. Owned by working set member created in this function if not null. + * expectedStatusOK - expected status of transformation + * expectedObjStr - expected object after successful projection. + * Ignored if expectedStatusOK is false. + */ + + void testTransform(const char* specStr, const char* queryStr, const char* objStr, + WorkingSetComputedData* data, + bool expectedStatusOK, const char* expectedObjStr) { + // Create projection exec object. + BSONObj spec = fromjson(specStr); + BSONObj query = fromjson(queryStr); + auto_ptr queryExpression(parseMatchExpression(query)); + ProjectionExec exec(spec, queryExpression.get()); + + // Create working set member. + WorkingSetMember wsm; + wsm.state = WorkingSetMember::OWNED_OBJ; + wsm.obj = fromjson(objStr); + if (data) { + wsm.addComputed(data); + } + + // Transform object + Status status = exec.transform(&wsm); + + // There are fewer checks to perform if we are expected a failed status. + if (!expectedStatusOK) { + if (status.isOK()) { + mongoutils::str::stream ss; + ss << "expected transform() to fail but got success instead." + << "\nprojection spec: " << specStr + << "\nquery: " << queryStr + << "\nobject before projection: " << objStr; + FAIL(ss); + } + return; + } + + // If we are expecting a successful transformation but got a failed status instead, + // print out status message in assertion message. + if (!status.isOK()) { + mongoutils::str::stream ss; + ss << "transform() test failed: unexpected failed status: " << status.toString() + << "\nprojection spec: " << specStr + << "\nquery: " << queryStr + << "\nobject before projection: " << objStr + << "\nexpected object after projection: " << expectedObjStr; + FAIL(ss); + } + + // Finally, we compare the projected object. + const BSONObj& obj = wsm.obj; + BSONObj expectedObj = fromjson(expectedObjStr); + if (obj != expectedObj) { + mongoutils::str::stream ss; + ss << "transform() test failed: unexpected projected object." + << "\nprojection spec: " << specStr + << "\nquery: " << queryStr + << "\nobject before projection: " << objStr + << "\nexpected object after projection: " << expectedObjStr + << "\nactual object after projection: " << obj.toString(); + FAIL(ss); + } + } + + /** + * testTransform without computed data argument. + */ + void testTransform(const char* specStr, const char* queryStr, const char* objStr, + bool expectedStatusOK, const char* expectedObjStr) { + testTransform(specStr, queryStr, objStr, NULL, expectedStatusOK, expectedObjStr); + } + + // + // position $ + // + + TEST(ProjectionExecTest, TransformPositionalDollar) { + // Valid position $ projections. + testTransform("{'a.$': 1}", "{a: 10}", "{a: [10, 20, 30]}", true, "{a: [10]}"); + testTransform("{'a.$': 1}", "{a: 20}", "{a: [10, 20, 30]}", true, "{a: [20]}"); + testTransform("{'a.$': 1}", "{a: 30}", "{a: [10, 20, 30]}", true, "{a: [30]}"); + testTransform("{'a.$': 1}", "{a: {$gt: 4}}", "{a: [5]}", true, "{a: [5]}"); + + // Invalid position $ projections. + testTransform("{'a.$': 1}", "{a: {$size: 1}}", "{a: [5]}", false, ""); + } + + // + // $elemMatch + // + + TEST(ProjectionExecTest, TransformElemMatch) { + const char* s = "{a: [{x: 1, y: 10}, {x: 1, y: 20}, {x: 2, y: 10}]}"; + + // Valid $elemMatch projections. + testTransform("{a: {$elemMatch: {x: 1}}}", "{}", s, true, "{a: [{x: 1, y: 10}]}"); + testTransform("{a: {$elemMatch: {x: 1, y: 20}}}", "{}", s, true, "{a: [{x: 1, y: 20}]}"); + testTransform("{a: {$elemMatch: {x: 2}}}", "{}", s, true, "{a: [{x: 2, y: 10}]}"); + testTransform("{a: {$elemMatch: {x: 3}}}", "{}", s, true, "{}"); + + // $elemMatch on unknown field z + testTransform("{a: {$elemMatch: {z: 1}}}", "{}", s, true, "{}"); + } + + // + // $slice + // + + TEST(ProjectionExecTest, TransformSliceCount) { + // Valid $slice projections using format {$slice: count}. + testTransform("{a: {$slice: -10}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4, 6, 8]}"); + testTransform("{a: {$slice: -3}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4, 6, 8]}"); + testTransform("{a: {$slice: -1}}", "{}", "{a: [4, 6, 8]}", true, "{a: [8]}"); + testTransform("{a: {$slice: 0}}", "{}", "{a: [4, 6, 8]}", true, "{a: []}"); + testTransform("{a: {$slice: 1}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4]}"); + testTransform("{a: {$slice: 3}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4, 6, 8]}"); + testTransform("{a: {$slice: 10}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4, 6, 8]}"); + } + + TEST(ProjectionExecTest, TransformSliceSkipLimit) { + // Valid $slice projections using format {$slice: [skip, limit]}. + // Non-positive limits are rejected at the query parser and therefore not handled by + // the projection execution stage. In fact, it will abort on an invalid limit. + testTransform("{a: {$slice: [-10, 10]}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4, 6, 8]}"); + testTransform("{a: {$slice: [-3, 5]}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4, 6, 8]}"); + testTransform("{a: {$slice: [-1, 1]}}", "{}", "{a: [4, 6, 8]}", true, "{a: [8]}"); + testTransform("{a: {$slice: [0, 2]}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4, 6]}"); + testTransform("{a: {$slice: [0, 1]}}", "{}", "{a: [4, 6, 8]}", true, "{a: [4]}"); + testTransform("{a: {$slice: [1, 1]}}", "{}", "{a: [4, 6, 8]}", true, "{a: [6]}"); + testTransform("{a: {$slice: [3, 5]}}", "{}", "{a: [4, 6, 8]}", true, "{a: []}"); + testTransform("{a: {$slice: [10, 10]}}", "{}", "{a: [4, 6, 8]}", true, "{a: []}"); + } + + // + // $meta + // $meta projections add computed values to the projected object. + // + + TEST(ProjectionExecTest, TransformMetaTextScore) { + // Query {} is ignored. + testTransform("{b: {$meta: 'textScore'}}", "{}", "{a: 'hello'}", + new mongo::TextScoreComputedData(100), + true, "{a: 'hello', b: 100}"); + // Projected meta field should overwrite existing field. + testTransform("{b: {$meta: 'textScore'}}", "{}", "{a: 'hello', b: -1}", + new mongo::TextScoreComputedData(100), + true, "{a: 'hello', b: 100}"); + } + +} // namespace diff --git a/src/mongo/db/exec/s2near.cpp b/src/mongo/db/exec/s2near.cpp new file mode 100644 index 00000000000..30770d4c22f --- /dev/null +++ b/src/mongo/db/exec/s2near.cpp @@ -0,0 +1,444 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/s2near.h" + +#include "mongo/db/client.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/exec/index_scan.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/geo/geoconstants.h" +#include "mongo/db/index/expression_index.h" +#include "third_party/s2/s2regionintersection.h" + +namespace mongo { + + S2NearStage::S2NearStage(const S2NearParams& params, WorkingSet* ws) + : _worked(false), + _params(params), + _ws(ws), + _indexScan(NULL), + _failed(false), + _initted(false) { + } + + void S2NearStage::init() { + _initted = true; + + // The field we're near-ing from is the n-th field. Figure out what that 'n' is. We + // put the cover for the search annulus in this spot in the bounds. + _nearFieldIndex = 0; + BSONObjIterator specIt(_params.indexKeyPattern); + while (specIt.more()) { + if (specIt.next().fieldName() == _params.nearQuery.field) { + break; + } + ++_nearFieldIndex; + } + + verify(_nearFieldIndex < _params.indexKeyPattern.nFields()); + + // FLAT implies the input distances are in radians. Convert to meters. + if (FLAT == _params.nearQuery.centroid.crs) { + _params.nearQuery.minDistance *= kRadiusOfEarthInMeters; + _params.nearQuery.maxDistance *= kRadiusOfEarthInMeters; + } + + // Make sure distances are sane. Possibly redundant given the checking during parsing. + _minDistance = max(0.0, _params.nearQuery.minDistance); + _maxDistance = min(M_PI * kRadiusOfEarthInMeters, _params.nearQuery.maxDistance); + _minDistance = min(_minDistance, _maxDistance); + + // We grow _outerRadius in nextAnnulus() below. + _innerRadius = _outerRadius = _minDistance; + _outerRadiusInclusive = false; + + // Grab the IndexDescriptor. + Database* db = cc().database(); + if (!db) { + _failed = true; + return; + } + + Collection* collection = db->getCollection(_params.ns); + if (!collection) { + _failed = true; + return; + } + + _descriptor = collection->getIndexCatalog()->findIndexByKeyPattern(_params.indexKeyPattern); + if (NULL == _descriptor) { + _failed = true; + return; + } + + // The user can override this so we honor it. We could ignore it though -- it's just used + // to set _radiusIncrement, not to do any covering. + int finestIndexedLevel; + BSONElement fl = _descriptor->infoObj()["finestIndexedLevel"]; + if (fl.isNumber()) { + finestIndexedLevel = fl.numberInt(); + } + else { + finestIndexedLevel = S2::kAvgEdge.GetClosestLevel(500.0 / kRadiusOfEarthInMeters); + } + + // Start with a conservative _radiusIncrement. When we're done searching a shell we + // increment the two radii by this. + _radiusIncrement = 5 * S2::kAvgEdge.GetValue(finestIndexedLevel) * kRadiusOfEarthInMeters; + } + + S2NearStage::~S2NearStage() { + // _annulus temporarily takes ownership of some member variables. + // Release them to avoid double-deleting _innerCap and _outerCap. + _annulus.Release(NULL); + } + + PlanStage::StageState S2NearStage::work(WorkingSetID* out) { + if (!_initted) { init(); } + + if (_failed) { + mongoutils::str::stream ss; + ss << "unable to load geo index " << _params.indexKeyPattern; + Status status(ErrorCodes::IndexNotFound, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + return PlanStage::FAILURE; + } + if (isEOF()) { return PlanStage::IS_EOF; } + ++_commonStats.works; + + // If we haven't opened up our very first ixscan+fetch children, do it. This is kind of + // heavy so we don't want to do it in the ctor. + if (!_worked) { + nextAnnulus(); + _worked = true; + } + + // If we're still reading results from the child, do that. + if (NULL != _child.get()) { + return addResultToQueue(out); + } + + // Not reading results. Perhaps we're returning buffered results. + if (!_results.empty()) { + Result result = _results.top(); + _results.pop(); + *out = result.id; + + // Remove from invalidation map. + WorkingSetMember* member = _ws->get(*out); + if (member->hasLoc()) { + unordered_map::iterator it + = _invalidationMap.find(member->loc); + verify(_invalidationMap.end() != it); + _invalidationMap.erase(it); + } + + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + + // Not EOF, not reading results, not returning any buffered results. Look in the next shell + // for results. + nextAnnulus(); + return PlanStage::NEED_TIME; + } + + /** + * A MatchExpression for seeing if an S2Cell-in-a-key is within an annulus. + */ + class GeoS2KeyMatchExpression : public MatchExpression { + public: + /** + * 'annulus' must outlive 'this'. + */ + GeoS2KeyMatchExpression(S2RegionIntersection* annulus, + StringData nearFieldPath) + : MatchExpression(INTERNAL_GEO_S2_KEYCHECK), + _annulus(annulus) { + + _elementPath.init(nearFieldPath); + } + + virtual ~GeoS2KeyMatchExpression(){} + + virtual bool matches(const MatchableDocument* doc, MatchDetails* details = 0) const { + MatchableDocument::IteratorHolder cursor(doc, &_elementPath); + + while (cursor->more()) { + ElementIterator::Context e = cursor->next(); + if (matchesSingleElement(e.element())) { + return true; + } + } + + return false; + } + + virtual bool matchesSingleElement(const BSONElement& e) const { + // Something has gone terribly wrong if this doesn't hold. + invariant(String == e.type()); + S2Cell keyCell = S2Cell(S2CellId::FromString(e.str())); + return _annulus->MayIntersect(keyCell); + } + + // + // These won't be called. + // + + virtual void debugString( StringBuilder& debug, int level = 0 ) const { + } + + virtual bool equivalent( const MatchExpression* other ) const { + return false; + } + + virtual MatchExpression* shallowClone() const { + return NULL; + } + + private: + // Not owned here. + S2RegionIntersection* _annulus; + + ElementPath _elementPath; + }; + + void S2NearStage::nextAnnulus() { + // Step 1: Grow the annulus. + _innerRadius = _outerRadius; + _outerRadius += _radiusIncrement; + if (_outerRadius >= _maxDistance) { + _outerRadius = _maxDistance; + _outerRadiusInclusive = true; + } + verify(_innerRadius <= _outerRadius); + + // We might have just grown our radius beyond anything reasonable. + if (isEOF()) { return; } + + // Step 2: Fill out bounds for the ixscan we use. + _innerCap = S2Cap::FromAxisAngle(_params.nearQuery.centroid.point, + S1Angle::Radians(_innerRadius / kRadiusOfEarthInMeters)); + _outerCap = S2Cap::FromAxisAngle(_params.nearQuery.centroid.point, + S1Angle::Radians(_outerRadius / kRadiusOfEarthInMeters)); + _innerCap = _innerCap.Complement(); + + vector regions; + regions.push_back(&_innerCap); + regions.push_back(&_outerCap); + + _annulus.Release(NULL); + _annulus.Init(®ions); + + // Step 3: Actually create the ixscan. + + IndexScanParams params; + params.descriptor = _descriptor; + _params.baseBounds.fields[_nearFieldIndex].intervals.clear(); + ExpressionMapping::cover2dsphere(_annulus, + params.descriptor->infoObj(), + &_params.baseBounds.fields[_nearFieldIndex]); + + params.bounds = _params.baseBounds; + params.direction = 1; + // We use a filter on the key. The filter rejects keys that don't intersect with the + // annulus. An object that is in the annulus might have a key that's not in it and a key + // that's in it. As such we can't just look at one key per object. + // + // This does force us to do our own deduping of results, though. + params.doNotDedup = true; + + // Owns geo filter. + _keyGeoFilter.reset(new GeoS2KeyMatchExpression( + &_annulus, _params.baseBounds.fields[_nearFieldIndex].name)); + _indexScan = new IndexScan(params, _ws, _keyGeoFilter.get()); + + // Owns '_indexScan'. + _child.reset(new FetchStage(_ws, _indexScan, _params.filter)); + _seenInScan.clear(); + } + + PlanStage::StageState S2NearStage::addResultToQueue(WorkingSetID* out) { + PlanStage::StageState state = _child->work(out); + + // All done reading from _child. + if (PlanStage::IS_EOF == state) { + // Aggregate stats from index scan used to get results for the annulus. + invariant(_indexScan); + _specificStats.nscanned += _indexScan->getSpecificStats()->keysExamined; + _specificStats.nscannedObjects += _indexScan->getCommonStats()->advanced; + if (_indexScan->getSpecificStats()->isMultiKey) { + _specificStats.isMultiKey = true; + } + _indexScan = NULL; + + _child.reset(); + _keyGeoFilter.reset(); + + // Adjust the annulus size depending on how many results we got. + if (_results.empty()) { + _radiusIncrement *= 2; + } else if (_results.size() < 300) { + _radiusIncrement *= 2; + } else if (_results.size() > 600) { + _radiusIncrement /= 2; + } + + // Make a new ixscan next time. + return PlanStage::NEED_TIME; + } + + // Nothing to do unless we advance. + if (PlanStage::ADVANCED != state) { return state; } + + WorkingSetMember* member = _ws->get(*out); + // Must have an object in order to get geometry out of it. + verify(member->hasObj()); + + // The scans we use don't dedup so we must dedup them ourselves. We only put locs into here + // if we know for sure whether or not we'll return them in this annulus. + if (member->hasLoc()) { + if (_seenInScan.end() != _seenInScan.find(member->loc)) { + return PlanStage::NEED_TIME; + } + } + + // Get all the fields with that name from the document. + BSONElementSet geom; + member->obj.getFieldsDotted(_params.nearQuery.field, geom, false); + if (geom.empty()) { + return PlanStage::NEED_TIME; + } + + // Some value that any distance we can calculate will be less than. + double minDistance = numeric_limits::max(); + BSONObj minDistanceObj; + for (BSONElementSet::iterator git = geom.begin(); git != geom.end(); ++git) { + if (!git->isABSONObj()) { + mongoutils::str::stream ss; + ss << "s2near stage read invalid geometry element " << *git << " from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + return PlanStage::FAILURE; + } + BSONObj obj = git->Obj(); + + double distToObj; + if (S2SearchUtil::distanceBetween(_params.nearQuery.centroid.point, obj, &distToObj)) { + if (distToObj < minDistance) { + minDistance = distToObj; + minDistanceObj = obj; + } + } + else { + warning() << "unknown geometry: " << obj.toString(); + } + } + + // If we're here we'll either include the doc in this annulus or reject it. It's safe to + // ignore it if it pops up again in this annulus. + if (member->hasLoc()) { + _seenInScan.insert(member->loc); + } + + // If the distance to the doc satisfies our distance criteria, add it to our buffered + // results. + if (minDistance >= _innerRadius && + (_outerRadiusInclusive ? minDistance <= _outerRadius : minDistance < _outerRadius)) { + _results.push(Result(*out, minDistance)); + if (_params.addDistMeta) { + // FLAT implies the output distances are in radians. Convert to meters. + if (FLAT == _params.nearQuery.centroid.crs) { + member->addComputed(new GeoDistanceComputedData(minDistance + / kRadiusOfEarthInMeters)); + } + else { + member->addComputed(new GeoDistanceComputedData(minDistance)); + } + } + if (_params.addPointMeta) { + member->addComputed(new GeoNearPointComputedData(minDistanceObj)); + } + if (member->hasLoc()) { + _invalidationMap[member->loc] = *out; + } + } + + return PlanStage::NEED_TIME; + } + + void S2NearStage::prepareToYield() { + if (NULL != _child.get()) { + _child->prepareToYield(); + } + } + + void S2NearStage::recoverFromYield() { + if (NULL != _child.get()) { + _child->recoverFromYield(); + } + } + + void S2NearStage::invalidate(const DiskLoc& dl, InvalidationType type) { + if (NULL != _child.get()) { + _child->invalidate(dl, type); + } + + // _results is a queue of results that we will return for the current shell we're on. + // If a result is in _results and has a DiskLoc it will be in _invalidationMap as well. + // It's safe to return the result w/o the DiskLoc. + unordered_map::iterator it + = _invalidationMap.find(dl); + + if (it != _invalidationMap.end()) { + WorkingSetMember* member = _ws->get(it->second); + verify(member->hasLoc()); + WorkingSetCommon::fetchAndInvalidateLoc(member); + verify(!member->hasLoc()); + // Don't keep it around in the invalidation map since there's no valid DiskLoc anymore. + _invalidationMap.erase(it); + } + } + + bool S2NearStage::isEOF() { + if (!_worked) { return false; } + if (_failed) { return true; } + // We're only done if we exhaust the search space. + return _innerRadius >= _maxDistance; + } + + PlanStageStats* S2NearStage::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_GEO_NEAR_2DSPHERE)); + ret->specific.reset(new S2NearStats(_specificStats)); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/s2near.h b/src/mongo/db/exec/s2near.h new file mode 100644 index 00000000000..9f9c1d07354 --- /dev/null +++ b/src/mongo/db/exec/s2near.h @@ -0,0 +1,165 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/exec/fetch.h" +#include "mongo/db/exec/index_scan.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/geo/s2common.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/query/index_bounds.h" +#include "mongo/platform/unordered_set.h" +#include "third_party/s2/s2cap.h" +#include "third_party/s2/s2regionintersection.h" + +namespace mongo { + + struct S2NearParams { + string ns; + BSONObj indexKeyPattern; + NearQuery nearQuery; + IndexBounds baseBounds; + MatchExpression* filter; + bool addPointMeta; + bool addDistMeta; + }; + + /** + * Executes a geoNear search. Is a leaf node. Output type is LOC_AND_UNOWNED_OBJ. + */ + class S2NearStage : public PlanStage { + public: + /** + * Takes: index to scan over, MatchExpression with near point, other MatchExpressions for + * covered data, + */ + S2NearStage(const S2NearParams& params, WorkingSet* ws); + + virtual ~S2NearStage(); + + StageState work(WorkingSetID* out); + bool isEOF(); + + void prepareToYield(); + void recoverFromYield(); + void invalidate(const DiskLoc& dl, InvalidationType type); + + PlanStageStats* getStats(); + + private: + void init(); + StageState addResultToQueue(WorkingSetID* out); + void nextAnnulus(); + + bool _worked; + + S2NearParams _params; + + WorkingSet* _ws; + + // This is the "array index" of the key field that is the near field. We use this to do + // cheap is-this-doc-in-the-annulus testing. We also need to know where to stuff the index + // bounds for the various annuluses/annuli. + int _nearFieldIndex; + + // Geo filter in index scan (which is owned by fetch stage in _child). + scoped_ptr _keyGeoFilter; + + // The child fetch stage for the current annulus. + scoped_ptr _child; + + // The child of '_child'. Not owned here. + IndexScan* _indexScan; + + // The S2 machinery that represents the search annulus. We keep this around after bounds + // generation to check for intersection. + S2Cap _innerCap; + S2Cap _outerCap; + S2RegionIntersection _annulus; + + // We use this to hold on to the results in an annulus. Results are sorted to have + // decreasing distance. + struct Result { + Result(WorkingSetID wsid, double dist) : id(wsid), distance(dist) { } + + bool operator<(const Result& other) const { + // We want increasing distance, not decreasing, so we reverse the <. + return distance > other.distance; + } + + WorkingSetID id; + double distance; + }; + + // Our index scans aren't deduped so we might see the same doc twice in a given + // annulus. + unordered_set _seenInScan; + + // We compute an annulus of results and cache it here. + priority_queue _results; + + // For fast invalidation. Perhaps not worth it. + unordered_map _invalidationMap; + + // Geo-related variables. + // At what min distance (arc length) do we start looking for results? + double _minDistance; + // What's the max distance (arc length) we're willing to look for results? + double _maxDistance; + + // These radii define the annulus we're currently looking at. + double _innerRadius; + double _outerRadius; + + // True if we are looking at last annulus + bool _outerRadiusInclusive; + + // When we search the next annulus, what to adjust our radius by? Grows when we search an + // annulus and find no results. + double _radiusIncrement; + + // Did we encounter an unrecoverable error? + bool _failed; + + // Have we init()'d yet? + bool _initted; + + // What index are we searching over? + IndexDescriptor* _descriptor; + + CommonStats _commonStats; + S2NearStats _specificStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/shard_filter.cpp b/src/mongo/db/exec/shard_filter.cpp new file mode 100644 index 00000000000..b2c817511ac --- /dev/null +++ b/src/mongo/db/exec/shard_filter.cpp @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/shard_filter.h" + +#include "mongo/db/keypattern.h" + +namespace mongo { + + ShardFilterStage::ShardFilterStage(const CollectionMetadataPtr& metadata, + WorkingSet* ws, + PlanStage* child) + : _ws(ws), _child(child), _metadata(metadata) { } + + ShardFilterStage::~ShardFilterStage() { } + + bool ShardFilterStage::isEOF() { return _child->isEOF(); } + + PlanStage::StageState ShardFilterStage::work(WorkingSetID* out) { + ++_commonStats.works; + + // If we've returned as many results as we're limited to, isEOF will be true. + if (isEOF()) { return PlanStage::IS_EOF; } + + StageState status = _child->work(out); + + if (PlanStage::ADVANCED == status) { + // If we're sharded make sure that we don't return any data that hasn't been migrated + // off of our shared yet. + if (_metadata) { + KeyPattern kp(_metadata->getKeyPattern()); + + WorkingSetMember* member = _ws->get(*out); + + // This performs excessive BSONObj creation but that's OK for now. + if (!_metadata->keyBelongsToMe(kp.extractSingleKey(member->obj))) { + _ws->free(*out); + ++_specificStats.chunkSkips; + return PlanStage::NEED_TIME; + } + } + + // If we're here either we have shard state and our doc passed, or we have no shard + // state. Either way, we advance. + ++_commonStats.advanced; + return status; + } + else { + if (PlanStage::NEED_FETCH == status) { + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == status) { + ++_commonStats.needTime; + } + return status; + } + } + + void ShardFilterStage::prepareToYield() { + ++_commonStats.yields; + _child->prepareToYield(); + } + + void ShardFilterStage::recoverFromYield() { + ++_commonStats.unyields; + _child->recoverFromYield(); + } + + void ShardFilterStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + _child->invalidate(dl, type); + } + + PlanStageStats* ShardFilterStage::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_SHARDING_FILTER)); + ret->children.push_back(_child->getStats()); + ret->specific.reset(new ShardingFilterStats(_specificStats)); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/shard_filter.h b/src/mongo/db/exec/shard_filter.h new file mode 100644 index 00000000000..6c01c1e1691 --- /dev/null +++ b/src/mongo/db/exec/shard_filter.h @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/s/chunk_version.h" +#include "mongo/s/d_logic.h" +#include "mongo/s/stale_exception.h" + +namespace mongo { + + /** + * This stage drops documents that didn't belong to the shard we're executing on at the time of + * construction. This matches the contract for sharded cursorids which guarantees that a + * StaleConfigException will be thrown early or the cursorid for its entire lifetime will return + * documents matching the shard version set on the connection at the time of cursorid creation. + * + * A related system will ensure that the data migrated away from a shard will not be deleted as + * long as there are active queries from before the migration. Currently, "active queries" is + * defined by cursorids so it is important that the metadata used in this stage uses the same + * version as the cursorid. Therefore, you must wrap any Runner using this Stage in a + * ClientCursor during the same lock grab as constructing the Runner. + * + * BEGIN NOTE FROM GREG + * + * There are three sharded query contracts: + * + * 0) Migration commit takes the db lock - i.e. is serialized with writes and reads. + * 1) No data should be returned from a query in ranges of migrations that committed after the + * query started, or from ranges not owned when the query began. + * 2) No migrated data should be removed from a shard while there are queries that were active + * before the migration. + * + * As implementation details, collection metadata is used to determine the ranges of all data + * not actively migrated (or orphaned). CursorIds are currently used to establish "active" + * queries before migration commit. + * + * Combining all this: if a query is started in a db lock and acquires in that (same) lock the + * collection metadata and a cursorId, the query will return results for exactly the ranges in + * the metadata (though of arbitrary staleness). This is the sharded collection query contract. + * + * END NOTE FROM GREG + * + * Preconditions: Child must be fetched. TODO: when covering analysis is in just build doc + * and check that against shard key. See SERVER-5022. + */ + class ShardFilterStage : public PlanStage { + public: + ShardFilterStage(const CollectionMetadataPtr& metadata, WorkingSet* ws, PlanStage* child); + virtual ~ShardFilterStage(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + WorkingSet* _ws; + scoped_ptr _child; + + // Stats + CommonStats _commonStats; + ShardingFilterStats _specificStats; + + // Note: it is important that this is the metadata from the time this stage is constructed. + // See class comment for details. + const CollectionMetadataPtr _metadata; + }; + +} // namespace mongo + diff --git a/src/mongo/db/exec/skip.cpp b/src/mongo/db/exec/skip.cpp new file mode 100644 index 00000000000..04eaf4ae140 --- /dev/null +++ b/src/mongo/db/exec/skip.cpp @@ -0,0 +1,110 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/exec/skip.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + SkipStage::SkipStage(int toSkip, WorkingSet* ws, PlanStage* child) + : _ws(ws), _child(child), _toSkip(toSkip) { } + + SkipStage::~SkipStage() { } + + bool SkipStage::isEOF() { return _child->isEOF(); } + + PlanStage::StageState SkipStage::work(WorkingSetID* out) { + ++_commonStats.works; + + WorkingSetID id = WorkingSet::INVALID_ID; + StageState status = _child->work(&id); + + if (PlanStage::ADVANCED == status) { + // If we're still skipping results... + if (_toSkip > 0) { + // ...drop the result. + --_toSkip; + _ws->free(id); + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + + *out = id; + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + else if (PlanStage::FAILURE == status) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "skip stage failed to read in results from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return status; + } + else { + if (PlanStage::NEED_FETCH == status) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == status) { + ++_commonStats.needTime; + } + // NEED_TIME/YIELD, ERROR, IS_EOF + return status; + } + } + + void SkipStage::prepareToYield() { + ++_commonStats.yields; + _child->prepareToYield(); + } + + void SkipStage::recoverFromYield() { + ++_commonStats.unyields; + _child->recoverFromYield(); + } + + void SkipStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + _child->invalidate(dl, type); + } + + PlanStageStats* SkipStage::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_SKIP)); + ret->children.push_back(_child->getStats()); + return ret.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/skip.h b/src/mongo/db/exec/skip.h new file mode 100644 index 00000000000..9c907ac8bb0 --- /dev/null +++ b/src/mongo/db/exec/skip.h @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" + +namespace mongo { + + /** + * This stage implements skip functionality. It drops the first 'toSkip' results from its child + * then returns the rest verbatim. + * + * Preconditions: None. + */ + class SkipStage : public PlanStage { + public: + SkipStage(int toSkip, WorkingSet* ws, PlanStage* child); + virtual ~SkipStage(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual PlanStageStats* getStats(); + + private: + WorkingSet* _ws; + scoped_ptr _child; + + // We drop the first _toSkip results that we would have returned. + int _toSkip; + + // Stats + CommonStats _commonStats; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/sort.cpp b/src/mongo/db/exec/sort.cpp new file mode 100644 index 00000000000..068dd1300d0 --- /dev/null +++ b/src/mongo/db/exec/sort.cpp @@ -0,0 +1,562 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/sort.h" + +#include + +#include "mongo/db/index_names.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/index/btree_key_generator.h" +#include "mongo/db/query/lite_parsed_query.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner.h" + +namespace mongo { + + using std::vector; + + const size_t kMaxBytes = 32 * 1024 * 1024; + + SortStageKeyGenerator::SortStageKeyGenerator(const BSONObj& sortSpec, const BSONObj& queryObj) { + _hasBounds = false; + _sortHasMeta = false; + _rawSortSpec = sortSpec; + + // 'sortSpec' can be a mix of $meta and index key expressions. We pick it apart so that + // we only generate Btree keys for the index key expressions. + + // The Btree key fields go in here. We pass this fake index key pattern to the Btree + // key generator below as part of generating sort keys for the docs. + BSONObjBuilder btreeBob; + + // The pattern we use to woCompare keys. Each field in 'sortSpec' will go in here with + // a value of 1 or -1. The Btree key fields are verbatim, meta fields have a default. + BSONObjBuilder comparatorBob; + + BSONObjIterator it(sortSpec); + while (it.more()) { + BSONElement elt = it.next(); + if (elt.isNumber()) { + // Btree key. elt (should be) foo: 1 or foo: -1. + comparatorBob.append(elt); + btreeBob.append(elt); + } + else if (LiteParsedQuery::isTextScoreMeta(elt)) { + // Sort text score decreasing by default. Field name doesn't matter but we choose + // something that a user shouldn't ever have. + comparatorBob.append("$metaTextScore", -1); + _sortHasMeta = true; + } + else { + // Sort spec. should have been validated before here. + verify(false); + } + } + + // Our pattern for woComparing keys. + _comparatorObj = comparatorBob.obj(); + + // The fake index key pattern used to generate Btree keys. + _btreeObj = btreeBob.obj(); + + // If we're just sorting by meta, don't bother with all the key stuff. + if (_btreeObj.isEmpty()) { + return; + } + + // We'll need to treat arrays as if we were to create an index over them. that is, + // we may need to unnest the first level and consider each array element to decide + // the sort order. + std::vector fieldNames; + std::vector fixed; + BSONObjIterator btreeIt(_btreeObj); + while (btreeIt.more()) { + BSONElement patternElt = btreeIt.next(); + fieldNames.push_back(patternElt.fieldName()); + fixed.push_back(BSONElement()); + } + + _keyGen.reset(new BtreeKeyGeneratorV1(fieldNames, fixed, false /* not sparse */)); + + // The bounds checker only works on the Btree part of the sort key. + getBoundsForSort(queryObj, _btreeObj); + + if (_hasBounds) { + _boundsChecker.reset(new IndexBoundsChecker(&_bounds, _btreeObj, 1 /* == order */)); + } + } + + Status SortStageKeyGenerator::getSortKey(const WorkingSetMember& member, + BSONObj* objOut) const { + BSONObj btreeKeyToUse; + + Status btreeStatus = getBtreeKey(member.obj, &btreeKeyToUse); + if (!btreeStatus.isOK()) { + return btreeStatus; + } + + if (!_sortHasMeta) { + *objOut = btreeKeyToUse; + return Status::OK(); + } + + BSONObjBuilder mergedKeyBob; + + // Merge metadata into the key. + BSONObjIterator it(_rawSortSpec); + BSONObjIterator btreeIt(btreeKeyToUse); + while (it.more()) { + BSONElement elt = it.next(); + if (elt.isNumber()) { + // Merge btree key elt. + mergedKeyBob.append(btreeIt.next()); + } + else if (LiteParsedQuery::isTextScoreMeta(elt)) { + // Add text score metadata + double score = 0.0; + if (member.hasComputed(WSM_COMPUTED_TEXT_SCORE)) { + const TextScoreComputedData* scoreData + = static_cast( + member.getComputed(WSM_COMPUTED_TEXT_SCORE)); + score = scoreData->getScore(); + } + mergedKeyBob.append("$metaTextScore", score); + } + } + + *objOut = mergedKeyBob.obj(); + return Status::OK(); + } + + Status SortStageKeyGenerator::getBtreeKey(const BSONObj& memberObj, BSONObj* objOut) const { + // Not sorting by anything in the key, just bail out early. + if (_btreeObj.isEmpty()) { + *objOut = BSONObj(); + return Status::OK(); + } + + // We will sort '_data' in the same order an index over '_pattern' would have. This is + // tricky. Consider the sort pattern {a:1} and the document {a:[1, 10]}. We have + // potentially two keys we could use to sort on. Here we extract these keys. + BSONObjCmp patternCmp(_btreeObj); + BSONObjSet keys(patternCmp); + + try { + _keyGen->getKeys(memberObj, &keys); + } + catch (const UserException& e) { + // Probably a parallel array. + if (BtreeKeyGenerator::ParallelArraysCode == e.getCode()) { + return Status(ErrorCodes::BadValue, + "cannot sort with keys that are parallel arrays"); + } + else { + return e.toStatus(); + } + } + catch (...) { + return Status(ErrorCodes::InternalError, "unknown error during sort key generation"); + } + + // Key generator isn't sparse so we should at least get an all-null key. + invariant(!keys.empty()); + + // No bounds? No problem! Use the first key. + if (!_hasBounds) { + // Note that we sort 'keys' according to the pattern '_btreeObj'. + *objOut = *keys.begin(); + return Status::OK(); + } + + // To decide which key to use in sorting, we must consider not only the sort pattern but + // the query. Assume we have the query {a: {$gte: 5}} and a document {a:1}. That + // document wouldn't match the query. As such, the key '1' in an array {a: [1, 10]} + // should not be considered as being part of the result set and thus that array cannot + // sort using the key '1'. To ensure that the keys we sort by are valid w.r.t. the + // query we use a bounds checker. + verify(NULL != _boundsChecker.get()); + for (BSONObjSet::const_iterator it = keys.begin(); it != keys.end(); ++it) { + if (_boundsChecker->isValidKey(*it)) { + *objOut = *it; + return Status::OK(); + } + } + + // No key is in our bounds. + // TODO: will this ever happen? don't think it should. + *objOut = *keys.begin(); + return Status::OK(); + } + + void SortStageKeyGenerator::getBoundsForSort(const BSONObj& queryObj, const BSONObj& sortObj) { + QueryPlannerParams params; + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + // We're creating a "virtual index" with key pattern equal to the sort order. + IndexEntry sortOrder(sortObj, IndexNames::BTREE, true, false, false, "doesnt_matter", + BSONObj()); + params.indices.push_back(sortOrder); + + CanonicalQuery* rawQueryForSort; + verify(CanonicalQuery::canonicalize("fake_ns", + queryObj, + &rawQueryForSort).isOK()); + auto_ptr queryForSort(rawQueryForSort); + + vector solns; + QLOG() << "Sort stage: Planning to obtain bounds for sort." << endl; + QueryPlanner::plan(*queryForSort, params, &solns); + + // TODO: are there ever > 1 solns? If so, do we look for a specific soln? + if (1 == solns.size()) { + IndexScanNode* ixScan = NULL; + QuerySolutionNode* rootNode = solns[0]->root.get(); + + if (rootNode->getType() == STAGE_FETCH) { + FetchNode* fetchNode = static_cast(rootNode); + if (fetchNode->children[0]->getType() != STAGE_IXSCAN) { + delete solns[0]; + // No bounds. + return; + } + ixScan = static_cast(fetchNode->children[0]); + } + else if (rootNode->getType() == STAGE_IXSCAN) { + ixScan = static_cast(rootNode); + } + + if (ixScan) { + _bounds.fields.swap(ixScan->bounds.fields); + _hasBounds = true; + } + } + + for (size_t i = 0; i < solns.size(); ++i) { + delete solns[i]; + } + } + + SortStage::WorkingSetComparator::WorkingSetComparator(BSONObj p) : pattern(p) { } + + bool SortStage::WorkingSetComparator::operator()(const SortableDataItem& lhs, const SortableDataItem& rhs) const { + // False means ignore field names. + int result = lhs.sortKey.woCompare(rhs.sortKey, pattern, false); + if (0 != result) { + return result < 0; + } + // Indices use DiskLoc as an additional sort key so we must as well. + return lhs.loc < rhs.loc; + } + + SortStage::SortStage(const SortStageParams& params, WorkingSet* ws, PlanStage* child) + : _ws(ws), + _child(child), + _pattern(params.pattern), + _query(params.query), + _limit(params.limit), + _sorted(false), + _resultIterator(_data.end()), + _memUsage(0) { + } + + SortStage::~SortStage() { } + + bool SortStage::isEOF() { + // We're done when our child has no more results, we've sorted the child's results, and + // we've returned all sorted results. + return _child->isEOF() && _sorted && (_data.end() == _resultIterator); + } + + PlanStage::StageState SortStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (NULL == _sortKeyGen) { + // This is heavy and should be done as part of work(). + _sortKeyGen.reset(new SortStageKeyGenerator(_pattern, _query)); + _sortKeyComparator.reset(new WorkingSetComparator(_sortKeyGen->getSortComparator())); + // If limit > 1, we need to initialize _dataSet here to maintain ordered + // set of data items while fetching from the child stage. + if (_limit > 1) { + const WorkingSetComparator& cmp = *_sortKeyComparator; + _dataSet.reset(new SortableDataItemSet(cmp)); + } + return PlanStage::NEED_TIME; + } + + if (_memUsage > kMaxBytes) { + mongoutils::str::stream ss; + ss << "sort stage buffered data usage of " << _memUsage + << " bytes exceeds internal limit of " << kMaxBytes << " bytes"; + Status status(ErrorCodes::Overflow, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + return PlanStage::FAILURE; + } + + if (isEOF()) { return PlanStage::IS_EOF; } + + // Still reading in results to sort. + if (!_sorted) { + WorkingSetID id = WorkingSet::INVALID_ID; + StageState code = _child->work(&id); + + if (PlanStage::ADVANCED == code) { + // Add it into the map for quick invalidation if it has a valid DiskLoc. + // A DiskLoc may be invalidated at any time (during a yield). We need to get into + // the WorkingSet as quickly as possible to handle it. + WorkingSetMember* member = _ws->get(id); + + // Planner must put a fetch before we get here. + verify(member->hasObj()); + + // We might be sorting something that was invalidated at some point. + if (member->hasLoc()) { + _wsidByDiskLoc[member->loc] = id; + } + + // The data remains in the WorkingSet and we wrap the WSID with the sort key. + SortableDataItem item; + Status sortKeyStatus = _sortKeyGen->getSortKey(*member, &item.sortKey); + if (!_sortKeyGen->getSortKey(*member, &item.sortKey).isOK()) { + *out = WorkingSetCommon::allocateStatusMember(_ws, sortKeyStatus); + return PlanStage::FAILURE; + } + item.wsid = id; + if (member->hasLoc()) { + // The DiskLoc breaks ties when sorting two WSMs with the same sort key. + item.loc = member->loc; + } + + addToBuffer(item); + + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::IS_EOF == code) { + // TODO: We don't need the lock for this. We could ask for a yield and do this work + // unlocked. Also, this is performing a lot of work for one call to work(...) + sortBuffer(); + _resultIterator = _data.begin(); + _sorted = true; + ++_commonStats.needTime; + return PlanStage::NEED_TIME; + } + else if (PlanStage::FAILURE == code) { + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "sort stage failed to read in results to sort from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + return code; + } + else { + if (PlanStage::NEED_FETCH == code) { + *out = id; + ++_commonStats.needFetch; + } + else if (PlanStage::NEED_TIME == code) { + ++_commonStats.needTime; + } + return code; + } + } + + // Returning results. + verify(_resultIterator != _data.end()); + verify(_sorted); + *out = _resultIterator->wsid; + _resultIterator++; + + // If we're returning something, take it out of our DL -> WSID map so that future + // calls to invalidate don't cause us to take action for a DL we're done with. + WorkingSetMember* member = _ws->get(*out); + if (member->hasLoc()) { + _wsidByDiskLoc.erase(member->loc); + } + + ++_commonStats.advanced; + return PlanStage::ADVANCED; + } + + void SortStage::prepareToYield() { + ++_commonStats.yields; + _child->prepareToYield(); + } + + void SortStage::recoverFromYield() { + ++_commonStats.unyields; + _child->recoverFromYield(); + } + + void SortStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + _child->invalidate(dl, type); + + // If we have a deletion, we can fetch and carry on. + // If we have a mutation, it's easier to fetch and use the previous document. + // So, no matter what, fetch and keep the doc in play. + + // _data contains indices into the WorkingSet, not actual data. If a WorkingSetMember in + // the WorkingSet needs to change state as a result of a DiskLoc invalidation, it will still + // be at the same spot in the WorkingSet. As such, we don't need to modify _data. + DataMap::iterator it = _wsidByDiskLoc.find(dl); + + // If we're holding on to data that's got the DiskLoc we're invalidating... + if (_wsidByDiskLoc.end() != it) { + // Grab the WSM that we're nuking. + WorkingSetMember* member = _ws->get(it->second); + verify(member->loc == dl); + + WorkingSetCommon::fetchAndInvalidateLoc(member); + + // Remove the DiskLoc from our set of active DLs. + _wsidByDiskLoc.erase(it); + ++_specificStats.forcedFetches; + } + } + + PlanStageStats* SortStage::getStats() { + _commonStats.isEOF = isEOF(); + _specificStats.memLimit = kMaxBytes; + _specificStats.memUsage = _memUsage; + + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_SORT)); + ret->specific.reset(new SortStats(_specificStats)); + ret->children.push_back(_child->getStats()); + return ret.release(); + } + + /** + * addToBuffer() and sortBuffer() work differently based on the + * configured limit. addToBuffer() is also responsible for + * performing some accounting on the overall memory usage to + * make sure we're not using too much memory. + * + * limit == 0: + * addToBuffer() - Adds item to vector. + * sortBuffer() - Sorts vector. + * limit == 1: + * addToBuffer() - Replaces first item in vector with max of + * current and new item. + * Updates memory usage if item was replaced. + * sortBuffer() - Does nothing. + * limit > 1: + * addToBuffer() - Does not update vector. Adds item to set. + * If size of set exceeds limit, remove item from set + * with lowest key. Updates memory usage accordingly. + * sortBuffer() - Copies items from set to vectors. + */ + void SortStage::addToBuffer(const SortableDataItem& item) { + // Holds ID of working set member to be freed at end of this function. + WorkingSetID wsidToFree = WorkingSet::INVALID_ID; + + if (_limit == 0) { + _data.push_back(item); + _memUsage += _ws->get(item.wsid)->getMemUsage(); + } + else if (_limit == 1) { + if (_data.empty()) { + _data.push_back(item); + _memUsage = _ws->get(item.wsid)->getMemUsage(); + return; + } + wsidToFree = item.wsid; + const WorkingSetComparator& cmp = *_sortKeyComparator; + // Compare new item with existing item in vector. + if (cmp(item, _data[0])) { + wsidToFree = _data[0].wsid; + _data[0] = item; + _memUsage = _ws->get(item.wsid)->getMemUsage(); + } + } + else { + // Update data item set instead of vector + // Limit not reached - insert and return + vector::size_type limit(_limit); + if (_dataSet->size() < limit) { + _dataSet->insert(item); + _memUsage += _ws->get(item.wsid)->getMemUsage(); + return; + } + // Limit will be exceeded - compare with item with lowest key + // If new item does not have a lower key value than last item, + // do nothing. + wsidToFree = item.wsid; + SortableDataItemSet::const_iterator lastItemIt = --(_dataSet->end()); + const SortableDataItem& lastItem = *lastItemIt; + const WorkingSetComparator& cmp = *_sortKeyComparator; + if (cmp(item, lastItem)) { + _memUsage -= _ws->get(lastItem.wsid)->getMemUsage(); + _memUsage += _ws->get(item.wsid)->getMemUsage(); + wsidToFree = lastItem.wsid; + // According to std::set iterator validity rules, + // it does not matter which of erase()/insert() happens first. + // Here, we choose to erase first to release potential resources + // used by the last item and to keep the scope of the iterator to a minimum. + _dataSet->erase(lastItemIt); + _dataSet->insert(item); + } + } + + // If the working set ID is valid, remove from + // DiskLoc invalidation map and free from working set. + if (wsidToFree != WorkingSet::INVALID_ID) { + WorkingSetMember* member = _ws->get(wsidToFree); + if (member->hasLoc()) { + _wsidByDiskLoc.erase(member->loc); + } + _ws->free(wsidToFree); + } + } + + void SortStage::sortBuffer() { + if (_limit == 0) { + const WorkingSetComparator& cmp = *_sortKeyComparator; + std::sort(_data.begin(), _data.end(), cmp); + } + else if (_limit == 1) { + // Buffer contains either 0 or 1 item so it is already in a sorted state. + return; + } + else { + // Set already contains items in sorted order, so we simply copy the items + // from the set to the vector. + // Release the memory for the set after the copy. + vector newData(_dataSet->begin(), _dataSet->end()); + _data.swap(newData); + _dataSet.reset(); + } + } + +} // namespace mongo diff --git a/src/mongo/db/exec/sort.h b/src/mongo/db/exec/sort.h new file mode 100644 index 00000000000..91fbb0826f4 --- /dev/null +++ b/src/mongo/db/exec/sort.h @@ -0,0 +1,252 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/index_bounds.h" +#include "mongo/platform/unordered_map.h" + +namespace mongo { + + class BtreeKeyGenerator; + + // Parameters that must be provided to a SortStage + class SortStageParams { + public: + SortStageParams() : limit(0) { } + + // How we're sorting. + BSONObj pattern; + + // The query. Used to create the IndexBounds for the sorting. + BSONObj query; + + // Equal to 0 for no limit. + size_t limit; + }; + + /** + * Maps a WSM value to a BSONObj key that can then be sorted via BSONObjCmp. + */ + class SortStageKeyGenerator { + public: + /** + * 'sortSpec' is the BSONObj in the .sort(...) clause. + * + * 'queryObj' is the BSONObj in the .find(...) clause. For multikey arrays we have to + * ensure that the value we select to sort by is within bounds generated by + * executing 'queryObj' using the virtual index with key pattern 'sortSpec'. + */ + SortStageKeyGenerator(const BSONObj& sortSpec, const BSONObj& queryObj); + + /** + * Returns the key used to sort 'member'. + */ + Status getSortKey(const WorkingSetMember& member, BSONObj* objOut) const; + + /** + * Passed to std::sort and used to sort the keys that are returned from getSortKey. + * + * Returned reference lives as long as 'this'. + */ + const BSONObj& getSortComparator() const { return _comparatorObj; } + + private: + Status getBtreeKey(const BSONObj& memberObj, BSONObj* objOut) const; + + /** + * In order to emulate the existing sort behavior we must make unindexed sort behavior as + * consistent as possible with indexed sort behavior. As such, we must only consider index + * keys that we would encounter if we were answering the query using the sort-providing + * index. + * + * Populates _hasBounds and _bounds. + */ + void getBoundsForSort(const BSONObj& queryObj, const BSONObj& sortObj); + + // The object that we use to call woCompare on our resulting key. Is equal to _rawSortSpec + // unless we have some $meta expressions. Each $meta expression has a default sort order. + BSONObj _comparatorObj; + + // The raw object in .sort() + BSONObj _rawSortSpec; + + // The sort pattern with any non-Btree sort pulled out. + BSONObj _btreeObj; + + // If we're not sorting with a $meta value we can short-cut some work. + bool _sortHasMeta; + + // True if the bounds are valid. + bool _hasBounds; + + // The bounds generated from the query we're sorting. + IndexBounds _bounds; + + // Helper to extract sorting keys from documents. + boost::scoped_ptr _keyGen; + + // Helper to filter keys, ensuring keys generated with _keyGen are within _bounds. + boost::scoped_ptr _boundsChecker; + }; + + /** + * Sorts the input received from the child according to the sort pattern provided. + * + * Preconditions: For each field in 'pattern', all inputs in the child must handle a + * getFieldDotted for that field. + */ + class SortStage : public PlanStage { + public: + SortStage(const SortStageParams& params, WorkingSet* ws, PlanStage* child); + + virtual ~SortStage(); + + virtual bool isEOF(); + virtual StageState work(WorkingSetID* out); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + PlanStageStats* getStats(); + + private: + void getBoundsForSort(const BSONObj& queryObj, const BSONObj& sortObj); + + // + // Query Stage + // + + // Not owned by us. + WorkingSet* _ws; + + // Where we're reading data to sort from. + boost::scoped_ptr _child; + + // The raw sort _pattern as expressed by the user + BSONObj _pattern; + + // The raw query as expressed by the user + BSONObj _query; + + // Equal to 0 for no limit. + size_t _limit; + + // + // Sort key generation + // + boost::scoped_ptr _sortKeyGen; + + // + // Data storage + // + + // Have we sorted our data? If so, we can access _resultIterator. If not, + // we're still populating _data. + bool _sorted; + + // Collection of working set members to sort with their respective sort key. + struct SortableDataItem { + WorkingSetID wsid; + BSONObj sortKey; + // Since we must replicate the behavior of a covered sort as much as possible we use the + // DiskLoc to break sortKey ties. + // See sorta.js. + DiskLoc loc; + }; + + // Comparison object for data buffers (vector and set). + // Items are compared on (sortKey, loc). This is also how the items are + // ordered in the indices. + // Keys are compared using BSONObj::woCompare() with DiskLoc as a tie-breaker. + struct WorkingSetComparator { + explicit WorkingSetComparator(BSONObj p); + + bool operator()(const SortableDataItem& lhs, const SortableDataItem& rhs) const; + + BSONObj pattern; + }; + + /** + * Inserts one item into data buffer (vector or set). + * If limit is exceeded, remove item with lowest key. + */ + void addToBuffer(const SortableDataItem& item); + + /** + * Sorts data buffer. + * Assumes no more items will be added to buffer. + * If data is stored in set, copy set + * contents to vector and clear set. + */ + void sortBuffer(); + + // Comparator for data buffer + // Initialization follows sort key generator + scoped_ptr _sortKeyComparator; + + // The data we buffer and sort. + // _data will contain sorted data when all data is gathered + // and sorted. + // When _limit is greater than 1 and not all data has been gathered from child stage, + // _dataSet is used instead to maintain an ordered set of the incomplete data set. + // When the data set is complete, we copy the items from _dataSet to _data which will + // be used to provide the results of this stage through _resultIterator. + vector _data; + typedef std::set SortableDataItemSet; + scoped_ptr _dataSet; + + // Iterates through _data post-sort returning it. + vector::iterator _resultIterator; + + // We buffer a lot of data and we want to look it up by DiskLoc quickly upon invalidation. + typedef unordered_map DataMap; + DataMap _wsidByDiskLoc; + + // + // Stats + // + + CommonStats _commonStats; + SortStats _specificStats; + + // The usage in bytes of all buffered data that we're sorting. + size_t _memUsage; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/sort_test.cpp b/src/mongo/db/exec/sort_test.cpp new file mode 100644 index 00000000000..c890fb4dd15 --- /dev/null +++ b/src/mongo/db/exec/sort_test.cpp @@ -0,0 +1,241 @@ +/** + * Copyright (C) 2013 mongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/exec/sort.cpp + */ + +#include "mongo/db/exec/sort.h" + +#include "mongo/db/json.h" +#include "mongo/db/exec/mock_stage.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + + TEST(SortStageTest, SortEmptyWorkingSet) { + WorkingSet ws; + + // MockStage will be owned by SortStage. + MockStage* ms = new MockStage(&ws); + SortStageParams params; + SortStage sort(params, &ws, ms); + + // Check initial EOF state. + ASSERT_TRUE(ms->isEOF()); + ASSERT_FALSE(sort.isEOF()); + + // First call to work() initializes sort key generator. + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState state = sort.work(&id); + ASSERT_EQUALS(state, PlanStage::NEED_TIME); + + // Second call to work() sorts data in vector. + state = sort.work(&id); + ASSERT_EQUALS(state, PlanStage::NEED_TIME); + + // Finally we hit EOF. + state = sort.work(&id); + ASSERT_EQUALS(state, PlanStage::IS_EOF); + + ASSERT_TRUE(sort.isEOF()); + } + + /** + * Test function to verify sort stage. + * SortStageParams will be initialized using patternStr, queryStr and limit. + * inputStr represents the input data set in a BSONObj. + * {input: [doc1, doc2, doc3, ...]} + * expectedStr represents the expected sorted data set. + * {output: [docA, docB, docC, ...]} + */ + void testWork(const char* patternStr, const char* queryStr, int limit, + const char* inputStr, const char* expectedStr) { + + // WorkingSet is not owned by stages + // so it's fine to declare + WorkingSet ws; + + // MockStage will be owned by SortStage. + MockStage* ms = new MockStage(&ws); + BSONObj inputObj = fromjson(inputStr); + BSONElement inputElt = inputObj.getField("input"); + ASSERT(inputElt.isABSONObj()); + BSONObjIterator inputIt(inputElt.embeddedObject()); + while (inputIt.more()) { + BSONElement elt = inputIt.next(); + ASSERT(elt.isABSONObj()); + BSONObj obj = elt.embeddedObject(); + + // Insert obj from input array into working set. + WorkingSetMember wsm; + wsm.state = WorkingSetMember::OWNED_OBJ; + wsm.obj = obj; + ms->pushBack(wsm); + } + + // Initialize SortStageParams + // Setting limit to 0 means no limit + SortStageParams params; + params.pattern = fromjson(patternStr); + params.query = fromjson(queryStr); + params.limit = limit; + + SortStage sort(params, &ws, ms); + + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState state = PlanStage::NEED_TIME; + + // Keep working sort stage until data is available. + while (state == PlanStage::NEED_TIME) { + state = sort.work(&id); + } + + // Child's state should be EOF when sort is ready to advance. + ASSERT_TRUE(ms->isEOF()); + + // While there's data to be retrieved, state should be equal to ADVANCED. + // Insert documents into BSON document in this format: + // {output: [docA, docB, docC, ...]} + BSONObjBuilder bob; + BSONArrayBuilder arr(bob.subarrayStart("output")); + while (state == PlanStage::ADVANCED) { + WorkingSetMember* member = ws.get(id); + const BSONObj& obj = member->obj; + arr.append(obj); + state = sort.work(&id); + } + arr.doneFast(); + BSONObj outputObj = bob.obj(); + + // Sort stage should be EOF after data is retrieved. + ASSERT_EQUALS(state, PlanStage::IS_EOF); + ASSERT_TRUE(sort.isEOF()); + + // Finally, we get to compare the sorted results against what we expect. + BSONObj expectedObj = fromjson(expectedStr); + if (outputObj != expectedObj) { + mongoutils::str::stream ss; + // Even though we have the original string representation of the expected output, + // we invoke BSONObj::toString() to get a format consistent with outputObj. + ss << "Unexpected sort result with query=" << queryStr << "; pattern=" << patternStr + << "; limit=" << limit << ":\n" + << "Expected: " << expectedObj.toString() << "\n" + << "Actual: " << outputObj.toString() << "\n"; + FAIL(ss); + } + } + + // + // Limit values + // The server interprets limit values from the user as follows: + // 0: no limit on query results. This is passed along unchanged to the sort stage. + // >0: soft limit. Also unchanged in sort stage. + // <0: hard limit. Absolute value is stored in parsed query and passed to sort stage. + // The sort stage treats both soft and hard limits in the same manner + + // + // Sort without limit + // Implementation should keep all items fetched from child. + // + + TEST(SortStageTest, SortAscending) { + testWork("{a: 1}", "{}", 0, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 1}, {a: 2}, {a: 3}]}"); + } + + TEST(SortStageTest, SortDescending) { + testWork("{a: -1}", "{}", 0, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 3}, {a: 2}, {a: 1}]}"); + } + + TEST(SortStageTest, SortIrrelevantSortKey) { + testWork("{b: 1}", "{}", 0, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 2}, {a: 1}, {a: 3}]}"); + } + + // + // Sorting with limit > 1 + // Implementation should retain top N items + // and discard the rest. + // + + TEST(SortStageTest, SortAscendingWithLimit) { + testWork("{a: 1}", "{}", 2, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 1}, {a: 2}]}"); + } + + TEST(SortStageTest, SortDescendingWithLimit) { + testWork("{a: -1}", "{}", 2, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 3}, {a: 2}]}"); + } + + // + // Sorting with limit > size of data set + // Implementation should retain top N items + // and discard the rest. + // + + TEST(SortStageTest, SortAscendingWithLimitGreaterThanInputSize) { + testWork("{a: 1}", "{}", 10, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 1}, {a: 2}, {a: 3}]}"); + } + + TEST(SortStageTest, SortDescendingWithLimitGreaterThanInputSize) { + testWork("{a: -1}", "{}", 10, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 3}, {a: 2}, {a: 1}]}"); + } + + // + // Sorting with limit 1 + // Implementation should optimize this into a running maximum. + // + + TEST(SortStageTest, SortAscendingWithLimitOfOne) { + testWork("{a: 1}", "{}", 1, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 1}]}"); + } + + TEST(SortStageTest, SortDescendingWithLimitOfOne) { + testWork("{a: -1}", "{}", 1, + "{input: [{a: 2}, {a: 1}, {a: 3}]}", + "{output: [{a: 3}]}"); + } + +} // namespace diff --git a/src/mongo/db/exec/stagedebug_cmd.cpp b/src/mongo/db/exec/stagedebug_cmd.cpp new file mode 100644 index 00000000000..55385fba4ce --- /dev/null +++ b/src/mongo/db/exec/stagedebug_cmd.cpp @@ -0,0 +1,390 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/exec/and_hash.h" +#include "mongo/db/exec/and_sorted.h" +#include "mongo/db/exec/collection_scan.h" +#include "mongo/db/exec/fetch.h" +#include "mongo/db/exec/index_scan.h" +#include "mongo/db/exec/limit.h" +#include "mongo/db/exec/merge_sort.h" +#include "mongo/db/exec/or.h" +#include "mongo/db/exec/skip.h" +#include "mongo/db/exec/sort.h" +#include "mongo/db/exec/text.h" +#include "mongo/db/index/fts_access_method.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/plan_executor.h" + +namespace mongo { + + /** + * A command for manually constructing a query tree and running it. + * + * db.runCommand({stageDebug: rootNode}) + * + * The value of the filter field is a BSONObj that specifies values that fields must have. What + * you'd pass to a matcher. + * + * Leaf Nodes: + * + * node -> {ixscan: {filter: {FILTER}, + * args: {name: "collectionname", indexKeyPattern: kpObj, start: startObj, + * stop: stopObj, endInclusive: true/false, direction: -1/1, + * limit: int}}} + * node -> {cscan: {filter: {filter}, args: {name: "collectionname", direction: -1/1}}} + * TODO: language for text. + * node -> {text: {filter: {filter}, args: {name: "collectionname", search: "searchstr"}}} + * + * Internal Nodes: + * + * node -> {andHash: {filter: {filter}, args: { nodes: [node, node]}}} + * node -> {andSorted: {filter: {filter}, args: { nodes: [node, node]}}} + * node -> {or: {filter: {filter}, args: { dedup:bool, nodes:[node, node]}}} + * node -> {fetch: {filter: {filter}, args: {node: node}}} + * node -> {limit: {args: {node: node, num: posint}}} + * node -> {skip: {args: {node: node, num: posint}}} + * node -> {sort: {args: {node: node, pattern: objWithSortCriterion }}} + * node -> {mergeSort: {args: {nodes: [node, node], pattern: objWithSortCriterion}}} + * node -> {cscan: {filter: {filter}, args: {name: "collectionname" }}} + * + * Forthcoming Nodes: + * + * node -> {dedup: {filter: {filter}, args: {node: node, field: field}}} + * node -> {unwind: {filter: filter}, args: {node: node, field: field}} + */ + class StageDebugCmd : public Command { + public: + StageDebugCmd() : Command("stageDebug") { } + + // Boilerplate for commands + virtual LockType locktype() const { return READ; } + bool slaveOk() const { return true; } + bool slaveOverrideOk() const { return true; } + void help(std::stringstream& h) const { } + + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector* out) { + ActionSet actions; + actions.addAction(ActionType::find); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); + } + + bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, + bool fromRepl) { + + BSONElement argElt = cmdObj["stageDebug"]; + if (argElt.eoo() || !argElt.isABSONObj()) { return false; } + BSONObj argObj = argElt.Obj(); + + OwnedPointerVector exprs; + auto_ptr ws(new WorkingSet()); + + PlanStage* userRoot = parseQuery(dbname, argObj, ws.get(), &exprs); + uassert(16911, "Couldn't parse plan from " + argObj.toString(), NULL != userRoot); + + // Add a fetch at the top for the user so we can get obj back for sure. + // TODO: Do we want to do this for the user? I think so. + PlanStage* rootFetch = new FetchStage(ws.get(), userRoot, NULL); + + PlanExecutor runner(ws.release(), rootFetch); + + BSONArrayBuilder resultBuilder(result.subarrayStart("results")); + + for (BSONObj obj; Runner::RUNNER_ADVANCED == runner.getNext(&obj, NULL); ) { + resultBuilder.append(obj); + } + + resultBuilder.done(); + return true; + } + + PlanStage* parseQuery(const string& dbname, BSONObj obj, WorkingSet* workingSet, + OwnedPointerVector* exprs) { + BSONElement firstElt = obj.firstElement(); + if (!firstElt.isABSONObj()) { return NULL; } + BSONObj paramObj = firstElt.Obj(); + + MatchExpression* matcher = NULL; + BSONObj nodeArgs; + + // Every node has these two fields. + const string filterTag = "filter"; + const string argsTag = "args"; + + BSONObjIterator it(paramObj); + while (it.more()) { + BSONElement e = it.next(); + if (!e.isABSONObj()) { return NULL; } + BSONObj argObj = e.Obj(); + if (filterTag == e.fieldName()) { + StatusWithMatchExpression swme = MatchExpressionParser::parse(argObj); + if (!swme.isOK()) { return NULL; } + // exprs is what will wind up deleting this. + matcher = swme.getValue(); + verify(NULL != matcher); + exprs->mutableVector().push_back(matcher); + } + else if (argsTag == e.fieldName()) { + nodeArgs = argObj; + } + else { + uasserted(16910, "Unknown fieldname " + string(e.fieldName()) + + " in query node " + obj.toString()); + return NULL; + } + } + + string nodeName = firstElt.fieldName(); + + if ("ixscan" == nodeName) { + + Database* db = cc().database(); + Collection* collection = db->getCollection( dbname + "." + nodeArgs["name"].String() ); + uassert(16913, "Can't find collection " + nodeArgs["name"].String(), collection); + + IndexDescriptor* desc = + collection->getIndexCatalog()->findIndexByKeyPattern(nodeArgs["keyPattern"].Obj()); + uassert(16890, "Can't find index: " + nodeArgs["keyPattern"].Obj().toString(), desc ); + + IndexScanParams params; + params.descriptor = desc; + params.bounds.isSimpleRange = true; + params.bounds.startKey = nodeArgs["startKey"].Obj(); + params.bounds.endKey = nodeArgs["endKey"].Obj(); + params.bounds.endKeyInclusive = nodeArgs["endKeyInclusive"].Bool(); + params.direction = nodeArgs["direction"].numberInt(); + + return new IndexScan(params, workingSet, matcher); + } + else if ("andHash" == nodeName) { + uassert(16921, "Nodes argument must be provided to AND", + nodeArgs["nodes"].isABSONObj()); + + auto_ptr andStage(new AndHashStage(workingSet, matcher)); + + int nodesAdded = 0; + BSONObjIterator it(nodeArgs["nodes"].Obj()); + while (it.more()) { + BSONElement e = it.next(); + uassert(16922, "node of AND isn't an obj?: " + e.toString(), + e.isABSONObj()); + + PlanStage* subNode = parseQuery(dbname, e.Obj(), workingSet, exprs); + uassert(16923, "Can't parse sub-node of AND: " + e.Obj().toString(), + NULL != subNode); + // takes ownership + andStage->addChild(subNode); + ++nodesAdded; + } + + uassert(16927, "AND requires more than one child", nodesAdded >= 2); + + return andStage.release(); + } + else if ("andSorted" == nodeName) { + uassert(16924, "Nodes argument must be provided to AND", + nodeArgs["nodes"].isABSONObj()); + + auto_ptr andStage(new AndSortedStage(workingSet, + matcher)); + + int nodesAdded = 0; + BSONObjIterator it(nodeArgs["nodes"].Obj()); + while (it.more()) { + BSONElement e = it.next(); + uassert(16925, "node of AND isn't an obj?: " + e.toString(), + e.isABSONObj()); + + PlanStage* subNode = parseQuery(dbname, e.Obj(), workingSet, exprs); + uassert(16926, "Can't parse sub-node of AND: " + e.Obj().toString(), + NULL != subNode); + // takes ownership + andStage->addChild(subNode); + ++nodesAdded; + } + + uassert(16928, "AND requires more than one child", nodesAdded >= 2); + + return andStage.release(); + } + else if ("or" == nodeName) { + uassert(16934, "Nodes argument must be provided to AND", + nodeArgs["nodes"].isABSONObj()); + uassert(16935, "Dedup argument must be provided to OR", + !nodeArgs["dedup"].eoo()); + BSONObjIterator it(nodeArgs["nodes"].Obj()); + auto_ptr orStage(new OrStage(workingSet, nodeArgs["dedup"].Bool(), + matcher)); + while (it.more()) { + BSONElement e = it.next(); + if (!e.isABSONObj()) { return NULL; } + PlanStage* subNode = parseQuery(dbname, e.Obj(), workingSet, exprs); + uassert(16936, "Can't parse sub-node of OR: " + e.Obj().toString(), + NULL != subNode); + // takes ownership + orStage->addChild(subNode); + } + + return orStage.release(); + } + else if ("fetch" == nodeName) { + uassert(16929, "Node argument must be provided to fetch", + nodeArgs["node"].isABSONObj()); + PlanStage* subNode = parseQuery(dbname, nodeArgs["node"].Obj(), workingSet, exprs); + return new FetchStage(workingSet, subNode, matcher); + } + else if ("limit" == nodeName) { + uassert(16937, "Limit stage doesn't have a filter (put it on the child)", + NULL == matcher); + uassert(16930, "Node argument must be provided to limit", + nodeArgs["node"].isABSONObj()); + uassert(16931, "Num argument must be provided to limit", + nodeArgs["num"].isNumber()); + PlanStage* subNode = parseQuery(dbname, nodeArgs["node"].Obj(), workingSet, exprs); + return new LimitStage(nodeArgs["num"].numberInt(), workingSet, subNode); + } + else if ("skip" == nodeName) { + uassert(16938, "Skip stage doesn't have a filter (put it on the child)", + NULL == matcher); + uassert(16932, "Node argument must be provided to skip", + nodeArgs["node"].isABSONObj()); + uassert(16933, "Num argument must be provided to skip", + nodeArgs["num"].isNumber()); + PlanStage* subNode = parseQuery(dbname, nodeArgs["node"].Obj(), workingSet, exprs); + return new SkipStage(nodeArgs["num"].numberInt(), workingSet, subNode); + } + else if ("cscan" == nodeName) { + CollectionScanParams params; + + // What collection? + params.ns = dbname + "." + nodeArgs["name"].String(); + uassert(16962, "Can't find collection " + nodeArgs["name"].String(), + NULL != cc().database()->getCollection(params.ns)); + + // What direction? + uassert(16963, "Direction argument must be specified and be a number", + nodeArgs["direction"].isNumber()); + if (1 == nodeArgs["direction"].numberInt()) { + params.direction = CollectionScanParams::FORWARD; + } + else { + params.direction = CollectionScanParams::BACKWARD; + } + + return new CollectionScan(params, workingSet, matcher); + } + // sort is disabled for now. +#if 0 + else if ("sort" == nodeName) { + uassert(16969, "Node argument must be provided to sort", + nodeArgs["node"].isABSONObj()); + uassert(16970, "Pattern argument must be provided to sort", + nodeArgs["pattern"].isABSONObj()); + PlanStage* subNode = parseQuery(dbname, nodeArgs["node"].Obj(), workingSet, exprs); + SortStageParams params; + params.pattern = nodeArgs["pattern"].Obj(); + return new SortStage(params, workingSet, subNode); + } +#endif + else if ("mergeSort" == nodeName) { + uassert(16971, "Nodes argument must be provided to sort", + nodeArgs["nodes"].isABSONObj()); + uassert(16972, "Pattern argument must be provided to sort", + nodeArgs["pattern"].isABSONObj()); + + MergeSortStageParams params; + params.pattern = nodeArgs["pattern"].Obj(); + // Dedup is true by default. + + auto_ptr mergeStage(new MergeSortStage(params, workingSet)); + + BSONObjIterator it(nodeArgs["nodes"].Obj()); + while (it.more()) { + BSONElement e = it.next(); + uassert(16973, "node of mergeSort isn't an obj?: " + e.toString(), + e.isABSONObj()); + + PlanStage* subNode = parseQuery(dbname, e.Obj(), workingSet, exprs); + uassert(16974, "Can't parse sub-node of mergeSort: " + e.Obj().toString(), + NULL != subNode); + // takes ownership + mergeStage->addChild(subNode); + } + return mergeStage.release(); + } + else if ("text" == nodeName) { + string ns = nodeArgs["name"].String(); + string search = nodeArgs["search"].String(); + Database* db = cc().database(); + Collection* collection = db->getCollection( ns ); + uassert(17193, "Can't find namespace " + ns, collection); + vector idxMatches; + collection->getIndexCatalog()->findIndexByType("text", idxMatches); + uassert(17194, "Expected exactly one text index", idxMatches.size() == 1); + + IndexDescriptor* index = idxMatches[0]; + FTSAccessMethod* fam = + dynamic_cast( collection->getIndexCatalog()->getIndex( index ) ); + TextStageParams params(fam->getSpec()); + params.ns = ns; + params.index = index; + + // TODO: Deal with non-empty filters. This is a hack to put in covering information + // that can only be checked for equality. We ignore this now. + Status s = fam->getSpec().getIndexPrefix(BSONObj(), ¶ms.indexPrefix); + if (!s.isOK()) { + // errmsg = s.toString(); + return NULL; + } + + params.spec = fam->getSpec(); + + if (!params.query.parse(search, + fam->getSpec().defaultLanguage().str().c_str(), + fam->getSpec().getTextIndexVersion()).isOK()) { + return NULL; + } + + return new TextStage(params, workingSet, matcher); + } + else { + return NULL; + } + } + } stageDebugCmd; + +} // namespace mongo diff --git a/src/mongo/db/exec/text.cpp b/src/mongo/db/exec/text.cpp new file mode 100644 index 00000000000..7c075587909 --- /dev/null +++ b/src/mongo/db/exec/text.cpp @@ -0,0 +1,362 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/text.h" + +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/exec/working_set_computed_data.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/query/internal_plans.h" + +namespace mongo { + + TextStage::TextStage(const TextStageParams& params, + WorkingSet* ws, + const MatchExpression* filter) + : _params(params), + _ftsMatcher(params.query, params.spec), + _ws(ws), + _filter(filter), + _internalState(INIT_SCANS), + _currentIndexScanner(0) { + + _scoreIterator = _scores.end(); + } + + TextStage::~TextStage() { } + + bool TextStage::isEOF() { + return _internalState == DONE; + } + + PlanStage::StageState TextStage::work(WorkingSetID* out) { + ++_commonStats.works; + + if (isEOF()) { return PlanStage::IS_EOF; } + invariant(_internalState != DONE); + + PlanStage::StageState stageState = PlanStage::IS_EOF; + + switch (_internalState) { + case INIT_SCANS: + stageState = initScans(out); + break; + case READING_TERMS: + stageState = readFromSubScanners(out); + break; + case RETURNING_RESULTS: + stageState = returnResults(out); + break; + case DONE: + // Handled above. + break; + } + + // Increment common stats counters that are specific to the return value of work(). + switch (stageState) { + case PlanStage::ADVANCED: + ++_commonStats.advanced; + break; + case PlanStage::NEED_TIME: + ++_commonStats.needTime; + break; + case PlanStage::NEED_FETCH: + ++_commonStats.needFetch; + break; + default: + break; + } + + return stageState; + } + + void TextStage::prepareToYield() { + ++_commonStats.yields; + + for (size_t i = 0; i < _scanners.size(); ++i) { + _scanners.mutableVector()[i]->prepareToYield(); + } + } + + void TextStage::recoverFromYield() { + ++_commonStats.unyields; + + for (size_t i = 0; i < _scanners.size(); ++i) { + _scanners.mutableVector()[i]->recoverFromYield(); + } + } + + void TextStage::invalidate(const DiskLoc& dl, InvalidationType type) { + ++_commonStats.invalidates; + + // Propagate invalidate to children. + for (size_t i = 0; i < _scanners.size(); ++i) { + _scanners.mutableVector()[i]->invalidate(dl, type); + } + + // We store the score keyed by DiskLoc. We have to toss out our state when the DiskLoc + // changes. + // TODO: If we're RETURNING_RESULTS we could somehow buffer the object. + ScoreMap::iterator scoreIt = _scores.find(dl); + if (scoreIt != _scores.end()) { + if (scoreIt == _scoreIterator) { + _scoreIterator++; + } + _scores.erase(scoreIt); + } + } + + PlanStageStats* TextStage::getStats() { + _commonStats.isEOF = isEOF(); + auto_ptr ret(new PlanStageStats(_commonStats, STAGE_TEXT)); + ret->specific.reset(new TextStats(_specificStats)); + return ret.release(); + } + + PlanStage::StageState TextStage::initScans(WorkingSetID* out) { + invariant(0 == _scanners.size()); + + _specificStats.parsedTextQuery = _params.query.toBSON(); + + // Get all the index scans for each term in our query. + for (size_t i = 0; i < _params.query.getTerms().size(); i++) { + const string& term = _params.query.getTerms()[i]; + IndexScanParams params; + params.bounds.startKey = FTSIndexFormat::getIndexKey(MAX_WEIGHT, + term, + _params.indexPrefix, + _params.spec.getTextIndexVersion()); + params.bounds.endKey = FTSIndexFormat::getIndexKey(0, + term, + _params.indexPrefix, + _params.spec.getTextIndexVersion()); + params.bounds.endKeyInclusive = true; + params.bounds.isSimpleRange = true; + params.descriptor = _params.index; + params.direction = -1; + _scanners.mutableVector().push_back(new IndexScan(params, _ws, NULL)); + } + + // If we have no terms we go right to EOF. + if (0 == _scanners.size()) { + _internalState = DONE; + return PlanStage::IS_EOF; + } + + // Transition to the next state. + _internalState = READING_TERMS; + return PlanStage::NEED_TIME; + } + + PlanStage::StageState TextStage::readFromSubScanners(WorkingSetID* out) { + // This should be checked before we get here. + invariant(_currentIndexScanner < _scanners.size()); + + // Read the next result from our current scanner. + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState childState = _scanners.vector()[_currentIndexScanner]->work(&id); + + if (PlanStage::ADVANCED == childState) { + WorkingSetMember* wsm = _ws->get(id); + invariant(1 == wsm->keyData.size()); + invariant(wsm->hasLoc()); + IndexKeyDatum& keyDatum = wsm->keyData.back(); + addTerm(keyDatum.keyData, wsm->loc); + _ws->free(id); + return PlanStage::NEED_TIME; + } + else if (PlanStage::IS_EOF == childState) { + // Done with this scan. + ++_currentIndexScanner; + + if (_currentIndexScanner < _scanners.size()) { + // We have another scan to read from. + return PlanStage::NEED_TIME; + } + + // If we're here we are done reading results. Move to the next state. + _scoreIterator = _scores.begin(); + _internalState = RETURNING_RESULTS; + + // Don't need to keep these around. + _scanners.clear(); + return PlanStage::NEED_TIME; + } + else { + if (PlanStage::FAILURE == childState) { + // Propagate failure from below. + *out = id; + // If a stage fails, it may create a status WSM to indicate why it + // failed, in which case 'id' is valid. If ID is invalid, we + // create our own error message. + if (WorkingSet::INVALID_ID == id) { + mongoutils::str::stream ss; + ss << "text stage failed to read in results from child"; + Status status(ErrorCodes::InternalError, ss); + *out = WorkingSetCommon::allocateStatusMember( _ws, status); + } + } + return childState; + } + } + + PlanStage::StageState TextStage::returnResults(WorkingSetID* out) { + if (_scoreIterator == _scores.end()) { + _internalState = DONE; + return PlanStage::IS_EOF; + } + + // Filter for phrases and negative terms, score and truncate. + DiskLoc loc = _scoreIterator->first; + double score = _scoreIterator->second; + _scoreIterator++; + + // Ignore non-matched documents. + if (score < 0) { + return PlanStage::NEED_TIME; + } + + // Filter for phrases and negated terms + if (_params.query.hasNonTermPieces()) { + if (!_ftsMatcher.matchesNonTerm(loc.obj())) { + return PlanStage::NEED_TIME; + } + } + + *out = _ws->allocate(); + WorkingSetMember* member = _ws->get(*out); + member->loc = loc; + member->obj = member->loc.obj(); + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + member->addComputed(new TextScoreComputedData(score)); + return PlanStage::ADVANCED; + } + + class TextMatchableDocument : public MatchableDocument { + public: + TextMatchableDocument(const BSONObj& keyPattern, + const BSONObj& key, + DiskLoc loc, + bool *fetched) + : _keyPattern(keyPattern), + _key(key), + _loc(loc), + _fetched(fetched) { } + + BSONObj toBSON() const { + *_fetched = true; + return _loc.obj(); + } + + virtual ElementIterator* allocateIterator(const ElementPath* path) const { + BSONObjIterator keyPatternIt(_keyPattern); + BSONObjIterator keyDataIt(_key); + + // Look in the key. + while (keyPatternIt.more()) { + BSONElement keyPatternElt = keyPatternIt.next(); + verify(keyDataIt.more()); + BSONElement keyDataElt = keyDataIt.next(); + + if (path->fieldRef().equalsDottedField(keyPatternElt.fieldName())) { + if (Array == keyDataElt.type()) { + return new SimpleArrayElementIterator(keyDataElt, true); + } + else { + return new SingleElementElementIterator(keyDataElt); + } + } + } + + // All else fails, fetch. + *_fetched = true; + return new BSONElementIterator(path, _loc.obj()); + } + + virtual void releaseIterator( ElementIterator* iterator ) const { + delete iterator; + } + + private: + BSONObj _keyPattern; + BSONObj _key; + DiskLoc _loc; + bool* _fetched; + }; + + void TextStage::addTerm(const BSONObj& key, const DiskLoc& loc) { + double *documentAggregateScore = &_scores[loc]; + + ++_specificStats.keysExamined; + + // Locate score within possibly compound key: {prefix,term,score,suffix}. + BSONObjIterator keyIt(key); + for (unsigned i = 0; i < _params.spec.numExtraBefore(); i++) { + keyIt.next(); + } + + keyIt.next(); // Skip past 'term'. + + BSONElement scoreElement = keyIt.next(); + double documentTermScore = scoreElement.number(); + + // Handle filtering. + if (*documentAggregateScore < 0) { + // We have already rejected this document. + return; + } + + if (*documentAggregateScore == 0) { + if (_filter) { + // We have not seen this document before and need to apply a filter. + bool fetched = false; + TextMatchableDocument tdoc(_params.index->keyPattern(), key, loc, &fetched); + + if (!_filter->matches(&tdoc)) { + // We had to fetch but we're not going to return it. + if (fetched) { + ++_specificStats.fetches; + } + *documentAggregateScore = -1; + return; + } + } + else { + // If we're here, we're going to return the doc, and we do a fetch later. + ++_specificStats.fetches; + } + } + + // Aggregate relevance score, term keys. + *documentAggregateScore += documentTermScore; + } + +} // namespace mongo diff --git a/src/mongo/db/exec/text.h b/src/mongo/db/exec/text.h new file mode 100644 index 00000000000..932e0de2e7c --- /dev/null +++ b/src/mongo/db/exec/text.h @@ -0,0 +1,171 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/fts/fts_index_format.h" +#include "mongo/db/fts/fts_matcher.h" +#include "mongo/db/fts/fts_query.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/db/fts/fts_util.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/platform/unordered_map.h" + +#include +#include +#include + +namespace mongo { + + using fts::FTSIndexFormat; + using fts::FTSMatcher; + using fts::FTSQuery; + using fts::FTSSpec; + using fts::MAX_WEIGHT; + + struct TextStageParams { + TextStageParams(const FTSSpec& s) : spec(s) {} + + // Namespace. + string ns; + + // Text index descriptor. IndexCatalog owns this. + IndexDescriptor* index; + + // Index spec. + FTSSpec spec; + + // Index keys that precede the "text" index key. + BSONObj indexPrefix; + + // The text query. + FTSQuery query; + }; + + /** + * Implements a blocking stage that returns text search results. + * + * Prerequisites: None; is a leaf node. + * Output type: LOC_AND_OBJ_UNOWNED. + */ + class TextStage : public PlanStage { + public: + /** + * The text stage has a few 'states' it transitions between. + */ + enum State { + // 1. Initialize the index scans we use to retrieve term/score info. + INIT_SCANS, + + // 2. Read the terms/scores from the text index. + READING_TERMS, + + // 3. Return results to our parent. + RETURNING_RESULTS, + + // 4. Done. + DONE, + }; + + TextStage(const TextStageParams& params, WorkingSet* ws, const MatchExpression* filter); + + virtual ~TextStage(); + + virtual StageState work(WorkingSetID* out); + virtual bool isEOF(); + + virtual void prepareToYield(); + virtual void recoverFromYield(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + PlanStageStats* getStats(); + + private: + /** + * Initializes sub-scanners. + */ + StageState initScans(WorkingSetID* out); + + /** + * Helper for buffering results array. Returns NEED_TIME (if any results were produced), + * IS_EOF, or FAILURE. + */ + StageState readFromSubScanners(WorkingSetID* out); + + /** + * Helper called from readFromSubScanners to update aggregate score with a new-found (term, + * score) pair for this document. Also rejects documents that don't match this stage's + * filter. + */ + void addTerm(const BSONObj& key, const DiskLoc& loc); + + /** + * Possibly return a result. FYI, this may perform a fetch directly if it is needed to + * evaluate all filters. + */ + StageState returnResults(WorkingSetID* out); + + // Parameters of this text stage. + TextStageParams _params; + + // Text-specific phrase and negated term matcher. + FTSMatcher _ftsMatcher; + + // Working set. Not owned by us. + WorkingSet* _ws; + + // Filter. Not owned by us. + const MatchExpression* _filter; + + // Stats. + CommonStats _commonStats; + TextStats _specificStats; + + // What state are we in? See the State enum above. + State _internalState; + + // Used in INIT_SCANS and READING_TERMS. The index scans we're using to retrieve text + // terms. + OwnedPointerVector _scanners; + + // Which _scanners are we currently reading from? + size_t _currentIndexScanner; + + // Temporary score data filled out by sub-scans. Used in READING_TERMS and + // RETURNING_RESULTS. + // Maps from diskloc -> aggregate score for doc. + typedef unordered_map ScoreMap; + ScoreMap _scores; + ScoreMap::const_iterator _scoreIterator; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/working_set.cpp b/src/mongo/db/exec/working_set.cpp new file mode 100644 index 00000000000..ec9eaca2453 --- /dev/null +++ b/src/mongo/db/exec/working_set.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/working_set.h" + +#include "mongo/db/index/index_descriptor.h" + +namespace mongo { + + WorkingSet::MemberHolder::MemberHolder() : member(NULL) { } + WorkingSet::MemberHolder::~MemberHolder() {} + + WorkingSet::WorkingSet() : _freeList(INVALID_ID) { } + + WorkingSet::~WorkingSet() { + for (size_t i = 0; i < _data.size(); i++) { + delete _data[i].member; + } + } + + WorkingSetID WorkingSet::allocate() { + if (_freeList == INVALID_ID) { + // The free list is empty so we need to make a single new WSM to return. This relies on + // vector::resize being amortized O(1) for efficient allocation. Note that the free list + // remains empty until something is returned by a call to free(). + WorkingSetID id = _data.size(); + _data.resize(_data.size() + 1); + _data.back().nextFreeOrSelf = id; + _data.back().member = new WorkingSetMember(); + return id; + } + + // Pop the head off the free list and return it. + WorkingSetID id = _freeList; + _freeList = _data[id].nextFreeOrSelf; + _data[id].nextFreeOrSelf = id; // set to self to mark as in-use + return id; + } + + void WorkingSet::free(const WorkingSetID& i) { + MemberHolder& holder = _data[i]; + verify(i < _data.size()); // ID has been allocated. + verify(holder.nextFreeOrSelf == i); // ID currently in use. + + // Free resources and push this WSM to the head of the freelist. + holder.member->clear(); + holder.nextFreeOrSelf = _freeList; + _freeList = i; + } + + void WorkingSet::flagForReview(const WorkingSetID& i) { + WorkingSetMember* member = get(i); + verify(WorkingSetMember::OWNED_OBJ == member->state); + _flagged.insert(i); + } + + const unordered_set& WorkingSet::getFlagged() const { + return _flagged; + } + + bool WorkingSet::isFlagged(WorkingSetID id) const { + invariant(id < _data.size()); + return _flagged.end() != _flagged.find(id); + } + + WorkingSetMember::WorkingSetMember() : state(WorkingSetMember::INVALID) { } + + WorkingSetMember::~WorkingSetMember() { } + + void WorkingSetMember::clear() { + for (size_t i = 0; i < WSM_COMPUTED_NUM_TYPES; i++) { + _computed[i].reset(); + } + + keyData.clear(); + obj = BSONObj(); + state = WorkingSetMember::INVALID; + } + + bool WorkingSetMember::hasLoc() const { + return state == LOC_AND_IDX || state == LOC_AND_UNOWNED_OBJ; + } + + bool WorkingSetMember::hasObj() const { + return hasOwnedObj() || hasUnownedObj(); + } + + bool WorkingSetMember::hasOwnedObj() const { + return state == OWNED_OBJ; + } + + bool WorkingSetMember::hasUnownedObj() const { + return state == LOC_AND_UNOWNED_OBJ; + } + + bool WorkingSetMember::hasComputed(const WorkingSetComputedDataType type) const { + return _computed[type]; + } + + const WorkingSetComputedData* WorkingSetMember::getComputed(const WorkingSetComputedDataType type) const { + verify(_computed[type]); + return _computed[type].get(); + } + + void WorkingSetMember::addComputed(WorkingSetComputedData* data) { + verify(!hasComputed(data->type())); + _computed[data->type()].reset(data); + } + + bool WorkingSetMember::getFieldDotted(const string& field, BSONElement* out) const { + // If our state is such that we have an object, use it. + if (hasObj()) { + *out = obj.getFieldDotted(field); + return true; + } + + // Our state should be such that we have index data/are covered. + for (size_t i = 0; i < keyData.size(); ++i) { + BSONObjIterator keyPatternIt(keyData[i].indexKeyPattern); + BSONObjIterator keyDataIt(keyData[i].keyData); + + while (keyPatternIt.more()) { + BSONElement keyPatternElt = keyPatternIt.next(); + verify(keyDataIt.more()); + BSONElement keyDataElt = keyDataIt.next(); + + if (field == keyPatternElt.fieldName()) { + *out = keyDataElt; + return true; + } + } + } + + return false; + } + + size_t WorkingSetMember::getMemUsage() const { + size_t memUsage = 0; + + if (hasLoc()) { + memUsage += sizeof(DiskLoc); + } + + // XXX: Unowned objects count towards current size. + // See SERVER-12579 + if (hasObj()) { + memUsage += obj.objsize(); + } + + for (size_t i = 0; i < keyData.size(); ++i) { + const IndexKeyDatum& keyDatum = keyData[i]; + memUsage += keyDatum.keyData.objsize(); + } + + return memUsage; + } + +} // namespace mongo diff --git a/src/mongo/db/exec/working_set.h b/src/mongo/db/exec/working_set.h new file mode 100644 index 00000000000..b71e4f01931 --- /dev/null +++ b/src/mongo/db/exec/working_set.h @@ -0,0 +1,260 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/unordered_map.h" + +namespace mongo { + + class WorkingSetMember; + + typedef size_t WorkingSetID; + + /** + * All data in use by a query. Data is passed through the stage tree by referencing the ID of + * an element of the working set. Stages can add elements to the working set, delete elements + * from the working set, or mutate elements in the working set. + * + * Concurrency Notes: + * flagForReview() can only be called with a write lock covering the collection this WorkingSet + * is for. All other methods should only be called by the thread owning this WorkingSet while + * holding the read lock covering the collection. + */ + class WorkingSet { + MONGO_DISALLOW_COPYING(WorkingSet); + public: + static const WorkingSetID INVALID_ID = WorkingSetID(-1); + + WorkingSet(); + ~WorkingSet(); + + /** + * Allocate a new query result and return the ID used to get and free it. + */ + WorkingSetID allocate(); + + /** + * Get the i-th mutable query result. The pointer will be valid for this id until freed. + * Do not delete the returned pointer as the WorkingSet retains ownership. Call free() to + * release it. + */ + WorkingSetMember* get(const WorkingSetID& i) const { + dassert(i < _data.size()); // ID has been allocated. + dassert(_data[i].nextFreeOrSelf == i); // ID currently in use. + return _data[i].member; + } + + /** + * Deallocate the i-th query result and release its resources. + */ + void free(const WorkingSetID& i); + + /** + * The DiskLoc in WSM 'i' was invalidated while being processed. Any predicates over the + * WSM could not be fully evaluated, so the WSM may or may not satisfy them. As such, if we + * wish to output the WSM, we must do some clean-up work later. Adds the WSM with id 'i' to + * the list of flagged WSIDs. + * + * The WSM must be in the state OWNED_OBJ. + */ + void flagForReview(const WorkingSetID& i); + + /** + * Return true if the provided ID is flagged. + */ + bool isFlagged(WorkingSetID id) const; + + /** + * Return the set of all WSIDs passed to flagForReview. + */ + const unordered_set& getFlagged() const; + + private: + struct MemberHolder { + MemberHolder(); + ~MemberHolder(); + + // Free list link if freed. Points to self if in use. + WorkingSetID nextFreeOrSelf; + + // Owning pointer + WorkingSetMember* member; + }; + + // All WorkingSetIDs are indexes into this, except for INVALID_ID. + // Elements are added to _freeList rather than removed when freed. + vector _data; + + // Index into _data, forming a linked-list using MemberHolder::nextFreeOrSelf as the next + // link. INVALID_ID is the list terminator since 0 is a valid index. + // If _freeList == INVALID_ID, the free list is empty and all elements in _data are in use. + WorkingSetID _freeList; + + // An insert-only set of WorkingSetIDs that have been flagged for review. + unordered_set _flagged; + }; + + /** + * The key data extracted from an index. Keeps track of both the key (currently a BSONObj) and + * the index that provided the key. The index key pattern is required to correctly interpret + * the key. + */ + struct IndexKeyDatum { + IndexKeyDatum(const BSONObj& keyPattern, const BSONObj& key) : indexKeyPattern(keyPattern), + keyData(key) { } + + // This is not owned and points into the IndexDescriptor's data. + BSONObj indexKeyPattern; + + // This is the BSONObj for the key that we put into the index. Owned by us. + BSONObj keyData; + }; + + /** + * What types of computed data can we have? + */ + enum WorkingSetComputedDataType { + // What's the score of the document retrieved from a $text query? + WSM_COMPUTED_TEXT_SCORE = 0, + + // What's the distance from a geoNear query point to the document? + WSM_COMPUTED_GEO_DISTANCE = 1, + + // The index key used to retrieve the document, for $returnKey query option. + WSM_INDEX_KEY = 2, + + // What point (of several possible points) was used to compute the distance to the document + // via geoNear? + WSM_GEO_NEAR_POINT = 3, + + // Must be last. + WSM_COMPUTED_NUM_TYPES, + }; + + /** + * Data that is a computed function of a WSM. + */ + class WorkingSetComputedData { + MONGO_DISALLOW_COPYING(WorkingSetComputedData); + public: + WorkingSetComputedData(const WorkingSetComputedDataType type) : _type(type) { } + virtual ~WorkingSetComputedData() { } + + WorkingSetComputedDataType type() const { return _type; } + + virtual WorkingSetComputedData* clone() const = 0; + + private: + WorkingSetComputedDataType _type; + }; + + /** + * The type of the data passed between query stages. In particular: + * + * Index scan stages return a WorkingSetMember in the LOC_AND_IDX state. + * + * Collection scan stages the LOC_AND_UNOWNED_OBJ state. + * + * A WorkingSetMember may have any of the data above. + */ + class WorkingSetMember { + MONGO_DISALLOW_COPYING(WorkingSetMember); + public: + WorkingSetMember(); + ~WorkingSetMember(); + + /** + * Reset to an "empty" state. + */ + void clear(); + + enum MemberState { + // Initial state. + INVALID, + + // Data is from 1 or more indices. + LOC_AND_IDX, + + // Data is from a collection scan, or data is from an index scan and was fetched. + LOC_AND_UNOWNED_OBJ, + + // DiskLoc has been invalidated, or the obj doesn't correspond to an on-disk document + // anymore (e.g. is a computed expression). + OWNED_OBJ, + }; + + // + // Core attributes + // + + DiskLoc loc; + BSONObj obj; + vector keyData; + MemberState state; + + bool hasLoc() const; + bool hasObj() const; + bool hasOwnedObj() const; + bool hasUnownedObj() const; + + // + // Computed data + // + + bool hasComputed(const WorkingSetComputedDataType type) const; + const WorkingSetComputedData* getComputed(const WorkingSetComputedDataType type) const; + void addComputed(WorkingSetComputedData* data); + + /** + * getFieldDotted uses its state (obj or index data) to produce the field with the provided + * name. + * + * Returns true if there is the element is in an index key or in an (owned or unowned) + * object. *out is set to the element if so. + * + * Returns false otherwise. Returning false indicates a query planning error. + */ + bool getFieldDotted(const string& field, BSONElement* out) const; + + /** + * Returns expected memory usage of working set member. + */ + size_t getMemUsage() const; + + private: + boost::scoped_ptr _computed[WSM_COMPUTED_NUM_TYPES]; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/working_set_common.cpp b/src/mongo/db/exec/working_set_common.cpp new file mode 100644 index 00000000000..8b9e07a974c --- /dev/null +++ b/src/mongo/db/exec/working_set_common.cpp @@ -0,0 +1,123 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/working_set.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/pdfile.h" + +namespace mongo { + + // static + bool WorkingSetCommon::fetchAndInvalidateLoc(WorkingSetMember* member) { + // Already in our desired state. + if (member->state == WorkingSetMember::OWNED_OBJ) { return true; } + + // We can't do anything without a DiskLoc. + if (!member->hasLoc()) { return false; } + + // Do the fetch, invalidate the DL. + member->obj = member->loc.obj().getOwned(); + + member->state = WorkingSetMember::OWNED_OBJ; + member->loc = DiskLoc(); + return true; + } + + // static + void WorkingSetCommon::initFrom(WorkingSetMember* dest, const WorkingSetMember& src) { + dest->loc = src.loc; + dest->obj = src.obj; + dest->keyData = src.keyData; + dest->state = src.state; + + // Merge computed data. + typedef WorkingSetComputedDataType WSCD; + for (WSCD i = WSCD(0); i < WSM_COMPUTED_NUM_TYPES; i = WSCD(i + 1)) { + if (src.hasComputed(i)) { + dest->addComputed(src.getComputed(i)->clone()); + } + } + } + + // static + WorkingSetID WorkingSetCommon::allocateStatusMember(WorkingSet* ws, const Status& status) { + invariant(ws); + + BSONObjBuilder bob; + bob.append("ok", status.isOK() ? 1.0 : 0.0); + bob.append("code", status.code()); + bob.append("errmsg", status.reason()); + + WorkingSetID wsid = ws->allocate(); + WorkingSetMember* member = ws->get(wsid); + member->state = WorkingSetMember::OWNED_OBJ; + member->obj = bob.obj(); + + return wsid; + } + + // static + bool WorkingSetCommon::isValidStatusMemberObject(const BSONObj& obj) { + return obj.nFields() == 3 && + obj.hasField("ok") && + obj.hasField("code") && + obj.hasField("errmsg"); + } + + // static + void WorkingSetCommon::getStatusMemberObject(const WorkingSet& ws, WorkingSetID wsid, + BSONObj* objOut) { + invariant(objOut); + + // Validate ID and working set member. + if (WorkingSet::INVALID_ID == wsid) { + return; + } + WorkingSetMember* member = ws.get(wsid); + if (!member->hasOwnedObj()) { + return; + } + BSONObj obj = member->obj; + if (!isValidStatusMemberObject(obj)) { + return; + } + *objOut = member->obj; + } + + // static + std::string WorkingSetCommon::toStatusString(const BSONObj& obj) { + if (!isValidStatusMemberObject(obj)) { + Status unknownStatus(ErrorCodes::UnknownError, "no details available"); + return unknownStatus.toString(); + } + Status status(ErrorCodes::fromInt(obj.getIntField("code")), + obj.getStringField("errmsg")); + return status.toString(); + } + +} // namespace mongo diff --git a/src/mongo/db/exec/working_set_common.h b/src/mongo/db/exec/working_set_common.h new file mode 100644 index 00000000000..a983ca055fe --- /dev/null +++ b/src/mongo/db/exec/working_set_common.h @@ -0,0 +1,82 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/exec/working_set.h" + +namespace mongo { + + class WorkingSetCommon { + public: + /** + * Get an owned copy of the BSONObj the WSM refers to. + * Requires either a valid BSONObj or valid DiskLoc. + * Returns true if the fetch and invalidate succeeded, false otherwise. + */ + static bool fetchAndInvalidateLoc(WorkingSetMember* member); + + /** + * Initialize the fields in 'dest' from 'src', creating copies of owned objects as needed. + */ + static void initFrom(WorkingSetMember* dest, const WorkingSetMember& src); + + + /** + * Allocate a new WSM and initialize it with + * the code and reason from the status. + * Owned BSON object will have the following layout: + * { + * ok: , // 1 for OK; 0 otherwise. + * code: , // Status::code() + * errmsg: // Status::reason() + * } + */ + static WorkingSetID allocateStatusMember(WorkingSet* ws, const Status& status); + + /** + * Returns true if object was created by allocateStatusMember(). + */ + static bool isValidStatusMemberObject(const BSONObj& obj); + + /** + * Returns object in working set member created with allocateStatusMember(). + * Does not assume isValidStatusMemberObject. + * If the WSID is invalid or the working set member is created by + * allocateStatusMember, objOut will not be updated. + */ + static void getStatusMemberObject(const WorkingSet& ws, WorkingSetID wsid, + BSONObj* objOut); + + /** + * Formats working set member object created with allocateStatusMember(). + */ + static std::string toStatusString(const BSONObj& obj); + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/working_set_computed_data.h b/src/mongo/db/exec/working_set_computed_data.h new file mode 100644 index 00000000000..53a74633764 --- /dev/null +++ b/src/mongo/db/exec/working_set_computed_data.h @@ -0,0 +1,99 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/exec/working_set.h" + +namespace mongo { + + class TextScoreComputedData : public WorkingSetComputedData { + public: + TextScoreComputedData(double score) + : WorkingSetComputedData(WSM_COMPUTED_TEXT_SCORE), + _score(score) { } + + double getScore() const { return _score; } + + virtual TextScoreComputedData* clone() const { + return new TextScoreComputedData(_score); + } + + private: + double _score; + }; + + class GeoDistanceComputedData : public WorkingSetComputedData { + public: + GeoDistanceComputedData(double dist) + : WorkingSetComputedData(WSM_COMPUTED_GEO_DISTANCE), + _dist(dist) { } + + double getDist() const { return _dist; } + + virtual GeoDistanceComputedData* clone() const { + return new GeoDistanceComputedData(_dist); + } + + private: + double _dist; + }; + + class IndexKeyComputedData : public WorkingSetComputedData { + public: + IndexKeyComputedData(BSONObj key) + : WorkingSetComputedData(WSM_INDEX_KEY), + _key(key.getOwned()) { } + + BSONObj getKey() const { return _key; } + + virtual IndexKeyComputedData* clone() const { + return new IndexKeyComputedData(_key); + } + + private: + BSONObj _key; + }; + + class GeoNearPointComputedData : public WorkingSetComputedData { + public: + GeoNearPointComputedData(BSONObj point) + : WorkingSetComputedData(WSM_GEO_NEAR_POINT), + _point(point.getOwned()) { } + + BSONObj getPoint() const { return _point; } + + virtual GeoNearPointComputedData* clone() const { + return new GeoNearPointComputedData(_point); + } + + private: + BSONObj _point; + }; + +} // namespace mongo diff --git a/src/mongo/db/exec/working_set_test.cpp b/src/mongo/db/exec/working_set_test.cpp new file mode 100644 index 00000000000..25e1f8e8778 --- /dev/null +++ b/src/mongo/db/exec/working_set_test.cpp @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/exec/working_set.cpp + */ + +#include + +#include "mongo/db/exec/working_set.h" +#include "mongo/db/json.h" +#include "mongo/db/jsobj.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +using namespace mongo; + +namespace { + + class WorkingSetFixture : public mongo::unittest::Test { + protected: + void setUp() { + ws.reset(new WorkingSet()); + WorkingSetID id = ws->allocate(); + ASSERT(id != WorkingSet::INVALID_ID); + member = ws->get(id); + ASSERT(NULL != member); + } + + void tearDown() { + ws.reset(); + member = NULL; + } + + boost::scoped_ptr ws; + WorkingSetMember* member; + }; + + TEST_F(WorkingSetFixture, noFieldToGet) { + BSONElement elt; + + // Make sure we're not getting anything out of an invalid WSM. + ASSERT_EQUALS(WorkingSetMember::INVALID, member->state); + ASSERT_FALSE(member->getFieldDotted("foo", &elt)); + + member->state = WorkingSetMember::LOC_AND_IDX; + ASSERT_FALSE(member->getFieldDotted("foo", &elt)); + + // Our state is that of a valid object. The getFieldDotted shouldn't throw; there's + // something to call getFieldDotted on, but there's no field there. + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + ASSERT_TRUE(member->getFieldDotted("foo", &elt)); + + member->state = WorkingSetMember::OWNED_OBJ; + ASSERT_TRUE(member->getFieldDotted("foo", &elt)); + } + + TEST_F(WorkingSetFixture, getFieldUnowned) { + string fieldName = "x"; + + BSONObj obj = BSON(fieldName << 5); + // Not truthful since the loc is bogus, but the loc isn't accessed anyway... + member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + member->obj = BSONObj(obj.objdata()); + ASSERT_TRUE(obj.isOwned()); + ASSERT_FALSE(member->obj.isOwned()); + + // Get out the field we put in. + BSONElement elt; + ASSERT_TRUE(member->getFieldDotted(fieldName, &elt)); + ASSERT_EQUALS(elt.numberInt(), 5); + } + + TEST_F(WorkingSetFixture, getFieldOwned) { + string fieldName = "x"; + + BSONObj obj = BSON(fieldName << 5); + member->obj = obj; + ASSERT_TRUE(member->obj.isOwned()); + member->state = WorkingSetMember::OWNED_OBJ; + BSONElement elt; + ASSERT_TRUE(member->getFieldDotted(fieldName, &elt)); + ASSERT_EQUALS(elt.numberInt(), 5); + } + + TEST_F(WorkingSetFixture, getFieldFromIndex) { + string firstName = "x"; + int firstValue = 5; + + string secondName = "y"; + int secondValue = 10; + + member->keyData.push_back(IndexKeyDatum(BSON(firstName << 1), BSON("" << firstValue))); + // Also a minor lie as loc is bogus. + member->state = WorkingSetMember::LOC_AND_IDX; + BSONElement elt; + ASSERT_TRUE(member->getFieldDotted(firstName, &elt)); + ASSERT_EQUALS(elt.numberInt(), firstValue); + // No foo field. + ASSERT_FALSE(member->getFieldDotted("foo", &elt)); + + // Add another index datum. + member->keyData.push_back(IndexKeyDatum(BSON(secondName << 1), BSON("" << secondValue))); + ASSERT_TRUE(member->getFieldDotted(secondName, &elt)); + ASSERT_EQUALS(elt.numberInt(), secondValue); + ASSERT_TRUE(member->getFieldDotted(firstName, &elt)); + ASSERT_EQUALS(elt.numberInt(), firstValue); + // Still no foo. + ASSERT_FALSE(member->getFieldDotted("foo", &elt)); + } + + TEST_F(WorkingSetFixture, getDottedFieldFromIndex) { + string firstName = "x.y"; + int firstValue = 5; + + member->keyData.push_back(IndexKeyDatum(BSON(firstName << 1), BSON("" << firstValue))); + member->state = WorkingSetMember::LOC_AND_IDX; + BSONElement elt; + ASSERT_TRUE(member->getFieldDotted(firstName, &elt)); + ASSERT_EQUALS(elt.numberInt(), firstValue); + ASSERT_FALSE(member->getFieldDotted("x", &elt)); + ASSERT_FALSE(member->getFieldDotted("y", &elt)); + } + +} // namespace diff --git a/src/mongo/db/explain.cpp b/src/mongo/db/explain.cpp deleted file mode 100644 index b0aae8fb1f0..00000000000 --- a/src/mongo/db/explain.cpp +++ /dev/null @@ -1,277 +0,0 @@ -// @file explain.cpp - Helper classes for generating query explain output. - -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "explain.h" - -#include "cmdline.h" -#include "../util/net/sock.h" -#include "../util/mongoutils/str.h" - -namespace mongo { - - // !!! TODO get rid of const casts - - ExplainPlanInfo::ExplainPlanInfo() : - _isMultiKey(), - _n(), - _nscannedObjects(), - _nscanned(), - _scanAndOrder(), - _indexOnly(), - _picked(), - _done() { - } - - void ExplainPlanInfo::notePlan( const Cursor &cursor, bool scanAndOrder, bool indexOnly ) { - _cursorName = const_cast(cursor).toString(); - _indexBounds = cursor.prettyIndexBounds().getOwned(); - _scanAndOrder = scanAndOrder; - _indexOnly = indexOnly; - noteCursorUpdate( cursor ); - } - - void ExplainPlanInfo::noteIterate( bool match, bool loadedRecord, const Cursor &cursor ) { - if ( match ) { - ++_n; - } - if ( loadedRecord ) { - ++_nscannedObjects; - } - noteCursorUpdate( cursor ); - } - - void ExplainPlanInfo::noteDone( const Cursor &cursor ) { - _done = true; - noteCursorUpdate( cursor ); - BSONObjBuilder bob; - const_cast(cursor).explainDetails( bob ); - _details = bob.obj(); - } - - void ExplainPlanInfo::notePicked() { - _picked = true; - } - - BSONObj ExplainPlanInfo::bson() const { - BSONObjBuilder bob; - bob.append( "cursor", _cursorName ); - bob.appendNumber( "n", _n ); - bob.appendNumber( "nscannedObjects", _nscannedObjects ); - bob.appendNumber( "nscanned", _nscanned ); - bob.append( "indexBounds", _indexBounds ); - return bob.obj(); - } - - BSONObj ExplainPlanInfo::pickedPlanBson( const ExplainClauseInfo &clauseInfo ) const { - BSONObjBuilder bob; - bob.append( "cursor", _cursorName ); - bob.append( "isMultiKey", _isMultiKey ); - bob.appendNumber( "n", clauseInfo.n() ); - bob.appendNumber( "nscannedObjects", clauseInfo.nscannedObjects() ); - bob.appendNumber( "nscanned", clauseInfo.nscanned() ); - bob.appendNumber( "nscannedObjectsAllPlans", clauseInfo.nscannedObjectsAllPlans() ); - bob.appendNumber( "nscannedAllPlans", clauseInfo.nscannedAllPlans() ); - bob.append( "scanAndOrder", _scanAndOrder ); - bob.append( "indexOnly", _indexOnly ); - bob.appendNumber( "nYields", clauseInfo.nYields() ); - bob.appendNumber( "nChunkSkips", clauseInfo.nChunkSkips() ); - bob.appendNumber( "millis", clauseInfo.millis() ); - bob.append( "indexBounds", _indexBounds ); - bob.appendElements( _details ); - return bob.obj(); - } - - void ExplainPlanInfo::noteCursorUpdate( const Cursor &cursor ) { - _isMultiKey = cursor.isMultiKey(); - _nscanned = const_cast(cursor).nscanned(); - } - - ExplainClauseInfo::ExplainClauseInfo() : - _n(), - _nscannedObjects(), - _nChunkSkips(), - _nYields() { - } - - BSONObj ExplainClauseInfo::bson() const { - BSONObjBuilder bb; - bb.appendElements( virtualPickedPlan().pickedPlanBson( *this ) ); - BSONArrayBuilder allPlans( bb.subarrayStart( "allPlans" ) ); - for( list >::const_iterator i = _plans.begin(); - i != _plans.end(); ++i ) { - allPlans << (*i)->bson(); - } - allPlans.done(); - return bb.obj(); - } - - void ExplainClauseInfo::addPlanInfo( const shared_ptr &info ) { - _plans.push_back( info ); - } - - void ExplainClauseInfo::noteYield() { ++_nYields; } - - void ExplainClauseInfo::noteIterate( bool match, bool loadedRecord, bool chunkSkip ) { - if ( match ) { - ++_n; - } - if ( loadedRecord ) { - ++_nscannedObjects; - } - if ( chunkSkip ) { - ++_nChunkSkips; - } - } - - void ExplainClauseInfo::reviseN( long long n ) { - _n = n; - } - - void ExplainClauseInfo::stopTimer() { - _timer.stop(); - } - - long long ExplainClauseInfo::nscannedObjects() const { - if ( _plans.empty() ) { - return 0; - } - return virtualPickedPlan().nscannedObjects(); - } - - long long ExplainClauseInfo::nscanned() const { - if ( _plans.empty() ) { - return 0; - } - return virtualPickedPlan().nscanned(); - } - - long long ExplainClauseInfo::nscannedAllPlans() const { - long long ret = 0; - for( list >::const_iterator i = _plans.begin(); - i != _plans.end(); ++i ) { - ret += (*i)->nscanned(); - } - return ret; - } - - const ExplainPlanInfo &ExplainClauseInfo::virtualPickedPlan() const { - // Return a picked plan if possible. - for( list >::const_iterator i = _plans.begin(); - i != _plans.end(); ++i ) { - if ( (*i)->picked() ) { - return **i; - } - } - // Return a done plan if possible. - for( list >::const_iterator i = _plans.begin(); - i != _plans.end(); ++i ) { - if ( (*i)->done() ) { - return **i; - } - } - // Return a plan with the highest match count. - long long maxN = -1; - shared_ptr ret; - for( list >::const_iterator i = _plans.begin(); - i != _plans.end(); ++i ) { - long long n = ( *i )->n(); - if ( n > maxN ) { - maxN = n; - ret = *i; - } - } - verify( ret ); - return *ret; - } - - void ExplainQueryInfo::noteIterate( bool match, bool loadedRecord, bool chunkSkip ) { - verify( !_clauses.empty() ); - _clauses.back()->noteIterate( match, loadedRecord, chunkSkip ); - } - - void ExplainQueryInfo::noteYield() { - verify( !_clauses.empty() ); - _clauses.back()->noteYield(); - } - - void ExplainQueryInfo::reviseN( long long n ) { - verify( !_clauses.empty() ); - _clauses.back()->reviseN( n ); - } - - void ExplainQueryInfo::setAncillaryInfo( const AncillaryInfo &ancillaryInfo ) { - _ancillaryInfo = ancillaryInfo; - } - - BSONObj ExplainQueryInfo::bson() const { - BSONObjBuilder bob; - if ( _clauses.size() == 1 ) { - bob.appendElements( _clauses.front()->bson() ); - } - else { - long long n = 0; - long long nscannedObjects = 0; - long long nscanned = 0; - long long nscannedObjectsAllPlans = 0; - long long nscannedAllPlans = 0; - BSONArrayBuilder clauseArray( bob.subarrayStart( "clauses" ) ); - for( list >::const_iterator i = _clauses.begin(); - i != _clauses.end(); ++i ) { - clauseArray << (*i)->bson(); - n += (*i)->n(); - nscannedObjects += (*i)->nscannedObjects(); - nscanned += (*i)->nscanned(); - nscannedObjectsAllPlans += (*i)->nscannedObjectsAllPlans(); - nscannedAllPlans += (*i)->nscannedAllPlans(); - } - clauseArray.done(); - bob.appendNumber( "n", n ); - bob.appendNumber( "nscannedObjects", nscannedObjects ); - bob.appendNumber( "nscanned", nscanned ); - bob.appendNumber( "nscannedObjectsAllPlans", nscannedObjectsAllPlans ); - bob.appendNumber( "nscannedAllPlans", nscannedAllPlans ); - bob.appendNumber( "millis", _timer.duration() ); - } - - if ( !_ancillaryInfo._oldPlan.isEmpty() ) { - bob.append( "oldPlan", _ancillaryInfo._oldPlan ); - } - bob.append( "server", server() ); - - return bob.obj(); - } - - void ExplainQueryInfo::addClauseInfo( const shared_ptr &info ) { - if ( !_clauses.empty() ) { - _clauses.back()->stopTimer(); - } - _clauses.push_back( info ); - } - - string ExplainQueryInfo::server() { - return mongoutils::str::stream() << getHostNameCached() << ":" << cmdLine.port; - } - - ExplainSinglePlanQueryInfo::ExplainSinglePlanQueryInfo() : - _planInfo( new ExplainPlanInfo() ), - _queryInfo( new ExplainQueryInfo() ) { - shared_ptr clauseInfo( new ExplainClauseInfo() ); - clauseInfo->addPlanInfo( _planInfo ); - _queryInfo->addClauseInfo( clauseInfo ); - } - -} // namespace mongo diff --git a/src/mongo/db/explain.h b/src/mongo/db/explain.h deleted file mode 100644 index bc9a7da0e96..00000000000 --- a/src/mongo/db/explain.h +++ /dev/null @@ -1,188 +0,0 @@ -// @file explain.h - Helper classes for generating query explain output. - -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "cursor.h" -#include "../util/timer.h" - -namespace mongo { - - /** - * Note: by default we filter out allPlans and oldPlan in the shell's - * explain() function. If you add any recursive structures, make sure to - * edit the JS to make sure everything gets filtered. - */ - - /** The timer starts on construction and provides the duration since then or until stopped. */ - class DurationTimer { - public: - DurationTimer() : _running( true ), _duration() {} - void stop() { _running = false; _duration = _timer.millis(); } - int duration() const { return _running ? _timer.millis() : _duration; } - private: - Timer _timer; - bool _running; - int _duration; - }; - - class ExplainClauseInfo; - - /** Data describing execution of a query plan. */ - class ExplainPlanInfo { - public: - ExplainPlanInfo(); - - /** Note information about the plan. */ - void notePlan( const Cursor &cursor, bool scanAndOrder, bool indexOnly ); - /** Note an iteration of the plan. */ - void noteIterate( bool match, bool loadedRecord, const Cursor &cursor ); - /** Note that the plan finished execution. */ - void noteDone( const Cursor &cursor ); - /** Note that the plan was chosen over others by the query optimizer. */ - void notePicked(); - - /** BSON summary of the plan. */ - BSONObj bson() const; - /** Combined details of both the plan and its clause. */ - BSONObj pickedPlanBson( const ExplainClauseInfo &clauseInfo ) const; - - bool picked() const { return _picked; } - bool done() const { return _done; } - long long n() const { return _n; } - long long nscannedObjects() const { return _nscannedObjects; } - long long nscanned() const { return _nscanned; } - - private: - void noteCursorUpdate( const Cursor &cursor ); - string _cursorName; - bool _isMultiKey; - long long _n; - long long _nscannedObjects; - long long _nscanned; - bool _scanAndOrder; - bool _indexOnly; - BSONObj _indexBounds; - bool _picked; - bool _done; - BSONObj _details; - }; - - /** Data describing execution of a query clause. */ - class ExplainClauseInfo { - public: - ExplainClauseInfo(); - - /** Note an iteration of the clause. */ - void noteIterate( bool match, bool loadedRecord, bool chunkSkip ); - /** Note a yield for the clause. */ - void noteYield(); - /** Revise the total number of documents returned to match an external count. */ - void reviseN( long long n ); - /** Stop the clauses's timer. */ - void stopTimer(); - - /** Add information about a plan to this clause. */ - void addPlanInfo( const shared_ptr &info ); - BSONObj bson() const; - - long long n() const { return _n; } - long long nscannedObjects() const; - long long nscanned() const; - long long nscannedObjectsAllPlans() const { return _nscannedObjects; } - long long nscannedAllPlans() const; - long long nChunkSkips() const { return _nChunkSkips; } - int nYields() const { return _nYields; } - int millis() const { return _timer.duration(); } - - private: - /** - * @return Plan explain information to be displayed at the top of the explain output. A - * picked() plan will be returned if one is available, otherwise a successful non picked() - * plan will be returned. - */ - const ExplainPlanInfo &virtualPickedPlan() const; - list > _plans; - long long _n; - long long _nscannedObjects; - long long _nChunkSkips; - int _nYields; - DurationTimer _timer; - }; - - /** Data describing execution of a query. */ - class ExplainQueryInfo { - public: - /** Note an iteration of the query's current clause. */ - void noteIterate( bool match, bool loadedRecord, bool chunkSkip ); - /** Note a yield of the query's current clause. */ - void noteYield(); - /** Revise the number of documents returned by the current clause. */ - void reviseN( long long n ); - - /* Additional information describing the query. */ - struct AncillaryInfo { - BSONObj _oldPlan; - }; - void setAncillaryInfo( const AncillaryInfo &ancillaryInfo ); - - /* Add information about a clause to this query. */ - void addClauseInfo( const shared_ptr &info ); - BSONObj bson() const; - - private: - static string server(); - - list > _clauses; - AncillaryInfo _ancillaryInfo; - DurationTimer _timer; - }; - - /** Data describing execution of a query with a single clause and plan. */ - class ExplainSinglePlanQueryInfo { - public: - ExplainSinglePlanQueryInfo(); - - /** Note information about the plan. */ - void notePlan( const Cursor &cursor, bool scanAndOrder, bool indexOnly ) { - _planInfo->notePlan( cursor, scanAndOrder, indexOnly ); - } - /** Note an iteration of the plan and the clause. */ - void noteIterate( bool match, bool loadedRecord, bool chunkSkip, const Cursor &cursor ) { - _planInfo->noteIterate( match, loadedRecord, cursor ); - _queryInfo->noteIterate( match, loadedRecord, chunkSkip ); - } - /** Note a yield for the clause. */ - void noteYield() { - _queryInfo->noteYield(); - } - /** Note that the plan finished execution. */ - void noteDone( const Cursor &cursor ) { - _planInfo->noteDone( cursor ); - } - - /** Return the corresponding ExplainQueryInfo for further use. */ - shared_ptr queryInfo() const { - return _queryInfo; - } - - private: - shared_ptr _planInfo; - shared_ptr _queryInfo; - }; - -} // namespace mongo diff --git a/src/mongo/db/extsort.cpp b/src/mongo/db/extsort.cpp index d955acb94cc..a25888c692e 100644 --- a/src/mongo/db/extsort.cpp +++ b/src/mongo/db/extsort.cpp @@ -14,343 +14,64 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "mongo/pch.h" +#include "mongo/platform/basic.h" #include "mongo/db/extsort.h" -#if defined(_WIN32) -# include -#endif - -#include -#include -#include -#include -#include -#include - #include "mongo/db/kill_current_op.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/platform/posix_fadvise.h" -#include "mongo/util/file.h" +#include "mongo/db/storage_options.h" namespace mongo { - HLMutex BSONObjExternalSorter::_extSortMutex("s"); - IndexInterface *BSONObjExternalSorter::extSortIdxInterface; - Ordering BSONObjExternalSorter::extSortOrder( Ordering::make(BSONObj()) ); - bool BSONObjExternalSorter::extSortMayInterrupt( false ); - unsigned long long BSONObjExternalSorter::_compares = 0; - unsigned long long BSONObjExternalSorter::_uniqueNumber = 0; - static SimpleMutex _uniqueNumberMutex( "uniqueNumberMutex" ); - - /*static*/ - int BSONObjExternalSorter::_compare(IndexInterface& i, const Data& l, const Data& r, const Ordering& order) { - _compares++; - int x = i.keyCompare(l.first, r.first, order); - if ( x ) - return x; - return l.second.compare( r.second ); - } - - /*static*/ - int BSONObjExternalSorter::extSortComp( const void *lv, const void *rv ) { - DEV RARELY { - _extSortMutex.dassertLocked(); // must be as we use a global var - } -#ifndef __sunos__ - // Some solaris gnu qsort implementations do not support callback exceptions. - RARELY killCurrentOp.checkForInterrupt(!extSortMayInterrupt); -#endif - Data * l = (Data*)lv; - Data * r = (Data*)rv; - return _compare(*extSortIdxInterface, *l, *r, extSortOrder); - }; - - BSONObjExternalSorter::BSONObjExternalSorter( IndexInterface &i, const BSONObj & order , long maxFileSize ) - : _idxi(i), _order( order.getOwned() ) , _maxFilesize( maxFileSize ) , - _arraySize(1000000), _cur(0), _curSizeSoFar(0), _sorted(0) { - - stringstream rootpath; - rootpath << dbpath; - if ( dbpath[dbpath.size()-1] != '/' ) - rootpath << "/"; - - unsigned long long thisUniqueNumber; - { - SimpleMutex::scoped_lock lk(_uniqueNumberMutex); - thisUniqueNumber = _uniqueNumber; - ++_uniqueNumber; - } - rootpath << "_tmp/esort." << time(0) << "." << thisUniqueNumber << "/"; - _root = rootpath.str(); - - LOG(1) << "external sort root: " << _root.string() << endl; - - create_directories( _root ); - _compares = 0; - } - - BSONObjExternalSorter::~BSONObjExternalSorter() { - if ( _cur ) { - delete _cur; - _cur = 0; - } - unsigned long removed = remove_all( _root ); - wassert( removed == 1 + _files.size() ); - } - - void BSONObjExternalSorter::_sortInMem( bool mayInterrupt ) { - // extSortComp needs to use glpbals - // qsort_r only seems available on bsd, which is what i really want to use - HLMutex::scoped_lock lk(_extSortMutex); - extSortIdxInterface = &_idxi; - extSortOrder = Ordering::make(_order); - extSortMayInterrupt = mayInterrupt; - _cur->sort( BSONObjExternalSorter::extSortComp ); - } - - void BSONObjExternalSorter::sort( bool mayInterrupt ) { - uassert( 10048 , "already sorted" , ! _sorted ); - - _sorted = true; - - if ( _cur && _files.size() == 0 ) { - _sortInMem( mayInterrupt ); - LOG(1) << "\t\t not using file. size:" << _curSizeSoFar << " _compares:" << _compares << endl; - return; - } - - if ( _cur ) { - finishMap( mayInterrupt ); - } - - if ( _cur ) { - delete _cur; - _cur = 0; - } - - if ( _files.size() == 0 ) - return; - - } - - void BSONObjExternalSorter::add( const BSONObj& o, const DiskLoc& loc, bool mayInterrupt ) { - uassert( 10049 , "sorted already" , ! _sorted ); - - if ( ! _cur ) { - _cur = new InMemory( _arraySize ); - } - - Data& d = _cur->getNext(); - d.first = o.getOwned(); - d.second = loc; - - long size = o.objsize(); - _curSizeSoFar += size + sizeof( DiskLoc ) + sizeof( BSONObj ); - - if ( _cur->hasSpace() == false || _curSizeSoFar > _maxFilesize ) { - finishMap( mayInterrupt ); - LOG(1) << "finishing map" << endl; - } - - } - - void BSONObjExternalSorter::finishMap( bool mayInterrupt ) { - uassert( 10050 , "bad" , _cur ); - - _curSizeSoFar = 0; - if ( _cur->size() == 0 ) - return; - - _sortInMem( mayInterrupt ); - - stringstream ss; - ss << _root.string() << "/file." << _files.size(); - string file = ss.str(); - - // todo: it may make sense to fadvise that this not be cached so that building the index doesn't - // eject other things the db is using from the file system cache. while we will soon be reading - // this back, if it fit in ram, there wouldn't have been a need for an external sort in the first - // place. - - ofstream out; - out.open( file.c_str() , ios_base::out | ios_base::binary ); - assertStreamGood( 10051 , (string)"couldn't open file: " + file , out ); - - int num = 0; - for ( InMemory::iterator i=_cur->begin(); i != _cur->end(); ++i ) { - Data p = *i; - out.write( p.first.objdata() , p.first.objsize() ); - out.write( (char*)(&p.second) , sizeof( DiskLoc ) ); - num++; - } - - _cur->clear(); + namespace { + class ComparatorWithInterruptCheck { + public: + typedef pair Data; - _files.push_back( file ); - out.close(); + ComparatorWithInterruptCheck(const ExternalSortComparison* comp, + boost::shared_ptr mayInterrupt) + : _comp(comp) + , _mayInterrupt(mayInterrupt) + {} - LOG(2) << "Added file: " << file << " with " << num << "objects for external sort" << endl; - } - - // --------------------------------- - - BSONObjExternalSorter::Iterator::Iterator( BSONObjExternalSorter * sorter ) : - _cmp( sorter->_idxi, sorter->_order ) , _in( 0 ) { - - for ( list::iterator i=sorter->_files.begin(); i!=sorter->_files.end(); i++ ) { - _files.push_back( new FileIterator( *i ) ); - _stash.push_back( pair( Data( BSONObj() , DiskLoc() ) , false ) ); - } - - if ( _files.size() == 0 && sorter->_cur ) { - _in = sorter->_cur; - _it = sorter->_cur->begin(); - } - } - - BSONObjExternalSorter::Iterator::~Iterator() { - for ( vector::iterator i=_files.begin(); i!=_files.end(); i++ ) - delete *i; - _files.clear(); - } - - bool BSONObjExternalSorter::Iterator::more() { - - if ( _in ) - return _it != _in->end(); + int operator() (const Data& l, const Data& r) const { + RARELY if (*_mayInterrupt) { + killCurrentOp.checkForInterrupt(!*_mayInterrupt); + } - for ( vector::iterator i=_files.begin(); i!=_files.end(); i++ ) - if ( (*i)->more() ) - return true; - for ( vector< pair >::iterator i=_stash.begin(); i!=_stash.end(); i++ ) - if ( i->second ) - return true; - return false; - } - - BSONObjExternalSorter::Data BSONObjExternalSorter::Iterator::next() { - - if ( _in ) { - Data& d = *_it; - ++_it; - return d; - } - - Data best; - int slot = -1; - - for ( unsigned i=0; i<_stash.size(); i++ ) { - - if ( ! _stash[i].second ) { - if ( _files[i]->more() ) - _stash[i] = pair( _files[i]->next() , true ); - else - continue; - } - - if ( slot == -1 || _cmp( best , _stash[i].first ) == 0 ) { - best = _stash[i].first; - slot = i; + return _comp->compare(l, r); } - } - - verify( slot >= 0 ); - _stash[slot].second = false; - - return best; - } - - // ----------------------------------- - - BSONObjExternalSorter::FileIterator::FileIterator( const std::string& file ) { -#ifdef _WIN32 - _file = ::_open( file.c_str(), _O_BINARY | _O_RDWR | _O_CREAT , _S_IREAD | _S_IWRITE ); -#else -#ifndef O_NOATIME -#define O_NOATIME 0 -#endif - _file = ::open( file.c_str(), O_CREAT | O_RDWR | O_NOATIME , S_IRUSR | S_IWUSR ); -#endif - massert( 16392, - str::stream() << "FileIterator can't open file: " - << file << errnoWithDescription(), - _file >= 0 ); - -#ifdef POSIX_FADV_SEQUENTIAL - int err = posix_fadvise(_file, 0, 0, POSIX_FADV_SEQUENTIAL ); - if ( err ) - log() << "posix_fadvise failed: " << err << endl; -#endif - - _length = (unsigned long long)boost::filesystem::file_size( file ); - _readSoFar = 0; - } - BSONObjExternalSorter::FileIterator::~FileIterator() { - if ( _file >= 0 ) { -#ifdef _WIN32 - _close( _file ); -#else - ::close( _file ); -#endif - } - } - - bool BSONObjExternalSorter::FileIterator::more() { - return _readSoFar < _length; - } - - - bool BSONObjExternalSorter::FileIterator::_read( char* buf, long long count ) { - long long total = 0; - while ( total < count ) { -#ifdef _WIN32 - long long now = ::_read( _file, buf, count ); -#else - long long now = ::read( _file, buf, count ); -#endif - if ( now < 0 ) { - log() << "read failed for BSONObjExternalSorter " << errnoWithDescription() << endl; - return false; - } - if ( now == 0 ) { - return false; - } - total += now; - buf += now; - } - return true; - } - - BSONObjExternalSorter::Data BSONObjExternalSorter::FileIterator::next() { - // read BSONObj - - int size; - verify( _read( reinterpret_cast(&size), 4 ) ); - char* buf = reinterpret_cast( malloc( sizeof(unsigned) + size ) ); - verify( buf ); - - memset( buf, 0, 4 ); // for Holder - memcpy( buf+sizeof(unsigned), reinterpret_cast(&size), sizeof(int) ); // size of doc - if ( ! _read( buf + sizeof(unsigned) + sizeof(int), size-sizeof(int) ) ) { // doc content - free( buf ); - msgasserted( 16394, std::string("reading doc for external sort failed:") + errnoWithDescription() ); - } - - // read DiskLoc - DiskLoc l; - if ( ! _read( reinterpret_cast(&l), 8 ) ) { - free( buf ); - msgasserted( 16393, std::string("reading DiskLoc for external sort failed") + errnoWithDescription() ); - } - _readSoFar += 8 + size; - - BSONObj::Holder* h = reinterpret_cast(buf); - return Data( BSONObj(h), l ); - } - + private: + const ExternalSortComparison* _comp; + boost::shared_ptr _mayInterrupt; + }; + } + + BSONObjExternalSorter::BSONObjExternalSorter(const ExternalSortComparison* comp, + long maxFileSize) + : _mayInterrupt(boost::make_shared(false)) + , _sorter(Sorter::make( + SortOptions().TempDir(storageGlobalParams.dbpath + "/_tmp") + .ExtSortAllowed() + .MaxMemoryUsageBytes(maxFileSize), + ComparatorWithInterruptCheck(comp, _mayInterrupt))) + {} } + +#include "mongo/db/sorter/sorter.cpp" +MONGO_CREATE_SORTER(mongo::BSONObj, mongo::DiskLoc, mongo::ComparatorWithInterruptCheck); diff --git a/src/mongo/db/extsort.h b/src/mongo/db/extsort.h index 92b7fea4d85..73a3f35f06a 100644 --- a/src/mongo/db/extsort.h +++ b/src/mongo/db/extsort.h @@ -14,129 +14,62 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "mongo/pch.h" - -#include "mongo/db/index.h" #include "mongo/db/jsobj.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/curop-inl.h" -#include "mongo/util/array.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/sorter/sorter.h" namespace mongo { + typedef pair ExternalSortDatum; + /** - for external (disk) sorting by BSONObj and attaching a value + * To external sort, you provide a pointer to an implementation of this class. + * The compare function follows the usual -1, 0, 1 semantics. */ - class BSONObjExternalSorter : boost::noncopyable { + class ExternalSortComparison { public: - BSONObjExternalSorter( IndexInterface &i, const BSONObj & order = BSONObj() , long maxFileSize = 1024 * 1024 * 100 ); - ~BSONObjExternalSorter(); - typedef pair Data; - /** @return the IndexInterface used to perform key comparisons. */ - const IndexInterface& getIndexInterface() const { return _idxi; } - - private: - static HLMutex _extSortMutex; - IndexInterface& _idxi; - - static int _compare(IndexInterface& i, const Data& l, const Data& r, const Ordering& order); - - class MyCmp { - public: - MyCmp( IndexInterface& i, BSONObj order = BSONObj() ) : _i(i), _order( Ordering::make(order) ) {} - bool operator()( const Data &l, const Data &r ) const { - return _compare(_i, l, r, _order) < 0; - }; - private: - IndexInterface& _i; - const Ordering _order; - }; - - static IndexInterface *extSortIdxInterface; - static Ordering extSortOrder; - static bool extSortMayInterrupt; - static int extSortComp( const void *lv, const void *rv ); - - class FileIterator : boost::noncopyable { - public: - FileIterator( const std::string& file ); - ~FileIterator(); - bool more(); - Data next(); - private: - bool _read( char* buf, long long count ); - - int _file; - unsigned long long _length; - unsigned long long _readSoFar; - }; + virtual ~ExternalSortComparison() { } + virtual int compare(const ExternalSortDatum& l, const ExternalSortDatum& r) const = 0; + }; + // TODO This class will probably disappear in the future or be replaced with a typedef + class BSONObjExternalSorter : boost::noncopyable { public: + typedef pair Data; + typedef SortIteratorInterface Iterator; - typedef FastArray InMemory; - - class Iterator : boost::noncopyable { - public: - - Iterator( BSONObjExternalSorter * sorter ); - ~Iterator(); - bool more(); - Data next(); + BSONObjExternalSorter(const ExternalSortComparison* comp, long maxFileSize=100*1024*1024); - private: - MyCmp _cmp; - vector _files; - vector< pair > _stash; - - InMemory * _in; - InMemory::iterator _it; - - }; - - void add( const BSONObj& o, const DiskLoc& loc, bool mayInterrupt ); - - /* call after adding values, and before fetching the iterator */ - void sort( bool mayInterrupt ); - - auto_ptr iterator() { - uassert( 10052 , "not sorted" , _sorted ); - return auto_ptr( new Iterator( this ) ); - } - - int numFiles() { - return _files.size(); + void add( const BSONObj& o, const DiskLoc& loc, bool mayInterrupt ) { + *_mayInterrupt = mayInterrupt; + _sorter->add(o.getOwned(), loc); } - long getCurSizeSoFar() { return _curSizeSoFar; } + auto_ptr iterator() { return auto_ptr(_sorter->done()); } - void hintNumObjects( long long numObjects ) { - if ( numObjects < _arraySize ) - _arraySize = (int)(numObjects + 100); - } + void sort( bool mayInterrupt ) { *_mayInterrupt = mayInterrupt; } + int numFiles() { return _sorter->numFiles(); } + long getCurSizeSoFar() { return _sorter->memUsed(); } + void hintNumObjects(long long) {} // unused private: - - void _sortInMem( bool mayInterrupt ); - - void sort( const std::string& file ); - void finishMap( bool mayInterrupt ); - - BSONObj _order; - long _maxFilesize; - boost::filesystem::path _root; - - int _arraySize; - InMemory * _cur; - long _curSizeSoFar; - - list _files; - bool _sorted; - - static unsigned long long _compares; - static unsigned long long _uniqueNumber; + shared_ptr _mayInterrupt; + scoped_ptr > _sorter; }; } diff --git a/src/mongo/db/field_parser-inl.h b/src/mongo/db/field_parser-inl.h new file mode 100644 index 00000000000..67a8ef4b436 --- /dev/null +++ b/src/mongo/db/field_parser-inl.h @@ -0,0 +1,367 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/field_parser.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + using mongoutils::str::stream; + + template + void _genFieldErrMsg(const BSONElement& elem, + const BSONField& field, + const string expected, + string* errMsg) + { + if (!errMsg) return; + *errMsg = stream() << "wrong type for '" << field() << "' field, expected " << expected + << ", found " << elem.toString(); + } + + template + FieldParser::FieldState FieldParser::extract(BSONObj doc, + const BSONField& field, + T* out, + string* errMsg) + { + BSONElement elem = doc[field.name()]; + if (elem.eoo()) { + if (field.hasDefault()) { + field.getDefault().cloneTo(out); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() != Object && elem.type() != Array) { + _genFieldErrMsg(elem, field, "Object/Array", errMsg); + return FIELD_INVALID; + } + + if (!out->parseBSON(elem.embeddedObject(), errMsg)) { + return FIELD_INVALID; + } + + return FIELD_SET; + } + + template + FieldParser::FieldState FieldParser::extract(BSONObj doc, + const BSONField& field, + T** out, + string* errMsg) + { + BSONElement elem = doc[field.name()]; + if (elem.eoo()) { + if (field.hasDefault()) { + auto_ptr temp(new T); + field.getDefault()->cloneTo(temp.get()); + + *out = temp.release(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() != Object && elem.type() != Array) { + _genFieldErrMsg(elem, field, "Object/Array", errMsg); + return FIELD_INVALID; + } + + auto_ptr temp(new T); + if (!temp->parseBSON(elem.embeddedObject(), errMsg)) { + return FIELD_INVALID; + } + + *out = temp.release(); + return FIELD_SET; + } + + template + FieldParser::FieldState FieldParser::extract(BSONObj doc, + const BSONField& field, + T** out, + string* errMsg) + { + BSONElement elem = doc[field.name()]; + if (elem.eoo()) { + if (field.hasDefault()) { + *out = new T; + field.getDefault().cloneTo(*out); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() != Object && elem.type() != Array) { + if (errMsg) { + *errMsg = stream() << "wrong type for '" << field() << "' field, expected " + << "vector or array" << ", found " + << doc[field.name()].toString(); + } + return FIELD_INVALID; + } + + auto_ptr temp(new T); + if (!temp->parseBSON(elem.embeddedObject(), errMsg)) { + return FIELD_INVALID; + } + + *out = temp.release(); + return FIELD_SET; + } + + // Extracts an array into a vector + template + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField >& field, + vector* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + template + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField >& field, + vector* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == Array) { + BSONArray arr = BSONArray(elem.embeddedObject()); + string elErrMsg; + + // Append all the new elements to the end of the vector + size_t initialSize = out->size(); + out->resize(initialSize + arr.nFields()); + + int i = 0; + BSONObjIterator objIt(arr); + while (objIt.more()) { + BSONElement next = objIt.next(); + BSONField fieldFor(next.fieldName(), out->at(initialSize + i)); + + if (!FieldParser::extract(next, + fieldFor, + &out->at(initialSize + i), + &elErrMsg)) + { + if (errMsg) { + *errMsg = stream() << "error parsing element " << i << " of field " + << field() << causedBy(elErrMsg); + } + return FIELD_INVALID; + } + i++; + } + + return FIELD_SET; + } + + if (errMsg) { + *errMsg = stream() << "wrong type for '" << field() << "' field, expected " + << "vector array" << ", found " << elem.toString(); + } + return FIELD_INVALID; + } + + template + FieldParser::FieldState FieldParser::extract(BSONObj doc, + const BSONField >& field, + vector* out, + string* errMsg) { + dassert(!field.hasDefault()); + + BSONElement elem = doc[field.name()]; + if (elem.eoo()) { + return FIELD_NONE; + } + + if (elem.type() != Array) { + if (errMsg) { + *errMsg = stream() << "wrong type for '" << field() << "' field, expected " + << "vector array" << ", found " << doc[field.name()].toString(); + } + return FIELD_INVALID; + } + + BSONArray arr = BSONArray(elem.embeddedObject()); + BSONObjIterator objIt(arr); + while (objIt.more()) { + + BSONElement next = objIt.next(); + + if (next.type() != Object) { + if (errMsg) { + *errMsg = stream() << "wrong type for '" << field() << "' field contents, " + << "expected object, found " << elem.type(); + } + return FIELD_INVALID; + } + + auto_ptr toInsert(new T); + + if ( !toInsert->parseBSON( next.embeddedObject(), errMsg ) + || !toInsert->isValid( errMsg ) ) { + return FIELD_INVALID; + } + + out->push_back(toInsert.release()); + } + + return FIELD_SET; + } + + template + void FieldParser::clearOwnedVector(vector* vec) { + for (typename vector::iterator it = vec->begin(); it != vec->end(); ++it) { + delete (*it); + } + } + + template + FieldParser::FieldState FieldParser::extract(BSONObj doc, + const BSONField >& field, + vector** out, + string* errMsg) { + dassert(!field.hasDefault()); + + BSONElement elem = doc[field.name()]; + if (elem.eoo()) { + return FIELD_NONE; + } + + if (elem.type() != Array) { + if (errMsg) { + *errMsg = stream() << "wrong type for '" << field() << "' field, expected " + << "vector array" << ", found " << doc[field.name()].toString(); + } + return FIELD_INVALID; + } + + auto_ptr > tempVector(new vector); + + BSONArray arr = BSONArray(elem.embeddedObject()); + BSONObjIterator objIt(arr); + while (objIt.more()) { + + BSONElement next = objIt.next(); + + if (next.type() != Object) { + if (errMsg) { + *errMsg = stream() << "wrong type for '" << field() << "' field contents, " + << "expected object, found " << elem.type(); + } + clearOwnedVector(tempVector.get()); + return FIELD_INVALID; + } + + auto_ptr toInsert(new T); + if (!toInsert->parseBSON(next.embeddedObject(), errMsg)) { + clearOwnedVector(tempVector.get()); + return FIELD_INVALID; + } + + tempVector->push_back(toInsert.release()); + } + + *out = tempVector.release(); + return FIELD_SET; + } + + // Extracts an object into a map + template + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField >& field, + map* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + template + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField >& field, + map* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == Object) { + BSONObj obj = elem.embeddedObject(); + string elErrMsg; + + BSONObjIterator objIt(obj); + while (objIt.more()) { + BSONElement next = objIt.next(); + T& value = (*out)[next.fieldName()]; + + BSONField fieldFor(next.fieldName(), value); + if (!FieldParser::extract(next, fieldFor, &value, &elErrMsg)) { + if (errMsg) { + *errMsg = stream() << "error parsing map element " << next.fieldName() + << " of field " << field() << causedBy(elErrMsg); + } + return FIELD_INVALID; + } + } + + return FIELD_SET; + } + + if (errMsg) { + *errMsg = stream() << "wrong type for '" << field() << "' field, expected " + << "vector array" << ", found " << elem.toString(); + } + return FIELD_INVALID; + } + +} // namespace mongo diff --git a/src/mongo/db/field_parser.cpp b/src/mongo/db/field_parser.cpp new file mode 100644 index 00000000000..80f04ab4e6e --- /dev/null +++ b/src/mongo/db/field_parser.cpp @@ -0,0 +1,411 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/field_parser.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + using mongoutils::str::stream; + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + bool* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + bool* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == Bool) { + *out = elem.boolean(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "boolean", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + BSONArray* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + BSONArray* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == Array) { + *out = BSONArray(elem.embeddedObject().getOwned()); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "array", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + BSONObj* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + BSONObj* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == Object) { + *out = elem.embeddedObject().getOwned(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "object", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + Date_t* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + Date_t* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == Date) { + *out = elem.date(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "date", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + OpTime* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + OpTime* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == Timestamp) { + *out = elem._opTime(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "timestamp", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + string* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + string* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == String) { + // Extract everything, including embedded null characters. + *out = string(elem.valuestr(), elem.valuestrsize() - 1); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "string", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + OID* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + OID* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == jstOID) { + *out = elem.__oid(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "OID", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + int* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + int* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == NumberInt) { + *out = elem.numberInt(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "integer", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extractNumber( BSONObj doc, + const BSONField& field, + int* out, + string* errMsg ) { + return extractNumber( doc[field.name()], field, out, errMsg ); + } + + FieldParser::FieldState FieldParser::extractNumber( BSONElement elem, + const BSONField& field, + int* out, + string* errMsg ) { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.isNumber()) { + *out = elem.numberInt(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "number", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extract( BSONObj doc, + const BSONField& field, + long long* out, + string* errMsg ) { + return extract( doc[field.name()], field, out, errMsg ); + } + + FieldParser::FieldState FieldParser::extract( BSONElement elem, + const BSONField& field, + long long* out, + string* errMsg ) + { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.type() == NumberLong) { + *out = elem.numberLong(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "long", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extractNumber( BSONObj doc, + const BSONField& field, + long long* out, + string* errMsg ) { + return extractNumber( doc[field.name()], field, out, errMsg ); + } + + FieldParser::FieldState FieldParser::extractNumber( BSONElement elem, + const BSONField& field, + long long* out, + string* errMsg ) { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault(); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if (elem.isNumber()) { + *out = elem.numberLong(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "number", errMsg); + return FIELD_INVALID; + } + + FieldParser::FieldState FieldParser::extractID( BSONObj doc, + const BSONField& field, + BSONObj* out, + string* errMsg ) { + return extractID( doc[field.name()], field, out, errMsg ); + } + + FieldParser::FieldState FieldParser::extractID( BSONElement elem, + const BSONField& field, + BSONObj* out, + string* errMsg ) { + if (elem.eoo()) { + if (field.hasDefault()) { + *out = field.getDefault().firstElement().wrap( "" ); + return FIELD_DEFAULT; + } + else { + return FIELD_NONE; + } + } + + if ( elem.type() != Array ) { + *out = elem.wrap( "" ).getOwned(); + return FIELD_SET; + } + + _genFieldErrMsg(elem, field, "id", errMsg); + return FIELD_INVALID; + } + +} // namespace mongo diff --git a/src/mongo/db/field_parser.h b/src/mongo/db/field_parser.h new file mode 100644 index 00000000000..cf3511cae92 --- /dev/null +++ b/src/mongo/db/field_parser.h @@ -0,0 +1,310 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/bson/bson_field.h" +#include "mongo/db/jsobj.h" +#include "mongo/s/bson_serializable.h" +#include "mongo/util/time_support.h" + +namespace mongo { + + class FieldParser { + public: + /** + * Returns true and fills in 'out' with the contents of the field described by 'field' + * or with the value in 'def', depending on whether the field is present and has the + * correct type in 'doc' or not, respectively. Otherwise, if the field exists but has + * the wrong type, returns false. + * + * NOTE ON BSON OWNERSHIP: + * + * The caller must assume that this class will point to data inside 'doc' without + * copying it. In practice this means that 'doc' MUST EXIST for as long as 'out' + * stays in scope. + */ + + enum FieldState { + // The field is present but has the wrong type + FIELD_INVALID = 0, + + // The field is present and has the correct type + FIELD_SET, + + // The field is absent in the BSON object but set from default + FIELD_DEFAULT, + + // The field is absent and no default was specified + FIELD_NONE + }; + + static FieldState extract( BSONObj doc, + const BSONField& field, + bool* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + bool* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + BSONArray* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + BSONArray* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + BSONObj* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + BSONObj* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + Date_t* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + Date_t* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + OpTime* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + OpTime* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + string* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + string* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + OID* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + OID* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + int* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + int* out, + string* errMsg = NULL ); + + static FieldState extract( BSONObj doc, + const BSONField& field, + long long* out, + string* errMsg = NULL ); + + static FieldState extract( BSONElement elem, + const BSONField& field, + long long* out, + string* errMsg = NULL ); + + /** + * The following extractNumber methods do implicit conversion between any numeric type and + * the BSONField type. This can be useful when an exact numeric type is not needed, for + * example if the field is sometimes modified from the shell which can change the type. + */ + static FieldState extractNumber( BSONObj doc, + const BSONField& field, + int* out, + string* errMsg = NULL ); + + static FieldState extractNumber( BSONElement elem, + const BSONField& field, + int* out, + string* errMsg = NULL ); + + static FieldState extractNumber( BSONObj doc, + const BSONField& field, + long long* out, + string* errMsg = NULL ); + + static FieldState extractNumber( BSONElement elem, + const BSONField& field, + long long* out, + string* errMsg = NULL ); + + /** + * Extracts a document id from a particular field name, which may be of any type but Array. + * Wraps the extracted id value in a BSONObj with one element and empty field name. + */ + static FieldState extractID( BSONObj doc, + const BSONField& field, + BSONObj* out, + string* errMsg = NULL ); + + static FieldState extractID( BSONElement elem, + const BSONField& field, + BSONObj* out, + string* errMsg = NULL ); + + // TODO: BSONElement extraction of types below + + /** + * Extracts a mandatory BSONSerializable structure 'field' from the object 'doc'. Write + * the extracted contents to '*out' if successful or fills '*errMsg', if exising, + * otherwise. This variant relies on T having a parseBSON, which all + * BSONSerializable's have. + * + * TODO: Tighten for BSONSerializable's only + */ + template + static FieldState extract(BSONObj doc, + const BSONField& field, + T* out, + string* errMsg = NULL); + + template + static FieldState extract(BSONObj doc, + const BSONField& field, + T** out, + string* errMsg = NULL); + + /** + * Similar to the mandatory 'extract' but on a optional field. '*out' would only be + * allocated if the field is present. The ownership of '*out' would be transferred to + * the caller, in that case. + * + * TODO: Tighten for BSONSerializable's only + */ + template + static FieldState extract(BSONObj doc, + const BSONField& field, + T** out, // alloc variation + string* errMsg = NULL); + + /** + * Extracts a mandatory repetition of BSONSerializable structures, 'field', from the + * object 'doc'. Write the extracted contents to '*out' if successful or fills + * '*errMsg', if exising, otherwise. This variant relies on T having a parseBSON, + * which all BSONSerializable's have. + * + * The vector owns the instances of T. + * + * TODO: Tighten for BSONSerializable's only + */ + template + static FieldState extract(BSONObj doc, + const BSONField >& field, + vector* out, + string* errMsg = NULL); + + /** + * Similar to the mandatory repetition' extract but on an optional field. '*out' would + * only be allocated if the field is present. The ownership of '*out' would be + * transferred to the caller, in that case. + * + * The vector owns the instances of T. + * + * TODO: Tighten for BSONSerializable's only + */ + template + static FieldState extract(BSONObj doc, + const BSONField >& field, + vector** out, + string* errMsg = NULL); + + // + // ==================== Below DEPRECATED; use types instead ==================== + // + + /** + * The following extract methods are templatized to handle extraction of vectors and + * maps of sub-objects. Keys in the map should be StringData compatible. + * + * It's possible to nest extraction of vectors and maps to any depth, i.e: + * + * vector > > val; + * FieldParser::extract(doc, field, val, &val); + */ + template + static FieldState extract( BSONObj doc, + const BSONField >& field, + vector* out, + string* errMsg = NULL ); + + template + static FieldState extract( BSONElement elem, + const BSONField >& field, + vector* out, + string* errMsg = NULL ); + + template + static FieldState extract( BSONObj doc, + const BSONField >& field, + map* out, + string* errMsg = NULL ); + + template + static FieldState extract( BSONElement elem, + const BSONField >& field, + map* out, + string* errMsg = NULL ); + + private: + template + static void clearOwnedVector(vector* vec); + }; + +} // namespace mongo + +// Inline functions for templating +#include "field_parser-inl.h" + diff --git a/src/mongo/db/field_parser_test.cpp b/src/mongo/db/field_parser_test.cpp new file mode 100644 index 00000000000..9e3f974c915 --- /dev/null +++ b/src/mongo/db/field_parser_test.cpp @@ -0,0 +1,444 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include +#include +#include + +#include "mongo/bson/bson_field.h" +#include "mongo/db/field_parser.h" +#include "mongo/db/jsobj.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/time_support.h" + +namespace { + + using mongo::BSONArray; + using mongo::BSONField; + using mongo::BSONObj; + using mongo::BSONObjBuilder; + using mongo::Date_t; + using mongo::FieldParser; + using mongo::OID; + using std::string; + using std::vector; + using std::map; + + class ExtractionFixture: public mongo::unittest::Test { + protected: + BSONObj doc; + + bool valBool; + BSONArray valArray; + BSONObj valObj; + Date_t valDate; + string valString; + OID valOID; + long long valLong; + + static BSONField aBool; + static BSONField anArray; + static BSONField anObj; + static BSONField aDate; + static BSONField aString; + static BSONField anOID; + static BSONField aLong; + + void setUp() { + valBool = true; + valArray = BSON_ARRAY(1 << 2 << 3); + valObj = BSON("a" << 1); + valDate = 1ULL; + valString = "a string"; + valOID = OID::gen(); + valLong = 1LL; + + doc = BSON(aBool(valBool) << + anArray(valArray) << + anObj(valObj) << + aDate(valDate) << + aString(valString) << + anOID(valOID) << + aLong(valLong)); + } + + void tearDown() { + } + }; + + BSONField ExtractionFixture::aBool("aBool"); + BSONField ExtractionFixture::anArray("anArray"); + BSONField ExtractionFixture::anObj("anObj"); + BSONField ExtractionFixture::aDate("aDate"); + BSONField ExtractionFixture::aString("aString"); + BSONField ExtractionFixture::anOID("anOID"); + BSONField ExtractionFixture::aLong("aLong"); + + TEST_F(ExtractionFixture, GetBool) { + BSONField notThere("otherBool", true); + BSONField wrongType(anObj.name()); + bool val; + ASSERT_TRUE(FieldParser::extract(doc, aBool, &val)); + ASSERT_EQUALS(val, valBool); + ASSERT_TRUE(FieldParser::extract(doc, notThere, &val)); + ASSERT_EQUALS(val, true); + ASSERT_FALSE(FieldParser::extract(doc, wrongType, &val)); + } + + TEST_F(ExtractionFixture, GetBSONArray) { + BSONField notThere("otherArray", BSON_ARRAY("a" << "b")); + BSONField wrongType(aString.name()); + BSONArray val; + ASSERT_TRUE(FieldParser::extract(doc, anArray, &val)); + ASSERT_EQUALS(val, valArray); + ASSERT_TRUE(FieldParser::extract(doc, notThere, &val)); + ASSERT_EQUALS(val, BSON_ARRAY("a" << "b")); + ASSERT_FALSE(FieldParser::extract(doc, wrongType, &val)); + } + + TEST_F(ExtractionFixture, GetBSONObj) { + BSONField notThere("otherObj", BSON("b" << 1)); + BSONField wrongType(aString.name()); + BSONObj val; + ASSERT_TRUE(FieldParser::extract(doc, anObj, &val)); + ASSERT_EQUALS(val, valObj); + ASSERT_TRUE(FieldParser::extract(doc, notThere, &val)); + ASSERT_EQUALS(val, BSON("b" << 1)); + ASSERT_FALSE(FieldParser::extract(doc, wrongType, &val)); + } + + TEST_F(ExtractionFixture, GetDate) { + BSONField notThere("otherDate", 99ULL); + BSONField wrongType(aString.name()); + Date_t val; + ASSERT_TRUE(FieldParser::extract(doc, aDate, &val)); + ASSERT_EQUALS(val, valDate); + ASSERT_TRUE(FieldParser::extract(doc, notThere, &val)); + ASSERT_EQUALS(val, 99ULL); + ASSERT_FALSE(FieldParser::extract(doc, wrongType, &val)); + } + + TEST_F(ExtractionFixture, GetString) { + BSONField notThere("otherString", "abc"); + BSONField wrongType(aBool.name()); + string val; + ASSERT_TRUE(FieldParser::extract(doc, aString, &val)); + ASSERT_EQUALS(val, valString); + ASSERT_TRUE(FieldParser::extract(doc, notThere, &val)); + ASSERT_EQUALS(val, "abc"); + ASSERT_FALSE(FieldParser::extract(doc, wrongType, &val)); + } + + TEST_F(ExtractionFixture, GetOID) { + OID defOID = OID::gen(); + BSONField notThere("otherOID", defOID); + BSONField wrongType(aString.name()); + OID val; + ASSERT_TRUE(FieldParser::extract(doc, anOID, &val)); + ASSERT_EQUALS(val, valOID); + ASSERT_TRUE(FieldParser::extract(doc, notThere, &val)); + ASSERT_EQUALS(val, defOID); + ASSERT_FALSE(FieldParser::extract(doc, wrongType, &val)); + } + + TEST_F(ExtractionFixture, GetLong) { + BSONField notThere("otherLong", 0); + BSONField wrongType(aString.name()); + long long val; + ASSERT_TRUE(FieldParser::extract(doc, aLong, &val)); + ASSERT_EQUALS(val, valLong); + ASSERT_TRUE(FieldParser::extract(doc, notThere, &val)); + ASSERT_EQUALS(val, 0); + ASSERT_FALSE(FieldParser::extract(doc, wrongType, &val)); + } + + TEST_F(ExtractionFixture, IsFound) { + bool bool_val; + BSONField aBoolMissing("aBoolMissing"); + ASSERT_EQUALS(FieldParser::extract(doc, aBool, &bool_val, NULL), + FieldParser::FIELD_SET); + ASSERT_EQUALS(FieldParser::extract(doc, aBoolMissing, &bool_val, NULL), + FieldParser::FIELD_NONE); + + Date_t Date_t_val; + BSONField aDateMissing("aDateMissing"); + ASSERT_EQUALS(FieldParser::extract(doc, aDate, &Date_t_val, NULL), + FieldParser::FIELD_SET); + ASSERT_EQUALS(FieldParser::extract(doc, aDateMissing, &Date_t_val, NULL), + FieldParser::FIELD_NONE); + + string string_val; + BSONField aStringMissing("aStringMissing"); + ASSERT_EQUALS(FieldParser::extract(doc, aString, &string_val, NULL), + FieldParser::FIELD_SET); + ASSERT_EQUALS(FieldParser::extract(doc, aStringMissing, &string_val, NULL), + FieldParser::FIELD_NONE); + + OID OID_val; + BSONField anOIDMissing("anOIDMissing"); + ASSERT_EQUALS(FieldParser::extract(doc, anOID, &OID_val, NULL), + FieldParser::FIELD_SET); + ASSERT_EQUALS(FieldParser::extract(doc, anOIDMissing, &OID_val, NULL), + FieldParser::FIELD_NONE); + + long long long_long_val; + BSONField aLongMissing("aLongMissing"); + ASSERT_EQUALS(FieldParser::extract(doc, aLong, &long_long_val, NULL), + FieldParser::FIELD_SET); + ASSERT_EQUALS(FieldParser::extract(doc, aLongMissing, &long_long_val, NULL), + FieldParser::FIELD_NONE); + } + + TEST(ComplexExtraction, GetStringVector) { + + // Test valid string vector extraction + BSONField > vectorField("testVector"); + + BSONObjBuilder bob; + bob << vectorField() << BSON_ARRAY("a" << "b" << "c"); + BSONObj obj = bob.obj(); + + vector parsedVector; + + ASSERT(FieldParser::extract(obj, vectorField, &parsedVector)); + ASSERT_EQUALS("a", parsedVector[0]); + ASSERT_EQUALS("b", parsedVector[1]); + ASSERT_EQUALS("c", parsedVector[2]); + ASSERT_EQUALS(parsedVector.size(), static_cast(3)); + } + + TEST(ComplexExtraction, GetObjectVector) { + + // Test valid BSONObj vector extraction + BSONField > vectorField("testVector"); + + BSONObjBuilder bob; + bob << vectorField() << BSON_ARRAY(BSON("a" << 1) << BSON("b" << 1) << BSON("c" << 1)); + BSONObj obj = bob.obj(); + + vector parsedVector; + + ASSERT(FieldParser::extract(obj, vectorField, &parsedVector)); + ASSERT_EQUALS(BSON("a" << 1), parsedVector[0]); + ASSERT_EQUALS(BSON("b" << 1), parsedVector[1]); + ASSERT_EQUALS(BSON("c" << 1), parsedVector[2]); + ASSERT_EQUALS(parsedVector.size(), static_cast(3)); + } + + TEST(ComplexExtraction, GetBadVector) { + + // Test invalid vector extraction + BSONField > vectorField("testVector"); + + BSONObjBuilder bob; + bob << vectorField() << BSON_ARRAY(BSON("a" << 1) << "XXX" << BSON("c" << 1)); + BSONObj obj = bob.obj(); + + vector parsedVector; + + string errMsg; + ASSERT(!FieldParser::extract(obj, vectorField, &parsedVector, &errMsg)); + ASSERT_NOT_EQUALS(errMsg, ""); + } + + TEST(ComplexExtraction, RoundTripVector) { + + // Test vector extraction after re-writing to BSON + BSONField > vectorField("testVector"); + + BSONObj obj; + { + BSONObjBuilder bob; + bob << vectorField() << BSON_ARRAY("a" << "b" << "c"); + obj = bob.obj(); + } + + vector parsedVector; + ASSERT(FieldParser::extract(obj, vectorField, &parsedVector)); + + { + BSONObjBuilder bob; + bob.append(vectorField(), parsedVector); + obj = bob.obj(); + } + + parsedVector.clear(); + ASSERT(FieldParser::extract(obj, vectorField, &parsedVector)); + + ASSERT_EQUALS("a", parsedVector[0]); + ASSERT_EQUALS("b", parsedVector[1]); + ASSERT_EQUALS("c", parsedVector[2]); + ASSERT_EQUALS(parsedVector.size(), static_cast(3)); + } + + TEST(ComplexExtraction, GetStringMap) { + + // Test valid string->string map extraction + BSONField > mapField("testMap"); + + BSONObjBuilder bob; + bob << mapField() << BSON("a" << "a" << "b" << "b" << "c" << "c"); + BSONObj obj = bob.obj(); + + map parsedMap; + + ASSERT(FieldParser::extract(obj, mapField, &parsedMap)); + ASSERT_EQUALS("a", parsedMap["a"]); + ASSERT_EQUALS("b", parsedMap["b"]); + ASSERT_EQUALS("c", parsedMap["c"]); + ASSERT_EQUALS(parsedMap.size(), static_cast(3)); + } + + TEST(ComplexExtraction, GetObjectMap) { + + // Test valid string->BSONObj map extraction + BSONField > mapField("testMap"); + + BSONObjBuilder bob; + bob << mapField() << BSON("a" << BSON("a" << "a") << + "b" << BSON("b" << "b") << + "c" << BSON("c" << "c")); + BSONObj obj = bob.obj(); + + map parsedMap; + + ASSERT(FieldParser::extract(obj, mapField, &parsedMap)); + ASSERT_EQUALS(BSON("a" << "a"), parsedMap["a"]); + ASSERT_EQUALS(BSON("b" << "b"), parsedMap["b"]); + ASSERT_EQUALS(BSON("c" << "c"), parsedMap["c"]); + ASSERT_EQUALS(parsedMap.size(), static_cast(3)); + } + + TEST(ComplexExtraction, GetBadMap) { + + // Test invalid map extraction + BSONField > mapField("testMap"); + + BSONObjBuilder bob; + bob << mapField() << BSON("a" << "a" << "b" << 123 << "c" << "c"); + BSONObj obj = bob.obj(); + + map parsedMap; + + string errMsg; + ASSERT(!FieldParser::extract(obj, mapField, &parsedMap, &errMsg)); + ASSERT_NOT_EQUALS(errMsg, ""); + } + + TEST(ComplexExtraction, RoundTripMap) { + + // Test map extraction after re-writing to BSON + BSONField > mapField("testMap"); + + BSONObj obj; + { + BSONObjBuilder bob; + bob << mapField() << BSON("a" << "a" << "b" << "b" << "c" << "c"); + obj = bob.obj(); + } + + map parsedMap; + ASSERT(FieldParser::extract(obj, mapField, &parsedMap)); + + { + BSONObjBuilder bob; + bob.append(mapField(), parsedMap); + obj = bob.obj(); + } + + parsedMap.clear(); + ASSERT(FieldParser::extract(obj, mapField, &parsedMap)); + + ASSERT_EQUALS("a", parsedMap["a"]); + ASSERT_EQUALS("b", parsedMap["b"]); + ASSERT_EQUALS("c", parsedMap["c"]); + ASSERT_EQUALS(parsedMap.size(), static_cast(3)); + } + + TEST(ComplexExtraction, GetNestedMap) { + + // Test extraction of complex nested vector and map + BSONField > > nestedField("testNested"); + + BSONObj nestedMapObj = BSON("a" << "a" << "b" << "b" << "c" << "c"); + + BSONObjBuilder bob; + bob << nestedField() << BSON_ARRAY(nestedMapObj << nestedMapObj << nestedMapObj); + BSONObj obj = bob.obj(); + + vector > parsed; + + ASSERT(FieldParser::extract(obj, nestedField, &parsed)); + ASSERT_EQUALS(parsed.size(), static_cast(3)); + for (int i = 0; i < 3; i++) { + map& parsedMap = parsed[i]; + ASSERT_EQUALS("a", parsedMap["a"]); + ASSERT_EQUALS("b", parsedMap["b"]); + ASSERT_EQUALS("c", parsedMap["c"]); + ASSERT_EQUALS(parsedMap.size(), static_cast(3)); + } + } + + TEST(ComplexExtraction, GetBadNestedMap) { + + // Test extraction of invalid complex nested vector and map + BSONField > > nestedField("testNested"); + + BSONObj nestedMapObj = BSON("a" << "a" << "b" << 123 << "c" << "c"); + + BSONObjBuilder bob; + bob << nestedField() << BSON_ARRAY(nestedMapObj << nestedMapObj << nestedMapObj); + BSONObj obj = bob.obj(); + + vector > parsed; + + string errMsg; + ASSERT(!FieldParser::extract(obj, nestedField, &parsed, &errMsg)); + ASSERT_NOT_EQUALS(errMsg, ""); + } + + TEST(EdgeCases, EmbeddedNullStrings) { + + // Test extraction of string values with embedded nulls. + BSONField field("testStr"); + + const char* str = "a\0c"; + const size_t strSize = 4; + BSONObjBuilder doc; + doc.append(field(), str, strSize); + BSONObj obj(doc.obj()); + + string parsed; + string errMsg; + ASSERT(FieldParser::extract(obj, field, &parsed, &errMsg)); + + ASSERT_EQUALS(0, memcmp(parsed.data(), str, strSize)); + ASSERT_EQUALS(errMsg, ""); + } + +} // unnamed namespace diff --git a/src/mongo/db/field_ref.cpp b/src/mongo/db/field_ref.cpp index 7b9f5631140..4d89695028d 100644 --- a/src/mongo/db/field_ref.cpp +++ b/src/mongo/db/field_ref.cpp @@ -12,37 +12,74 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/db/field_ref.h" + +#include // for min + +#include "mongo/util/log.h" #include "mongo/util/assert_util.h" namespace mongo { - void FieldRef::parse(const StringData& dottedField) { - if (dottedField.size() == 0) { + FieldRef::FieldRef() : _size(0) {} + + FieldRef::FieldRef(const StringData& path) : _size(0) { + parse(path); + } + + void FieldRef::parse(const StringData& path) { + if (path.size() == 0) { return; } + if (_size != 0) { + clear(); + } + // We guarantee that accesses through getPart() will be valid while 'this' is. So we - // take a copy. We're going to be "chopping" up the copy into c-strings. - _fieldBase.reset(new char[dottedField.size()+1]); - dottedField.copyTo( _fieldBase.get(), true ); + // keep a copy in a local sting. + + _dotted = path.toString(); // Separate the field parts using '.' as a delimiter. - char* beg = _fieldBase.get(); - char* cur = beg; - char* end = beg + dottedField.size(); + std::string::iterator beg = _dotted.begin(); + std::string::iterator cur = beg; + const std::string::iterator end = _dotted.end(); while (true) { if (cur != end && *cur != '.') { cur++; continue; } - appendPart(StringData(beg, cur - beg)); + // If cur != beg then we advanced cur in the loop above, so we have a real sequence + // of characters to add as a new part. Otherwise, we may be parsing something odd, + // like "..", and we need to add an empty StringData piece to represent the "part" + // in-between the dots. This also handles the case where 'beg' and 'cur' are both + // at 'end', which can happen if we are parsing anything with a terminal "." + // character. In that case, we still need to add an empty part, but we will break + // out of the loop below since we will not execute the guarded 'continue' and will + // instead reach the break statement. + + if (cur != beg) + appendPart(StringData(&*beg, cur - beg)); + else + appendPart(StringData()); if (cur != end) { - *cur = '\0'; beg = ++cur; continue; } @@ -77,6 +114,43 @@ namespace mongo { return ++_size; } + void FieldRef::reserialize() const { + std::string nextDotted; + // Reserve some space in the string. We know we will have, at minimum, a character for + // each component we are writing, and a dot for each component, less one. We don't want + // to reserve more, since we don't want to forfeit the SSO if it is applicable. + nextDotted.reserve((_size * 2) - 1); + + // Concatenate the fields to a new string + for (size_t i = 0; i != _size; ++i) { + if (i > 0) + nextDotted.append(1, '.'); + const StringData part = getPart(i); + nextDotted.append(part.rawData(), part.size()); + } + + // Make the new string our contents + _dotted.swap(nextDotted); + + // Fixup the parts to refer to the new string + std::string::const_iterator where = _dotted.begin(); + const std::string::const_iterator end = _dotted.end(); + for (size_t i = 0; i != _size; ++i) { + StringData& part = (i < kReserveAhead) ? _fixed[i] : _variable[getIndex(i)]; + const size_t size = part.size(); + part = StringData(&*where, size); + where += size; + // skip over '.' unless we are at the end. + if (where != end) { + dassert(*where == '.'); + ++where; + } + } + + // Drop any replacements + _replacements.clear(); + } + StringData FieldRef::getPart(size_t i) const { dassert(i < _size); @@ -88,36 +162,118 @@ namespace mongo { } } - void FieldRef::clear() { - _size = 0; - _variable.clear(); - _fieldBase.reset(); - _replacements.clear(); + bool FieldRef::isPrefixOf( const FieldRef& other ) const { + // Can't be a prefix if the size is equal to or larger. + if ( _size >= other._size ) { + return false; + } + + // Empty FieldRef is not a prefix of anything. + if ( _size == 0 ) { + return false; + } + + size_t common = commonPrefixSize( other ); + return common == _size && other._size > common; } - std::string FieldRef::dottedField() const { - std::string res; - if (_size == 0) { - return res; + size_t FieldRef::commonPrefixSize( const FieldRef& other ) const { + if (_size == 0 || other._size == 0) { + return 0; } - res.append(_fixed[0].rawData(), _fixed[0].size()); - for (size_t i=1; i<_size; i++) { - res.append(1, '.'); - StringData part = getPart(i); - res.append(part.rawData(), part.size()); + size_t maxPrefixSize = std::min( _size-1, other._size-1 ); + size_t prefixSize = 0; + + while ( prefixSize <= maxPrefixSize ) { + if ( getPart( prefixSize ) != other.getPart( prefixSize ) ) { + break; + } + prefixSize++; } - return res; + + return prefixSize; } - size_t FieldRef::numReplaced() const { - size_t res = 0; - for (size_t i = 0; i < _replacements.size(); i++) { - if (!_replacements[i].empty()) { - res++; + StringData FieldRef::dottedField( size_t offset ) const { + if (_size == 0 || offset >= numParts() ) + return StringData(); + + if (!_replacements.empty()) + reserialize(); + dassert(_replacements.empty()); + + // Assume we want the whole thing + StringData result(_dotted); + + // Strip off any leading parts we were asked to ignore + for (size_t i = 0; i < offset; ++i) { + const StringData part = getPart(i); + result = StringData( + result.rawData() + part.size() + 1, + result.size() - part.size() - 1); + } + + return result; + } + + bool FieldRef::equalsDottedField( const StringData& other ) const { + StringData rest = other; + + for ( size_t i = 0; i < _size; i++ ) { + + StringData part = getPart( i ); + + if ( !rest.startsWith( part ) ) + return false; + + if ( i == _size - 1 ) + return rest.size() == part.size(); + + // make sure next thing is a dot + if ( rest.size() == part.size() ) + return false; + + if ( rest[part.size()] != '.' ) + return false; + + rest = rest.substr( part.size() + 1 ); + } + + return false; + } + + int FieldRef::compare(const FieldRef& other) const { + const size_t toCompare = std::min(_size, other._size); + for (size_t i = 0; i < toCompare; i++) { + if (getPart(i) == other.getPart(i)) { + continue; } + return getPart(i) < other.getPart(i) ? -1 : 1; + } + + const size_t rest = _size - toCompare; + const size_t otherRest = other._size - toCompare; + if ((rest == 0) && (otherRest == 0)) { + return 0; } - return res; + else if (rest < otherRest ) { + return -1; + } + else { + return 1; + } + } + + void FieldRef::clear() { + _size = 0; + _variable.clear(); + _dotted.clear(); + _replacements.clear(); + } + + std::ostream& operator<<(std::ostream& stream, const FieldRef& field) { + return stream << field.dottedField(); } } // namespace mongo diff --git a/src/mongo/db/field_ref.h b/src/mongo/db/field_ref.h index ea7b7dab34d..d35a94d284d 100644 --- a/src/mongo/db/field_ref.h +++ b/src/mongo/db/field_ref.h @@ -12,11 +12,24 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once #include +#include #include #include @@ -38,7 +51,9 @@ namespace mongo { class FieldRef { MONGO_DISALLOW_COPYING(FieldRef); public: - FieldRef() : _size(0) {} + FieldRef(); + + explicit FieldRef(const StringData& path); /** * Field parts accessed through getPart() calls no longer would be valid, after the @@ -62,11 +77,33 @@ namespace mongo { */ StringData getPart(size_t i) const; + /** + * Returns true when 'this' FieldRef is a prefix of 'other'. Equality is not considered + * a prefix. + */ + bool isPrefixOf( const FieldRef& other ) const; + + /** + * Returns the number of field parts in the prefix that 'this' and 'other' share. + */ + size_t commonPrefixSize( const FieldRef& other ) const; + /** * Returns a copy of the full dotted field in its current state (i.e., some parts may * have been replaced since the parse() call). */ - std::string dottedField() const; + StringData dottedField( size_t offsetFromStart = 0 ) const; + + /** + * Compares the full dotted path represented by this FieldRef to other + */ + bool equalsDottedField( const StringData& other ) const; + + /** + * Return 0 if 'this' is equal to 'other' lexicographically, -1 if is it less than or + * +1 if it is greater than. + */ + int compare( const FieldRef& other ) const; /** * Resets the internal state. See note in parse() call. @@ -82,26 +119,19 @@ namespace mongo { */ size_t numParts() const { return _size; } - /** - * Returns the number of fields parts that were replaced so far. Replacing the same - * fields several times only counts for 1. - */ - size_t numReplaced() const; + bool empty() const { return numParts() == 0; } private: - // Dotted fields are most often not longer than three parts. We use a mixed structure + // Dotted fields are most often not longer than four parts. We use a mixed structure // here that will not require any extra memory allocation when that is the case. And // handle larger dotted fields if it is. The idea is not to penalize the common case // with allocations. static const size_t kReserveAhead = 4; - size_t _size; // # of field parts stored - StringData _fixed[kReserveAhead]; // first kResevedAhead field components - std::vector _variable; // remaining field components - - // Areas that _fixed and _variable point to. - boost::scoped_array _fieldBase; // concatenation of null-terminated parts - std::vector _replacements; // added with the setPart call + /** + * Parses 'path' into parts. + */ + void _parse(const StringData& path); /** Converts the field part index to the variable part equivalent */ size_t getIndex(size_t i) const { return i-kReserveAhead; } @@ -112,6 +142,55 @@ namespace mongo { */ size_t appendPart(const StringData& part); + /** + * Re-assemble _dotted from components, including any replacements in _replacements, + * and update the StringData components in _fixed and _variable to refer to the parts + * of the new _dotted. This is used to make the storage for the current value of this + * FieldRef contiguous so it can be returned as a StringData from the dottedField + * method above. + */ + void reserialize() const; + + // number of field parts stored + size_t _size; + + // first kResevedAhead field components + mutable StringData _fixed[kReserveAhead]; + + // remaining field components + mutable std::vector _variable; + + // cached dotted name + mutable std::string _dotted; + + // back memory added with the setPart call pointed to by _fized and _variable + mutable std::vector _replacements; }; + inline bool operator==(const FieldRef& lhs, const FieldRef& rhs) { + return lhs.compare(rhs) == 0; + } + + inline bool operator!=(const FieldRef& lhs, const FieldRef& rhs) { + return lhs.compare(rhs) != 0; + } + + inline bool operator<(const FieldRef& lhs, const FieldRef& rhs) { + return lhs.compare(rhs) < 0; + } + + inline bool operator<=(const FieldRef& lhs, const FieldRef& rhs) { + return lhs.compare(rhs) <= 0; + } + + inline bool operator>(const FieldRef& lhs, const FieldRef& rhs) { + return lhs.compare(rhs) > 0; + } + + inline bool operator>=(const FieldRef& lhs, const FieldRef& rhs) { + return lhs.compare(rhs) >= 0; + } + + std::ostream& operator<<(std::ostream& stream, const FieldRef& value); + } // namespace mongo diff --git a/src/mongo/db/field_ref_set.cpp b/src/mongo/db/field_ref_set.cpp new file mode 100644 index 00000000000..509de2845fc --- /dev/null +++ b/src/mongo/db/field_ref_set.cpp @@ -0,0 +1,155 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/field_ref_set.h" + +#include "mongo/util/assert_util.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + namespace { + + // For legacy purposes, we must handle empty fieldnames, which FieldRef clearly + // prohibits. It is preferrable to have FieldRef keep that constraint and relax it here + // -- stricly in update code. The rationale is that, if we want to ban data with no + // field names, we must allow that data to be updated. + StringData safeFirstPart(const FieldRef* fieldRef) { + if (fieldRef->numParts() == 0) { + return StringData(); + } + else { + return fieldRef->getPart(0); + } + } + + } + + bool FieldRefSet::FieldRefPtrLessThan::operator()(const FieldRef* l, const FieldRef* r) const { + return *l < *r; + } + + FieldRefSet::FieldRefSet() { + } + + bool FieldRefSet::findConflicts(const FieldRef* toCheck, FieldRefSet* conflicts) const { + bool foundConflict = false; + + // If the set is empty, there is no work to do. + if (_fieldSet.empty()) + return foundConflict; + + StringData prefixStr = safeFirstPart(toCheck); + FieldRef prefixField(prefixStr); + + FieldSet::iterator it = _fieldSet.lower_bound(&prefixField); + // Now, iterate over all the present fields in the set that have the same prefix. + + while (it != _fieldSet.end() && safeFirstPart(*it) == prefixStr) { + size_t common = (*it)->commonPrefixSize(*toCheck); + if ((*it)->numParts() == common || toCheck->numParts() == common) { + if (!conflicts) + return true; + + conflicts->_fieldSet.insert(*it); + foundConflict = true; + } + ++it; + } + + return foundConflict; + } + + void FieldRefSet::keepShortest(const FieldRef* toInsert) { + const FieldRef* conflict; + if ( !insert(toInsert, &conflict) && (toInsert->numParts() < (conflict->numParts()))) { + _fieldSet.erase(conflict); + keepShortest(toInsert); + } + } + + void FieldRefSet::fillFrom(const std::vector& fields) { + dassert(_fieldSet.empty()); + _fieldSet.insert(fields.begin(), fields.end()); + } + + bool FieldRefSet::insert(const FieldRef* toInsert, const FieldRef** conflict) { + + // We can determine if two fields conflict by checking their common prefix. + // + // If each field is exactly of the size of the common prefix, this means the fields are + // the same. If one of the fields is greater than the common prefix and the other + // isn't, the latter is a prefix of the former. And vice-versa. + // + // Example: + // + // inserted > | a a.c + // exiting v | (0) (+1) + // ----------------|------------------------ + // a (0) | equal prefix < + // a.b (+1) | prefix ^ * + // + // * Disjoint sub-trees + + // At each insertion, we only need to bother checking the fields in the set that have + // at least some common prefix with the 'toInsert' field. + StringData prefixStr = safeFirstPart(toInsert); + FieldRef prefixField(prefixStr); + FieldSet::iterator it = _fieldSet.lower_bound(&prefixField); + + // Now, iterate over all the present fields in the set that have the same prefix. + while (it != _fieldSet.end() && safeFirstPart(*it) == prefixStr) { + size_t common = (*it)->commonPrefixSize(*toInsert); + if ((*it)->numParts() == common || toInsert->numParts() == common) { + *conflict = *it; + return false; + } + ++it; + } + + _fieldSet.insert(it, toInsert); + *conflict = NULL; + return true; + } + + const std::string FieldRefSet::toString() const { + str::stream res; + res << "Fields:[ "; + FieldRefSet::const_iterator where = _fieldSet.begin(); + const FieldRefSet::const_iterator end = _fieldSet.end(); + for( ; where != end; ++where ) { + const FieldRef& current = **where; + res << current.dottedField() << ","; + } + res << "]"; + return res; + } + +} // namespace mongo diff --git a/src/mongo/db/field_ref_set.h b/src/mongo/db/field_ref_set.h new file mode 100644 index 00000000000..e7258c2a184 --- /dev/null +++ b/src/mongo/db/field_ref_set.h @@ -0,0 +1,116 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/base/status.h" +#include "mongo/db/field_ref.h" + +namespace mongo { + + /** + * A FieldRefSet holds a set of FieldRefs's that do not conflict with one another, that is, + * they target different subtrees of a given document. Two fieldRef's would conflict if they + * are equal or one is prefix of the other. + */ + class FieldRefSet { + MONGO_DISALLOW_COPYING(FieldRefSet); + + struct FieldRefPtrLessThan { + bool operator()(const FieldRef* lhs, const FieldRef* rhs) const; + }; + + typedef std::set FieldSet; + + public: + typedef FieldSet::iterator iterator; + typedef FieldSet::const_iterator const_iterator; + + FieldRefSet(); + + /** Returns 'true' if the set is empty */ + bool empty() const { + return _fieldSet.empty(); + } + + inline const_iterator begin() const { + return _fieldSet.begin(); + } + + inline const_iterator end() const { + return _fieldSet.end(); + } + + /** + * Returns true if the field 'toInsert' can be added in the set without + * conflicts. Otherwise returns false and fill in '*conflict' with the field 'toInsert' + * clashed with. + * + * There is no ownership transfer of 'toInsert'. The caller is responsible for + * maintaining it alive for as long as the FieldRefSet is so. By the same token + * 'conflict' can only be referred to while the FieldRefSet can. + */ + bool insert(const FieldRef* toInsert, const FieldRef** conflict); + + /** + * Fills the set with the supplied FieldRef*s + */ + void fillFrom(const std::vector& fields); + + /** + * Replace any existing conflicting FieldRef with the shortest (closest to root) one + */ + void keepShortest(const FieldRef* toInsert); + + /** + * Find all inserted fields which conflict with the FieldRef 'toCheck' by the semantics + * of 'insert', and add those fields to the 'conflicts' set. + * + * Return true if conflicts were found. + */ + bool findConflicts(const FieldRef* toCheck, FieldRefSet* conflicts) const; + + void clear() { + _fieldSet.clear(); + } + + /** + * A debug/log-able string + */ + const std::string toString() const; + + private: + // A set of field_ref pointers, none of which is owned here. + FieldSet _fieldSet; + }; + +} // namespace mongo diff --git a/src/mongo/db/field_ref_set_test.cpp b/src/mongo/db/field_ref_set_test.cpp new file mode 100644 index 00000000000..99abf360234 --- /dev/null +++ b/src/mongo/db/field_ref_set_test.cpp @@ -0,0 +1,132 @@ +/** + * Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/field_ref_set.h" + +#include "mongo/db/field_ref.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::FieldRef; + using mongo::FieldRefSet; + + TEST(EmptySet, Normal) { + // insert "b" + FieldRefSet fieldSet; + FieldRef bSimple("b"); + const FieldRef* conflict; + ASSERT_TRUE(fieldSet.insert(&bSimple, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + // insert "a", OK + FieldRef aSimple("a"); + ASSERT_TRUE(fieldSet.insert(&aSimple, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + // insert "c", OK + FieldRef cSimple("c"); + ASSERT_TRUE(fieldSet.insert(&cSimple, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + } + + TEST(EmptySet, Conflict) { + // insert "a.b" + FieldRefSet fieldSet; + FieldRef aDotB("a.b"); + const FieldRef* conflict; + ASSERT_TRUE(fieldSet.insert(&aDotB, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + // insert "a", conflicts with "a.b" + FieldRef prefix("a"); + ASSERT_FALSE(fieldSet.insert(&prefix, &conflict)); + ASSERT_EQUALS(aDotB, *conflict); + + // insert "a.b.c", conflicts with "a.b" + FieldRef superSet("a.b.c"); + ASSERT_FALSE(fieldSet.insert(&superSet, &conflict)); + ASSERT_EQUALS(aDotB, *conflict); + } + + TEST(EmptySet, EmptyField) { + // Old data may have empty field names. We test that we can catch conflicts if we try + // to insert an empty field twice. + FieldRefSet fieldSet; + FieldRef empty; + const FieldRef* conflict; + ASSERT_TRUE(fieldSet.insert(&empty, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + ASSERT_FALSE(fieldSet.insert(&empty, &conflict)); + ASSERT_EQUALS(empty, *conflict); + } + + TEST(NotEmptySet, Normal) { + // insert "b.c" and "b.e" + FieldRefSet fieldSet; + FieldRef bDotC("b.c"); + FieldRef bDotE("b.e"); + const FieldRef* conflict; + ASSERT_TRUE(fieldSet.insert(&bDotC, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + ASSERT_TRUE(fieldSet.insert(&bDotE, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + // insert "a" before, OK + FieldRef aSimple("a"); + ASSERT_TRUE(fieldSet.insert(&aSimple, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + // insert "b.d" in the middle, OK + FieldRef bDotD("b.d"); + ASSERT_TRUE(fieldSet.insert(&bDotD, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + // insert "c" after, OK + FieldRef cSimple("c"); + ASSERT_TRUE(fieldSet.insert(&cSimple, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + } + + TEST(NotEmpty, Conflict) { + // insert "b.c" and "b.e" + FieldRefSet fieldSet; + FieldRef bDotC("b.c"); + FieldRef bDotE("b.e"); + const FieldRef* conflict; + ASSERT_TRUE(fieldSet.insert(&bDotC, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + ASSERT_TRUE(fieldSet.insert(&bDotE, &conflict)); + ASSERT_EQUALS(static_cast(NULL), conflict); + + // insert "b" before, conflicts "b.c" + FieldRef bSimple("b"); + ASSERT_FALSE(fieldSet.insert(&bSimple, &conflict)); + ASSERT_EQUALS(bDotC, *conflict); + + // insert: "b.c.d" in the "middle", conflicts "b.c" + FieldRef bDotCDotD("b.c.d"); + ASSERT_FALSE(fieldSet.insert(&bDotCDotD, &conflict)); + ASSERT_EQUALS(bDotC, *conflict); + + // insert: "b.e.f" at the end, conflicts "b.e" + FieldRef bDotEDotF("b.e.f"); + ASSERT_FALSE(fieldSet.insert(&bDotEDotF, &conflict)); + ASSERT_EQUALS(bDotE, *conflict); + } + +} // unnamed namespace diff --git a/src/mongo/db/field_ref_test.cpp b/src/mongo/db/field_ref_test.cpp index e84a9d1158a..e672a23e855 100644 --- a/src/mongo/db/field_ref_test.cpp +++ b/src/mongo/db/field_ref_test.cpp @@ -30,26 +30,33 @@ namespace { using std::string; TEST(Empty, NoFields) { - FieldRef fieldRef; - fieldRef.parse(""); + FieldRef fieldRef(""); ASSERT_EQUALS(fieldRef.numParts(), 0U); ASSERT_EQUALS(fieldRef.dottedField(), ""); } TEST(Empty, NoFieldNames) { string field = "."; - FieldRef fieldRef; - fieldRef.parse(field); + FieldRef fieldRef(field); ASSERT_EQUALS(fieldRef.numParts(), 2U); ASSERT_EQUALS(fieldRef.getPart(0), ""); ASSERT_EQUALS(fieldRef.getPart(1), ""); ASSERT_EQUALS(fieldRef.dottedField(), field); } + TEST(Empty, NoFieldNames2) { + string field = ".."; + FieldRef fieldRef(field); + ASSERT_EQUALS(fieldRef.numParts(), 3U); + ASSERT_EQUALS(fieldRef.getPart(0), ""); + ASSERT_EQUALS(fieldRef.getPart(1), ""); + ASSERT_EQUALS(fieldRef.getPart(2), ""); + ASSERT_EQUALS(fieldRef.dottedField(), field); + } + TEST(Empty, EmptyFieldName) { string field = ".b."; - FieldRef fieldRef; - fieldRef.parse(field); + FieldRef fieldRef(field); ASSERT_EQUALS(fieldRef.numParts(), 3U); ASSERT_EQUALS(fieldRef.getPart(0), ""); ASSERT_EQUALS(fieldRef.getPart(1), "b"); @@ -59,13 +66,23 @@ namespace { TEST(Normal, SinglePart) { string field = "a"; - FieldRef fieldRef; - fieldRef.parse(field); + FieldRef fieldRef(field); ASSERT_EQUALS(fieldRef.numParts(), 1U); ASSERT_EQUALS(fieldRef.getPart(0), field); ASSERT_EQUALS(fieldRef.dottedField(), field); } + TEST(Normal, ParseTwice) { + string field = "a"; + FieldRef fieldRef; + for (int i = 0; i < 2; i++) { + fieldRef.parse(field); + ASSERT_EQUALS(fieldRef.numParts(), 1U); + ASSERT_EQUALS(fieldRef.getPart(0), field); + ASSERT_EQUALS(fieldRef.dottedField(), field); + } + } + TEST(Normal, MulitplePartsVariable) { const char* parts[] = {"a", "b", "c", "d", "e"}; size_t size = sizeof(parts)/sizeof(char*); @@ -75,8 +92,7 @@ namespace { field.append(parts[i]); } - FieldRef fieldRef; - fieldRef.parse(field); + FieldRef fieldRef(field); ASSERT_EQUALS(fieldRef.numParts(), size); for (size_t i=0; i a ); + ASSERT_TRUE( a >= a ); + } + + TEST( Comparison, EqualInSize ) { + FieldRef a( "a.b.c" ), b( "a.d.c" ); + ASSERT_FALSE( a == b ); + ASSERT_TRUE( a != b ); + ASSERT_TRUE( a < b ); + ASSERT_TRUE( a <= b ); + ASSERT_FALSE( a > b ); + ASSERT_FALSE( a >= b ); + } + + TEST( Comparison, NonEqual ) { + FieldRef a( "a.b.c" ), b( "b.d" ); + ASSERT_FALSE( a == b ); + ASSERT_TRUE( a != b ); + ASSERT_TRUE( a < b ); + ASSERT_TRUE( a <= b ); + ASSERT_FALSE( a > b ); + ASSERT_FALSE( a >= b ); + } + + TEST( Comparison, MixedEmtpyAndNot ) { + FieldRef a( "a" ), b; + ASSERT_FALSE( a == b ); + ASSERT_TRUE( a != b ); + ASSERT_FALSE( a < b ); + ASSERT_FALSE( a <= b ); + ASSERT_TRUE( a > b ); + ASSERT_TRUE( a >= b ); + } + + TEST( DottedField, Simple1 ) { + FieldRef a( "a.b.c.d.e" ); + ASSERT_EQUALS( "a.b.c.d.e", a.dottedField() ); + ASSERT_EQUALS( "a.b.c.d.e", a.dottedField(0) ); + ASSERT_EQUALS( "b.c.d.e", a.dottedField(1) ); + ASSERT_EQUALS( "c.d.e", a.dottedField(2) ); + ASSERT_EQUALS( "d.e", a.dottedField(3) ); + ASSERT_EQUALS( "e", a.dottedField(4) ); + ASSERT_EQUALS( "", a.dottedField(5) ); + ASSERT_EQUALS( "", a.dottedField(6) ); + } + +} // namespace diff --git a/src/mongo/db/filever.h b/src/mongo/db/filever.h index e89a8243dcf..05b58c1734c 100644 --- a/src/mongo/db/filever.h +++ b/src/mongo/db/filever.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once diff --git a/src/mongo/db/flushtest.cpp b/src/mongo/db/flushtest.cpp index 2009d922950..e057fe870c3 100644 --- a/src/mongo/db/flushtest.cpp +++ b/src/mongo/db/flushtest.cpp @@ -12,12 +12,26 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include -#include "../util/goodies.h" +#include "mongo/pch.h" + #include +#include + +#include "mongo/util/goodies.h" namespace mongo { diff --git a/src/mongo/db/fts/SConscript b/src/mongo/db/fts/SConscript index 6534b73bdc1..c922c15d14c 100644 --- a/src/mongo/db/fts/SConscript +++ b/src/mongo/db/fts/SConscript @@ -2,7 +2,7 @@ Import("env") -stop_word_lanages = [ +stop_word_languages = [ 'danish', 'dutch', 'english', @@ -21,38 +21,39 @@ stop_word_lanages = [ ] env.Command( [ "stop_words_list.h", "stop_words_list.cpp"], - [ "generate_stop_words.py"] + [ 'stop_words_%s.txt' % x for x in stop_word_lanages ], + [ "generate_stop_words.py"] + [ 'stop_words_%s.txt' % x for x in stop_word_languages ], "$PYTHON $SOURCES $TARGETS" ) -env.StaticLibrary('base', [ +env.Library('base', [ 'fts_index_format.cpp', 'fts_matcher.cpp', 'fts_query.cpp', 'fts_spec.cpp', + 'fts_spec_legacy.cpp', + 'fts_language.cpp', 'fts_util.cpp', + 'fts_element_iterator.cpp', 'stemmer.cpp', 'stop_words.cpp', 'stop_words_list.cpp', 'tokenizer.cpp', ], LIBDEPS=["$BUILD_DIR/mongo/base/base", - "$BUILD_DIR/mongo/bson", + "$BUILD_DIR/mongo/db/common", "$BUILD_DIR/mongo/platform/platform", - "$BUILD_DIR/third_party/libstemmer_c/stemmer" + "$BUILD_DIR/third_party/shim_stemmer" ]) -env.StaticLibrary( 'server_common', [ - 'fts_command.cpp', - 'fts_enabled.cpp' - ] ) +env.Library( 'server_common', [ + 'fts_command.cpp', + 'fts_enabled.cpp' + ] ) -env.StaticLibrary('ftsmongod', [ +env.Library('ftsmongod', [ 'fts_command_mongod.cpp', - 'fts_index.cpp', - 'fts_search.cpp', ], LIBDEPS=["base","server_common"]) -env.StaticLibrary('ftsmongos', [ +env.Library('ftsmongos', [ 'fts_command_mongos.cpp', ], LIBDEPS=["server_common"]) @@ -75,8 +76,14 @@ env.CppUnitTest( "fts_query_test", "fts_query_test.cpp", env.CppUnitTest( "fts_spec_test", "fts_spec_test.cpp", LIBDEPS=["base"] ) +env.CppUnitTest( "fts_language_test", "fts_language_test.cpp", + LIBDEPS=["base"] ) + env.CppUnitTest( "fts_matcher_test", "fts_matcher_test.cpp", LIBDEPS=["base"] ) +env.CppUnitTest( "fts_element_iterator_test", "fts_element_iterator_test.cpp", + LIBDEPS=["base"] ) + env.CppUnitTest( "fts_util_test", "fts_util_test.cpp", LIBDEPS=["base","$BUILD_DIR/mongo/mongohasher"] ) diff --git a/src/mongo/db/fts/fts_command.cpp b/src/mongo/db/fts/fts_command.cpp index 45be13d198f..644024c8e17 100644 --- a/src/mongo/db/fts/fts_command.cpp +++ b/src/mongo/db/fts/fts_command.cpp @@ -14,14 +14,24 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include #include #include "mongo/db/fts/fts_command.h" -#include "mongo/db/fts/fts_enabled.h" -#include "mongo/db/fts/fts_search.h" #include "mongo/db/fts/fts_util.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/timer.h" @@ -43,7 +53,7 @@ namespace mongo { std::vector* out) { ActionSet actions; actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } @@ -54,9 +64,9 @@ namespace mongo { BSONObjBuilder& result, bool fromRepl) { - if ( !isTextSearchEnabled() ) { - errmsg = "text search not enabled"; - return false; + ONCE { + log() << "The text command will be removed in a future release. Use the $text " + "query operator instead." << startupWarningsLog; } string ns = dbname + "." + cmdObj.firstElement().String(); diff --git a/src/mongo/db/fts/fts_command.h b/src/mongo/db/fts/fts_command.h index cbd92758ecb..81a4a3d6043 100644 --- a/src/mongo/db/fts/fts_command.h +++ b/src/mongo/db/fts/fts_command.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -23,6 +35,9 @@ #include "mongo/db/commands.h" +// mongo::fts::FTSCommand is deprecated: the "text" command is deprecated in favor of the $text +// query operator. + namespace mongo { namespace fts { diff --git a/src/mongo/db/fts/fts_command_mongod.cpp b/src/mongo/db/fts/fts_command_mongod.cpp index 6fe534ad681..b01205a23c8 100644 --- a/src/mongo/db/fts/fts_command_mongod.cpp +++ b/src/mongo/db/fts/fts_command_mongod.cpp @@ -14,17 +14,27 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include #include -#include #include "mongo/db/fts/fts_command.h" -#include "mongo/db/fts/fts_search.h" #include "mongo/db/fts/fts_util.h" #include "mongo/db/pdfile.h" -#include "mongo/db/projection.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/type_explain.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/timer.h" @@ -60,104 +70,100 @@ namespace mongo { Timer comm; - scoped_ptr pr; - if ( !projection.isEmpty() ) { - pr.reset( new Projection() ); - pr->init( projection ); - } - - // priority queue for results - Results results; - - NamespaceDetails * d = nsdetails( ns.c_str() ); - if ( !d ) { - errmsg = "can't find ns"; - return false; - } - - vector idxMatches; - d->findIndexByType( INDEX_NAME, idxMatches ); - if ( idxMatches.size() == 0 ) { - errmsg = str::stream() << "no text index for: " << ns; - return false; - } - if ( idxMatches.size() > 1 ) { - errmsg = str::stream() << "too many text index for: " << ns; - return false; - } - - const IndexDetails& id = d->idx( idxMatches[0] ); - BSONObj indexPrefix; + // Rewrite the cmd as a normal query. + BSONObjBuilder queryBob; + queryBob.appendElements(filter); - FTSIndex* ftsIndex = static_cast(id.getSpec().getType()); - if ( language == "" ) { - language = ftsIndex->getFtsSpec().defaultLanguage(); + BSONObjBuilder textBob; + textBob.append("$search", searchString); + if (!language.empty()) { + textBob.append("$language", language); } - Status s = ftsIndex->getFtsSpec().getIndexPrefix( filter, &indexPrefix ); - if ( !s.isOK() ) { - errmsg = s.toString(); + queryBob.append("$text", textBob.obj()); + + // This is the query we exec. + BSONObj queryObj = queryBob.obj(); + + // We sort by the score. + BSONObj sortSpec = BSON("$s" << BSON("$meta" << LiteParsedQuery::metaTextScore)); + + // We also project the score into the document and strip it out later during the reformatting + // of the results. + BSONObjBuilder projBob; + projBob.appendElements(projection); + projBob.appendElements(sortSpec); + BSONObj projObj = projBob.obj(); + + CanonicalQuery* cq; + Status canonicalizeStatus = CanonicalQuery::canonicalize(ns, queryObj, sortSpec, + projObj, 0, limit, BSONObj(), + &cq); + if (!canonicalizeStatus.isOK()) { + errmsg = canonicalizeStatus.reason(); return false; } - - FTSQuery query; - if ( !query.parse( searchString, language ).isOK() ) { - errmsg = "can't parse search"; + Runner* rawRunner; + Status getRunnerStatus = getRunner(cq, &rawRunner, 0); + if (!getRunnerStatus.isOK()) { + errmsg = getRunnerStatus.reason(); return false; } - result.append( "queryDebugString", query.debugString() ); - result.append( "language", language ); - - FTSSearch search( d, id, indexPrefix, query, filter ); - search.go( &results, limit ); - - // grab underlying container inside priority queue - vector r( results.dangerous() ); - // sort results by score (not always in correct order, especially w.r.t. multiterm) - sort( r.begin(), r.end() ); + auto_ptr runner(rawRunner); - // build the results bson array shown to user - BSONArrayBuilder a( result.subarrayStart( "results" ) ); + BSONArrayBuilder resultBuilder(result.subarrayStart("results")); - int tempSize = 1024 * 1024; // leave a mb for other things - long long numReturned = 0; + // Quoth: "leave a mb for other things" + int resultSize = 1024 * 1024; - for ( unsigned n = 0; n < r.size(); n++ ) { - BSONObj obj = BSONObj::make(r[n].rec); - BSONObj toSendBack = obj; + int numReturned = 0; - if ( pr ) { - toSendBack = pr->transform(obj); - } - - if ( ( tempSize + toSendBack.objsize() ) >= BSONObjMaxUserSize ) { + BSONObj obj; + while (Runner::RUNNER_ADVANCED == runner->getNext(&obj, NULL)) { + if ((resultSize + obj.objsize()) >= BSONObjMaxUserSize) { break; } - - BSONObjBuilder x( a.subobjStart() ); - x.append( "score" , r[n].score ); - x.append( "obj", toSendBack ); - - BSONObj xobj = x.done(); - tempSize += xobj.objsize(); - + // We return an array of results. Add another element. + BSONObjBuilder oneResultBuilder(resultBuilder.subobjStart()); + oneResultBuilder.append("score", obj["$s"].number()); + + // Strip out the score from the returned obj. + BSONObjIterator resIt(obj); + BSONObjBuilder resBob; + while (resIt.more()) { + BSONElement elt = resIt.next(); + if (!mongoutils::str::equals("$s", elt.fieldName())) { + resBob.append(elt); + } + } + oneResultBuilder.append("obj", resBob.obj()); + BSONObj addedArrayObj = oneResultBuilder.done(); + resultSize += addedArrayObj.objsize(); numReturned++; } - a.done(); + resultBuilder.done(); // returns some stats to the user - BSONObjBuilder bb( result.subobjStart( "stats" ) ); - bb.appendNumber( "nscanned" , search.getKeysLookedAt() ); - bb.appendNumber( "nscannedObjects" , search.getObjLookedAt() ); - bb.appendNumber( "n" , numReturned ); - bb.appendNumber( "nfound" , r.size() ); - bb.append( "timeMicros", (int)comm.micros() ); - bb.done(); + BSONObjBuilder stats(result.subobjStart("stats")); + + // Fill in nscanned from the explain. + TypeExplain* bareExplain; + Status res = runner->getInfo(&bareExplain, NULL); + if (res.isOK()) { + auto_ptr explain(bareExplain); + stats.append("nscanned", explain->getNScanned()); + stats.append("nscannedObjects", explain->getNScannedObjects()); + } + + stats.appendNumber( "n" , numReturned ); + stats.append( "timeMicros", (int)comm.micros() ); + stats.done(); return true; } - } -} + } // namespace fts + +} // namespace mongo diff --git a/src/mongo/db/fts/fts_command_mongos.cpp b/src/mongo/db/fts/fts_command_mongos.cpp index 8a2e52c5671..4f39eb1543f 100644 --- a/src/mongo/db/fts/fts_command_mongos.cpp +++ b/src/mongo/db/fts/fts_command_mongos.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include @@ -24,6 +36,7 @@ #include "mongo/db/fts/fts_command.h" #include "mongo/s/strategy.h" +#include "mongo/util/timer.h" namespace mongo { @@ -59,8 +72,8 @@ namespace mongo { Timer timer; - map results; - SHARDED->commandOp( dbName, cmdObj, cmdOptions, ns, filter, results ); + vector results; + STRATEGY->commandOp( dbName, cmdObj, cmdOptions, ns, filter, &results ); vector all; long long nscanned = 0; @@ -68,13 +81,14 @@ namespace mongo { BSONObjBuilder shardStats; - for ( map::const_iterator i = results.begin(); i != results.end(); ++i ) { - BSONObj r = i->second; + for ( vector::const_iterator i = results.begin(); + i != results.end(); ++i ) { + BSONObj r = i->result; - LOG(2) << "fts result for shard: " << i->first << "\n" << r << endl; + LOG(2) << "fts result for shard: " << i->shardTarget << "\n" << r << endl; if ( !r["ok"].trueValue() ) { - errmsg = str::stream() << "failure on shard: " << i->first.toString() + errmsg = str::stream() << "failure on shard: " << i->shardTarget.toString() << ": " << r["errmsg"]; result.append( "rawresult", r ); return false; @@ -85,7 +99,7 @@ namespace mongo { nscanned += x["nscanned"].numberLong(); nscannedObjects += x["nscannedObjects"].numberLong(); - shardStats.append( i->first.getName(), x ); + shardStats.append( i->shardTarget.getName(), x ); } if ( r["results"].isABSONObj() ) { diff --git a/src/mongo/db/fts/fts_element_iterator.cpp b/src/mongo/db/fts/fts_element_iterator.cpp new file mode 100644 index 00000000000..477822371a4 --- /dev/null +++ b/src/mongo/db/fts/fts_element_iterator.cpp @@ -0,0 +1,186 @@ +// fts_element_iterator.cpp +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/fts/fts_element_iterator.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/db/fts/fts_util.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/stringutils.h" + +#include + +namespace mongo { + + namespace fts { + + extern const double DEFAULT_WEIGHT; + extern const double MAX_WEIGHT; + + std::ostream& operator<<( std::ostream& os, FTSElementIterator::FTSIteratorFrame& frame ) { + BSONObjIterator it = frame._it; + return os << "FTSIteratorFrame[" + " element=" << (*it).toString() << + ", _language=" << frame._language->str() << + ", _parentPath=" << frame._parentPath << + ", _isArray=" << frame._isArray << "]"; + } + + FTSElementIterator::FTSElementIterator( const FTSSpec& spec, const BSONObj& obj ) + : _frame( obj, spec, &spec.defaultLanguage(), "", false ), + _spec( spec ), + _currentValue( advance() ) + { } + + namespace { + /** Check for exact match or path prefix match. */ + inline bool _matchPrefix( const string& dottedName, const string& weight ) { + if ( weight == dottedName ) { + return true; + } + return mongoutils::str::startsWith( weight, dottedName + '.' ); + } + } + + bool FTSElementIterator::more() { + //_currentValue = advance(); + return _currentValue.valid(); + } + + FTSIteratorValue FTSElementIterator::next() { + FTSIteratorValue result = _currentValue; + _currentValue = advance(); + return result; + } + + /** + * Helper method: + * if (current object iterator not exhausted) return true; + * while (frame stack not empty) { + * resume object iterator popped from stack; + * if (resumed iterator not exhausted) return true; + * } + * return false; + */ + bool FTSElementIterator::moreFrames() { + if (_frame._it.more()) return true; + while (!_frameStack.empty()) { + _frame = _frameStack.top(); + _frameStack.pop(); + if (_frame._it.more()) { + return true; + } + } + return false; + } + + FTSIteratorValue FTSElementIterator::advance() { + while ( moreFrames() ) { + + BSONElement elem = _frame._it.next(); + string fieldName = elem.fieldName(); + + // Skip "language" specifier fields if wildcard. + if ( _spec.wildcard() && _spec.languageOverrideField() == fieldName ) { + continue; + } + + // Compose the dotted name of the current field: + // 1. parent path empty (top level): use the current field name + // 2. parent path non-empty and obj is an array: use the parent path + // 3. parent path non-empty and obj is a sub-doc: append field name to parent path + string dottedName = ( _frame._parentPath.empty() ? fieldName + : _frame._isArray ? _frame._parentPath + : _frame._parentPath + '.' + fieldName ); + + // Find lower bound of dottedName in _weights. lower_bound leaves us at the first + // weight that could possibly match or be a prefix of dottedName. And if this + // element fails to match, then no subsequent weight can match, since the weights + // are lexicographically ordered. + Weights::const_iterator i = _spec.weights().lower_bound( elem.type() == Object + ? dottedName + '.' + : dottedName ); + + // possibleWeightMatch is set if the weight map contains either a match or some item + // lexicographically larger than fieldName. This boolean acts as a guard on + // dereferences of iterator 'i'. + bool possibleWeightMatch = ( i != _spec.weights().end() ); + + // Optimize away two cases, when not wildcard: + // 1. lower_bound seeks to end(): no prefix match possible + // 2. lower_bound seeks to a name which is not a prefix + if ( !_spec.wildcard() ) { + if ( !possibleWeightMatch ) { + continue; + } + else if ( !_matchPrefix( dottedName, i->first ) ) { + continue; + } + } + + // Is the current field an exact match on a weight? + bool exactMatch = ( possibleWeightMatch && i->first == dottedName ); + double weight = ( possibleWeightMatch ? i->second : DEFAULT_WEIGHT ); + + switch ( elem.type() ) { + case String: + // Only index strings on exact match or wildcard. + if ( exactMatch || _spec.wildcard() ) { + return FTSIteratorValue( elem.valuestr(), _frame._language, weight ); + } + break; + + case Object: + // Only descend into a sub-document on proper prefix or wildcard. Note that + // !exactMatch is a sufficient test for proper prefix match, because of + // if ( !matchPrefix( dottedName, i->first ) ) continue; + // block above. + if ( !exactMatch || _spec.wildcard() ) { + _frameStack.push( _frame ); + _frame = FTSIteratorFrame( elem.Obj(), _spec, _frame._language, dottedName, false ); + } + break; + + case Array: + // Only descend into arrays from non-array parents or on wildcard. + if ( !_frame._isArray || _spec.wildcard() ) { + _frameStack.push( _frame ); + _frame = FTSIteratorFrame( elem.Obj(), _spec, _frame._language, dottedName, true ); + } + break; + + default: + // Skip over all other BSON types. + break; + } + } + return FTSIteratorValue(); // valid()==false + } + + } // namespace fts +} // namespace mongo diff --git a/src/mongo/db/fts/fts_element_iterator.h b/src/mongo/db/fts/fts_element_iterator.h new file mode 100644 index 00000000000..3ca2372d1be --- /dev/null +++ b/src/mongo/db/fts/fts_element_iterator.h @@ -0,0 +1,173 @@ +// fts_element_iterator.h + +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/db/fts/fts_language.h" +#include "mongo/db/fts/fts_spec.h" + +#include +#include +#include + +namespace mongo { + + namespace fts { + + /** + * Encapsulates data fields returned by FTSElementIterator + */ + struct FTSIteratorValue { + + FTSIteratorValue( const char* text, + const FTSLanguage* language, + double weight ) + : _text(text), + _language(language), + _weight(weight), + _valid(true) + {} + + FTSIteratorValue() + : _text(NULL), + _language(), + _weight(0.0), + _valid(false) + {} + + bool valid() const { return _valid; } + + const char* _text; + const FTSLanguage* _language; + double _weight; + bool _valid; + }; + + /** + * Iterator pattern for walking through text-indexed fields of a + * BSON document. + * + * Example usage: + * FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + * FTSElementIterator it( spec, obj ); + * while ( it.more() ) { + * FTSIteratorValue val = it.next(); + * std::cout << val._text << '[' << val._language.str() + * << ',' << val._weight << ']' << std::endl; + * } + * + */ + class FTSElementIterator { + public: + /** + * Iterator constructor + * + * Note: Caller must ensure that the constructed FTSElementIterator + * does >not< outlive either spec or obj. + * + * @arg spec text index specifier + * @arg obj document that the iterator will traverse + */ + FTSElementIterator( const FTSSpec& spec, const BSONObj& obj); + + /** + * Iterator interface: returns false iff there are no further text-indexable fields. + */ + bool more(); + + /** + * Iterator interface: advances to the next text-indexable field. + */ + FTSIteratorValue next(); + + /** + * Iterator frame needed for iterative implementation of + * recursive sub-documents. + */ + struct FTSIteratorFrame { + FTSIteratorFrame( const BSONObj& obj, + const FTSSpec& spec, + const FTSLanguage* parentLanguage, + const std::string& parentPath, + bool isArray ) + : _it( obj ), + _language( spec._getLanguageToUseV2( obj, parentLanguage ) ), + _parentPath( parentPath ), + _isArray( isArray ) + {} + + friend std::ostream& operator<<(std::ostream&, FTSIteratorFrame&); + + BSONObjIterator _it; + const FTSLanguage* _language; + std::string _parentPath; + bool _isArray; + }; + + private: + /** + * Helper method: + * returns false iff all FTSIteratorFrames on _frameStack are exhausted. + */ + bool moreFrames(); + + /** + * Helper method: + * advances to the next text-indexable field, possibly pushing frames as + * needed for recursive sub-documents. + */ + FTSIteratorValue advance(); + + /** + * Stack used by iterative implementation of recursive sub-document traversal. + */ + std::stack _frameStack; + + /** + * Current frame, not yet pushed to stack. + */ + FTSIteratorFrame _frame; + + /** + * Constructor input parameter: text index specification. + */ + const FTSSpec& _spec; + + /** + * Current iterator return value, computed by 'more()', returned by 'next()'. + */ + FTSIteratorValue _currentValue; + }; + + } // namespace fts +} // namespace mongo + diff --git a/src/mongo/db/fts/fts_element_iterator_test.cpp b/src/mongo/db/fts/fts_element_iterator_test.cpp new file mode 100644 index 00000000000..1d0eb88841a --- /dev/null +++ b/src/mongo/db/fts/fts_element_iterator_test.cpp @@ -0,0 +1,310 @@ +// fts_element_iterator_test.cpp +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/pch.h" + +#include "mongo/db/fts/fts_element_iterator.h" +#include "mongo/db/json.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { + namespace fts { + + TEST( FTSElementIterator, Test1 ) { + + BSONObj obj = fromjson( + "{ b : \"walking\"," + " c : { e: \"walked\" }," + " d : \"walker\"" + " }" ); + + BSONObj indexSpec = fromjson( + "{ key : { a : \"text\" }, weights : { b : 10, d : 5 } }" ); + + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + Weights::const_iterator itt = spec.weights().begin(); + ASSERT( itt != spec.weights().end() ); + ASSERT_EQUALS( "a", itt->first ); + ASSERT_EQUALS( 1, itt->second ); + ++itt; + ASSERT( itt != spec.weights().end() ); + ASSERT_EQUALS( "b", itt->first ); + ASSERT_EQUALS( 10, itt->second ); + ++itt; + ASSERT( itt != spec.weights().end() ); + ASSERT_EQUALS( "d", itt->first ); + ASSERT_EQUALS( 5, itt->second ); + ++itt; + + FTSElementIterator it( spec, obj ); + + ASSERT( it.more() ); + FTSIteratorValue val = it.next(); + ASSERT_EQUALS( "walking", string(val._text) ); + ASSERT_EQUALS( "english", val._language->str() ); + ASSERT_EQUALS( 10, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "walker", string(val._text) ); + ASSERT_EQUALS( "english", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + } + + // Multi-language : test + TEST( FTSElementIterator, Test2 ) { + + BSONObj obj = fromjson( + "{ a :" + " { b :" + " [ { c : \"walked\", language : \"english\" }," + " { c : \"camminato\", language : \"italian\" }," + " { c : \"ging\", language : \"german\" } ]" + " }," + " d : \"Feliz Año Nuevo!\"," + " language : \"spanish\"" + " }" ); + + BSONObj indexSpec = fromjson( + "{ key : { \"a.b.c\" : \"text\", d : \"text\" } }" ); + + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + + FTSElementIterator it( spec, obj ); + + ASSERT( it.more() ); + FTSIteratorValue val = it.next(); + ASSERT_EQUALS( "walked", string(val._text) ); + ASSERT_EQUALS( "english", val._language->str() ); + ASSERT_EQUALS( 1, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "camminato", string(val._text) ); + ASSERT_EQUALS( "italian", val._language->str() ); + ASSERT_EQUALS( 1, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "ging", string(val._text) ); + ASSERT_EQUALS( "german", val._language->str() ); + ASSERT_EQUALS( 1, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "Feliz Año Nuevo!", string(val._text) ); + ASSERT_EQUALS( "spanish", val._language->str() ); + ASSERT_EQUALS( 1, val._weight ); + } + + // Multi-language : test nested stemming per sub-document + TEST( FTSElementIterator, Test3 ) { + + BSONObj obj = fromjson( + "{ language : \"english\"," + " a :" + " { language : \"danish\"," + " b :" + " [ { c : \"foredrag\" }," + " { c : \"foredragsholder\" }," + " { c : \"lector\" } ]" + " }" + "}" ); + + BSONObj indexSpec = fromjson( + "{ key : { a : \"text\", \"a.b.c\" : \"text\" }, weights : { \"a.b.c\" : 5 } }" ); + + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + Weights::const_iterator itt = spec.weights().begin(); + ASSERT( itt != spec.weights().end() ); + ASSERT_EQUALS( "a", itt->first ); + ASSERT_EQUALS( 1, itt->second ); + ++itt; + ASSERT( itt != spec.weights().end() ); + ASSERT_EQUALS( "a.b.c", itt->first ); + ASSERT_EQUALS( 5, itt->second ); + + FTSElementIterator it( spec, obj ); + + ASSERT( it.more() ); + FTSIteratorValue val = it.next(); + ASSERT_EQUALS( "foredrag", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "foredragsholder", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "lector", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + } + + // Multi-language : test nested arrays + TEST( FTSElementIterator, Test4 ) { + + BSONObj obj = fromjson( + "{ language : \"english\"," + " a : [" + " { language : \"danish\"," + " b :" + " [ { c : [\"foredrag\"] }," + " { c : [\"foredragsholder\"] }," + " { c : [\"lector\"] } ]" + " } ]" + "}" ); + + BSONObj indexSpec = fromjson( + "{ key : { \"a.b.c\" : \"text\" }, weights : { \"a.b.c\" : 5 } }" ); + + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + FTSElementIterator it( spec, obj ); + + ASSERT( it.more() ); + FTSIteratorValue val = it.next(); + ASSERT_EQUALS( "foredrag", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "foredragsholder", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "lector", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + } + + // Multi-language : test wildcard spec + TEST( FTSElementIterator, Test5 ) { + + BSONObj obj = fromjson( + "{ language : \"english\"," + " b : \"these boots were made for walking\"," + " c : { e: \"I walked half way to the market before seeing the sunrise\" }," + " d : " + " { language : \"danish\"," + " e :" + " [ { f : \"foredrag\", g : 12 }," + " { f : \"foredragsholder\", g : 13 }," + " { f : \"lector\", g : 14 } ]" + " }" + "}" ); + + BSONObj indexSpec = fromjson( + "{ key : { a : \"text\" }, weights : { b : 20, c : 10, \"d.e.f\" : 5 } }" ); + + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + FTSElementIterator it( spec, obj ); + + ASSERT( it.more() ); + FTSIteratorValue val = it.next(); + ASSERT_EQUALS( "these boots were made for walking", string(val._text) ); + ASSERT_EQUALS( "english", val._language->str() ); + ASSERT_EQUALS( 20, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "foredrag", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "foredragsholder", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "lector", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + } + + // Multi-language : test wildcard spec + TEST( FTSElementIterator, Test6 ) { + + BSONObj obj = fromjson( + "{ language : \"english\"," + " b : \"these boots were made for walking\"," + " c : { e: \"I walked half way to the market before seeing the sunrise\" }," + " d : " + " { language : \"danish\"," + " e :" + " [ { f : \"foredrag\", g : 12 }," + " { f : \"foredragsholder\", g : 13 }," + " { f : \"lector\", g : 14 } ]" + " }" + "}" ); + + BSONObj indexSpec = fromjson( + "{ key : { a : \"text\" }, weights : { b : 20, c : 10, \"d.e.f\" : 5 } }" ); + + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + FTSElementIterator it( spec, obj ); + + ASSERT( it.more() ); + FTSIteratorValue val = it.next(); + ASSERT_EQUALS( "these boots were made for walking", string(val._text) ); + ASSERT_EQUALS( "english", val._language->str() ); + ASSERT_EQUALS( 20, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "foredrag", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "foredragsholder", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + + ASSERT( it.more() ); + val = it.next(); + ASSERT_EQUALS( "lector", string(val._text) ); + ASSERT_EQUALS( "danish", val._language->str() ); + ASSERT_EQUALS( 5, val._weight ); + } + } +} + diff --git a/src/mongo/db/fts/fts_enabled.cpp b/src/mongo/db/fts/fts_enabled.cpp index 7a11e394f6a..25a4619eae1 100644 --- a/src/mongo/db/fts/fts_enabled.cpp +++ b/src/mongo/db/fts/fts_enabled.cpp @@ -1,4 +1,4 @@ -// fts_enabled.h +// fts_enabled.cpp /** * Copyright (C) 2012 10gen Inc. @@ -14,15 +14,63 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/server_parameters.h" namespace mongo { namespace fts { - MONGO_EXPORT_SERVER_PARAMETER( textSearchEnabled, bool, false ); - bool isTextSearchEnabled() { - return textSearchEnabled; + namespace { + + bool dummyEnabledFlag = true; // Unused, needed for server parameter. + + /** + * Declaration for the "textSearchEnabled" server parameter, which is now deprecated. + * Note that: + * - setting to true performs a no-op and logs a deprecation message. + * - setting to false will fail. + */ + class ExportedTextSearchEnabledParameter : public ExportedServerParameter { + public: + ExportedTextSearchEnabledParameter() : + ExportedServerParameter( ServerParameterSet::getGlobal(), + "textSearchEnabled", + &dummyEnabledFlag, + true, + true ) {} + + virtual Status validate( const bool& potentialNewValue ) { + if ( !potentialNewValue ) { + return Status( ErrorCodes::BadValue, + "textSearchEnabled cannot be set to false"); + } + + log() << "Attempted to set textSearchEnabled server parameter."; + log() << "Text search is enabled by default and cannot be disabled."; + log() << "The following are now deprecated and will be removed in a future " + << "release:"; + log() << "- the \"textSearchEnabled\" server parameter (setting it has no " + << "effect)"; + log() << "- the \"text\" command (has been replaced by the $text query " + "operator)"; + + return Status::OK(); + } + + } exportedTextSearchEnabledParam; + } } } diff --git a/src/mongo/db/fts/fts_enabled.h b/src/mongo/db/fts/fts_enabled.h deleted file mode 100644 index d3f733dc49f..00000000000 --- a/src/mongo/db/fts/fts_enabled.h +++ /dev/null @@ -1,25 +0,0 @@ -// fts_enabled.h - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -namespace mongo { - namespace fts { - bool isTextSearchEnabled(); - } -} diff --git a/src/mongo/db/fts/fts_index.cpp b/src/mongo/db/fts/fts_index.cpp deleted file mode 100644 index 2e667d0d1f7..00000000000 --- a/src/mongo/db/fts/fts_index.cpp +++ /dev/null @@ -1,104 +0,0 @@ -// fts_index.cpp - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/base/init.h" -#include "mongo/db/client.h" -#include "mongo/db/fts/fts_enabled.h" -#include "mongo/db/fts/fts_index.h" -#include "mongo/db/fts/fts_index_format.h" -#include "mongo/util/mongoutils/str.h" -#include "mongo/util/stringutils.h" -#include "mongo/util/timer.h" -#include "mongo/db/pdfile.h" - -namespace mongo { - - namespace fts { - - using namespace mongoutils; - - /* - * extrapolates the weights vector - * and extra information from the spec - * @param plugin the index plugin for FTS - * @param spec the index specification - */ - FTSIndex::FTSIndex( const IndexPlugin* plugin, const IndexSpec* spec ) - : IndexType( plugin, spec ), _ftsSpec( spec->info ) { - } - - void FTSIndex::getKeys( const BSONObj& obj, BSONObjSet& keys) const { - FTSIndexFormat::getKeys( _ftsSpec, obj, &keys ); - } - - shared_ptr FTSIndex::newCursor( const BSONObj& query, - const BSONObj& order, - int numWanted ) const { - shared_ptr c; - verify(0); - return c; - } - - - FTSIndexPlugin::FTSIndexPlugin() : IndexPlugin( INDEX_NAME ) {} - - - /* - * Adjusts spec by appending information relative to the - * FTS Index (such as weights, index name, etc) - * @param spec, specification object - * - */ - BSONObj FTSIndexPlugin::adjustIndexSpec( const BSONObj& spec ) const { - StringData desc = cc().desc(); - if ( desc.find( "conn" ) == 0 ) { - // this is to make sure we only complain for users - // if you do get a text index created an a primary - // want it to index on the secondary as well - massert( 16633, "text search not enabled", isTextSearchEnabled() ); - } - return FTSSpec::fixSpec( spec ); - } - - /* - * Generates an FTSIndex with a spec and this plugin - * @param spec, specification to be used - */ - IndexType* FTSIndexPlugin::generate( const IndexSpec* spec ) const { - return new FTSIndex( this, spec ); - } - - void FTSIndexPlugin::postBuildHook( const IndexSpec& spec ) const { - string ns = spec.getDetails()->parentNS(); - NamespaceDetails* nsd = nsdetails( ns ); - if ( nsd->setUserFlag( NamespaceDetails::Flag_UsePowerOf2Sizes ) ) { - nsd->syncUserFlags( ns ); - } - } - - FTSIndexPlugin* ftsPlugin; - MONGO_INITIALIZER(FTSIndexPlugin)(InitializerContext* context) { - ftsPlugin = new FTSIndexPlugin(); - return Status::OK(); - } - - } - -} diff --git a/src/mongo/db/fts/fts_index.h b/src/mongo/db/fts/fts_index.h deleted file mode 100644 index 2eff9207ab1..00000000000 --- a/src/mongo/db/fts/fts_index.h +++ /dev/null @@ -1,69 +0,0 @@ -// fts_index.h - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include -#include - -#include "mongo/db/fts/fts_spec.h" -#include "mongo/db/fts/fts_util.h" -#include "mongo/db/fts/stemmer.h" -#include "mongo/db/fts/stop_words.h" -#include "mongo/db/fts/tokenizer.h" -#include "mongo/db/index.h" - -namespace mongo { - - namespace fts { - - class FTSIndex : public IndexType { - public: - - // index constructor, called when user enters ensureIndex command with fts flag - FTSIndex(const IndexPlugin *plugin, const IndexSpec* spec); - - void getKeys( const BSONObj& obj, BSONObjSet& keys) const; - - /* newCursor is pure Virtual in IndexType so it has to be redefined in FTSIndex */ - shared_ptr newCursor( const BSONObj& query, - const BSONObj& order, - int numWanted ) const; - - const FTSSpec& getFtsSpec() const { return _ftsSpec; } - - private: - - FTSSpec _ftsSpec; - }; - - - class FTSIndexPlugin : public IndexPlugin { - public: - FTSIndexPlugin(); - - IndexType* generate( const IndexSpec* spec ) const; - - BSONObj adjustIndexSpec( const BSONObj& spec ) const; - - void postBuildHook( const IndexSpec& spec ) const; - - }; - - } //namespace fts -} //namespace mongo diff --git a/src/mongo/db/fts/fts_index_format.cpp b/src/mongo/db/fts/fts_index_format.cpp index f153dc02864..cff983f01ad 100644 --- a/src/mongo/db/fts/fts_index_format.cpp +++ b/src/mongo/db/fts/fts_index_format.cpp @@ -14,12 +14,27 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" +#include + #include "mongo/base/init.h" #include "mongo/db/fts/fts_index_format.h" +#include "mongo/util/hex.h" #include "mongo/util/mongoutils/str.h" namespace mongo { @@ -29,6 +44,37 @@ namespace mongo { namespace { BSONObj nullObj; BSONElement nullElt; + + // New in textIndexVersion 2. + // If the term is longer than 32 characters, it may + // result in the generated key being too large + // for the index. In that case, we generate a 64-character key + // from the concatenation of the first 32 characters + // and the hex string of the murmur3 hash value of the entire + // term value. + const size_t termKeyPrefixLength = 32U; + // 128-bit hash value expressed in hex = 32 characters + const size_t termKeySuffixLength = 32U; + const size_t termKeyLength = termKeyPrefixLength + termKeySuffixLength; + + /** + * Returns size of buffer required to store term in index key. + * In version 1, terms are stored verbatim in key. + * In version 2, terms longer than 32 characters are hashed and combined + * with a prefix. + */ + int guessTermSize( const std::string& term, TextIndexVersion textIndexVersion ) { + if ( TEXT_INDEX_VERSION_1 == textIndexVersion ) { + return term.size(); + } + else { + invariant( TEXT_INDEX_VERSION_2 == textIndexVersion ); + if ( term.size() <= termKeyPrefixLength ) { + return term.size(); + } + return termKeyLength; + } + } } MONGO_INITIALIZER( FTSIndexFormat )( InitializerContext* context ) { @@ -80,9 +126,7 @@ namespace mongo { long long keyBSONSize = 0; const int MaxKeyBSONSizeMB = 4; - for ( TermFrequencyMap::const_iterator i = term_freqs.begin(); - i != term_freqs.end(); - ++i ) { + for ( TermFrequencyMap::const_iterator i = term_freqs.begin(); i != term_freqs.end(); ++i ) { const string& term = i->first; double weight = i->second; @@ -92,21 +136,23 @@ namespace mongo { 5 /* bson overhead */ + 10 /* weight */ + 8 /* term overhead */ + - term.size() + + /* term size (could be truncated/hashed) */ + guessTermSize( term, spec.getTextIndexVersion() ) + extraSize; BSONObjBuilder b(guess); // builds a BSON object with guess length. - for ( unsigned k = 0; k < extrasBefore.size(); k++ ) + for ( unsigned k = 0; k < extrasBefore.size(); k++ ) { b.appendAs( extrasBefore[k], "" ); - _appendIndexKey( b, weight, term ); - for ( unsigned k = 0; k < extrasAfter.size(); k++ ) + } + _appendIndexKey( b, weight, term, spec.getTextIndexVersion() ); + for ( unsigned k = 0; k < extrasAfter.size(); k++ ) { b.appendAs( extrasAfter[k], "" ); + } BSONObj res = b.obj(); verify( guess >= res.objsize() ); keys->insert( res ); - keyBSONSize += res.objsize(); uassert( 16733, @@ -120,21 +166,48 @@ namespace mongo { BSONObj FTSIndexFormat::getIndexKey( double weight, const string& term, - const BSONObj& indexPrefix ) { + const BSONObj& indexPrefix, + TextIndexVersion textIndexVersion ) { BSONObjBuilder b; BSONObjIterator i( indexPrefix ); - while ( i.more() ) + while ( i.more() ) { b.appendAs( i.next(), "" ); + } - _appendIndexKey( b, weight, term ); + _appendIndexKey( b, weight, term, textIndexVersion ); return b.obj(); } - void FTSIndexFormat::_appendIndexKey( BSONObjBuilder& b, double weight, const string& term ) { + void FTSIndexFormat::_appendIndexKey( BSONObjBuilder& b, double weight, const string& term, + TextIndexVersion textIndexVersion ) { verify( weight >= 0 && weight <= MAX_WEIGHT ); // FTSmaxweight = defined in fts_header - b.append( "", term ); - b.append( "", weight ); + // Terms are added to index key verbatim. + if ( TEXT_INDEX_VERSION_1 == textIndexVersion ) { + b.append( "", term ); + b.append( "", weight ); + } + // See comments at the top of file for termKeyPrefixLength. + // Apply hash for text index version 2 to long terms (longer than 32 characters). + else { + invariant( TEXT_INDEX_VERSION_2 == textIndexVersion ); + if ( term.size() <= termKeyPrefixLength ) { + b.append( "", term ); + } + else { + union { + uint64_t hash[2]; + char data[16]; + } t; + uint32_t seed = 0; + MurmurHash3_x64_128( term.data(), term.size(), seed, t.hash ); + string keySuffix = mongo::toHexLower( t.data, sizeof( t.data ) ); + invariant( termKeySuffixLength == keySuffix.size() ); + b.append( "", term.substr( 0, termKeyPrefixLength ) + + keySuffix ); + } + b.append( "", weight ); + } } } } diff --git a/src/mongo/db/fts/fts_index_format.h b/src/mongo/db/fts/fts_index_format.h index eeb225e756f..e7d64bc0479 100644 --- a/src/mongo/db/fts/fts_index_format.h +++ b/src/mongo/db/fts/fts_index_format.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -36,10 +48,12 @@ namespace mongo { * @param weight, the weight of the term in the entry * @param term, the string term in the entry * @param indexPrefix, the fields that go in the index first + * @param textIndexVersion, index version. affects key format. */ static BSONObj getIndexKey( double weight, const string& term, - const BSONObj& indexPrefix ); + const BSONObj& indexPrefix, + TextIndexVersion textIndexVersion ); private: /* @@ -47,8 +61,10 @@ namespace mongo { * @param b, reference to the BSONOBjBuilder * @param weight, the weight of the term in the entry * @param term, the string term in the entry + * @param textIndexVersion, index version. affects key format. */ - static void _appendIndexKey( BSONObjBuilder& b, double weight, const string& term ); + static void _appendIndexKey( BSONObjBuilder& b, double weight, const string& term, + TextIndexVersion textIndexVersion ); }; } diff --git a/src/mongo/db/fts/fts_index_format_test.cpp b/src/mongo/db/fts/fts_index_format_test.cpp index 5ef27dd83f5..09a3766f94e 100644 --- a/src/mongo/db/fts/fts_index_format_test.cpp +++ b/src/mongo/db/fts/fts_index_format_test.cpp @@ -14,18 +14,35 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" +#include + #include "mongo/db/fts/fts_index_format.h" +#include "mongo/util/mongoutils/str.h" #include "mongo/unittest/unittest.h" namespace mongo { namespace fts { + using std::string; + TEST( FTSIndexFormat, Simple1 ) { FTSSpec spec( FTSSpec::fixSpec( BSON( "key" << BSON( "data" << "text" ) ) ) ); BSONObjSet keys; @@ -103,6 +120,84 @@ namespace mongo { ASSERT_EQUALS( 1U, keys2.size() ); } + /** + * Helper function to compare keys returned in getKeys() result + * with expected values. + */ + void assertEqualsIndexKeys( std::set& expectedKeys, const BSONObjSet& keys ) { + ASSERT_EQUALS( expectedKeys.size(), keys.size() ); + for ( BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i ) { + BSONObj key = *i; + ASSERT_EQUALS( 2, key.nFields() ); + ASSERT_EQUALS( String, key.firstElement().type() ); + string s = key.firstElement().String(); + std::set::const_iterator j = expectedKeys.find(s); + if (j == expectedKeys.end()) { + mongoutils::str::stream ss; + ss << "unexpected key " << s << " in FTSIndexFormat::getKeys result. " + << "expected keys:"; + for (std::set::const_iterator k = expectedKeys.begin(); + k != expectedKeys.end(); ++k) { + ss << "\n " << *k; + } + FAIL(ss); + } + } + } + + /** + * Tests keys for long terms using text index version 1. + * Terms that are too long are not truncated in version 1. + */ + TEST( FTSIndexFormat, LongWordsTextIndexVersion1 ) { + FTSSpec spec( FTSSpec::fixSpec( BSON( "key" << BSON( "data" << "text" ) << + "textIndexVersion" << 1 ) ) ); + BSONObjSet keys; + string longPrefix( 1024U, 'a' ); + // "aaa...aaacat" + string longWordCat = longPrefix + "cat"; + // "aaa...aaasat" + string longWordSat = longPrefix + "sat"; + string text = mongoutils::str::stream() << longWordCat << " " << longWordSat; + FTSIndexFormat::getKeys( spec, BSON( "data" << text ), &keys ); + + // Hard-coded expected computed keys for future-proofing. + std::set expectedKeys; + // cat + expectedKeys.insert( longWordCat ); + // sat + expectedKeys.insert( longWordSat ); + + assertEqualsIndexKeys( expectedKeys, keys); + } + + /** + * Tests keys for long terms using text index version 2. + * In version 2, long terms (longer than 32 characters) + * are hashed with murmur3 and appended to the first 32 + * characters of the term to form the index key. + */ + TEST( FTSIndexFormat, LongWordTextIndexVersion2 ) { + FTSSpec spec( FTSSpec::fixSpec( BSON( "key" << BSON( "data" << "text" ) << + "textIndexVersion" << 2 ) ) ); + BSONObjSet keys; + string longPrefix( 1024U, 'a' ); + // "aaa...aaacat" + string longWordCat = longPrefix + "cat"; + // "aaa...aaasat" + string longWordSat = longPrefix + "sat"; + string text = mongoutils::str::stream() << longWordCat << " " << longWordSat; + FTSIndexFormat::getKeys( spec, BSON( "data" << text ), &keys ); + + // Hard-coded expected computed keys for future-proofing. + std::set expectedKeys; + // cat + expectedKeys.insert( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab8e78455d827ebb87cbe87f392bf45f6" ); + // sat + expectedKeys.insert( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaf2d6f58bb3b81b97e611ae7ccac6dea7" ); + + assertEqualsIndexKeys( expectedKeys, keys); + } } } diff --git a/src/mongo/db/fts/fts_language.cpp b/src/mongo/db/fts/fts_language.cpp new file mode 100644 index 00000000000..21474038f06 --- /dev/null +++ b/src/mongo/db/fts/fts_language.cpp @@ -0,0 +1,262 @@ +// fts_language.cpp + +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/fts/fts_language.h" + +#include + +#include "mongo/base/init.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/string_map.h" +#include "mongo/util/stringutils.h" + +namespace mongo { + + namespace fts { + + namespace { + + /** + * Case-insensitive StringData comparator. + */ + struct LanguageStringCompare { + /** Returns true if lhs < rhs. */ + bool operator()( const StringData& lhs, const StringData& rhs ) const { + size_t minSize = std::min( lhs.size(), rhs.size() ); + + for ( size_t x = 0; x < minSize; x++ ) { + char a = tolower( lhs[x] ); + char b = tolower( rhs[x] ); + if ( a < b ) { + return true; + } + if ( a > b ) { + return false; + } + } + + return lhs.size() < rhs.size(); + } + }; + + // Lookup table from user language string (case-insensitive) to FTSLanguage. Populated + // by initializers in group FTSAllLanguagesRegistered and initializer + // FTSRegisterLanguageAliases. For use with TEXT_INDEX_VERSION_2 text indexes only. + typedef std::map LanguageMapV2; + LanguageMapV2 languageMapV2; + + // Like languageMapV2, but for use with TEXT_INDEX_VERSION_1 text indexes. + // Case-sensitive by lookup key. + typedef std::map LanguageMapV1; + LanguageMapV1 languageMapV1; + } + + MONGO_INITIALIZER_GROUP( FTSAllLanguagesRegistered, MONGO_NO_PREREQUISITES, + MONGO_NO_DEPENDENTS ); + + // + // Register supported languages' canonical names for TEXT_INDEX_VERSION_2. + // + + MONGO_FTS_LANGUAGE_DECLARE( languageNoneV2, "none", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDanishV2, "danish", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDutchV2, "dutch", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageEnglishV2, "english", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFinnishV2, "finnish", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFrenchV2, "french", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageGermanV2, "german", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageHungarianV2, "hungarian", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageItalianV2, "italian", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageNorwegianV2, "norwegian", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languagePortugueseV2, "portuguese", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRomanianV2, "romanian", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRussianV2, "russian", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageSpanishV2, "spanish", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageSwedishV2, "swedish", TEXT_INDEX_VERSION_2 ); + MONGO_FTS_LANGUAGE_DECLARE( languageTurkishV2, "turkish", TEXT_INDEX_VERSION_2 ); + + // + // Register all Snowball language modules for TEXT_INDEX_VERSION_1. Note that only the full + // names are recognized by the StopWords class (as such, the language string "dan" in + // TEXT_INDEX_VERSION_1 will generate the Danish stemmer and the empty stopword list). + // + + MONGO_FTS_LANGUAGE_DECLARE( languageNoneV1, "none", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDaV1, "da", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDanV1, "dan", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDanishV1, "danish", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDeV1, "de", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDeuV1, "deu", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDutV1, "dut", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageDutchV1, "dutch", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageEnV1, "en", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageEngV1, "eng", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageEnglishV1, "english", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageEsV1, "es", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageEslV1, "esl", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFiV1, "fi", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFinV1, "fin", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFinnishV1, "finnish", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFrV1, "fr", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFraV1, "fra", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFreV1, "fre", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageFrenchV1, "french", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageGerV1, "ger", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageGermanV1, "german", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageHuV1, "hu", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageHunV1, "hun", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageHungarianV1, "hungarian", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageItV1, "it", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageItaV1, "ita", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageItalianV1, "italian", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageNlV1, "nl", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageNldV1, "nld", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageNoV1, "no", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageNorV1, "nor", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageNorwegianV1, "norwegian", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languagePorV1, "por", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languagePorterV1, "porter", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languagePortugueseV1, "portuguese", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languagePtV1, "pt", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRoV1, "ro", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRomanianV1, "romanian", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRonV1, "ron", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRuV1, "ru", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRumV1, "rum", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRusV1, "rus", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageRussianV1, "russian", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageSpaV1, "spa", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageSpanishV1, "spanish", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageSvV1, "sv", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageSweV1, "swe", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageSwedishV1, "swedish", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageTrV1, "tr", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageTurV1, "tur", TEXT_INDEX_VERSION_1 ); + MONGO_FTS_LANGUAGE_DECLARE( languageTurkishV1, "turkish", TEXT_INDEX_VERSION_1 ); + + MONGO_INITIALIZER_WITH_PREREQUISITES( FTSRegisterLanguageAliases, + ( "FTSAllLanguagesRegistered" ) ) + ( InitializerContext* context ) { + // Register language aliases for TEXT_INDEX_VERSION_2. + FTSLanguage::registerLanguageAlias( &languageDanishV2, "da", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageDutchV2, "nl", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageEnglishV2, "en", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageFinnishV2, "fi", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageFrenchV2, "fr", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageGermanV2, "de", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageHungarianV2, "hu", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageItalianV2, "it", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageNorwegianV2, "nb", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languagePortugueseV2, "pt", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageRomanianV2, "ro", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageRussianV2, "ru", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageSpanishV2, "es", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageSwedishV2, "sv", TEXT_INDEX_VERSION_2 ); + FTSLanguage::registerLanguageAlias( &languageTurkishV2, "tr", TEXT_INDEX_VERSION_2 ); + return Status::OK(); + } + + // static + void FTSLanguage::registerLanguage( const StringData& languageName, + TextIndexVersion textIndexVersion, + FTSLanguage* language ) { + verify( !languageName.empty() ); + language->_canonicalName = languageName.toString(); + switch ( textIndexVersion ) { + case TEXT_INDEX_VERSION_2: + verify( languageMapV2.find( languageName ) == languageMapV2.end() ); + languageMapV2[ languageName ] = language; + return; + case TEXT_INDEX_VERSION_1: + verify( languageMapV1.find( languageName ) == languageMapV1.end() ); + languageMapV1[ languageName ] = language; + return; + } + verify( false ); + } + + // static + void FTSLanguage::registerLanguageAlias( const FTSLanguage* language, + const StringData& alias, + TextIndexVersion textIndexVersion ) { + switch ( textIndexVersion ) { + case TEXT_INDEX_VERSION_2: + verify( languageMapV2.find( alias ) == languageMapV2.end() ); + languageMapV2[ alias ] = language; + return; + case TEXT_INDEX_VERSION_1: + verify( languageMapV1.find( alias ) == languageMapV1.end() ); + languageMapV1[ alias ] = language; + return; + } + verify( false ); + } + + FTSLanguage::FTSLanguage() : _canonicalName() { + } + + const std::string& FTSLanguage::str() const { + verify( !_canonicalName.empty() ); + return _canonicalName; + } + + // static + StatusWithFTSLanguage FTSLanguage::make( const StringData& langName, + TextIndexVersion textIndexVersion ) { + switch ( textIndexVersion ) { + case TEXT_INDEX_VERSION_2: { + LanguageMapV2::const_iterator it = languageMapV2.find( langName ); + if ( it == languageMapV2.end() ) { + // TEXT_INDEX_VERSION_2 rejects unrecognized language strings. + Status status = Status( ErrorCodes::BadValue, + mongoutils::str::stream() << + "unsupported language: \"" << langName << + "\"" ); + return StatusWithFTSLanguage( status ); + } + + return StatusWithFTSLanguage( it->second ); + } + case TEXT_INDEX_VERSION_1: { + LanguageMapV1::const_iterator it = languageMapV1.find( langName ); + if ( it == languageMapV1.end() ) { + // TEXT_INDEX_VERSION_1 treats unrecognized language strings as "none". + return StatusWithFTSLanguage( &languageNoneV1 ); + } + return StatusWithFTSLanguage( it->second ); + } + } + + verify( false ); + return StatusWithFTSLanguage( Status::OK() ); + } + } +} diff --git a/src/mongo/db/fts/fts_language.h b/src/mongo/db/fts/fts_language.h new file mode 100644 index 00000000000..3a7a471bc32 --- /dev/null +++ b/src/mongo/db/fts/fts_language.h @@ -0,0 +1,128 @@ +// fts_language.h + +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/fts/fts_util.h" +#include "mongo/base/status_with.h" + +#include + +namespace mongo { + + namespace fts { + + #define MONGO_FTS_LANGUAGE_DECLARE( language, name, version ) \ + FTSLanguage language; \ + MONGO_INITIALIZER_GENERAL( language, MONGO_NO_PREREQUISITES, \ + ( "FTSAllLanguagesRegistered" ) ) \ + ( ::mongo::InitializerContext* context ) { \ + FTSLanguage::registerLanguage( name, version, &language ); \ + return Status::OK(); \ + } + + /** + * A FTSLanguage represents a language for a text-indexed document or a text search. + * FTSLanguage objects are not copyable. + * + * Recommended usage: + * + * StatusWithFTSLanguage swl = FTSLanguage::make( "en", TEXT_INDEX_VERSION_2 ); + * if ( !swl.getStatus().isOK() ) { + * // Error. + * } + * else { + * const FTSLanguage* language = swl.getValue(); + * // Use language. + * } + */ + class FTSLanguage { + // Use make() instead of copying. + MONGO_DISALLOW_COPYING( FTSLanguage ); + public: + /** Create an uninitialized language. */ + FTSLanguage(); + + /** + * Returns the language as a string in canonical form (lowercased English name). It is + * an error to call str() on an uninitialized language. + */ + const std::string& str() const; + + /** + * Register string 'languageName' as a new language with text index version + * 'textIndexVersion'. Saves the resulting language to out-argument 'languageOut'. + * Subsequent calls to FTSLanguage::make() will recognize the newly-registered language + * string. + */ + static void registerLanguage( const StringData& languageName, + TextIndexVersion textIndexVersion, + FTSLanguage *languageOut ); + + /** + * Register 'alias' as an alias for 'language' with text index version + * 'textIndexVersion'. Subsequent calls to FTSLanguage::make() will recognize the + * newly-registered alias. + */ + static void registerLanguageAlias( const FTSLanguage* language, + const StringData& alias, + TextIndexVersion textIndexVersion ); + + /** + * Return the FTSLanguage associated with the given language string. Returns an error + * Status if an invalid language string is passed. + * + * For textIndexVersion=TEXT_INDEX_VERSION_2, language strings are + * case-insensitive, and need to be in one of the two following forms: + * - English name, like "spanish". + * - Two-letter code, like "es". + * + * For textIndexVersion=TEXT_INDEX_VERSION_1, no validation or normalization of + * language strings is performed. This is necessary to preserve indexing behavior for + * documents with language strings like "en": for compatibility, text data in these + * documents needs to be processed with the English stemmer and the empty stopword list + * (since "en" is recognized by Snowball but not the stopword processing logic). + */ + static StatusWith make( const StringData& langName, + TextIndexVersion textIndexVersion ); + + private: + // String representation of language in canonical form. + std::string _canonicalName; + }; + + typedef StatusWith StatusWithFTSLanguage; + + extern FTSLanguage languagePorterV1; + extern FTSLanguage languageEnglishV2; + extern FTSLanguage languageFrenchV2; + + } +} diff --git a/src/mongo/db/fts/fts_language_test.cpp b/src/mongo/db/fts/fts_language_test.cpp new file mode 100644 index 00000000000..141bbce27d9 --- /dev/null +++ b/src/mongo/db/fts/fts_language_test.cpp @@ -0,0 +1,137 @@ +// fts_language_test.cpp + +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" +#include "mongo/db/fts/fts_language.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { + + namespace fts { + + // Positive tests for FTSLanguage::make() with TEXT_INDEX_VERSION_2. + + TEST( FTSLanguageV2, ExactLanguage ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "spanish", TEXT_INDEX_VERSION_2 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "spanish" ); + } + + TEST( FTSLanguageV2, ExactCode ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "es", TEXT_INDEX_VERSION_2 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "spanish" ); + } + + TEST( FTSLanguageV2, UpperCaseLanguage ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "SPANISH", TEXT_INDEX_VERSION_2 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "spanish" ); + } + + TEST( FTSLanguageV2, UpperCaseCode ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "ES", TEXT_INDEX_VERSION_2 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "spanish" ); + } + + TEST( FTSLanguageV2, NoneLanguage ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "none", TEXT_INDEX_VERSION_2 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "none" ); + } + + // Negative tests for FTSLanguage::make() with TEXT_INDEX_VERSION_2. + + TEST( FTSLanguageV2, Unknown ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "spanglish", TEXT_INDEX_VERSION_2 ); + ASSERT( !swl.getStatus().isOK() ); + } + + TEST( FTSLanguageV2, Empty ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "", TEXT_INDEX_VERSION_2 ); + ASSERT( !swl.getStatus().isOK() ); + } + + // Positive tests for FTSLanguage::make() with TEXT_INDEX_VERSION_1. + + TEST( FTSLanguageV1, ExactLanguage ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "spanish", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "spanish" ); + } + + TEST( FTSLanguageV1, DeprecatedLanguage ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "porter", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "porter" ); + } + + TEST( FTSLanguageV1, StemmerOnlyLanguage1 ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "en", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "en" ); + } + + TEST( FTSLanguageV1, StemmerOnlyLanguage2 ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "eng", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "eng" ); + } + + TEST( FTSLanguageV1, NoneLanguage ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "none", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "none" ); + } + + // Negative tests for FTSLanguage::make() with TEXT_INDEX_VERSION_1. + + TEST( FTSLanguageV1, CaseSensitive ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "SPANISH", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "none" ); + } + + TEST( FTSLanguageV1, Unknown ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "asdf", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "none" ); + } + + TEST( FTSLanguageV1, Empty ) { + StatusWithFTSLanguage swl = FTSLanguage::make( "", TEXT_INDEX_VERSION_1 ); + ASSERT( swl.getStatus().isOK() ); + ASSERT_EQUALS( swl.getValue()->str(), "none" ); + } + + } +} diff --git a/src/mongo/db/fts/fts_matcher.cpp b/src/mongo/db/fts/fts_matcher.cpp index ee462bbb009..c9fad05f77f 100644 --- a/src/mongo/db/fts/fts_matcher.cpp +++ b/src/mongo/db/fts/fts_matcher.cpp @@ -14,22 +14,33 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" #include "mongo/db/fts/fts_matcher.h" +#include "mongo/db/fts/fts_element_iterator.h" #include "mongo/platform/strcasestr.h" namespace mongo { namespace fts { - FTSMatcher::FTSMatcher( const FTSQuery& query, const FTSSpec& spec ) : _query( query ), - _spec( spec ), - _stemmer( query.getLanguage() ){ + _spec( spec ) { } /* @@ -41,67 +52,19 @@ namespace mongo { // flagged for exclusion, i.e. "hello -world" we want to remove all // results that include "world" - if ( _query.getNegatedTerms().size() == 0 ) + if ( _query.getNegatedTerms().size() == 0 ) { return false; - - if ( _spec.wildcard() ) { - return _hasNegativeTerm_recurse(obj); } - /* otherwise look at fields where weights are defined */ - for ( Weights::const_iterator i = _spec.weights().begin(); - i != _spec.weights().end(); - i++ ) { - const char * leftOverName = i->first.c_str(); - BSONElement e = obj.getFieldDottedOrArray(leftOverName); - - if ( e.type() == Array ) { - BSONObjIterator j( e.Obj() ); - while ( j.more() ) { - BSONElement x = j.next(); - if ( leftOverName[0] && x.isABSONObj() ) - x = x.Obj().getFieldDotted( leftOverName ); - if ( x.type() == String ) - if ( _hasNegativeTerm_string( x.String() ) ) - return true; - } - } - else if ( e.type() == String ) { - if ( _hasNegativeTerm_string( e.String() ) ) - return true; - } - } - return false; - } - - bool FTSMatcher::_hasNegativeTerm_recurse(const BSONObj& obj ) const { - BSONObjIterator j( obj ); - while ( j.more() ) { - BSONElement x = j.next(); - - if ( _spec.languageOverrideField() == x.fieldName()) - continue; + FTSElementIterator it( _spec, obj); - if (x.type() == String) { - if ( _hasNegativeTerm_string( x.String() ) ) - return true; - } - else if ( x.isABSONObj() ) { - BSONObjIterator k( x.Obj() ); - while ( k.more() ) { - // check if k.next() is a obj/array or not - BSONElement y = k.next(); - if ( y.type() == String ) { - if ( _hasNegativeTerm_string( y.String() ) ) - return true; - } - else if ( y.isABSONObj() ) { - if ( _hasNegativeTerm_recurse( y.Obj() ) ) - return true; - } - } + while ( it.more() ) { + FTSIteratorValue val = it.next(); + if (_hasNegativeTerm_string( val._language, val._text )) { + return true; } } + return false; } @@ -109,21 +72,22 @@ namespace mongo { * Checks if any of the negTerms is in the tokenized string * @param raw, the raw string to be tokenized */ - bool FTSMatcher::_hasNegativeTerm_string( const string& raw ) const { + bool FTSMatcher::_hasNegativeTerm_string( const FTSLanguage* language, + const string& raw ) const { - Tokenizer i( _query.getLanguage(), raw ); + Tokenizer i( *language, raw ); + Stemmer stemmer( *language ); while ( i.more() ) { Token t = i.next(); if ( t.type != Token::TEXT ) continue; - string word = tolowerString( _stemmer.stem( t.data ) ); + string word = stemmer.stem( tolowerString( t.data ) ); if ( _query.getNegatedTerms().count( word ) > 0 ) return true; } return false; } - bool FTSMatcher::phrasesMatch( const BSONObj& obj ) const { for (unsigned i = 0; i < _query.getPhr().size(); i++ ) { if ( !phraseMatch( _query.getPhr()[i], obj ) ) { @@ -140,90 +104,24 @@ namespace mongo { return true; } - /** * Checks if phrase is exactly matched in obj, returns true if so, false otherwise * @param phrase, the string to be matched * @param obj, document in the collection to match against */ bool FTSMatcher::phraseMatch( const string& phrase, const BSONObj& obj ) const { + FTSElementIterator it( _spec, obj); - if ( _spec.wildcard() ) { - // case where everything is indexed (all fields) - return _phraseRecurse( phrase, obj ); - } - - for ( Weights::const_iterator i = _spec.weights().begin(); - i != _spec.weights().end(); - ++i ) { - - // figure out what the indexed field is.. ie. is it "field" or "field.subfield" etc. - const char * leftOverName = i->first.c_str(); - BSONElement e = obj.getFieldDottedOrArray(leftOverName); - - if ( e.type() == Array ) { - BSONObjIterator j( e.Obj() ); - while ( j.more() ) { - BSONElement x = j.next(); - - if ( leftOverName[0] && x.isABSONObj() ) - x = x.Obj().getFieldDotted( leftOverName ); - - if ( x.type() == String ) - if ( _phraseMatches( phrase, x.String() ) ) - return true; - } - } - else if ( e.type() == String ) { - if ( _phraseMatches( phrase, e.String() ) ) - return true; - } - } - return false; - } - - - /* - * Recurses over all fields in the obj to match against phrase - * @param phrase, string to be matched - * @param obj, object to matched against - */ - bool FTSMatcher::_phraseRecurse( const string& phrase, const BSONObj& obj ) const { - BSONObjIterator j( obj ); - while ( j.more() ) { - BSONElement x = j.next(); - - if ( _spec.languageOverrideField() == x.fieldName() ) - continue; - - if ( x.type() == String ) { - if ( _phraseMatches( phrase, x.String() ) ) - return true; - } - else if ( x.isABSONObj() ) { - BSONObjIterator k( x.Obj() ); - - while ( k.more() ) { - - BSONElement y = k.next(); - - if ( y.type() == mongo::String ) { - if ( _phraseMatches( phrase, y.String() ) ) - return true; - } - else if ( y.isABSONObj() ) { - if ( _phraseRecurse( phrase, y.Obj() ) ) - return true; - } - } - + while ( it.more() ) { + FTSIteratorValue val = it.next(); + if (_phraseMatches( phrase, val._text )) { + return true; } } return false; } - /* * Looks for phrase in a raw string * @param phrase, phrase to match @@ -232,7 +130,5 @@ namespace mongo { bool FTSMatcher::_phraseMatches( const string& phrase, const string& haystack ) const { return strcasestr( haystack.c_str(), phrase.c_str() ) > 0; } - - } } diff --git a/src/mongo/db/fts/fts_matcher.h b/src/mongo/db/fts/fts_matcher.h index c5478d63b78..d32a92d11f9 100644 --- a/src/mongo/db/fts/fts_matcher.h +++ b/src/mongo/db/fts/fts_matcher.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -40,7 +52,6 @@ namespace mongo { * so all full phrases and no negated */ bool phrasesMatch( const BSONObj& obj ) const; - bool phraseMatch( const string& phrase, const BSONObj& obj ) const; bool matchesNonTerm( const BSONObj& obj ) const { @@ -48,19 +59,18 @@ namespace mongo { } private: - bool _hasNegativeTerm_recurse(const BSONObj& obj ) const; - /** * @return true if raw has a negated term */ - bool _hasNegativeTerm_string( const string& raw ) const; + bool _hasNegativeTerm_string( const FTSLanguage* language, const string& raw ) const; - bool _phraseRecurse( const string& phrase, const BSONObj& obj ) const; - bool _phraseMatches( const string& phrase, const string& haystack ) const; + /** + * @return true if raw has a phrase + */ + bool _phraseMatches( const string& phrase, const string& raw ) const; FTSQuery _query; - FTSSpec _spec; - Stemmer _stemmer; + FTSSpec _spec; }; } diff --git a/src/mongo/db/fts/fts_matcher_test.cpp b/src/mongo/db/fts/fts_matcher_test.cpp index 15369980885..d4eb9eccdf7 100644 --- a/src/mongo/db/fts/fts_matcher_test.cpp +++ b/src/mongo/db/fts/fts_matcher_test.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -26,19 +38,30 @@ namespace mongo { TEST( FTSMatcher, NegWild1 ) { FTSQuery q; - q.parse( "foo -bar", "english" ); + ASSERT_OK( q.parse( "foo -bar", "english", TEXT_INDEX_VERSION_2 ) ); FTSMatcher m( q, - FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "$**" << "fts" ) ) ) ) ); + FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "$**" << "text" ) ) ) ) ); ASSERT( m.hasNegativeTerm( BSON( "x" << BSON( "y" << "bar" ) ) ) ); ASSERT( m.hasNegativeTerm( BSON( "x" << BSON( "y" << "bar" ) ) ) ); } + // Regression test for SERVER-11994. + TEST( FTSMatcher, NegWild2 ) { + FTSQuery q; + ASSERT_OK( q.parse( "pizza -restaurant", "english", TEXT_INDEX_VERSION_2 ) ); + FTSMatcher m( q, + FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "$**" << "text" ) ) ) ) ); + + ASSERT( m.hasNegativeTerm( BSON( "x" << BSON( "y" << "pizza restaurant" ) ) ) ); + ASSERT( m.hasNegativeTerm( BSON( "x" << BSON( "y" << "PIZZA RESTAURANT" ) ) ) ); + } + TEST( FTSMatcher, Phrase1 ) { FTSQuery q; - q.parse( "foo \"table top\"", "english" ); + ASSERT_OK( q.parse( "foo \"table top\"", "english", TEXT_INDEX_VERSION_2 ) ); FTSMatcher m( q, - FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "$**" << "fts" ) ) ) ) ); + FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "$**" << "text" ) ) ) ) ); ASSERT( m.phraseMatch( "table top", BSON( "x" << "table top" ) ) ); ASSERT( m.phraseMatch( "table top", BSON( "x" << " asd table top asd" ) ) ); @@ -52,12 +75,25 @@ namespace mongo { TEST( FTSMatcher, Phrase2 ) { FTSQuery q; - q.parse( "foo \"table top\"", "english" ); + ASSERT_OK( q.parse( "foo \"table top\"", "english", TEXT_INDEX_VERSION_2 ) ); FTSMatcher m( q, - FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "x" << "fts" ) ) ) ) ); + FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "x" << "text" ) ) ) ) ); ASSERT( m.phraseMatch( "table top", BSON( "x" << BSON_ARRAY( "table top" ) ) ) ); } + // Test that the matcher parses the document with the document language, not the search + // language. + TEST( FTSMatcher, ParsesUsingDocLanguage ) { + FTSQuery q; + ASSERT_OK( q.parse( "-glad", "none", TEXT_INDEX_VERSION_2 ) ); + FTSMatcher m( q, + FTSSpec( FTSSpec::fixSpec( BSON( "key" << BSON( "x" << "text" ) ) ) ) ); + + // Even though the search language is "none", the document {x: "gladly"} should be + // parsed using the English stemmer, and as such should match the negated term "glad". + ASSERT( m.hasNegativeTerm( BSON( "x" << "gladly" ) ) ); + } + } } diff --git a/src/mongo/db/fts/fts_query.cpp b/src/mongo/db/fts/fts_query.cpp index 662b6a6fe9d..0269864e126 100644 --- a/src/mongo/db/fts/fts_query.cpp +++ b/src/mongo/db/fts/fts_query.cpp @@ -14,11 +14,24 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" #include "mongo/db/fts/fts_query.h" +#include "mongo/db/fts/fts_spec.h" #include "mongo/db/fts/tokenizer.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/stringutils.h" @@ -29,19 +42,24 @@ namespace mongo { using namespace mongoutils; - Status FTSQuery::parse(const string& query, const string& language) { + Status FTSQuery::parse(const string& query, const StringData& language, + TextIndexVersion textIndexVersion) { _search = query; - _language = language; + StatusWithFTSLanguage swl = FTSLanguage::make( language, textIndexVersion ); + if ( !swl.getStatus().isOK() ) { + return swl.getStatus(); + } + _language = swl.getValue(); - const StopWords* stopWords = StopWords::getStopWords( language ); - Stemmer stemmer( language ); + const StopWords* stopWords = StopWords::getStopWords( *_language ); + Stemmer stemmer( *_language ); bool inNegation = false; bool inPhrase = false; unsigned quoteOffset = 0; - Tokenizer i( _language, query ); + Tokenizer i( *_language, query ); while ( i.more() ) { Token t = i.next(); @@ -61,8 +79,11 @@ namespace mongo { else if ( t.type == Token::DELIMITER ) { char c = t.data[0]; if ( c == '-' ) { - if ( t.previousWhiteSpace ) + if ( !inPhrase && t.previousWhiteSpace ) { + // phrases can be negated, and terms not in phrases can be negated. + // terms in phrases can not be negated. inNegation = true; + } } else if ( c == '"' ) { if ( inPhrase ) { @@ -121,11 +142,6 @@ namespace mongo { _debugHelp( ss, s, sep ); } - void _debugHelp( stringstream& ss, const unordered_set& v, const string& sep ) { - set s( v.begin(), v.end() ); - _debugHelp( ss, s, sep ); - } - } string FTSQuery::toString() const { @@ -167,5 +183,14 @@ namespace mongo { return ss.str(); } + + BSONObj FTSQuery::toBSON() const { + BSONObjBuilder bob; + bob.append( "terms", getTerms() ); + bob.append( "negatedTerms", getNegatedTerms() ); + bob.append( "phrases", getPhr() ); + bob.append( "negatedPhrases", getNegatedPhr() ); + return bob.obj(); + } } } diff --git a/src/mongo/db/fts/fts_query.h b/src/mongo/db/fts/fts_query.h index 7022760b3a7..5fe14bc7e84 100644 --- a/src/mongo/db/fts/fts_query.h +++ b/src/mongo/db/fts/fts_query.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -24,7 +36,6 @@ #include "mongo/base/status.h" #include "mongo/db/fts/stemmer.h" #include "mongo/db/fts/stop_words.h" -#include "mongo/platform/unordered_set.h" #include "mongo/util/stringutils.h" namespace mongo { @@ -38,10 +49,16 @@ namespace mongo { class FTSQuery { public: - Status parse(const string& query, const string& language); + // Initializes an FTSQuery. Note that the parsing of "language" depends on the text + // index version, since a query which doesn't specify a language and is against a + // version 1 text index with a version 1 default language string needs to be parsed as + // version 1 (see fts_language.cpp for a list of language strings specific to version + // 1). + Status parse(const string& query, const StringData& language, + TextIndexVersion textIndexVersion); const vector& getTerms() const { return _terms; } - const unordered_set& getNegatedTerms() const { return _negatedTerms; } + const set& getNegatedTerms() const { return _negatedTerms; } const vector& getPhr() const { return _phrases; } const vector& getNegatedPhr() const { return _negatedPhrases; } @@ -57,17 +74,19 @@ namespace mongo { } string getSearch() const { return _search; } - string getLanguage() const { return _language; } + const FTSLanguage& getLanguage() const { return *_language; } string toString() const; string debugString() const; + BSONObj toBSON() const; + protected: string _search; - string _language; + const FTSLanguage* _language; vector _terms; - unordered_set _negatedTerms; + set _negatedTerms; vector _phrases; vector _negatedPhrases; diff --git a/src/mongo/db/fts/fts_query_test.cpp b/src/mongo/db/fts/fts_query_test.cpp index a87dc8932ef..1e5318a1592 100644 --- a/src/mongo/db/fts/fts_query_test.cpp +++ b/src/mongo/db/fts/fts_query_test.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ @@ -25,7 +37,7 @@ namespace mongo { TEST( FTSQuery, Basic1 ) { FTSQuery q; - ASSERT( q.parse( "this is fun", "english" ).isOK() ); + ASSERT( q.parse( "this is fun", "english", TEXT_INDEX_VERSION_2 ).isOK() ); ASSERT_EQUALS( 1U, q.getTerms().size() ); ASSERT_EQUALS( "fun", q.getTerms()[0] ); @@ -36,7 +48,7 @@ namespace mongo { TEST( FTSQuery, Neg1 ) { FTSQuery q; - ASSERT( q.parse( "this is -really fun", "english" ).isOK() ); + ASSERT( q.parse( "this is -really fun", "english", TEXT_INDEX_VERSION_2 ).isOK() ); ASSERT_EQUALS( 1U, q.getTerms().size() ); ASSERT_EQUALS( "fun", q.getTerms()[0] ); @@ -46,7 +58,8 @@ namespace mongo { TEST( FTSQuery, Phrase1 ) { FTSQuery q; - ASSERT( q.parse( "doing a \"phrase test\" for fun", "english" ).isOK() ); + ASSERT( q.parse( "doing a \"phrase test\" for fun", "english", + TEXT_INDEX_VERSION_2 ).isOK() ); ASSERT_EQUALS( 3U, q.getTerms().size() ); ASSERT_EQUALS( 0U, q.getNegatedTerms().size() ); @@ -59,22 +72,109 @@ namespace mongo { TEST( FTSQuery, Phrase2 ) { FTSQuery q; - ASSERT( q.parse( "doing a \"phrase-test\" for fun", "english" ).isOK() ); + ASSERT( q.parse( "doing a \"phrase-test\" for fun", "english", + TEXT_INDEX_VERSION_2 ).isOK() ); ASSERT_EQUALS( 1U, q.getPhr().size() ); ASSERT_EQUALS( "phrase-test", q.getPhr()[0] ); } TEST( FTSQuery, NegPhrase1 ) { FTSQuery q; - ASSERT( q.parse( "doing a -\"phrase test\" for fun", "english" ).isOK() ); + ASSERT( q.parse( "doing a -\"phrase test\" for fun", "english", + TEXT_INDEX_VERSION_2 ).isOK() ); ASSERT_EQUALS( "fun||||||phrase test", q.debugString() ); } TEST( FTSQuery, Mix1 ) { FTSQuery q; - ASSERT( q.parse( "\"industry\" -Melbourne -Physics", "english" ).isOK() ); + ASSERT( q.parse( "\"industry\" -Melbourne -Physics", "english", + TEXT_INDEX_VERSION_2 ).isOK() ); ASSERT_EQUALS( "industri||melbourn|physic||industry||", q.debugString() ); } + TEST( FTSQuery, NegPhrase2) { + FTSQuery q1, q2, q3; + ASSERT( q1.parse( "foo \"bar\"", "english", TEXT_INDEX_VERSION_2 ).isOK() ); + ASSERT( q2.parse( "foo \"-bar\"", "english", TEXT_INDEX_VERSION_2 ).isOK() ); + ASSERT( q3.parse( "foo \" -bar\"", "english", TEXT_INDEX_VERSION_2 ).isOK() ); + + ASSERT_EQUALS( 2U, q1.getTerms().size() ); + ASSERT_EQUALS( 2U, q2.getTerms().size() ); + ASSERT_EQUALS( 2U, q3.getTerms().size() ); + + ASSERT_EQUALS( 0U, q1.getNegatedTerms().size() ); + ASSERT_EQUALS( 0U, q2.getNegatedTerms().size() ); + ASSERT_EQUALS( 0U, q3.getNegatedTerms().size() ); + + ASSERT_EQUALS( 1U, q1.getPhr().size() ); + ASSERT_EQUALS( 1U, q2.getPhr().size() ); + ASSERT_EQUALS( 1U, q3.getPhr().size() ); + + ASSERT_EQUALS( 0U, q1.getNegatedPhr().size() ); + ASSERT_EQUALS( 0U, q2.getNegatedPhr().size() ); + ASSERT_EQUALS( 0U, q3.getNegatedPhr().size() ); + } + + TEST( FTSQuery, NegPhrase3) { + FTSQuery q1, q2, q3; + ASSERT( q1.parse( "foo -\"bar\"", "english", TEXT_INDEX_VERSION_2 ).isOK() ); + ASSERT( q2.parse( "foo -\"-bar\"", "english", TEXT_INDEX_VERSION_2 ).isOK() ); + ASSERT( q3.parse( "foo -\" -bar\"", "english", TEXT_INDEX_VERSION_2 ).isOK() ); + + ASSERT_EQUALS( 1U, q1.getTerms().size() ); + ASSERT_EQUALS( 1U, q2.getTerms().size() ); + ASSERT_EQUALS( 1U, q3.getTerms().size() ); + + ASSERT_EQUALS( 0U, q1.getNegatedTerms().size() ); + ASSERT_EQUALS( 0U, q2.getNegatedTerms().size() ); + ASSERT_EQUALS( 0U, q3.getNegatedTerms().size() ); + + ASSERT_EQUALS( 0U, q1.getPhr().size() ); + ASSERT_EQUALS( 0U, q2.getPhr().size() ); + ASSERT_EQUALS( 0U, q3.getPhr().size() ); + + ASSERT_EQUALS( 1U, q1.getNegatedPhr().size() ); + ASSERT_EQUALS( 1U, q2.getNegatedPhr().size() ); + ASSERT_EQUALS( 1U, q3.getNegatedPhr().size() ); + } + + // Test textIndexVersion:1 query with language "english". This invokes the standard English + // stemmer and stopword list. + TEST( FTSQuery, TextIndexVersion1LanguageEnglish ) { + FTSQuery q; + ASSERT( q.parse( "the running", "english", TEXT_INDEX_VERSION_1 ).isOK() ); + ASSERT_EQUALS( 1U, q.getTerms().size() ); + ASSERT_EQUALS( "run", q.getTerms()[0] ); + ASSERT_EQUALS( 0U, q.getNegatedTerms().size() ); + ASSERT_EQUALS( 0U, q.getPhr().size() ); + ASSERT_EQUALS( 0U, q.getNegatedPhr().size() ); + } + + // Test textIndexVersion:1 query with language "eng". "eng" uses the English stemmer, and + // no stopword list. + TEST( FTSQuery, TextIndexVersion1LanguageEng ) { + FTSQuery q; + ASSERT( q.parse( "the running", "eng", TEXT_INDEX_VERSION_1 ).isOK() ); + ASSERT_EQUALS( 2U, q.getTerms().size() ); + ASSERT_EQUALS( 1, std::count( q.getTerms().begin(), q.getTerms().end(), "the" ) ); + ASSERT_EQUALS( 1, std::count( q.getTerms().begin(), q.getTerms().end(), "run" ) ); + ASSERT_EQUALS( 0U, q.getNegatedTerms().size() ); + ASSERT_EQUALS( 0U, q.getPhr().size() ); + ASSERT_EQUALS( 0U, q.getNegatedPhr().size() ); + } + + // Test textIndexVersion:1 query with language "invalid". No stemming will be performed, + // and no stopword list will be used. + TEST( FTSQuery, TextIndexVersion1LanguageInvalid ) { + FTSQuery q; + ASSERT( q.parse( "the running", "invalid", TEXT_INDEX_VERSION_1 ).isOK() ); + ASSERT_EQUALS( 2U, q.getTerms().size() ); + ASSERT_EQUALS( 1, std::count( q.getTerms().begin(), q.getTerms().end(), "the" ) ); + ASSERT_EQUALS( 1, std::count( q.getTerms().begin(), q.getTerms().end(), "running" ) ); + ASSERT_EQUALS( 0U, q.getNegatedTerms().size() ); + ASSERT_EQUALS( 0U, q.getPhr().size() ); + ASSERT_EQUALS( 0U, q.getNegatedPhr().size() ); + } + } } diff --git a/src/mongo/db/fts/fts_search.cpp b/src/mongo/db/fts/fts_search.cpp deleted file mode 100644 index 0dd955444f6..00000000000 --- a/src/mongo/db/fts/fts_search.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// fts_search.cpp - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/btreecursor.h" -#include "mongo/db/fts/fts_index_format.h" -#include "mongo/db/fts/fts_search.h" -#include "mongo/db/kill_current_op.h" -#include "mongo/db/pdfile.h" - -namespace mongo { - - namespace fts { - - /* - * Constructor generates query and term dictionaries - * @param ns, namespace - * @param idxNum, index number - * @param search, query string - * @param language, language of the query - * @param filter, filter object - */ - FTSSearch::FTSSearch( NamespaceDetails* ns, - const IndexDetails& id, - const BSONObj& indexPrefix, - const FTSQuery& query, - const BSONObj& filter ) - : _ns( ns ), - _id( id ), - _fts( static_cast(_id.getSpec().getType()) ), - _indexPrefix( indexPrefix ), - _query( query ), - _ftsMatcher( query, static_cast(_id.getSpec().getType())->getFtsSpec() ) { - - if ( !filter.isEmpty() ) - _matcher.reset( new CoveredIndexMatcher( filter, _fts->keyPattern() ) ); - - _keysLookedAt = 0; - _objectsLookedAt = 0; - } - - bool FTSSearch::_ok( Record* record ) const { - if ( !_query.hasNonTermPieces() ) - return true; - return _ftsMatcher.matchesNonTerm( BSONObj::make( record ) ); - } - - /* - * GO: sets the tree cursors on each term in terms, processes the terms by advancing - * the terms cursors and storing the partial - * results and lastly calculates the top results - * @param results, the priority queue containing the top results - * @param limit, number of results in the priority queue - */ - void FTSSearch::go(Results* results, unsigned limit ) { - vector< shared_ptr > cursors; - - for ( unsigned i = 0; i < _query.getTerms().size(); i++ ) { - const string& term = _query.getTerms()[i]; - BSONObj min = FTSIndexFormat::getIndexKey( MAX_WEIGHT, term, _indexPrefix ); - BSONObj max = FTSIndexFormat::getIndexKey( 0, term, _indexPrefix ); - shared_ptr c( BtreeCursor::make( _ns, _id, min, max, true, -1 ) ); - cursors.push_back( c ); - } - - while ( !inShutdown() ) { - bool gotAny = false; - for ( unsigned i = 0; i < cursors.size(); i++ ) { - if ( cursors[i]->eof() ) - continue; - gotAny = true; - _process( cursors[i].get() ); - cursors[i]->advance(); - } - - if ( !gotAny ) - break; - - RARELY killCurrentOp.checkForInterrupt(); - } - - - // priority queue using a compare that grabs the lowest of two ScoredLocations by score. - for ( Scores::iterator i = _scores.begin(); i != _scores.end(); ++i ) { - - if ( i->second < 0 ) - continue; - - // priority queue - if ( results->size() < limit ) { // case a: queue unfilled - - if ( !_ok( i->first ) ) - continue; - - results->push( ScoredLocation( i->first, i->second ) ); - - } - else if ( i->second > results->top().score ) { // case b: queue filled - - if ( !_ok( i->first ) ) - continue; - - results->pop(); - results->push( ScoredLocation( i->first, i->second ) ); - } - else { - // else do nothing (case c) - } - - } - - } - - /* - * Takes a cursor and updates the partial score for said cursor in _scores map - * @param cursor, btree cursor pointing to the current document to be scored - */ - void FTSSearch::_process( BtreeCursor* cursor ) { - _keysLookedAt++; - - BSONObj key = cursor->currKey(); - - BSONObjIterator i( key ); - for ( unsigned j = 0; j < _fts->getFtsSpec().numExtraBefore(); j++ ) - i.next(); - i.next(); // move pase indexToken - BSONElement scoreElement = i.next(); - - double score = scoreElement.number(); - - double& cur = _scores[(cursor->currLoc()).rec()]; - - if ( cur < 0 ) { - // already been rejected - return; - } - - if ( cur == 0 && _matcher.get() ) { - // we haven't seen this before and we have a matcher - MatchDetails d; - if ( !_matcher->matchesCurrent( cursor, &d ) ) { - cur = -1; - } - - if ( d.hasLoadedRecord() ) - _objectsLookedAt++; - - if ( cur == -1 ) - return; - } - - if ( cur ) - cur += score * (1 + 1 / score); - else - cur += score; - - } - - } - -} diff --git a/src/mongo/db/fts/fts_search.h b/src/mongo/db/fts/fts_search.h deleted file mode 100644 index 82e5b66f3b2..00000000000 --- a/src/mongo/db/fts/fts_search.h +++ /dev/null @@ -1,103 +0,0 @@ -// fts_search.h - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include -#include -#include -#include - -#include "mongo/base/disallow_copying.h" -#include "mongo/db/fts/fts_index.h" -#include "mongo/db/fts/fts_matcher.h" -#include "mongo/db/fts/fts_query.h" -#include "mongo/db/fts/fts_util.h" -#include "mongo/db/matcher.h" - -namespace mongo { - - class BtreeCursor; - - namespace fts { - - // priority queue template, for use when we're populating results - // vector returned to the user. extends the default priority_queue - // by providing direct access to the underlying vector, which should - // be used CAREFULLY because you can get into trouble.. - template - class a_priority_queue : public std::priority_queue { - public: - // return the value of an element at position n when we call pq[n] - T operator[](const int &n) { return this->c[n]; } - // return underlying data structure. called dangerous because it is. - S dangerous() { return this->c; } - }; - - typedef a_priority_queue, ScoredLocationComp> Results; - - class FTSSearch { - MONGO_DISALLOW_COPYING(FTSSearch); - public: - - typedef std::map Scores; - - FTSSearch( NamespaceDetails* ns, - const IndexDetails& id, - const BSONObj& indexPrefix, - const FTSQuery& query, - const BSONObj& filter ); - - - void go(Results* results, unsigned limit ); - - const FTSIndex * getIndex() const { return _fts; } - - long long getKeysLookedAt() const { return _keysLookedAt; } - long long getObjLookedAt() const { return _objectsLookedAt; } - - private: - - void _process( BtreeCursor* cursor ); - - /** - * checks not index pieces - * i.e. prhases & negated terms - */ - bool _ok( Record* record ) const; - - NamespaceDetails* _ns; - const IndexDetails& _id; - FTSIndex* _fts; - BSONObj _indexPrefix; - FTSQuery _query; - FTSMatcher _ftsMatcher; - - scoped_ptr _matcher; - - long long _keysLookedAt; - long long _objectsLookedAt; - - Scores _scores; - - }; - - } // namespace fts - -} // namespace mongo - diff --git a/src/mongo/db/fts/fts_spec.cpp b/src/mongo/db/fts/fts_spec.cpp index eb534f10a3e..8ad22119a90 100644 --- a/src/mongo/db/fts/fts_spec.cpp +++ b/src/mongo/db/fts/fts_spec.cpp @@ -1,5 +1,4 @@ // fts_spec.cpp - /** * Copyright (C) 2012 10gen Inc. * @@ -14,13 +13,29 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" #include "mongo/db/fts/fts_spec.h" + +#include "mongo/db/field_ref.h" +#include "mongo/db/fts/fts_element_iterator.h" #include "mongo/db/fts/fts_util.h" #include "mongo/util/mongoutils/str.h" +#include "mongo/util/stringutils.h" namespace mongo { @@ -28,21 +43,57 @@ namespace mongo { using namespace mongoutils; - const double MAX_WEIGHT = 1000000000.0; + const double DEFAULT_WEIGHT = 1; + const double MAX_WEIGHT = 1000000000; const double MAX_WORD_WEIGHT = MAX_WEIGHT / 10000; + namespace { + // Default language. Used for new indexes. + const std::string moduleDefaultLanguage( "english" ); + + /** Validate the given language override string. */ + bool validateOverride( const string& override ) { + // The override field can't be empty, can't be prefixed with a dollar sign, and + // can't contain a dot. + return !override.empty() && + override[0] != '$' && + override.find('.') == std::string::npos; + } + } + FTSSpec::FTSSpec( const BSONObj& indexInfo ) { + // indexInfo is a text index spec. Text index specs pass through fixSpec() before + // being saved to the system.indexes collection. fixSpec() enforces a schema, such that + // required fields must exist and be of the correct type (e.g. weights, + // textIndexVersion). massert( 16739, "found invalid spec for text index", indexInfo["weights"].isABSONObj() ); + BSONElement textIndexVersionElt = indexInfo["textIndexVersion"]; + massert( 17367, + "found invalid spec for text index, expected number for textIndexVersion", + textIndexVersionElt.isNumber() ); + + // We currently support TEXT_INDEX_VERSION_1 (deprecated) and TEXT_INDEX_VERSION_2. + // Reject all other values. + massert( 17364, + str::stream() << "attempt to use unsupported textIndexVersion " << + textIndexVersionElt.numberInt() << "; versions supported: " << + TEXT_INDEX_VERSION_2 << ", " << TEXT_INDEX_VERSION_1, + textIndexVersionElt.numberInt() == TEXT_INDEX_VERSION_2 || + textIndexVersionElt.numberInt() == TEXT_INDEX_VERSION_1 ); + + _textIndexVersion = ( textIndexVersionElt.numberInt() == TEXT_INDEX_VERSION_2 ) ? + TEXT_INDEX_VERSION_2 : TEXT_INDEX_VERSION_1; + + // Initialize _defaultLanguage. Note that the FTSLanguage constructor requires + // textIndexVersion, since language parsing is version-specific. + StatusWithFTSLanguage swl = + FTSLanguage::make( indexInfo["default_language"].String(), _textIndexVersion ); + verify( swl.getStatus().isOK() ); // should not fail, since validated by fixSpec(). + _defaultLanguage = swl.getValue(); - _defaultLanguage = indexInfo["default_language"].valuestrsafe(); _languageOverrideField = indexInfo["language_override"].valuestrsafe(); - if ( _defaultLanguage.size() == 0 ) - _defaultLanguage = "english"; - if ( _languageOverrideField.size() == 0 ) - _languageOverrideField = "language"; - _wildcard = false; // in this block we fill in the _weights map @@ -89,120 +140,41 @@ namespace mongo { } } - bool FTSSpec::weight( const StringData& field, double* out ) const { - Weights::const_iterator i = _weights.find( field.toString() ); - if ( i == _weights.end() ) - return false; - *out = i->second; - return true; - } - - string FTSSpec::getLanguageToUse( const BSONObj& userDoc ) const { + const FTSLanguage* FTSSpec::_getLanguageToUseV2( const BSONObj& userDoc, + const FTSLanguage* currentLanguage ) const { BSONElement e = userDoc[_languageOverrideField]; - if ( e.type() == String ) { - const char * x = e.valuestrsafe(); - if ( strlen( x ) > 0 ) - return x; + if ( e.eoo() ) { + return currentLanguage; } - return _defaultLanguage; + uassert( 17261, + "found language override field in document with non-string type", + e.type() == mongo::String ); + StatusWithFTSLanguage swl = FTSLanguage::make( e.String(), TEXT_INDEX_VERSION_2 ); + uassert( 17262, + "language override unsupported: " + e.String(), + swl.getStatus().isOK() ); + return swl.getValue(); } - - /* - * Calculates the score for all terms in a document of a collection - * @param obj, the document in the collection being parsed - * @param term_freqs, map to fill up - */ void FTSSpec::scoreDocument( const BSONObj& obj, TermFrequencyMap* term_freqs ) const { - - string language = getLanguageToUse( obj ); - - Stemmer stemmer(language); - Tools tools(language); - tools.stemmer = &stemmer; - tools.stopwords = StopWords::getStopWords( language ); - - if ( wildcard() ) { - // if * is specified for weight, we can recurse over all fields. - _scoreRecurse(tools, obj, term_freqs); - return; + if ( _textIndexVersion == TEXT_INDEX_VERSION_1 ) { + return _scoreDocumentV1( obj, term_freqs ); } - // otherwise, we need to remember the different weights for each field - // and act accordingly (in other words, call _score) - for ( Weights::const_iterator i = _weights.begin(); i != _weights.end(); i++ ) { - const char * leftOverName = i->first.c_str(); - // name of field - BSONElement e = obj.getFieldDottedOrArray(leftOverName); - // weight associated to name of field - double weight = i->second; - - if ( e.eoo() ) { - // do nothing - } - else if ( e.type() == Array ) { - BSONObjIterator j( e.Obj() ); - while ( j.more() ) { - BSONElement x = j.next(); - if ( leftOverName[0] && x.isABSONObj() ) - x = x.Obj().getFieldDotted( leftOverName ); - if ( x.type() == String ) - _scoreString( tools, x.valuestr(), term_freqs, weight ); - } - } - else if ( e.type() == String ) { - _scoreString( tools, e.valuestr(), term_freqs, weight ); - } + FTSElementIterator it( *this, obj ); + while ( it.more() ) { + FTSIteratorValue val = it.next(); + Stemmer stemmer( *val._language ); + Tools tools( *val._language, &stemmer, StopWords::getStopWords( *val._language ) ); + _scoreStringV2( tools, val._text, term_freqs, val._weight ); } } - - /* - * Recurses over all fields of an obj (document in collection) - * and fills term,score map term_freqs - * @param tokenizer, tokenizer to tokenize a string into terms - * @param obj, object being parsed - * term_freqs, map to be filled up - */ - void FTSSpec::_scoreRecurse(const Tools& tools, - const BSONObj& obj, - TermFrequencyMap* term_freqs ) const { - BSONObjIterator j( obj ); - while ( j.more() ) { - BSONElement x = j.next(); - - if ( languageOverrideField() == x.fieldName() ) - continue; - - if (x.type() == String) { - double w = 1; - weight( x.fieldName(), &w ); - _scoreString(tools, x.valuestr(), term_freqs, w); - } - else if ( x.isABSONObj() ) { - _scoreRecurse( tools, x.Obj(), term_freqs); - } - - } - } - - namespace { - struct ScoreHelperStruct { - ScoreHelperStruct() - : freq(0), count(0), exp(0){ - } - double freq; - double count; - double exp; - }; - typedef unordered_map ScoreHelperMap; - } - - void FTSSpec::_scoreString( const Tools& tools, - const StringData& raw, - TermFrequencyMap* docScores, - double weight ) const { + void FTSSpec::_scoreStringV2( const Tools& tools, + const StringData& raw, + TermFrequencyMap* docScores, + double weight ) const { ScoreHelperMap terms; @@ -216,19 +188,21 @@ namespace mongo { string term = t.data.toString(); makeLower( &term ); - if ( tools.stopwords->isStopWord( term ) ) + if ( tools.stopwords->isStopWord( term ) ) { continue; + } term = tools.stemmer->stem( term ); ScoreHelperStruct& data = terms[term]; - if ( data.exp ) + if ( data.exp ) { data.exp *= 2; - else + } + else { data.exp = 1; + } data.count += 1; data.freq += ( 1 / data.exp ); - numTokens++; } @@ -283,59 +257,126 @@ namespace mongo { return Status::OK(); } - void _addFTSStuff( BSONObjBuilder* b ) { - b->append( "_fts", INDEX_NAME ); - b->append( "_ftsx", 1 ); + namespace { + void _addFTSStuff( BSONObjBuilder* b ) { + b->append( "_fts", INDEX_NAME ); + b->append( "_ftsx", 1 ); + } + + void verifyFieldNameNotReserved( StringData s ) { + uassert( 17289, + "text index with reserved fields _fts/_ftsx not allowed", + s != "_fts" && s != "_ftsx" ); + } } BSONObj FTSSpec::fixSpec( const BSONObj& spec ) { + if ( spec["textIndexVersion"].numberInt() == TEXT_INDEX_VERSION_1 ) { + return _fixSpecV1( spec ); + } + map m; BSONObj keyPattern; { BSONObjBuilder b; - bool addedFtsStuff = false; - - BSONObjIterator i( spec["key"].Obj() ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( str::equals( e.fieldName(), "_fts" ) || - str::equals( e.fieldName(), "_ftsx" ) ) { - addedFtsStuff = true; - b.append( e ); - } - else if ( e.type() == String && - ( str::equals( "fts", e.valuestr() ) || - str::equals( "text", e.valuestr() ) ) ) { - if ( !addedFtsStuff ) { - _addFTSStuff( &b ); + // Populate m and keyPattern. + { + bool addedFtsStuff = false; + BSONObjIterator i( spec["key"].Obj() ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( str::equals( e.fieldName(), "_fts" ) ) { + uassert( 17271, + "expecting _fts:\"text\"", + INDEX_NAME == e.valuestrsafe() ); addedFtsStuff = true; + b.append( e ); + } + else if ( str::equals( e.fieldName(), "_ftsx" ) ) { + uassert( 17272, "expecting _ftsx:1", e.numberInt() == 1 ); + b.append( e ); + } + else if ( e.type() == String && INDEX_NAME == e.valuestr() ) { + + if ( !addedFtsStuff ) { + _addFTSStuff( &b ); + addedFtsStuff = true; + } + + m[e.fieldName()] = 1; + } + else { + uassert( 17273, + "expected value 1 or -1 for non-text key in compound index", + e.numberInt() == 1 || e.numberInt() == -1 ); + b.append( e ); } + } + verify( addedFtsStuff ); + } + keyPattern = b.obj(); - m[e.fieldName()] = 1; + // Verify that index key is in the correct format: extraBefore fields, then text + // fields, then extraAfter fields. + { + BSONObjIterator i( spec["key"].Obj() ); + verify( i.more() ); + BSONElement e = i.next(); + + // extraBefore fields + while ( String != e.type() ) { + verifyFieldNameNotReserved( e.fieldNameStringData() ); + verify( i.more() ); + e = i.next(); + } + + // text fields + bool alreadyFixed = str::equals( e.fieldName(), "_fts" ); + if ( alreadyFixed ) { + uassert( 17288, "expected _ftsx after _fts", i.more() ); + e = i.next(); + uassert( 17274, + "expected _ftsx after _fts", + str::equals( e.fieldName(), "_ftsx" ) ); + e = i.next(); } else { - b.append( e ); + do { + verifyFieldNameNotReserved( e.fieldNameStringData() ); + e = i.next(); + } while ( !e.eoo() && e.type() == String ); } - } - if ( !addedFtsStuff ) - _addFTSStuff( &b ); + // extraAfterFields + while ( !e.eoo() ) { + uassert( 17389, + "'text' fields in index must all be adjacent", + e.type() != String ); + verifyFieldNameNotReserved( e.fieldNameStringData() ); + e = i.next(); + } + } - keyPattern = b.obj(); } - if ( spec["weights"].isABSONObj() ) { + if ( spec["weights"].type() == Object ) { BSONObjIterator i( spec["weights"].Obj() ); while ( i.more() ) { BSONElement e = i.next(); + uassert( 17283, + "weight for text index needs numeric type", + e.isNumber() ); m[e.fieldName()] = e.numberInt(); } } else if ( spec["weights"].str() == WILDCARD ) { m[WILDCARD] = 1; } + else if ( !spec["weights"].eoo() ) { + uasserted( 17284, "text index option 'weights' must be an object" ); + } BSONObj weights; { @@ -343,21 +384,58 @@ namespace mongo { for ( map::iterator i = m.begin(); i != m.end(); ++i ) { uassert( 16674, "score for word too high", i->second > 0 && i->second < MAX_WORD_WEIGHT ); + + // Verify weight refers to a valid field. + if ( i->first != "$**" ) { + FieldRef keyField( i->first ); + uassert( 17294, + "weight cannot be on an empty field", + keyField.numParts() != 0 ); + for ( size_t partNum = 0; partNum < keyField.numParts(); partNum++ ) { + StringData part = keyField.getPart(partNum); + uassert( 17291, + "weight cannot have empty path component", + !part.empty() ); + uassert( 17292, + "weight cannot have path component with $ prefix", + !part.startsWith( "$" ) ); + } + } + b.append( i->first, i->second ); } weights = b.obj(); } - string default_language(spec.getStringField("default_language")); - if ( default_language.empty() ) - default_language = "english"; - - string language_override(spec.getStringField("language_override")); - if ( language_override.empty() ) + BSONElement default_language_elt = spec["default_language"]; + string default_language( default_language_elt.str() ); + if ( default_language_elt.eoo() ) { + default_language = moduleDefaultLanguage; + } + else { + uassert( 17263, + "default_language needs a string type", + default_language_elt.type() == String ); + } + uassert( 17264, + "default_language is not valid", + FTSLanguage::make( default_language, + TEXT_INDEX_VERSION_2 ).getStatus().isOK() ); + + BSONElement language_override_elt = spec["language_override"]; + string language_override( language_override_elt.str() ); + if ( language_override_elt.eoo() ) { language_override = "language"; + } + else { + uassert( 17136, + "language_override is not valid", + language_override_elt.type() == String + && validateOverride( language_override ) ); + } int version = -1; - int textIndexVersion = 1; + int textIndexVersion = TEXT_INDEX_VERSION_2; BSONObjBuilder b; BSONObjIterator i( spec ); @@ -382,30 +460,34 @@ namespace mongo { version = e.numberInt(); } else if ( str::equals( e.fieldName(), "textIndexVersion" ) ) { + uassert( 17293, + "text index option 'textIndexVersion' must be a number", + e.isNumber() ); textIndexVersion = e.numberInt(); uassert( 16730, str::stream() << "bad textIndexVersion: " << textIndexVersion, - textIndexVersion == 1 ); + textIndexVersion == TEXT_INDEX_VERSION_2 ); } else { b.append( e ); } } - if ( !weights.isEmpty() ) + if ( !weights.isEmpty() ) { b.append( "weights", weights ); - if ( !default_language.empty() ) + } + if ( !default_language.empty() ) { b.append( "default_language", default_language); - if ( !language_override.empty() ) + } + if ( !language_override.empty() ) { b.append( "language_override", language_override); - - if ( version >= 0 ) + } + if ( version >= 0 ) { b.append( "v", version ); - + } b.append( "textIndexVersion", textIndexVersion ); return b.obj(); - } } diff --git a/src/mongo/db/fts/fts_spec.h b/src/mongo/db/fts/fts_spec.h index e3ebf24f76b..8bc47fcbaad 100644 --- a/src/mongo/db/fts/fts_spec.h +++ b/src/mongo/db/fts/fts_spec.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -22,6 +34,7 @@ #include #include +#include "mongo/db/fts/fts_language.h" #include "mongo/db/fts/fts_util.h" #include "mongo/db/fts/stemmer.h" #include "mongo/db/fts/stop_words.h" @@ -33,18 +46,33 @@ namespace mongo { namespace fts { extern const double MAX_WEIGHT; + extern const double MAX_WORD_WEIGHT; + extern const double DEFAULT_WEIGHT; typedef std::map Weights; // TODO cool map - typedef unordered_map TermFrequencyMap; + struct ScoreHelperStruct { + ScoreHelperStruct() + : freq(0), count(0), exp(0){ + } + double freq; + double count; + double exp; + }; + typedef unordered_map ScoreHelperMap; class FTSSpec { struct Tools { - Tools( string language ) - : language( language ){} - const std::string& language; + Tools( const FTSLanguage& _language, + const Stemmer* _stemmer, + const StopWords* _stopwords ) + : language( _language ) + , stemmer( _stemmer ) + , stopwords( _stopwords ) {} + + const FTSLanguage& language; const Stemmer* stemmer; const StopWords* stopwords; }; @@ -53,7 +81,7 @@ namespace mongo { FTSSpec( const BSONObj& indexInfo ); bool wildcard() const { return _wildcard; } - const string& defaultLanguage() const { return _defaultLanguage; } + const FTSLanguage& defaultLanguage() const { return *_defaultLanguage; } const string& languageOverrideField() const { return _languageOverrideField; } size_t numExtraBefore() const { return _extraBefore.size(); } @@ -62,9 +90,12 @@ namespace mongo { size_t numExtraAfter() const { return _extraAfter.size(); } const std::string& extraAfter( unsigned i ) const { return _extraAfter[i]; } - string getLanguageToUse( const BSONObj& userDoc ) const; - - void scoreDocument( const BSONObj& obj, TermFrequencyMap* scores ) const; + /** + * Calculates term/score pairs for a BSONObj as applied to this spec. + * @arg obj document to traverse; can be a subdocument or array + * @arg term_freqs output parameter to store (term,score) results + */ + void scoreDocument( const BSONObj& obj, TermFrequencyMap* term_freqs ) const; /** * given a query, pulls out the pieces (in order) that go in the index first @@ -72,35 +103,74 @@ namespace mongo { Status getIndexPrefix( const BSONObj& filter, BSONObj* out ) const; const Weights& weights() const { return _weights; } + static BSONObj fixSpec( const BSONObj& spec ); /** - * @param out - untouched if field isn't present - * @return if field is here + * Returns text index version. */ - bool weight( const StringData& field, double* out ) const; + TextIndexVersion getTextIndexVersion() const { return _textIndexVersion; } + private: + // + // Helper methods. Invoked for TEXT_INDEX_VERSION_2 spec objects only. + // + + /** + * Calculate the term scores for 'raw' and update 'term_freqs' with the result. Parses + * 'raw' using 'tools', and weights term scores based on 'weight'. + */ + void _scoreStringV2( const Tools& tools, + const StringData& raw, + TermFrequencyMap* term_freqs, + double weight ) const; + + public: + /** + * Get the language override for the given BSON doc. If no language override is + * specified, returns currentLanguage. + */ + const FTSLanguage* _getLanguageToUseV2( const BSONObj& userDoc, + const FTSLanguage* currentLanguage ) const; - static BSONObj fixSpec( const BSONObj& spec ); private: - void _scoreRecurse(const Tools& tools, - const BSONObj& obj, - TermFrequencyMap* term_freqs ) const; + // + // Deprecated helper methods. Invoked for TEXT_INDEX_VERSION_1 spec objects only. + // + + void _scoreStringV1( const Tools& tools, + const StringData& raw, + TermFrequencyMap* docScores, + double weight ) const; + + bool _weightV1( const StringData& field, double* out ) const; - void _scoreString( const Tools& tools, - const StringData& raw, - TermFrequencyMap* term_freqs, - double weight ) const; + void _scoreRecurseV1( const Tools& tools, + const BSONObj& obj, + TermFrequencyMap* term_freqs ) const; - string _defaultLanguage; + void _scoreDocumentV1( const BSONObj& obj, TermFrequencyMap* term_freqs ) const; + + const FTSLanguage& _getLanguageToUseV1( const BSONObj& userDoc ) const; + + static BSONObj _fixSpecV1( const BSONObj& spec ); + + // + // Instance variables. + // + + TextIndexVersion _textIndexVersion; + + const FTSLanguage* _defaultLanguage; string _languageOverrideField; bool _wildcard; - // _weights stores a mapping between the fields and the value as a double - // basically, how much should an occurence of (query term) in (field) be worth + // mapping : fieldname -> weight Weights _weights; - // other fields to index + // Prefix compound key - used to partition search index std::vector _extraBefore; + + // Suffix compound key - used for covering index behavior std::vector _extraAfter; }; diff --git a/src/mongo/db/fts/fts_spec_legacy.cpp b/src/mongo/db/fts/fts_spec_legacy.cpp new file mode 100644 index 00000000000..556cac1e091 --- /dev/null +++ b/src/mongo/db/fts/fts_spec_legacy.cpp @@ -0,0 +1,320 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/fts/fts_spec.h" + +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace fts { + + // + // This file contains functionality specific to indexing documents from TEXT_INDEX_VERSION_1 + // text indexes. + // + + using namespace mongoutils; + + namespace { + void _addFTSStuff( BSONObjBuilder* b ) { + b->append( "_fts", INDEX_NAME ); + b->append( "_ftsx", 1 ); + } + } + + const FTSLanguage& FTSSpec::_getLanguageToUseV1( const BSONObj& userDoc ) const { + BSONElement e = userDoc[_languageOverrideField]; + if ( e.type() == String ) { + const char * x = e.valuestrsafe(); + if ( strlen( x ) > 0 ) { + StatusWithFTSLanguage swl = FTSLanguage::make( x, TEXT_INDEX_VERSION_1 ); + dassert( swl.isOK() ); // make() w/ TEXT_INDEX_VERSION_1 guaranteed to not fail. + return *swl.getValue(); + } + } + return *_defaultLanguage; + } + + void FTSSpec::_scoreStringV1( const Tools& tools, + const StringData& raw, + TermFrequencyMap* docScores, + double weight ) const { + + ScoreHelperMap terms; + + unsigned numTokens = 0; + + Tokenizer i( tools.language, raw ); + while ( i.more() ) { + Token t = i.next(); + if ( t.type != Token::TEXT ) + continue; + + string term = t.data.toString(); + makeLower( &term ); + if ( tools.stopwords->isStopWord( term ) ) + continue; + term = tools.stemmer->stem( term ); + + ScoreHelperStruct& data = terms[term]; + + if ( data.exp ) + data.exp *= 2; + else + data.exp = 1; + data.count += 1; + data.freq += ( 1 / data.exp ); + + numTokens++; + } + + for ( ScoreHelperMap::const_iterator i = terms.begin(); i != terms.end(); ++i ) { + + const string& term = i->first; + const ScoreHelperStruct& data = i->second; + + // in order to adjust weights as a function of term count as it + // relates to total field length. ie. is this the only word or + // a frequently occuring term? or does it only show up once in + // a long block of text? + + double coeff = ( 0.5 * data.count / numTokens ) + 0.5; + + // if term is identical to the raw form of the + // field (untokenized) give it a small boost. + double adjustment = 1; + if ( raw.size() == term.length() && raw.equalCaseInsensitive( term ) ) + adjustment += 0.1; + + double& score = (*docScores)[term]; + score += ( weight * data.freq * coeff * adjustment ); + verify( score <= MAX_WEIGHT ); + } + } + + bool FTSSpec::_weightV1( const StringData& field, double* out ) const { + Weights::const_iterator i = _weights.find( field.toString() ); + if ( i == _weights.end() ) + return false; + *out = i->second; + return true; + } + + /* + * Recurses over all fields of an obj (document in collection) + * and fills term,score map term_freqs + * @param tokenizer, tokenizer to tokenize a string into terms + * @param obj, object being parsed + * term_freqs, map to be filled up + */ + void FTSSpec::_scoreRecurseV1( const Tools& tools, + const BSONObj& obj, + TermFrequencyMap* term_freqs ) const { + BSONObjIterator j( obj ); + while ( j.more() ) { + BSONElement x = j.next(); + + if ( languageOverrideField() == x.fieldName() ) + continue; + + if (x.type() == String) { + double w = 1; + _weightV1( x.fieldName(), &w ); + _scoreStringV1(tools, x.valuestr(), term_freqs, w); + } + else if ( x.isABSONObj() ) { + _scoreRecurseV1( tools, x.Obj(), term_freqs); + } + + } + } + + void FTSSpec::_scoreDocumentV1( const BSONObj& obj, + TermFrequencyMap* term_freqs ) const { + + const FTSLanguage& language = _getLanguageToUseV1( obj ); + + Stemmer stemmer(language); + Tools tools(language, &stemmer, StopWords::getStopWords( language )); + + if ( wildcard() ) { + // if * is specified for weight, we can recurse over all fields. + _scoreRecurseV1(tools, obj, term_freqs); + return; + } + + // otherwise, we need to remember the different weights for each field + // and act accordingly (in other words, call _score) + for ( Weights::const_iterator i = _weights.begin(); i != _weights.end(); i++ ) { + const char * leftOverName = i->first.c_str(); + // name of field + BSONElement e = obj.getFieldDottedOrArray(leftOverName); + // weight associated to name of field + double weight = i->second; + + if ( e.eoo() ) { + // do nothing + } + else if ( e.type() == Array ) { + BSONObjIterator j( e.Obj() ); + while ( j.more() ) { + BSONElement x = j.next(); + if ( leftOverName[0] && x.isABSONObj() ) + x = x.Obj().getFieldDotted( leftOverName ); + if ( x.type() == String ) + _scoreStringV1( tools, x.valuestr(), term_freqs, weight ); + } + } + else if ( e.type() == String ) { + _scoreStringV1( tools, e.valuestr(), term_freqs, weight ); + } + + } + } + + BSONObj FTSSpec::_fixSpecV1( const BSONObj& spec ) { + map m; + + BSONObj keyPattern; + { + BSONObjBuilder b; + bool addedFtsStuff = false; + + BSONObjIterator i( spec["key"].Obj() ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( str::equals( e.fieldName(), "_fts" ) || + str::equals( e.fieldName(), "_ftsx" ) ) { + addedFtsStuff = true; + b.append( e ); + } + else if ( e.type() == String && + ( str::equals( "fts", e.valuestr() ) || + str::equals( "text", e.valuestr() ) ) ) { + + if ( !addedFtsStuff ) { + _addFTSStuff( &b ); + addedFtsStuff = true; + } + + m[e.fieldName()] = 1; + } + else { + b.append( e ); + } + } + + if ( !addedFtsStuff ) + _addFTSStuff( &b ); + + keyPattern = b.obj(); + } + + if ( spec["weights"].isABSONObj() ) { + BSONObjIterator i( spec["weights"].Obj() ); + while ( i.more() ) { + BSONElement e = i.next(); + m[e.fieldName()] = e.numberInt(); + } + } + else if ( spec["weights"].str() == WILDCARD ) { + m[WILDCARD] = 1; + } + + BSONObj weights; + { + BSONObjBuilder b; + for ( map::iterator i = m.begin(); i != m.end(); ++i ) { + uassert( 17365, "score for word too high", + i->second > 0 && i->second < MAX_WORD_WEIGHT ); + b.append( i->first, i->second ); + } + weights = b.obj(); + } + + string default_language(spec.getStringField("default_language")); + if ( default_language.empty() ) + default_language = "english"; + + string language_override(spec.getStringField("language_override")); + if ( language_override.empty() ) + language_override = "language"; + + int version = -1; + int textIndexVersion = 1; + + BSONObjBuilder b; + BSONObjIterator i( spec ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( str::equals( e.fieldName(), "key" ) ) { + b.append( "key", keyPattern ); + } + else if ( str::equals( e.fieldName(), "weights" ) ) { + b.append( "weights", weights ); + weights = BSONObj(); + } + else if ( str::equals( e.fieldName(), "default_language" ) ) { + b.append( "default_language", default_language); + default_language = ""; + } + else if ( str::equals( e.fieldName(), "language_override" ) ) { + b.append( "language_override", language_override); + language_override = ""; + } + else if ( str::equals( e.fieldName(), "v" ) ) { + version = e.numberInt(); + } + else if ( str::equals( e.fieldName(), "textIndexVersion" ) ) { + textIndexVersion = e.numberInt(); + uassert( 17366, + str::stream() << "bad textIndexVersion: " << textIndexVersion, + textIndexVersion == 1 ); + } + else { + b.append( e ); + } + } + + if ( !weights.isEmpty() ) + b.append( "weights", weights ); + if ( !default_language.empty() ) + b.append( "default_language", default_language); + if ( !language_override.empty() ) + b.append( "language_override", language_override); + + if ( version >= 0 ) + b.append( "v", version ); + + b.append( "textIndexVersion", textIndexVersion ); + + return b.obj(); + } + } +} diff --git a/src/mongo/db/fts/fts_spec_test.cpp b/src/mongo/db/fts/fts_spec_test.cpp index df4ff719f02..dce90fbe2d0 100644 --- a/src/mongo/db/fts/fts_spec_test.cpp +++ b/src/mongo/db/fts/fts_spec_test.cpp @@ -14,29 +14,173 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" #include "mongo/db/fts/fts_spec.h" +#include "mongo/db/json.h" #include "mongo/unittest/unittest.h" namespace mongo { namespace fts { - TEST( FTSSpec, Fix1 ) { - BSONObj user = BSON( "key" << BSON( "title" << "fts" << - "text" << "fts" ) << - "weights" << BSON( "title" << 10 ) ); + /** + * Assert that fixSpec() accepts the provided text index spec. + */ + void assertFixSuccess( const std::string& s ) { + BSONObj user = fromjson( s ); - BSONObj fixed = FTSSpec::fixSpec( user ); - BSONObj fixed2 = FTSSpec::fixSpec( fixed ); - ASSERT_EQUALS( fixed, fixed2 ); + try { + // fixSpec() should not throw on a valid spec. + BSONObj fixed = FTSSpec::fixSpec( user ); + + // fixSpec() on an already-fixed spec shouldn't change it. + BSONObj fixed2 = FTSSpec::fixSpec( fixed ); + ASSERT_EQUALS( fixed, fixed2 ); + } + catch ( UserException& ) { + ASSERT( false ); + } + } + + /** + * Assert that fixSpec() rejects the provided text index spec. + */ + void assertFixFailure( const std::string& s ) { + BSONObj user = fromjson( s ); + + try { + // fixSpec() on an invalid spec should uassert. + BSONObj fixed = FTSSpec::fixSpec( user ); + } + catch ( UserException& ) { + return; + } + ASSERT( false ); + } + + TEST( FTSSpec, FixNormalKey1 ) { + assertFixSuccess("{key: {a: 'text'}}"); + assertFixSuccess("{key: {a: 'text', b: 'text'}}"); + assertFixSuccess("{key: {a: 'text', b: 'text', c: 'text'}}"); + + assertFixFailure("{key: {_fts: 'text'}}"); // not allowed to index reserved field + assertFixFailure("{key: {_ftsx: 'text'}}"); + } + + TEST( FTSSpec, FixCompoundKey1 ) { + assertFixSuccess("{key: {a: 'text', b: 1.0}}"); + assertFixSuccess("{key: {a: 'text', b: NumberInt(1)}}"); + assertFixSuccess("{key: {a: 'text', b: NumberLong(1)}}"); + assertFixSuccess("{key: {a: 'text', b: -1.0}}"); + assertFixSuccess("{key: {a: 'text', b: NumberInt(-1)}}"); + assertFixSuccess("{key: {a: 'text', b: NumberLong(-1)}}"); + assertFixSuccess("{key: {a: 1.0, b: 'text'}}"); + assertFixSuccess("{key: {a: NumberInt(1), b: 'text'}}"); + assertFixSuccess("{key: {a: NumberLong(1), b: 'text'}}"); + assertFixSuccess("{key: {a: -1, b: 'text'}}"); + assertFixSuccess("{key: {a: 1, b: 1, c: 'text'}}"); + assertFixSuccess("{key: {a: 1, b: -1, c: 'text'}}"); + assertFixSuccess("{key: {a: -1, b: 1, c: 'text'}}"); + assertFixSuccess("{key: {a: 1, b: 'text', c: 1}}"); + assertFixSuccess("{key: {a: 'text', b: 1, c: 1}}"); + assertFixSuccess("{key: {a: 'text', b: 1, c: -1}}"); + assertFixSuccess("{key: {a: 'text', b: 'text', c: 1}}"); + assertFixSuccess("{key: {a: 1, b: 'text', c: 'text'}}"); + + assertFixFailure("{key: {a: 'text', b: 0}}"); + assertFixFailure("{key: {a: 'text', b: '2d'}}"); // not allowed to mix special indexes + assertFixFailure("{key: {a: 'text', b: '1'}}"); + assertFixFailure("{key: {a: 'text', _fts: 1}}"); + assertFixFailure("{key: {a: 'text', _fts: 'text'}}"); + assertFixFailure("{key: {a: 'text', _ftsx: 1}}"); + assertFixFailure("{key: {a: 'text', _ftsx: 'text'}}"); + assertFixFailure("{key: {_fts: 1, a: 'text'}}"); + assertFixFailure("{key: {_fts: 'text', a: 'text'}}"); + assertFixFailure("{key: {_ftsx: 1, a: 'text'}}"); + assertFixFailure("{key: {_ftsx: 'text', a: 'text'}}"); + assertFixFailure("{key: {a: 'text', b: 1, c: 'text'}}"); // 'text' must all be adjacent + assertFixFailure("{key: {a: 'text', b: 1, c: 'text', d: 1}}"); + assertFixFailure("{key: {a: 1, b: 'text', c: 1, d: 'text', e: 1}}"); + } + + TEST( FTSSpec, FixDefaultLanguage1 ) { + assertFixSuccess("{key: {a: 'text'}, default_language: 'english'}"); + assertFixSuccess("{key: {a: 'text'}, default_language: 'engLISH'}"); + assertFixSuccess("{key: {a: 'text'}, default_language: 'en'}"); + assertFixSuccess("{key: {a: 'text'}, default_language: 'eN'}"); + assertFixSuccess("{key: {a: 'text'}, default_language: 'spanish'}"); + assertFixSuccess("{key: {a: 'text'}, default_language: 'none'}"); + + assertFixFailure("{key: {a: 'text'}, default_language: 'engrish'}"); + assertFixFailure("{key: {a: 'text'}, default_language: ' english'}"); + assertFixFailure("{key: {a: 'text'}, default_language: ''}"); + } + + TEST( FTSSpec, FixWeights1 ) { + assertFixSuccess("{key: {a: 'text'}, weights: {}}"); + assertFixSuccess("{key: {a: 'text'}, weights: {a: 1.0}}"); + assertFixSuccess("{key: {a: 'text'}, weights: {a: NumberInt(1)}}"); + assertFixSuccess("{key: {a: 'text'}, weights: {a: NumberLong(1)}}"); + assertFixSuccess("{key: {a: 'text'}, weights: {a: 99999}}"); + assertFixSuccess("{key: {'$**': 'text'}, weights: {'a.b': 2}}"); + assertFixSuccess("{key: {'$**': 'text'}, weights: {a: 2, b: 2}}"); + assertFixSuccess("{key: {'$**': 'text'}, weights: {'$**': 2}}"); + + assertFixFailure("{key: {a: 'text'}, weights: 0}"); + assertFixFailure("{key: {a: 'text'}, weights: []}"); + assertFixFailure("{key: {a: 'text'}, weights: 'x'}"); + assertFixFailure("{key: {a: 'text'}, weights: {a: 0}}"); + assertFixFailure("{key: {a: 'text'}, weights: {a: -1}}"); + assertFixFailure("{key: {a: 'text'}, weights: {a: 100000}}"); // above max weight + assertFixFailure("{key: {a: 'text'}, weights: {a: '1'}}"); + assertFixFailure("{key: {a: 'text'}, weights: {'': 1}}"); // "invalid" path + assertFixFailure("{key: {a: 'text'}, weights: {'a.': 1}}"); + assertFixFailure("{key: {a: 'text'}, weights: {'.a': 1}}"); + assertFixFailure("{key: {a: 'text'}, weights: {'a..a': 1}}"); + assertFixFailure("{key: {a: 'text'}, weights: {$a: 1}}"); + assertFixFailure("{key: {a: 'text'}, weights: {'a.$a': 1}}"); + assertFixFailure("{key: {a: 'text'}, weights: {'a.$**': 1}}"); + } + + TEST( FTSSpec, FixLanguageOverride1 ) { + assertFixSuccess("{key: {a: 'text'}, language_override: 'foo'}"); + assertFixSuccess("{key: {a: 'text'}, language_override: 'foo$bar'}"); + + assertFixFailure("{key: {a: 'text'}, language_override: 'foo.bar'}"); // can't have '.' + assertFixFailure("{key: {a: 'text'}, language_override: ''}"); + assertFixFailure("{key: {a: 'text'}, language_override: '$foo'}"); + } + + TEST( FTSSpec, FixTextIndexVersion1 ) { + assertFixSuccess("{key: {a: 'text'}, textIndexVersion: 1.0}}"); + assertFixSuccess("{key: {a: 'text'}, textIndexVersion: NumberInt(1)}}"); + assertFixSuccess("{key: {a: 'text'}, textIndexVersion: NumberLong(1)}}"); + assertFixSuccess("{key: {a: 'text'}, textIndexVersion: 2.0}}"); + assertFixSuccess("{key: {a: 'text'}, textIndexVersion: NumberInt(2)}}"); + assertFixSuccess("{key: {a: 'text'}, textIndexVersion: NumberLong(2)}}"); + + assertFixFailure("{key: {a: 'text'}, textIndexVersion: 3}"); + assertFixFailure("{key: {a: 'text'}, textIndexVersion: '2'}"); + assertFixFailure("{key: {a: 'text'}, textIndexVersion: {}}"); } TEST( FTSSpec, ScoreSingleField1 ) { - BSONObj user = BSON( "key" << BSON( "title" << "fts" << - "text" << "fts" ) << + BSONObj user = BSON( "key" << BSON( "title" << "text" << + "text" << "text" ) << "weights" << BSON( "title" << 10 ) ); FTSSpec spec( FTSSpec::fixSpec( user ) ); @@ -50,16 +194,14 @@ namespace mongo { } TEST( FTSSpec, ScoreMultipleField1 ) { - BSONObj user = BSON( "key" << BSON( "title" << "fts" << - "text" << "fts" ) << + BSONObj user = BSON( "key" << BSON( "title" << "text" << + "text" << "text" ) << "weights" << BSON( "title" << 10 ) ); FTSSpec spec( FTSSpec::fixSpec( user ) ); TermFrequencyMap m; - spec.scoreDocument( BSON( "title" << "cat sat run" - << "text" << "cat book" ), - &m ); + spec.scoreDocument( BSON( "title" << "cat sat run" << "text" << "cat book" ), &m ); ASSERT_EQUALS( 4U, m.size() ); ASSERT_EQUALS( m["sat"], m["run"] ); @@ -71,10 +213,20 @@ namespace mongo { ASSERT( m["book"] < m["sat"] ); } + TEST( FTSSpec, ScoreMultipleField2 ) { + // Test where one indexed field is a parent component of another indexed field. + BSONObj user = BSON( "key" << BSON( "a" << "text" << "a.b" << "text" ) ); + + FTSSpec spec( FTSSpec::fixSpec( user ) ); + + TermFrequencyMap m; + spec.scoreDocument( BSON( "a" << BSON( "b" << "term" ) ), &m ); + ASSERT_EQUALS( 1U, m.size() ); + } TEST( FTSSpec, ScoreRepeatWord ) { - BSONObj user = BSON( "key" << BSON( "title" << "fts" << - "text" << "fts" ) << + BSONObj user = BSON( "key" << BSON( "title" << "text" << + "text" << "text" ) << "weights" << BSON( "title" << 10 ) ); FTSSpec spec( FTSSpec::fixSpec( user ) ); @@ -89,14 +241,14 @@ namespace mongo { } TEST( FTSSpec, Extra1 ) { - BSONObj user = BSON( "key" << BSON( "data" << "fts" ) ); + BSONObj user = BSON( "key" << BSON( "data" << "text" ) ); FTSSpec spec( FTSSpec::fixSpec( user ) ); ASSERT_EQUALS( 0U, spec.numExtraBefore() ); ASSERT_EQUALS( 0U, spec.numExtraAfter() ); } TEST( FTSSpec, Extra2 ) { - BSONObj user = BSON( "key" << BSON( "data" << "fts" << "x" << 1 ) ); + BSONObj user = BSON( "key" << BSON( "data" << "text" << "x" << 1 ) ); BSONObj fixed = FTSSpec::fixSpec( user ); FTSSpec spec( fixed ); ASSERT_EQUALS( 0U, spec.numExtraBefore() ); @@ -108,7 +260,7 @@ namespace mongo { } TEST( FTSSpec, Extra3 ) { - BSONObj user = BSON( "key" << BSON( "x" << 1 << "data" << "fts" ) ); + BSONObj user = BSON( "key" << BSON( "x" << 1 << "data" << "text" ) ); BSONObj fixed = FTSSpec::fixSpec( user ); ASSERT_EQUALS( BSON( "x" << 1 << @@ -139,5 +291,282 @@ namespace mongo { ASSERT( !spec.getIndexPrefix( BSONObj(), &prefix ).isOK() ); } + // Test for correct behavior when encountering nested arrays (both directly nested and + // indirectly nested). + + TEST( FTSSpec, NestedArraysPos1 ) { + BSONObj user = BSON( "key" << BSON( "a.b" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( user ) ); + + // The following document matches {"a.b": {$type: 2}}, so "term" should be indexed. + BSONObj obj = fromjson("{a: [{b: ['term']}]}"); // indirectly nested arrays + TermFrequencyMap m; + spec.scoreDocument( obj, &m ); + ASSERT_EQUALS( 1U, m.size() ); + } + + TEST( FTSSpec, NestedArraysPos2 ) { + BSONObj user = BSON( "key" << BSON( "$**" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( user ) ); + + // The wildcard spec implies a full recursive traversal, so "term" should be indexed. + BSONObj obj = fromjson("{a: {b: [['term']]}}"); // directly nested arrays + TermFrequencyMap m; + spec.scoreDocument( obj, &m ); + ASSERT_EQUALS( 1U, m.size() ); + } + + TEST( FTSSpec, NestedArraysNeg1 ) { + BSONObj user = BSON( "key" << BSON( "a.b" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( user ) ); + + // The following document does not match {"a.b": {$type: 2}}, so "term" should not be + // indexed. + BSONObj obj = fromjson("{a: {b: [['term']]}}"); // directly nested arrays + TermFrequencyMap m; + spec.scoreDocument( obj, &m ); + ASSERT_EQUALS( 0U, m.size() ); + } + + // Multi-language test_1: test independent stemming per sub-document + TEST( FTSSpec, NestedLanguages_PerArrayItemStemming ) { + BSONObj indexSpec = BSON( "key" << BSON( "a.b.c" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + + BSONObj obj = fromjson( + "{ a :" + " { b :" + " [ { c : \"walked\", language : \"english\" }," + " { c : \"camminato\", language : \"italian\" }," + " { c : \"ging\", language : \"german\" } ]" + " }" + " }" ); + + spec.scoreDocument( obj, &tfm ); + + set hits; + hits.insert("walk"); + hits.insert("cammin"); + hits.insert("ging"); + + for (TermFrequencyMap::const_iterator i = tfm.begin(); i!=tfm.end(); ++i) { + string term = i->first; + ASSERT_EQUALS( 1U, hits.count( term ) ); + } + + } + + // Multi-language test_2: test nested stemming per sub-document + TEST( FTSSpec, NestedLanguages_PerSubdocStemming ) { + BSONObj indexSpec = BSON( "key" << BSON( "a.b.c" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + + BSONObj obj = fromjson( + "{ language : \"english\"," + " a :" + " { language : \"danish\"," + " b :" + " [ { c : \"foredrag\" }," + " { c : \"foredragsholder\" }," + " { c : \"lector\" } ]" + " }" + "}" ); + + spec.scoreDocument( obj, &tfm ); + + set hits; + hits.insert("foredrag"); + hits.insert("foredragshold"); + hits.insert("lector"); + + for (TermFrequencyMap::const_iterator i = tfm.begin(); i!=tfm.end(); ++i) { + string term = i->first; + ASSERT_EQUALS( 1U, hits.count( term ) ); + } + + } + + // Multi-language test_3: test nested arrays + TEST( FTSSpec, NestedLanguages_NestedArrays ) { + BSONObj indexSpec = BSON( "key" << BSON( "a.b.c" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + + BSONObj obj = fromjson( + "{ language : \"english\"," + " a : [" + " { language : \"danish\"," + " b :" + " [ { c : [\"foredrag\"] }," + " { c : [\"foredragsholder\"] }," + " { c : [\"lector\"] } ]" + " } ]" + "}" ); + + spec.scoreDocument( obj, &tfm ); + + set hits; + hits.insert("foredrag"); + hits.insert("foredragshold"); + hits.insert("lector"); + + for (TermFrequencyMap::const_iterator i = tfm.begin(); i!=tfm.end(); ++i) { + string term = i->first; + ASSERT_EQUALS( 1U, hits.count( term ) ); + } + + } + + // Multi-language test_4: test pruning + TEST( FTSSpec, NestedLanguages_PathPruning ) { + BSONObj indexSpec = BSON( "key" << BSON( "a.b.c" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + + BSONObj obj = fromjson( + "{ language : \"english\"," + " a : " + " { language : \"danish\"," + " bc : \"foo\"," + " b : { d: \"bar\" }," + " b :" + " [ { c : \"foredrag\" }," + " { c : \"foredragsholder\" }," + " { c : \"lector\" } ]" + " }" + "}" ); + + spec.scoreDocument( obj, &tfm ); + + set hits; + hits.insert("foredrag"); + hits.insert("foredragshold"); + hits.insert("lector"); + + for (TermFrequencyMap::const_iterator i = tfm.begin(); i!=tfm.end(); ++i) { + string term = i->first; + ASSERT_EQUALS( 1U, hits.count( term ) ); + } + + } + + // Multi-language test_5: test wildcard spec + TEST( FTSSpec, NestedLanguages_Wildcard ) { + BSONObj indexSpec = BSON( "key" << BSON( "$**" << "text" ) ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + + BSONObj obj = fromjson( + "{ language : \"english\"," + " b : \"walking\"," + " c : { e: \"walked\" }," + " d : " + " { language : \"danish\"," + " e :" + " [ { f : \"foredrag\" }," + " { f : \"foredragsholder\" }," + " { f : \"lector\" } ]" + " }" + "}" ); + + spec.scoreDocument( obj, &tfm ); + + set hits; + hits.insert("foredrag"); + hits.insert("foredragshold"); + hits.insert("lector"); + hits.insert("walk"); + + for (TermFrequencyMap::const_iterator i = tfm.begin(); i!=tfm.end(); ++i) { + string term = i->first; + ASSERT_EQUALS( 1U, hits.count( term ) ); + } + + } + + // Multi-language test_6: test wildcard spec with override + TEST( FTSSpec, NestedLanguages_WildcardOverride ) { + BSONObj indexSpec = BSON( "key" << BSON( "$**" << "text" ) << + "weights" << BSON( "d.e.f" << 20 ) ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + + BSONObj obj = fromjson( + "{ language : \"english\"," + " b : \"walking\"," + " c : { e: \"walked\" }," + " d : " + " { language : \"danish\"," + " e :" + " [ { f : \"foredrag\" }," + " { f : \"foredragsholder\" }," + " { f : \"lector\" } ]" + " }" + "}" ); + + spec.scoreDocument( obj, &tfm ); + + set hits; + hits.insert("foredrag"); + hits.insert("foredragshold"); + hits.insert("lector"); + hits.insert("walk"); + + for (TermFrequencyMap::const_iterator i = tfm.begin(); i!=tfm.end(); ++i) { + string term = i->first; + ASSERT_EQUALS( 1U, hits.count( term ) ); + } + + } + + /** Test differences across textIndexVersion values in handling of nested arrays. */ + TEST( FTSSpec, TextIndexLegacyNestedArrays ) { + BSONObj obj = fromjson( "{a: [{b: ['hello']}]}" ); + + // textIndexVersion=1 FTSSpec objects do not index nested arrays. + { + BSONObj indexSpec = fromjson( "{key: {'a.b': 'text'}, textIndexVersion: 1}" ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + spec.scoreDocument( obj, &tfm ); + ASSERT_EQUALS( tfm.size(), 0U ); + } + + // textIndexVersion=2 FTSSpec objects do index nested arrays. + { + BSONObj indexSpec = fromjson( "{key: {'a.b': 'text'}, textIndexVersion: 2}" ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + spec.scoreDocument( obj, &tfm ); + ASSERT_EQUALS( tfm.size(), 1U ); + } + } + + /** Test differences across textIndexVersion values in handling of language annotations. */ + TEST( FTSSpec, TextIndexLegacyLanguageRecognition) { + BSONObj obj = fromjson( "{a: 'the', language: 'EN'}" ); + + // textIndexVersion=1 FTSSpec objects treat two-letter language annotations as "none" + // for purposes of stopword processing. + { + BSONObj indexSpec = fromjson( "{key: {'a': 'text'}, textIndexVersion: 1}" ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + spec.scoreDocument( obj, &tfm ); + ASSERT_EQUALS( tfm.size(), 1U ); // "the" not recognized as stopword + } + + // textIndexVersion=2 FTSSpec objects recognize two-letter codes. + { + BSONObj indexSpec = fromjson( "{key: {'a': 'text'}, textIndexVersion: 2}" ); + FTSSpec spec( FTSSpec::fixSpec( indexSpec ) ); + TermFrequencyMap tfm; + spec.scoreDocument( obj, &tfm ); + ASSERT_EQUALS( tfm.size(), 0U ); // "the" recognized as stopword + } + } + } } diff --git a/src/mongo/db/fts/fts_util.cpp b/src/mongo/db/fts/fts_util.cpp index ace11b67409..f2bd4e50905 100644 --- a/src/mongo/db/fts/fts_util.cpp +++ b/src/mongo/db/fts/fts_util.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/fts/fts_util.h" diff --git a/src/mongo/db/fts/fts_util.h b/src/mongo/db/fts/fts_util.h index 3df5a0c5ee2..fe9ad0de341 100644 --- a/src/mongo/db/fts/fts_util.h +++ b/src/mongo/db/fts/fts_util.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -22,7 +34,7 @@ #include "mongo/db/hasher.h" #include "mongo/db/jsobj.h" -#include "mongo/db/record.h" +#include "mongo/db/storage/record.h" #include "mongo/util/unordered_fast_key_table.h" namespace mongo { @@ -32,6 +44,12 @@ namespace mongo { extern const std::string WILDCARD; extern const std::string INDEX_NAME; + enum TextIndexVersion { + TEXT_INDEX_VERSION_1 = 1, // Legacy index format. Deprecated. + TEXT_INDEX_VERSION_2 = 2 // Current index format. + }; + + /** * destructive! */ diff --git a/src/mongo/db/fts/fts_util_test.cpp b/src/mongo/db/fts/fts_util_test.cpp index 7d959dca08a..381ed1bf86a 100644 --- a/src/mongo/db/fts/fts_util_test.cpp +++ b/src/mongo/db/fts/fts_util_test.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ diff --git a/src/mongo/db/fts/generate_stop_words.py b/src/mongo/db/fts/generate_stop_words.py index 2434e30fdc7..e0dc801ca92 100644 --- a/src/mongo/db/fts/generate_stop_words.py +++ b/src/mongo/db/fts/generate_stop_words.py @@ -10,13 +10,13 @@ def generate( header, source, language_files ): out = open( header, "wb" ) out.write( """ #pragma once -#include #include #include +#include "mongo/util/string_map.h" namespace mongo { namespace fts { - void loadStopWordMap( std::map< std::string, std::set< std::string > >* m ); + void loadStopWordMap( StringMap< std::set< std::string > >* m ); } } """ ) @@ -30,7 +30,7 @@ namespace fts { namespace mongo { namespace fts { - void loadStopWordMap( std::map< std::string, std::set< std::string > >* m ) { + void loadStopWordMap( StringMap< std::set< std::string > >* m ) { """ ) diff --git a/src/mongo/db/fts/stemmer.cpp b/src/mongo/db/fts/stemmer.cpp index c04d05c87ca..a86cfda8015 100644 --- a/src/mongo/db/fts/stemmer.cpp +++ b/src/mongo/db/fts/stemmer.cpp @@ -14,20 +14,34 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ +#include #include #include "mongo/db/fts/stemmer.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { namespace fts { - Stemmer::Stemmer( const string& language ) { + Stemmer::Stemmer( const FTSLanguage& language ) { _stemmer = NULL; - if ( language != "none" ) - _stemmer = sb_stemmer_new(language.c_str(), "UTF_8"); + if ( language.str() != "none" ) + _stemmer = sb_stemmer_new(language.str().c_str(), "UTF_8"); } Stemmer::~Stemmer() { diff --git a/src/mongo/db/fts/stemmer.h b/src/mongo/db/fts/stemmer.h index d212cc01fce..fe028e2aba7 100644 --- a/src/mongo/db/fts/stemmer.h +++ b/src/mongo/db/fts/stemmer.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ @@ -21,9 +33,9 @@ #include -#include "libstemmer.h" - #include "mongo/base/string_data.h" +#include "mongo/db/fts/fts_language.h" +#include "third_party/libstemmer_c/include/libstemmer.h" namespace mongo { @@ -36,7 +48,7 @@ namespace mongo { */ class Stemmer { public: - Stemmer( const std::string& language ); + Stemmer( const FTSLanguage& language ); ~Stemmer(); std::string stem( const StringData& word ) const; diff --git a/src/mongo/db/fts/stemmer_test.cpp b/src/mongo/db/fts/stemmer_test.cpp index 808b8141a64..9037715d4da 100644 --- a/src/mongo/db/fts/stemmer_test.cpp +++ b/src/mongo/db/fts/stemmer_test.cpp @@ -14,29 +14,40 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/unittest/unittest.h" +#include "mongo/db/fts/fts_spec.h" #include "mongo/db/fts/stemmer.h" namespace mongo { namespace fts { TEST( English, Stemmer1 ) { - Stemmer s( "english" ); + Stemmer s( languageEnglishV2 ); ASSERT_EQUALS( "run", s.stem( "running" ) ); ASSERT_EQUALS( "Run", s.stem( "Running" ) ); } - TEST( English, Caps ) { - Stemmer s( "porter" ); + Stemmer s( languagePorterV1 ); ASSERT_EQUALS( "unit", s.stem( "united" ) ); ASSERT_EQUALS( "Unite", s.stem( "United" ) ); } - } } diff --git a/src/mongo/db/fts/stop_words.cpp b/src/mongo/db/fts/stop_words.cpp index 0d664caf1bf..bc0240600c1 100644 --- a/src/mongo/db/fts/stop_words.cpp +++ b/src/mongo/db/fts/stop_words.cpp @@ -14,16 +14,27 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include #include #include #include "mongo/db/fts/stop_words.h" #include "mongo/base/init.h" -#include "mongo/platform/unordered_map.h" +#include "mongo/util/string_map.h" @@ -31,10 +42,10 @@ namespace mongo { namespace fts { - void loadStopWordMap( std::map< std::string, std::set< std::string > >* m ); + void loadStopWordMap( StringMap< std::set< std::string > >* m ); namespace { - unordered_map STOP_WORDS; + StringMap STOP_WORDS; StopWords* empty = NULL; } @@ -47,8 +58,8 @@ namespace mongo { _words.insert( *i ); } - const StopWords* StopWords::getStopWords( const std::string& langauge ) { - unordered_map::const_iterator i = STOP_WORDS.find( langauge ); + const StopWords* StopWords::getStopWords( const FTSLanguage& language ) { + StringMap::const_iterator i = STOP_WORDS.find( language.str() ); if ( i == STOP_WORDS.end() ) return empty; return i->second; @@ -58,9 +69,9 @@ namespace mongo { MONGO_INITIALIZER(StopWords)(InitializerContext* context) { empty = new StopWords(); - std::map< std::string, std::set< std::string > > raw; + StringMap< std::set< std::string > > raw; loadStopWordMap( &raw ); - for ( std::map< std::string, std::set< std::string > >::const_iterator i = raw.begin(); + for ( StringMap< std::set< std::string > >::const_iterator i = raw.begin(); i != raw.end(); ++i ) { STOP_WORDS[i->first] = new StopWords( i->second ); diff --git a/src/mongo/db/fts/stop_words.h b/src/mongo/db/fts/stop_words.h index 5816afa560c..22ec22f3fa8 100644 --- a/src/mongo/db/fts/stop_words.h +++ b/src/mongo/db/fts/stop_words.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ @@ -22,6 +34,7 @@ #include #include +#include "mongo/db/fts/fts_language.h" #include "mongo/platform/unordered_set.h" namespace mongo { @@ -39,7 +52,7 @@ namespace mongo { size_t numStopWords() const { return _words.size(); } - static const StopWords* getStopWords( const std::string& langauge ); + static const StopWords* getStopWords( const FTSLanguage& langauge ); private: ~StopWords(){} unordered_set _words; diff --git a/src/mongo/db/fts/stop_words_test.cpp b/src/mongo/db/fts/stop_words_test.cpp index 8d70600ce8e..0edf4e2540c 100644 --- a/src/mongo/db/fts/stop_words_test.cpp +++ b/src/mongo/db/fts/stop_words_test.cpp @@ -14,8 +14,21 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ +#include "mongo/db/fts/fts_spec.h" #include "mongo/db/fts/stop_words.h" #include "mongo/unittest/unittest.h" @@ -23,9 +36,9 @@ namespace mongo { namespace fts { TEST( English, Basic1 ) { - const StopWords* english = StopWords::getStopWords( "english" ); - ASSERT( english->isStopWord( "the" ) ); - ASSERT( !english->isStopWord( "computer" ) ); + const StopWords* englishStopWords = StopWords::getStopWords( languageEnglishV2 ); + ASSERT( englishStopWords->isStopWord( "the" ) ); + ASSERT( !englishStopWords->isStopWord( "computer" ) ); } } diff --git a/src/mongo/db/fts/tokenizer.cpp b/src/mongo/db/fts/tokenizer.cpp index 73f485901f6..53580fca4be 100644 --- a/src/mongo/db/fts/tokenizer.cpp +++ b/src/mongo/db/fts/tokenizer.cpp @@ -14,20 +14,33 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include #include "mongo/db/fts/tokenizer.h" +#include "mongo/util/mongoutils/str.h" #include "mongo/util/stringutils.h" namespace mongo { namespace fts { - Tokenizer::Tokenizer( const string& language, const StringData& str ) + Tokenizer::Tokenizer( const FTSLanguage& language, const StringData& str ) : _pos(0), _raw( str ) { - _english = language == "english"; + _english = ( language.str() == "english" ); _skipWhitespace(); _previousWhiteSpace = true; } diff --git a/src/mongo/db/fts/tokenizer.h b/src/mongo/db/fts/tokenizer.h index 8c0e3a15bbf..06e63f6aa2f 100644 --- a/src/mongo/db/fts/tokenizer.h +++ b/src/mongo/db/fts/tokenizer.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ @@ -22,6 +34,7 @@ #include #include "mongo/base/string_data.h" +#include "mongo/db/fts/fts_language.h" #include "mongo/platform/unordered_map.h" #include "mongo/platform/unordered_set.h" @@ -48,7 +61,7 @@ namespace mongo { class Tokenizer { public: - Tokenizer( const std::string& language, const StringData& str ); + Tokenizer( const FTSLanguage& language, const StringData& str ); bool more() const; Token next(); diff --git a/src/mongo/db/fts/tokenizer_test.cpp b/src/mongo/db/fts/tokenizer_test.cpp index 1502b2f4390..29153a329a6 100644 --- a/src/mongo/db/fts/tokenizer_test.cpp +++ b/src/mongo/db/fts/tokenizer_test.cpp @@ -14,8 +14,21 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ +#include "mongo/db/fts/fts_spec.h" #include "mongo/db/fts/tokenizer.h" #include "mongo/unittest/unittest.h" @@ -23,12 +36,12 @@ namespace mongo { namespace fts { TEST( Tokenizer, Empty1 ) { - Tokenizer i( "english", "" ); + Tokenizer i( languageEnglishV2, "" ); ASSERT( !i.more() ); } TEST( Tokenizer, Basic1 ) { - Tokenizer i( "english", "blue red green" ); + Tokenizer i( languageEnglishV2, "blue red green" ); ASSERT( i.more() ); ASSERT_EQUALS( i.next().data.toString(), "blue" ); @@ -43,7 +56,7 @@ namespace mongo { } TEST( Tokenizer, Basic2 ) { - Tokenizer i( "english", "blue-red" ); + Tokenizer i( languageEnglishV2, "blue-red" ); Token a = i.next(); Token b = i.next(); @@ -65,7 +78,7 @@ namespace mongo { } TEST( Tokenizer, Basic3 ) { - Tokenizer i( "english", "blue -red" ); + Tokenizer i( languageEnglishV2, "blue -red" ); Token a = i.next(); Token b = i.next(); @@ -92,7 +105,7 @@ namespace mongo { } TEST( Tokenizer, Quote1English ) { - Tokenizer i( "english", "eliot's car" ); + Tokenizer i( languageEnglishV2, "eliot's car" ); Token a = i.next(); Token b = i.next(); @@ -102,7 +115,7 @@ namespace mongo { } TEST( Tokenizer, Quote1French ) { - Tokenizer i( "french", "eliot's car" ); + Tokenizer i( languageFrenchV2, "eliot's car" ); Token a = i.next(); Token b = i.next(); diff --git a/src/mongo/db/geo/2d.cpp b/src/mongo/db/geo/2d.cpp deleted file mode 100644 index e1e21eee4f7..00000000000 --- a/src/mongo/db/geo/2d.cpp +++ /dev/null @@ -1,2732 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" - -#include - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/index.h" -#include "mongo/util/startup_test.h" -#include "mongo/db/commands.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/btreecursor.h" -#include "mongo/db/curop-inl.h" -#include "mongo/db/matcher.h" -#include "mongo/db/queryutil.h" -#include "mongo/db/geo/core.h" -#include "mongo/db/geo/geonear.h" -#include "mongo/db/geo/hash.h" -#include "mongo/db/geo/shapes.h" -#include "mongo/util/timer.h" - -// Note: we use indexinterface herein to talk to the btree code. In the future it would be nice to -// be able to use the V1 key class (see key.h) instead of toBson() which has some cost. -// toBson() is new with v1 so this could be slower than it used to be? a quick profiling -// might make sense. - -namespace mongo { - - class GeoKeyNode { - GeoKeyNode(); - public: - GeoKeyNode(DiskLoc bucket, int keyOfs, DiskLoc r, BSONObj k) - : _bucket(bucket), _keyOfs(keyOfs), recordLoc(r), _key(k) { } - const DiskLoc _bucket; - const int _keyOfs; - const DiskLoc recordLoc; - const BSONObj _key; - }; - - enum GeoDistType { - GEO_PLAIN, - GEO_SPHERE - }; - - inline double computeXScanDistance(double y, double maxDistDegrees) { - // TODO: this overestimates for large madDistDegrees far from the equator - return maxDistDegrees / min(cos(deg2rad(min(+89.0, y + maxDistDegrees))), - cos(deg2rad(max(-89.0, y - maxDistDegrees)))); - } - - const string GEO2DNAME = "2d"; - - class Geo2dType : public IndexType { - public: - virtual ~Geo2dType() { } - - Geo2dType(const IndexPlugin *plugin, const IndexSpec* spec) : IndexType(plugin, spec) { - BSONObjIterator i(spec->keyPattern); - while (i.more()) { - BSONElement e = i.next(); - if (e.type() == String && GEO2DNAME == e.valuestr()) { - uassert(13022, "can't have 2 geo field", _geo.size() == 0); - uassert(13023, "2d has to be first in index", _other.size() == 0); - _geo = e.fieldName(); - } else { - int order = 1; - if (e.isNumber()) { - order = static_cast(e.Number()); - } - _other.push_back(make_pair(e.fieldName(), order)); - } - } - uassert(13024, "no geo field specified", _geo.size()); - - double bits = configValueWithDefault(spec, "bits", 26); // for lat/long, ~ 1ft - uassert(13028, "bits in geo index must be between 1 and 32", bits > 0 && bits <= 32); - - GeoHashConverter::Parameters params; - params.bits = static_cast(bits); - params.max = configValueWithDefault(spec, "max", 180.0); - params.min = configValueWithDefault(spec, "min", -180.0); - double numBuckets = (1024 * 1024 * 1024 * 4.0); - params.scaling = numBuckets / (params.max - params.min); - - _geoHashConverter.reset(new GeoHashConverter(params)); - } - - // XXX: what does this do - virtual BSONObj fixKey(const BSONObj& in) { - if (in.firstElement().type() == BinData) - return in; - - BSONObjBuilder b(in.objsize() + 16); - - if (in.firstElement().isABSONObj()) - _geoHashConverter->hash(in.firstElement().embeddedObject()).appendToBuilder(&b, ""); - else if (in.firstElement().type() == String) - GeoHash(in.firstElement().valuestr()).appendToBuilder(&b, ""); - else if (in.firstElement().type() == RegEx) - GeoHash(in.firstElement().regex()).appendToBuilder(&b, ""); - else - return in; - - BSONObjIterator i(in); - i.next(); - while (i.more()) - b.append(i.next()); - return b.obj(); - } - - /** Finds the key objects to put in an index */ - virtual void getKeys(const BSONObj& obj, BSONObjSet& keys) const { - getKeys(obj, &keys, NULL); - } - - /** Finds all locations in a geo-indexed object */ - // TODO: Can we just return references to the locs, if they won't change? - void getKeys(const BSONObj& obj, vector& locs) const { - getKeys(obj, NULL, &locs); - } - - /** Finds the key objects and/or locations for a geo-indexed object */ - void getKeys(const BSONObj &obj, BSONObjSet* keys, vector* locs) const { - BSONElementMSet bSet; - - // Get all the nested location fields, but don't return individual elements from - // the last array, if it exists. - obj.getFieldsDotted(_geo.c_str(), bSet, false); - - if (bSet.empty()) - return; - - for (BSONElementMSet::iterator setI = bSet.begin(); setI != bSet.end(); ++setI) { - BSONElement geo = *setI; - - GEODEBUG("Element " << geo << " found for query " << _geo.c_str()); - - if (geo.eoo() || !geo.isABSONObj()) - continue; - - // - // Grammar for location lookup: - // locs ::= [loc,loc,...,loc]|{:loc,:loc,...,:loc}|loc - // loc ::= { : #, : # }|[#, #]|{} - // - // Empty locations are ignored, preserving single-location semantics - // - - BSONObj embed = geo.embeddedObject(); - if (embed.isEmpty()) - continue; - - // Differentiate between location arrays and locations - // by seeing if the first element value is a number - bool singleElement = embed.firstElement().isNumber(); - - BSONObjIterator oi(embed); - - while (oi.more()) { - BSONObj locObj; - - if (singleElement) { - locObj = embed; - } else { - BSONElement locElement = oi.next(); - - uassert(13654, str::stream() << "location object expected, location " - "array not in correct format", - locElement.isABSONObj()); - - locObj = locElement.embeddedObject(); - if(locObj.isEmpty()) - continue; - } - - BSONObjBuilder b(64); - - // Remember the actual location object if needed - if (locs) - locs->push_back(locObj); - - // Stop if we don't need to get anything but location objects - if (!keys) { - if (singleElement) break; - else continue; - } - - _geoHashConverter->hash(locObj, &obj).appendToBuilder(&b, ""); - - // Go through all the other index keys - for (vector >::const_iterator i = _other.begin(); - i != _other.end(); ++i) { - // Get *all* fields for the index key - BSONElementSet eSet; - obj.getFieldsDotted(i->first, eSet); - - if (eSet.size() == 0) - b.appendAs(_spec->missingField(), ""); - else if (eSet.size() == 1) - b.appendAs(*(eSet.begin()), ""); - else { - // If we have more than one key, store as an array of the objects - BSONArrayBuilder aBuilder; - - for (BSONElementSet::iterator ei = eSet.begin(); ei != eSet.end(); - ++ei) { - aBuilder.append(*ei); - } - - b.append("", aBuilder.arr()); - } - } - keys->insert(b.obj()); - if(singleElement) break; - } - } - } - - const IndexDetails* getDetails() const { - return _spec->getDetails(); - } - - virtual shared_ptr newCursor(const BSONObj& query, const BSONObj& order, - int numWanted) const; - - virtual IndexSuitability suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const { - BSONObj query = queryConstraints.originalQuery(); - - BSONElement e = query.getFieldDotted(_geo.c_str()); - switch (e.type()) { - case Object: { - BSONObj sub = e.embeddedObject(); - switch (sub.firstElement().getGtLtOp()) { - case BSONObj::opNEAR: - return OPTIMAL; - case BSONObj::opWITHIN: { - // Don't return optimal if it's $within: {$geometry: ... } - // because we will error out in that case, but the matcher - // or 2dsphere index may handle it. - BSONElement elt = sub.firstElement(); - if (Object == elt.type()) { - BSONObjIterator it(elt.embeddedObject()); - while (it.more()) { - BSONElement elt = it.next(); - if (mongoutils::str::equals("$geometry", elt.fieldName())) { - return USELESS; - } - } - } - return OPTIMAL; - } - default: - // We can try to match if there's no other indexing defined, - // this is assumed a point - return HELPFUL; - } - } - case Array: - // We can try to match if there's no other indexing defined, - // this is assumed a point - return HELPFUL; - default: - return USELESS; - } - } - - const GeoHashConverter& getConverter() const { return *_geoHashConverter; } - - // XXX: make private with a getter - string _geo; - vector > _other; - private: - double configValueWithDefault(const IndexSpec* spec, const string& name, double def) { - BSONElement e = spec->info[name]; - if (e.isNumber()) { - return e.numberDouble(); - } - return def; - } - - scoped_ptr _geoHashConverter; - }; - - class Geo2dPlugin : public IndexPlugin { - public: - Geo2dPlugin() : IndexPlugin(GEO2DNAME) { } - - virtual IndexType* generate(const IndexSpec* spec) const { - return new Geo2dType(this, spec); - } - } geo2dplugin; - - void __forceLinkGeoPlugin() { - geo2dplugin.getName(); - } - - class GeoHopper; - - class GeoPoint { - public: - GeoPoint() : _distance(-1), _exact(false), _dirty(false) { } - - //// Distance not used //// - - GeoPoint(const GeoKeyNode& node) - : _key(node._key), _loc(node.recordLoc), _o(node.recordLoc.obj()), - _distance(-1), _exact(false), _dirty(false), _bucket(node._bucket), - _pos(node._keyOfs) { } - - //// Immediate initialization of distance //// - - GeoPoint(const GeoKeyNode& node, double distance, bool exact) - : _key(node._key), _loc(node.recordLoc), _o(node.recordLoc.obj()), _distance(distance), _exact(exact), _dirty(false) { - } - - GeoPoint(const GeoPoint& pt, double distance, bool exact) - : _key(pt.key()), _loc(pt.loc()), _o(pt.obj()), _distance(distance), _exact(exact), _dirty(false) { - } - - bool operator<(const GeoPoint& other) const { - if(_distance != other._distance) return _distance < other._distance; - if(_exact != other._exact) return _exact < other._exact; - return _loc < other._loc; - } - - double distance() const { - return _distance; - } - - bool isExact() const { - return _exact; - } - - BSONObj key() const { - return _key; - } - - bool hasLoc() const { - return _loc.isNull(); - } - - DiskLoc loc() const { - verify(! _dirty); - return _loc; - } - - BSONObj obj() const { - return _o; - } - - BSONObj pt() const { - return _pt; - } - - bool isEmpty() { - return _o.isEmpty(); - } - - bool isCleanAndEmpty() { - return isEmpty() && ! isDirty(); - } - - string toString() const { - return str::stream() << "Point from " << _key << " - " << _o << " dist : " << _distance << (_exact ? " (ex)" : " (app)"); - } - - - // TODO: Recover from yield by finding all the changed disk locs here, modifying the _seenPts array. - // Not sure yet the correct thing to do about _seen. - // Definitely need to re-find our current max/min locations too - bool unDirty(const Geo2dType* g, DiskLoc& oldLoc){ - - verify(_dirty); - verify(! _id.isEmpty()); - - oldLoc = _loc; - _loc = DiskLoc(); - - // Fast undirty - IndexInterface& ii = g->getDetails()->idxInterface(); - // Check this position and the one immediately preceding - for(int i = 0; i < 2; i++){ - if(_pos - i < 0) continue; - - // log() << "bucket : " << _bucket << " pos " << _pos << endl; - - BSONObj key; - DiskLoc loc; - ii.keyAt(_bucket, _pos - i, key, loc); - - // log() << "Loc: " << loc << " Key : " << key << endl; - - if(loc.isNull()) continue; - - if(key.binaryEqual(_key) && loc.obj()["_id"].wrap("").binaryEqual(_id)){ - _pos = _pos - i; - _loc = loc; - _dirty = false; - _o = loc.obj(); - return true; - } - } - - // Slow undirty - scoped_ptr cursor(BtreeCursor::make(nsdetails(g->getDetails()->parentNS()), - *(g->getDetails()), _key, _key, true, 1)); - - int count = 0; - while(cursor->ok()){ - count++; - if(cursor->current()["_id"].wrap("").binaryEqual(_id)){ - _bucket = cursor->getBucket(); - _pos = cursor->getKeyOfs(); - _loc = cursor->currLoc(); - _o = _loc.obj(); - break; - } - else{ - LOG(CDEBUG + 1) << "Key doesn't match : " << cursor->current()["_id"] << " saved : " << _id << endl; - } - cursor->advance(); - } - - if(! count) { LOG(CDEBUG) << "No key found for " << _key << endl; } - - _dirty = false; - - return _loc == oldLoc; - } - - bool isDirty(){ - return _dirty; - } - - bool makeDirty(){ - if(! _dirty){ - verify(! obj()["_id"].eoo()); - verify(! _bucket.isNull()); - verify(_pos >= 0); - - if(_id.isEmpty()){ - _id = obj()["_id"].wrap("").getOwned(); - } - _o = BSONObj(); - _key = _key.getOwned(); - _pt = _pt.getOwned(); - _dirty = true; - - return true; - } - - return false; - } - - BSONObj _key; - DiskLoc _loc; - BSONObj _o; - BSONObj _pt; - - double _distance; - bool _exact; - - BSONObj _id; - bool _dirty; - DiskLoc _bucket; - int _pos; - }; - - // GeoBrowse subclasses this - class GeoAccumulator { - public: - GeoAccumulator(const Geo2dType * g, const BSONObj& filter, bool uniqueDocs, bool needDistance) - : _g(g), - _lookedAt(0), - _matchesPerfd(0), - _objectsLoaded(0), - _pointsLoaded(0), - _found(0), - _uniqueDocs(uniqueDocs), - _needDistance(needDistance) - { - if (! filter.isEmpty()) { - _matcher.reset(new CoveredIndexMatcher(filter, g->keyPattern())); - GEODEBUG("Matcher is now " << _matcher->docMatcher().toString()); - } - } - - virtual ~GeoAccumulator() { } - - enum KeyResult { BAD, BORDER, GOOD }; - - virtual void add(const GeoKeyNode& node) { - - GEODEBUG("\t\t\t\t checking key " << node._key.toString()) - - _lookedAt++; - - //// - // Approximate distance check using key data - //// - double keyD = 0; - //Point keyP(_g, GeoHash(node._key.firstElement(), _g->_bits)); - Point keyP(_g->getConverter().unhashToPoint(node._key.firstElement())); - KeyResult keyOk = approxKeyCheck(keyP, keyD); - if (keyOk == BAD) { - GEODEBUG("\t\t\t\t bad distance : " << node.recordLoc.obj() << "\t" << keyD); - return; - } - GEODEBUG("\t\t\t\t good distance : " << node.recordLoc.obj() << "\t" << keyD); - - //// - // Check for match using other key (and potentially doc) criteria - //// - // Remember match results for each object - map::iterator match = _matched.find(node.recordLoc); - bool newDoc = match == _matched.end(); - if(newDoc) { - - GEODEBUG("\t\t\t\t matching new doc with " << (_matcher ? _matcher->docMatcher().toString() : "(empty)")); - - // matcher - MatchDetails details; - if (_matcher.get()) { - bool good = _matcher->matchesWithSingleKeyIndex(node._key, node.recordLoc, &details); - - _matchesPerfd++; - - if (details.hasLoadedRecord()) - _objectsLoaded++; - - if (! good) { - GEODEBUG("\t\t\t\t didn't match : " << node.recordLoc.obj()["_id"]); - _matched[ node.recordLoc ] = false; - return; - } - } - - _matched[ node.recordLoc ] = true; - - if (! details.hasLoadedRecord()) // don't double count - _objectsLoaded++; - - } - else if(!((*match).second)) { - GEODEBUG("\t\t\t\t previously didn't match : " << node.recordLoc.obj()["_id"]); - return; - } - - //// - // Exact check with particular data fields - //// - // Can add multiple points - int diff = addSpecific(node, keyP, keyOk == BORDER, keyD, newDoc); - if(diff > 0) _found += diff; - else _found -= -diff; - - } - - virtual void getPointsFor(const BSONObj& key, const BSONObj& obj, - vector &locsForNode, bool allPoints = false) { - // Find all the location objects from the keys - vector locs; - _g->getKeys(obj, allPoints ? locsForNode : locs); - ++_pointsLoaded; - - if (allPoints) return; - if (locs.size() == 1){ - locsForNode.push_back(locs[0]); - return; - } - - // Find the particular location we want - GeoHash keyHash(key.firstElement(), _g->getConverter().getBits()); - - for(vector< BSONObj >::iterator i = locs.begin(); i != locs.end(); ++i) { - // Ignore all locations not hashed to the key's hash, since we may see - // those later - if(_g->getConverter().hash(*i) != keyHash) continue; - locsForNode.push_back(*i); - } - } - - virtual int addSpecific(const GeoKeyNode& node, const Point& p, bool inBounds, double d, - bool newDoc) = 0; - virtual KeyResult approxKeyCheck(const Point& p, double& keyD) = 0; - virtual bool exactDocCheck(const Point& p, double& d) = 0; - virtual bool expensiveExactCheck(){ return false; } - - long long found() const { return _found; } - - const Geo2dType * _g; - map _matched; - shared_ptr _matcher; - - long long _lookedAt; - long long _matchesPerfd; - long long _objectsLoaded; - long long _pointsLoaded; - long long _found; - - bool _uniqueDocs; - bool _needDistance; - }; - - struct BtreeLocation { - BtreeLocation() { } - - scoped_ptr _cursor; - scoped_ptr _frs; - scoped_ptr _spec; - - BSONObj key() { - return _cursor->currKey(); - } - - bool hasPrefix(const GeoHash& hash) { - BSONObj k = key(); - BSONElement e = k.firstElement(); - if (e.eoo()) - return false; - return GeoHash(e).hasPrefix(hash); - } - - bool checkAndAdvance(const GeoHash& hash, int& totalFound, GeoAccumulator* all){ - if(! _cursor->ok() || ! hasPrefix(hash)) return false; - - if(all){ - totalFound++; - GeoKeyNode n(_cursor->getBucket(), _cursor->getKeyOfs(), _cursor->currLoc(), _cursor->currKey()); - all->add(n); - } - _cursor->advance(); - - return true; - } - - void save(){ - _cursor->noteLocation(); - } - - void restore(){ - _cursor->checkLocation(); - } - - string toString() { - stringstream ss; - ss << "bucket: " << _cursor->getBucket().toString() << " pos: " << _cursor->getKeyOfs() << - (_cursor->ok() ? (str::stream() << " k: " << _cursor->currKey() << " o : " << _cursor->current()["_id"]) : (string)"[none]") << endl; - return ss.str(); - } - - // Returns the min and max keys which bound a particular location. - // The only time these may be equal is when we actually equal the location - // itself, otherwise our expanding algorithm will fail. - static bool initial(const IndexDetails& id, const Geo2dType * spec, - BtreeLocation& min, BtreeLocation& max, - GeoHash start, - int & found, GeoAccumulator * hopper) { - - //Ordering ordering = Ordering::make(spec->_order); - - // Would be nice to build this directly, but bug in max/min queries SERVER-3766 and lack of interface - // makes this easiest for now. - BSONObj minQuery = BSON(spec->_geo << BSON("$gt" << MINKEY << start.wrap("$lte").firstElement())); - BSONObj maxQuery = BSON(spec->_geo << BSON("$lt" << MAXKEY << start.wrap("$gt").firstElement())); - - // log() << "MinQuery: " << minQuery << endl; - // log() << "MaxQuery: " << maxQuery << endl; - - min._frs.reset(new FieldRangeSet(spec->getDetails()->parentNS().c_str(), - minQuery, - true, - false)); - - max._frs.reset(new FieldRangeSet(spec->getDetails()->parentNS().c_str(), - maxQuery, - true, - false)); - - - BSONObjBuilder bob; - bob.append(spec->_geo, 1); - for(vector >::const_iterator i = spec->_other.begin(); - i != spec->_other.end(); i++){ - bob.append(i->first, i->second); - } - BSONObj iSpec = bob.obj(); - - min._spec.reset(new IndexSpec(iSpec)); - max._spec.reset(new IndexSpec(iSpec)); - - shared_ptr frvMin(new FieldRangeVector(*(min._frs), *(min._spec), -1)); - shared_ptr frvMax(new FieldRangeVector(*(max._frs), *(max._spec), 1)); - - min._cursor.reset( - BtreeCursor::make(nsdetails(spec->getDetails()->parentNS()), - *(spec->getDetails()), - frvMin, - 0, - -1) - ); - - max._cursor.reset( - BtreeCursor::make(nsdetails(spec->getDetails()->parentNS()), - *(spec->getDetails()), - frvMax, - 0, - 1) - ); - - // if(hopper) min.checkCur(found, hopper); - // if(hopper) max.checkCur(found, hopper); - - return min._cursor->ok() || max._cursor->ok(); - } - }; - - - class GeoCursorBase : public Cursor { - public: - - static const shared_ptr< CoveredIndexMatcher > emptyMatcher; - - GeoCursorBase(const Geo2dType * spec) - : _spec(spec), _id(_spec->getDetails()) { - - } - - virtual DiskLoc refLoc() { return DiskLoc(); } - - virtual BSONObj indexKeyPattern() { - return _spec->keyPattern(); - } - - virtual void noteLocation() { - // no-op since these are meant to be safe - } - - /* called before query getmore block is iterated */ - virtual void checkLocation() { - // no-op since these are meant to be safe - } - - virtual bool supportGetMore() { return false; } - virtual bool supportYields() { return false; } - - virtual bool getsetdup(DiskLoc loc) { return false; } - virtual bool modifiedKeys() const { return true; } - virtual bool isMultiKey() const { return false; } - - virtual bool autoDedup() const { return false; } - - const Geo2dType * _spec; - const IndexDetails * _id; - }; - - const shared_ptr< CoveredIndexMatcher > GeoCursorBase::emptyMatcher(new CoveredIndexMatcher(BSONObj(), BSONObj())); - - // TODO: Pull out the cursor bit from the browse, have GeoBrowse as field of cursor to clean up - // this hierarchy a bit. Also probably useful to look at whether GeoAccumulator can be a member instead - // of a superclass. - class GeoBrowse : public GeoCursorBase, public GeoAccumulator { - public: - - // The max points which should be added to an expanding box at one time - static const int maxPointsHeuristic = 50; - - // Expand states - enum State { - START, - DOING_EXPAND, - DONE_NEIGHBOR, - DONE - } _state; - - GeoBrowse(const Geo2dType * g, string type, BSONObj filter = BSONObj(), bool uniqueDocs = true, bool needDistance = false) - : GeoCursorBase(g), GeoAccumulator(g, filter, uniqueDocs, needDistance), - _type(type), _filter(filter), _firstCall(true), _noted(false), _nscanned(), _nDirtied(0), _nChangedOnYield(0), _nRemovedOnYield(0), _centerPrefix(0, 0, 0) { - - // Set up the initial expand state - _state = START; - _neighbor = -1; - _foundInExp = 0; - - } - - virtual string toString() { - return (string)"GeoBrowse-" + _type; - } - - virtual bool ok() { - - bool filled = false; - - LOG(CDEBUG) << "Checking cursor, in state " << (int) _state << ", first call " << _firstCall << - ", empty : " << _cur.isEmpty() << ", dirty : " << _cur.isDirty() << ", stack : " << _stack.size() << endl; - - bool first = _firstCall; - if (_firstCall) { - fillStack(maxPointsHeuristic); - filled = true; - _firstCall = false; - } - if (! _cur.isCleanAndEmpty() || _stack.size()) { - if (first) { - ++_nscanned; - } - - if(_noted && filled) noteLocation(); - return true; - } - - while (moreToDo()) { - - LOG(CDEBUG) << "Refilling stack..." << endl; - - fillStack(maxPointsHeuristic); - filled = true; - - if (! _cur.isCleanAndEmpty()) { - if (first) { - ++_nscanned; - } - - if(_noted && filled) noteLocation(); - return true; - } - } - - if(_noted && filled) noteLocation(); - return false; - } - - virtual bool advance() { - _cur._o = BSONObj(); - - if (_stack.size()) { - _cur = _stack.front(); - _stack.pop_front(); - ++_nscanned; - return true; - } - - if (! moreToDo()) - return false; - - bool filled = false; - while (_cur.isCleanAndEmpty() && moreToDo()){ - fillStack(maxPointsHeuristic); - filled = true; - } - - if(_noted && filled) noteLocation(); - return ! _cur.isCleanAndEmpty() && ++_nscanned; - } - - virtual void noteLocation() { - _noted = true; - - LOG(CDEBUG) << "Noting location with " << _stack.size() << (_cur.isEmpty() ? "" : " + 1 ") << " points " << endl; - - // Make sure we advance past the point we're at now, - // since the current location may move on an update/delete - // if(_state == DOING_EXPAND){ - // if(_min.hasPrefix(_prefix)){ _min.advance(-1, _foundInExp, this); } - // if(_max.hasPrefix(_prefix)){ _max.advance( 1, _foundInExp, this); } - // } - - // Remember where our _max, _min are - _min.save(); - _max.save(); - - LOG(CDEBUG) << "Min " << _min.toString() << endl; - LOG(CDEBUG) << "Max " << _max.toString() << endl; - - // Dirty all our queued stuff - for(list::iterator i = _stack.begin(); i != _stack.end(); i++){ - - LOG(CDEBUG) << "Undirtying stack point with id " << i->_id << endl; - - if(i->makeDirty()) _nDirtied++; - verify(i->isDirty()); - } - - // Check current item - if(! _cur.isEmpty()){ - if(_cur.makeDirty()) _nDirtied++; - } - - // Our cached matches become invalid now - _matched.clear(); - } - - void fixMatches(DiskLoc oldLoc, DiskLoc newLoc){ - map::iterator match = _matched.find(oldLoc); - if(match != _matched.end()){ - bool val = match->second; - _matched.erase(oldLoc); - _matched[ newLoc ] = val; - } - } - - /* called before query getmore block is iterated */ - virtual void checkLocation() { - - LOG(CDEBUG) << "Restoring location with " << _stack.size() << (! _cur.isDirty() ? "" : " + 1 ") << " points " << endl; - - // We can assume an error was thrown earlier if this database somehow disappears - - // Recall our _max, _min - _min.restore(); - _max.restore(); - - LOG(CDEBUG) << "Min " << _min.toString() << endl; - LOG(CDEBUG) << "Max " << _max.toString() << endl; - - // If the current key moved, we may have been advanced past the current point - need to check this - // if(_state == DOING_EXPAND){ - // if(_min.hasPrefix(_prefix)){ _min.advance(-1, _foundInExp, this); } - // if(_max.hasPrefix(_prefix)){ _max.advance( 1, _foundInExp, this); } - //} - - // Undirty all the queued stuff - // Dirty all our queued stuff - list::iterator i = _stack.begin(); - while(i != _stack.end()){ - - LOG(CDEBUG) << "Undirtying stack point with id " << i->_id << endl; - - DiskLoc oldLoc; - if(i->unDirty(_spec, oldLoc)){ - // Document is in same location - LOG(CDEBUG) << "Undirtied " << oldLoc << endl; - - i++; - } - else if(! i->loc().isNull()){ - - // Re-found document somewhere else - LOG(CDEBUG) << "Changed location of " << i->_id << " : " << i->loc() << " vs " << oldLoc << endl; - - _nChangedOnYield++; - fixMatches(oldLoc, i->loc()); - i++; - } - else { - - // Can't re-find document - LOG(CDEBUG) << "Removing document " << i->_id << endl; - - _nRemovedOnYield++; - _found--; - verify(_found >= 0); - - // Can't find our key again, remove - i = _stack.erase(i); - } - } - - if(_cur.isDirty()){ - LOG(CDEBUG) << "Undirtying cur point with id : " << _cur._id << endl; - } - - // Check current item - DiskLoc oldLoc; - if(_cur.isDirty() && ! _cur.unDirty(_spec, oldLoc)){ - if(_cur.loc().isNull()){ - - // Document disappeared! - LOG(CDEBUG) << "Removing cur point " << _cur._id << endl; - - _nRemovedOnYield++; - advance(); - } - else{ - - // Document moved - LOG(CDEBUG) << "Changed location of cur point " << _cur._id << " : " << _cur.loc() << " vs " << oldLoc << endl; - - _nChangedOnYield++; - fixMatches(oldLoc, _cur.loc()); - } - } - - _noted = false; - } - - virtual Record* _current() { verify(ok()); LOG(CDEBUG + 1) << "_current " << _cur._loc.obj()["_id"] << endl; return _cur._loc.rec(); } - virtual BSONObj current() { verify(ok()); LOG(CDEBUG + 1) << "current " << _cur._o << endl; return _cur._o; } - virtual DiskLoc currLoc() { verify(ok()); LOG(CDEBUG + 1) << "currLoc " << _cur._loc << endl; return _cur._loc; } - virtual BSONObj currKey() const { return _cur._key; } - - virtual CoveredIndexMatcher* matcher() const { - if(_matcher.get()) return _matcher.get(); - else return GeoCursorBase::emptyMatcher.get(); - } - - // Are we finished getting points? - virtual bool moreToDo() { - return _state != DONE; - } - - virtual bool supportGetMore() { return true; } - - // XXX: make private, move into geohashconverter - Box makeBox(const GeoHash &hash) const { - double sizeEdge = _g->getConverter().sizeEdge(hash); - Point min(_g->getConverter().unhashToPoint(hash)); - Point max(min.x + sizeEdge, min.y + sizeEdge); - return Box(min, max); - } - - // Fills the stack, but only checks a maximum number of maxToCheck points at a time. - // Further calls to this function will continue the expand/check neighbors algorithm. - virtual void fillStack(int maxToCheck, int maxToAdd = -1, bool onlyExpand = false) { - -#ifdef GEODEBUGGING - log() << "Filling stack with maximum of " << maxToCheck << ", state : " << (int) _state << endl; -#endif - - if(maxToAdd < 0) maxToAdd = maxToCheck; - int maxFound = _foundInExp + maxToCheck; - verify(maxToCheck > 0); - verify(maxFound > 0); - verify(_found <= 0x7fffffff); // conversion to int - int maxAdded = static_cast(_found) + maxToAdd; - verify(maxAdded >= 0); // overflow check - - bool isNeighbor = _centerPrefix.constrains(); - - // Starting a box expansion - if (_state == START) { - - // Get the very first hash point, if required - if(! isNeighbor) - _prefix = expandStartHash(); - GEODEBUG("initializing btree"); - -#ifdef GEODEBUGGING - log() << "Initializing from b-tree with hash of " << _prefix << " @ " << Box(_g, _prefix) << endl; -#endif - - if (! BtreeLocation::initial(*_id, _spec, _min, _max, _prefix, _foundInExp, this)) { - _state = isNeighbor ? DONE_NEIGHBOR : DONE; - } else { - _state = DOING_EXPAND; - _lastPrefix.reset(); - } - - GEODEBUG((_state == DONE_NEIGHBOR || _state == DONE ? "not initialized" : "initializedFig")); - - } - - // Doing the actual box expansion - if (_state == DOING_EXPAND) { - while (true) { - - GEODEBUG("box prefix [" << _prefix << "]"); -#ifdef GEODEBUGGING - if(_prefix.constrains()) { - log() << "current expand box : " << Box(_g, _prefix).toString() << endl; - } - else { - log() << "max expand box." << endl; - } -#endif - - GEODEBUG("expanding box points... "); - - // Record the prefix we're actively exploring... - _expPrefix.reset(new GeoHash(_prefix)); - - // Find points inside this prefix - while (_min.checkAndAdvance(_prefix, _foundInExp, this) && _foundInExp < maxFound && _found < maxAdded) {} - while (_max.checkAndAdvance(_prefix, _foundInExp, this) && _foundInExp < maxFound && _found < maxAdded) {} - -#ifdef GEODEBUGGING - - log() << "finished expand, checked : " << (maxToCheck - (maxFound - _foundInExp)) - << " found : " << (maxToAdd - (maxAdded - _found)) - << " max : " << maxToCheck << " / " << maxToAdd << endl; - -#endif - - GEODEBUG("finished expand, found : " << (maxToAdd - (maxAdded - _found))); - if(_foundInExp >= maxFound || _found >= maxAdded) return; - - // We've searched this prefix fully, remember - _lastPrefix.reset(new GeoHash(_prefix)); - - // If we've searched the entire space, we're finished. - if (! _prefix.constrains()) { - GEODEBUG("box exhausted"); - _state = DONE; - notePrefix(); - return; - } - - // If we won't fit in the box, and we're not doing a sub-scan, increase the size - if (! fitsInBox(_g->getConverter().sizeEdge(_prefix)) && _fringe.size() == 0) { - // If we're still not expanded bigger than the box size, expand again - // TODO: Is there an advantage to scanning prior to expanding? - _prefix = _prefix.up(); - continue; - } - - // log() << "finished box prefix [" << _prefix << "]" << endl; - - // We're done and our size is large enough - _state = DONE_NEIGHBOR; - - // Go to the next sub-box, if applicable - if(_fringe.size() > 0) _fringe.pop_back(); - // Go to the next neighbor if this was the last sub-search - if(_fringe.size() == 0) _neighbor++; - - break; - } - - notePrefix(); - } - - // If we doeighbors - if(onlyExpand) return; - - // If we're done expanding the current box... - if(_state == DONE_NEIGHBOR) { - // Iterate to the next neighbor - // Loop is useful for cases where we want to skip over boxes entirely, - // otherwise recursion increments the neighbors. - for (; _neighbor < 9; _neighbor++) { - // If we have no fringe for the neighbor, make sure we have the default fringe - if(_fringe.size() == 0) _fringe.push_back(""); - - if(! isNeighbor) { - _centerPrefix = _prefix; - _centerBox = makeBox(_centerPrefix); - isNeighbor = true; - } - - int i = (_neighbor / 3) - 1; - int j = (_neighbor % 3) - 1; - - if ((i == 0 && j == 0) || - (i < 0 && _centerPrefix.atMinX()) || - (i > 0 && _centerPrefix.atMaxX()) || - (j < 0 && _centerPrefix.atMinY()) || - (j > 0 && _centerPrefix.atMaxY())) { - - //log() << "not moving to neighbor " << _neighbor << " @ " << i << ", " << j << " fringe : " << _fringe.size() << " " << _centerPrefix << endl; - //log() << _centerPrefix.atMinX() << " " - // << _centerPrefix.atMinY() << " " - // << _centerPrefix.atMaxX() << " " - // << _centerPrefix.atMaxY() << " " << endl; - - continue; // main box or wrapped edge - // TODO: We may want to enable wrapping in future, probably best as layer on top of - // this search. - } - - // Make sure we've got a reasonable center - verify(_centerPrefix.constrains()); - - GeoHash _neighborPrefix = _centerPrefix; - _neighborPrefix.move(i, j); - - //log() << "moving to neighbor " << _neighbor << " @ " << i << ", " << j << " fringe : " << _fringe.size() << " " << _centerPrefix << " " << _neighborPrefix << endl; - - GEODEBUG("moving to neighbor " << _neighbor << " @ " << i << ", " << j - << " fringe : " << _fringe.size()); - PREFIXDEBUG(_centerPrefix, _g); - PREFIXDEBUG(_neighborPrefix, _g); - - while(_fringe.size() > 0) { - _prefix = _neighborPrefix + _fringe.back(); - Box cur(makeBox(_prefix)); - - PREFIXDEBUG(_prefix, _g); - - double intAmt = intersectsBox(cur); - - // No intersection - if(intAmt <= 0) { - GEODEBUG("skipping box" << cur.toString()); - _fringe.pop_back(); - continue; - } - // Small intersection, refine search - else if(intAmt < 0.5 && _prefix.canRefine() && _fringe.back().size() < 4 /* two bits */) { - - GEODEBUG("Intersection small : " << intAmt << ", adding to fringe: " << _fringe.back() << " curr prefix : " << _prefix << " bits : " << _prefix.getBits()); - - // log() << "Diving to level : " << (_fringe.back().size() / 2 + 1) << endl; - - string lastSuffix = _fringe.back(); - _fringe.pop_back(); - _fringe.push_back(lastSuffix + "00"); - _fringe.push_back(lastSuffix + "01"); - _fringe.push_back(lastSuffix + "11"); - _fringe.push_back(lastSuffix + "10"); - - continue; - } - - // Restart our search from a diff box. - _state = START; - - verify(! onlyExpand); - - verify(_found <= 0x7fffffff); - fillStack(maxFound - _foundInExp, maxAdded - static_cast(_found)); - - // When we return from the recursive fillStack call, we'll either have checked enough points or - // be entirely done. Max recurse depth is < 8 * 16. - - // If we're maxed out on points, return - if(_foundInExp >= maxFound || _found >= maxAdded) { - // Make sure we'll come back to add more points - verify(_state == DOING_EXPAND); - return; - } - - // Otherwise we must be finished to return - verify(_state == DONE); - return; - - } - - } - - // Finished with neighbors - _state = DONE; - } - - } - - // The initial geo hash box for our first expansion - virtual GeoHash expandStartHash() = 0; - - // Whether the current box width is big enough for our search area - virtual bool fitsInBox(double width) = 0; - - // The amount the current box overlaps our search area - virtual double intersectsBox(Box& cur) = 0; - - bool remembered(BSONObj o){ - BSONObj seenId = o["_id"].wrap("").getOwned(); - if(_seenIds.find(seenId) != _seenIds.end()){ - LOG(CDEBUG + 1) << "Object " << o["_id"] << " already seen." << endl; - return true; - } - else{ - _seenIds.insert(seenId); - LOG(CDEBUG + 1) << "Object " << o["_id"] << " remembered." << endl; - return false; - } - } - - virtual int addSpecific(const GeoKeyNode& node, const Point& keyP, bool onBounds, double keyD, bool potentiallyNewDoc) { - - int found = 0; - - // We need to handle every possible point in this method, even those not in the key value, to - // avoid us tracking which hashes we've already seen. - if(! potentiallyNewDoc){ - // log() << "Already handled doc!" << endl; - return 0; - } - - // Final check for new doc - // OK to touch, since we're probably returning this object now - if(remembered(node.recordLoc.obj())) return 0; - - if(_uniqueDocs && ! onBounds) { - //log() << "Added ind to " << _type << endl; - _stack.push_front(GeoPoint(node)); - found++; - } - else { - // We now handle every possible point in the document, even those not in the key value, - // since we're iterating through them anyway - prevents us from having to save the hashes - // we've seen per-doc - - // If we're filtering by hash, get the original - bool expensiveExact = expensiveExactCheck(); - - vector< BSONObj > locs; - getPointsFor(node._key, node.recordLoc.obj(), locs, true); - for(vector< BSONObj >::iterator i = locs.begin(); i != locs.end(); ++i){ - - double d = -1; - Point p(*i); - - // We can avoid exact document checks by redoing approx checks, - // if the exact checks are more expensive. - bool needExact = true; - if(expensiveExact){ - verify(false); - KeyResult result = approxKeyCheck(p, d); - if(result == BAD) continue; - else if(result == GOOD) needExact = false; - } - - if(! needExact || exactDocCheck(p, d)){ - //log() << "Added mult to " << _type << endl; - _stack.push_front(GeoPoint(node)); - found++; - // If returning unique, just exit after first point is added - if(_uniqueDocs) break; - } - } - } - - while(_cur.isCleanAndEmpty() && _stack.size() > 0){ - _cur = _stack.front(); - _stack.pop_front(); - } - - return found; - } - - virtual long long nscanned() { - if (_firstCall) { - ok(); - } - return _nscanned; - } - - virtual void explainDetails(BSONObjBuilder& b){ - b << "lookedAt" << _lookedAt; - b << "matchesPerfd" << _matchesPerfd; - b << "objectsLoaded" << _objectsLoaded; - b << "pointsLoaded" << _pointsLoaded; - b << "pointsSavedForYield" << _nDirtied; - b << "pointsChangedOnYield" << _nChangedOnYield; - b << "pointsRemovedOnYield" << _nRemovedOnYield; - } - - virtual BSONObj prettyIndexBounds() const { - - vector::const_iterator i = _expPrefixes.end(); - if(_expPrefixes.size() > 0 && *(--i) != *(_expPrefix.get())) - _expPrefixes.push_back(*(_expPrefix.get())); - - BSONObjBuilder bob; - BSONArrayBuilder bab; - for(i = _expPrefixes.begin(); i != _expPrefixes.end(); ++i){ - bab << makeBox(*i).toBSON(); - } - bob << _g->_geo << bab.arr(); - - return bob.obj(); - - } - - void notePrefix() { - _expPrefixes.push_back(_prefix); - } - - string _type; - BSONObj _filter; - list _stack; - set _seenIds; - - GeoPoint _cur; - bool _firstCall; - bool _noted; - - long long _nscanned; - long long _nDirtied; - long long _nChangedOnYield; - long long _nRemovedOnYield; - - // The current box we're expanding (-1 is first/center box) - int _neighbor; - - // The points we've found so far - // TODO: Long long? - int _foundInExp; - - // The current hash prefix we're expanding and the center-box hash prefix - GeoHash _prefix; - shared_ptr _lastPrefix; - GeoHash _centerPrefix; - list _fringe; - int recurseDepth; - Box _centerBox; - - // Start and end of our search range in the current box - BtreeLocation _min; - BtreeLocation _max; - - shared_ptr _expPrefix; - mutable vector _expPrefixes; - - }; - - - class GeoHopper : public GeoBrowse { - public: - typedef multiset Holder; - - GeoHopper(const Geo2dType * g, - unsigned max, - const Point& n, - const BSONObj& filter = BSONObj(), - double maxDistance = numeric_limits::max(), - GeoDistType type = GEO_PLAIN, - bool uniqueDocs = false, - bool needDistance = true) - : GeoBrowse(g, "search", filter, uniqueDocs, needDistance), - _max(max), - _near(n), - _maxDistance(maxDistance), - _type(type), - _distError(type == GEO_PLAIN ? g->getConverter().getError() - : g->getConverter().getErrorSphere()), - _farthest(0) - {} - - virtual KeyResult approxKeyCheck(const Point& p, double& d) { - // Always check approximate distance, since it lets us avoid doing - // checks of the rest of the object if it succeeds - switch (_type) { - case GEO_PLAIN: - d = distance(_near, p); - break; - case GEO_SPHERE: - checkEarthBounds(p); - d = spheredist_deg(_near, p); - break; - default: verify(false); - } - verify(d >= 0); - - GEODEBUG("\t\t\t\t\t\t\t checkDistance " << _near.toString() - << "\t" << p.toString() << "\t" << d - << " farthest: " << farthest()); - - // If we need more points - double borderDist = (_points.size() < _max ? _maxDistance : farthest()); - - if (d >= borderDist - 2 * _distError && d <= borderDist + 2 * _distError) return BORDER; - else return d < borderDist ? GOOD : BAD; - } - - virtual bool exactDocCheck(const Point& p, double& d){ - bool within = false; - - // Get the appropriate distance for the type - switch (_type) { - case GEO_PLAIN: - d = distance(_near, p); - within = distanceWithin(_near, p, _maxDistance); - break; - case GEO_SPHERE: - checkEarthBounds(p); - d = spheredist_deg(_near, p); - within = (d <= _maxDistance); - break; - default: verify(false); - } - - return within; - } - - // Always in distance units, whether radians or normal - double farthest() const { - return _farthest; - } - - virtual int addSpecific(const GeoKeyNode& node, const Point& keyP, bool onBounds, double keyD, bool potentiallyNewDoc) { - - // Unique documents - - GeoPoint newPoint(node, keyD, false); - - int prevSize = _points.size(); - - // STEP 1 : Remove old duplicate points from the set if needed - if(_uniqueDocs){ - - // Lookup old point with same doc - map< DiskLoc, Holder::iterator >::iterator oldPointIt = _seenPts.find(newPoint.loc()); - - if(oldPointIt != _seenPts.end()){ - const GeoPoint& oldPoint = *(oldPointIt->second); - // We don't need to care if we've already seen this same approx pt or better, - // or we've already gone to disk once for the point - if(oldPoint < newPoint){ - GEODEBUG("\t\tOld point closer than new point"); - return 0; - } - GEODEBUG("\t\tErasing old point " << oldPointIt->first.obj()); - _points.erase(oldPointIt->second); - } - } - - Holder::iterator newIt = _points.insert(newPoint); - if(_uniqueDocs) _seenPts[ newPoint.loc() ] = newIt; - - GEODEBUG("\t\tInserted new point " << newPoint.toString() << " approx : " << keyD); - - verify(_max > 0); - - Holder::iterator lastPtIt = _points.end(); - lastPtIt--; - _farthest = lastPtIt->distance() + 2 * _distError; - - return _points.size() - prevSize; - - } - - // Removes extra points from end of _points set. - // Check can be a bit costly if we have lots of exact points near borders, - // so we'll do this every once and awhile. - void processExtraPoints(){ - - if(_points.size() == 0) return; - - int prevSize = _points.size(); - - // Erase all points from the set with a position >= _max *and* - // whose distance isn't close to the _max - 1 position distance - - int numToErase = _points.size() - _max; - if(numToErase < 0) numToErase = 0; - - // Get the first point definitely in the _points array - Holder::iterator startErase = _points.end(); - for(int i = 0; i < numToErase + 1; i++) startErase--; - _farthest = startErase->distance() + 2 * _distError; - - GEODEBUG("\t\tPotentially erasing " << numToErase << " points, " << " size : " << _points.size() << " max : " << _max << " dist : " << startErase->distance() << " farthest dist : " << _farthest << " from error : " << _distError); - - startErase++; - while(numToErase > 0 && startErase->distance() <= _farthest){ - GEODEBUG("\t\tNot erasing point " << startErase->toString()); - numToErase--; - startErase++; - verify(startErase != _points.end() || numToErase == 0); - } - - if(_uniqueDocs){ - for(Holder::iterator i = startErase; i != _points.end(); ++i) - _seenPts.erase(i->loc()); - } - - _points.erase(startErase, _points.end()); - - int diff = _points.size() - prevSize; - if(diff > 0) _found += diff; - else _found -= -diff; - - } - - unsigned _max; - Point _near; - Holder _points; - double _maxDistance; - GeoDistType _type; - double _distError; - double _farthest; - - // Safe to use currently since we don't yield in $near searches. If we do start to yield, we may need to - // replace dirtied disklocs in our holder / ensure our logic is correct. - map< DiskLoc, Holder::iterator > _seenPts; - - }; - - - - class GeoSearch : public GeoHopper { - public: - GeoSearch(const Geo2dType * g, - const Point& startPt, - int numWanted = 100, - BSONObj filter = BSONObj(), - double maxDistance = numeric_limits::max(), - GeoDistType type = GEO_PLAIN, - bool uniqueDocs = false, - bool needDistance = false) - : GeoHopper(g, numWanted, startPt, filter, maxDistance, type, uniqueDocs, needDistance), - _start(g->getConverter().hash(startPt.x, startPt.y)), - // TODO: Remove numWanted... - _numWanted(numWanted), - _type(type) - { - - verify(g->getDetails()); - _nscanned = 0; - _found = 0; - - if(_maxDistance < 0){ - _scanDistance = numeric_limits::max(); - } - else if (type == GEO_PLAIN) { - _scanDistance = maxDistance + _spec->getConverter().getError(); - } - else if (type == GEO_SPHERE) { - checkEarthBounds(startPt); - // TODO: consider splitting into x and y scan distances - _scanDistance = computeXScanDistance(startPt.y, - rad2deg(_maxDistance) + _spec->getConverter().getError()); - } - - verify(_scanDistance > 0); - - } - - - /** Check if we've already looked at a key. ALSO marks as seen, anticipating a follow-up call - to add(). This is broken out to avoid some work extracting the key bson if it's an - already seen point. - */ - private: - set< pair > _seen; - public: - - void exec() { - - if(_numWanted == 0) return; - - /* - * Search algorithm - * 1) use geohash prefix to find X items - * 2) compute max distance from want to an item - * 3) find optimal set of boxes that complete circle - * 4) use regular btree cursors to scan those boxes - */ - -#ifdef GEODEBUGGING - - log() << "start near search for " << _numWanted << " points near " << _near << " (max dist " << _maxDistance << ")" << endl; - -#endif - - // Part 1 - { - do { - long long f = found(); - verify(f <= 0x7fffffff); - fillStack(maxPointsHeuristic, _numWanted - static_cast(f), true); - processExtraPoints(); - } while(_state != DONE && _state != DONE_NEIGHBOR && - found() < _numWanted && - (!_prefix.constrains() || - _g->getConverter().sizeEdge(_prefix) <= _scanDistance)); - - // If we couldn't scan or scanned everything, we're done - if(_state == DONE){ - expandEndPoints(); - return; - } - } - -#ifdef GEODEBUGGING - - log() << "part 1 of near search completed, found " << found() << " points (out of " << _foundInExp << " scanned)" - << " in expanded region " << _prefix << " @ " << Box(_g, _prefix) - << " with furthest distance " << farthest() << endl; - -#endif - - // Part 2 - { - // Find farthest distance for completion scan - double farDist = farthest(); - if(found() < _numWanted) { - // Not enough found in Phase 1 - farDist = _scanDistance; - } - else if (_type == GEO_PLAIN) { - // Enough found, but need to search neighbor boxes - farDist += _spec->getConverter().getError(); - } - else if (_type == GEO_SPHERE) { - // Enough found, but need to search neighbor boxes - farDist = std::min(_scanDistance, - computeXScanDistance(_near.y, - rad2deg(farDist)) + 2 * _spec->getConverter().getError()); - } - verify(farDist >= 0); - GEODEBUGPRINT(farDist); - - // Find the box that includes all the points we need to return - _want = Box(_near.x - farDist, _near.y - farDist, farDist * 2); - GEODEBUGPRINT(_want.toString()); - - // log() << "Found : " << found() << " wanted : " << _numWanted << " Far distance : " << farDist << " box : " << _want << endl; - - // Remember the far distance for further scans - _scanDistance = farDist; - - // Reset the search, our distances have probably changed - if(_state == DONE_NEIGHBOR){ - _state = DOING_EXPAND; - _neighbor = -1; - } - -#ifdef GEODEBUGGING - - log() << "resetting search with start at " << _start << " (edge length " << _g->sizeEdge(_start) << ")" << endl; - -#endif - - // Do regular search in the full region - do { - fillStack(maxPointsHeuristic); - processExtraPoints(); - } - while(_state != DONE); - - } - - GEODEBUG("done near search with " << _points.size() << " points "); - - expandEndPoints(); - - } - - void addExactPoints(const GeoPoint& pt, Holder& points, bool force){ - int before, after; - addExactPoints(pt, points, before, after, force); - } - - void addExactPoints(const GeoPoint& pt, Holder& points, int& before, int& after, bool force){ - - before = 0; - after = 0; - - GEODEBUG("Adding exact points for " << pt.toString()); - - if(pt.isExact()){ - if(force) points.insert(pt); - return; - } - - vector locs; - getPointsFor(pt.key(), pt.obj(), locs, _uniqueDocs); - - GeoPoint nearestPt(pt, -1, true); - - for(vector::iterator i = locs.begin(); i != locs.end(); i++){ - - Point loc(*i); - - double d; - if(! exactDocCheck(loc, d)) continue; - - if(_uniqueDocs && (nearestPt.distance() < 0 || d < nearestPt.distance())){ - nearestPt._distance = d; - nearestPt._pt = *i; - continue; - } - else if(! _uniqueDocs){ - GeoPoint exactPt(pt, d, true); - exactPt._pt = *i; - GEODEBUG("Inserting exact pt " << exactPt.toString() << " for " << pt.toString() << " exact : " << d << " is less? " << (exactPt < pt) << " bits : " << _g->_bits); - points.insert(exactPt); - exactPt < pt ? before++ : after++; - } - - } - - if(_uniqueDocs && nearestPt.distance() >= 0){ - GEODEBUG("Inserting unique exact pt " << nearestPt.toString() << " for " << pt.toString() << " exact : " << nearestPt.distance() << " is less? " << (nearestPt < pt) << " bits : " << _g->_bits); - points.insert(nearestPt); - if(nearestPt < pt) before++; - else after++; - } - - } - - // TODO: Refactor this back into holder class, allow to run periodically when we are seeing a lot of pts - void expandEndPoints(bool finish = true){ - - processExtraPoints(); - - // All points in array *could* be in maxDistance - - // Step 1 : Trim points to max size - // TODO: This check will do little for now, but is skeleton for future work in incremental $near - // searches - if(_max > 0){ - - int numToErase = _points.size() - _max; - - if(numToErase > 0){ - - Holder tested; - - // Work backward through all points we're not sure belong in the set - Holder::iterator maybePointIt = _points.end(); - maybePointIt--; - double approxMin = maybePointIt->distance() - 2 * _distError; - - GEODEBUG("\t\tNeed to erase " << numToErase << " max : " << _max << " min dist " << approxMin << " error : " << _distError << " starting from : " << (*maybePointIt).toString()); - - // Insert all - int erased = 0; - while(_points.size() > 0 && (maybePointIt->distance() >= approxMin || erased < numToErase)){ - - Holder::iterator current = maybePointIt--; - - addExactPoints(*current, tested, true); - _points.erase(current); - erased++; - - if(tested.size()) - approxMin = tested.begin()->distance() - 2 * _distError; - - } - - GEODEBUG("\t\tEnding search at point " << (_points.size() == 0 ? "(beginning)" : maybePointIt->toString())); - - int numToAddBack = erased - numToErase; - verify(numToAddBack >= 0); - - GEODEBUG("\t\tNum tested valid : " << tested.size() << " erased : " << erased << " added back : " << numToAddBack); - -#ifdef GEODEBUGGING - for(Holder::iterator it = tested.begin(); it != tested.end(); it++){ - log() << "Tested Point: " << *it << endl; - } -#endif - Holder::iterator testedIt = tested.begin(); - for(int i = 0; i < numToAddBack && testedIt != tested.end(); i++){ - _points.insert(*testedIt); - testedIt++; - } - } - } - -#ifdef GEODEBUGGING - for(Holder::iterator it = _points.begin(); it != _points.end(); it++){ - log() << "Point: " << *it << endl; - } -#endif - // We've now trimmed first set of unneeded points - - GEODEBUG("\t\t Start expanding, num points : " << _points.size() << " max : " << _max); - - // Step 2: iterate through all points and add as needed - unsigned expandedPoints = 0; - Holder::iterator it = _points.begin(); - double expandWindowEnd = -1; - - while(it != _points.end()){ - const GeoPoint& currPt = *it; - // TODO: If one point is exact, maybe not 2 * _distError - - // See if we're in an expand window - bool inWindow = currPt.distance() <= expandWindowEnd; - // If we're not, and we're done with points, break - if(! inWindow && expandedPoints >= _max) break; - - bool expandApprox = !currPt.isExact() && - (!_uniqueDocs || (finish && _needDistance) || inWindow); - - if (expandApprox) { - // Add new point(s). These will only be added in a radius of 2 * _distError - // around the current point, so should not affect previously valid points. - int before, after; - addExactPoints(currPt, _points, before, after, false); - expandedPoints += before; - - if(_max > 0 && expandedPoints < _max) - expandWindowEnd = currPt.distance() + 2 * _distError; - - // Iterate to the next point - Holder::iterator current = it++; - // Erase the current point - _points.erase(current); - - } - else{ - expandedPoints++; - it++; - } - } - - GEODEBUG("\t\tFinished expanding, num points : " << _points.size() - << " max : " << _max); - - // Finish - // TODO: Don't really need to trim? - for(; expandedPoints > _max; expandedPoints--) it--; - _points.erase(it, _points.end()); - -#ifdef GEODEBUGGING - for(Holder::iterator it = _points.begin(); it != _points.end(); it++){ - log() << "Point: " << *it << endl; - } -#endif - } - - virtual GeoHash expandStartHash(){ - return _start; - } - - // Whether the current box width is big enough for our search area - virtual bool fitsInBox(double width){ - return width >= _scanDistance; - } - - // Whether the current box overlaps our search area - virtual double intersectsBox(Box& cur){ - return cur.intersects(_want); - } - - GeoHash _start; - int _numWanted; - double _scanDistance; - - long long _nscanned; - int _found; - GeoDistType _type; - - Box _want; - }; - - class GeoSearchCursor : public GeoCursorBase { - public: - GeoSearchCursor(shared_ptr s) - : GeoCursorBase(s->_spec), - _s(s), _cur(s->_points.begin()), _end(s->_points.end()), _nscanned() { - if (_cur != _end) { - ++_nscanned; - } - } - - virtual ~GeoSearchCursor() {} - - virtual bool ok() { - return _cur != _end; - } - - virtual Record* _current() { verify(ok()); return _cur->_loc.rec(); } - virtual BSONObj current() { verify(ok()); return _cur->_o; } - virtual DiskLoc currLoc() { verify(ok()); return _cur->_loc; } - virtual bool advance() { - if(ok()){ - _cur++; - incNscanned(); - return ok(); - } - return false; - } - virtual BSONObj currKey() const { return _cur->_key; } - - virtual string toString() { - return "GeoSearchCursor"; - } - - - virtual BSONObj prettyStartKey() const { - return BSON(_s->_g->_geo << _s->_prefix.toString()); - } - virtual BSONObj prettyEndKey() const { - GeoHash temp = _s->_prefix; - temp.move(1, 1); - return BSON(_s->_g->_geo << temp.toString()); - } - - virtual long long nscanned() { return _nscanned; } - - virtual CoveredIndexMatcher* matcher() const { - if(_s->_matcher.get()) return _s->_matcher.get(); - else return emptyMatcher.get(); - } - - shared_ptr _s; - GeoHopper::Holder::iterator _cur; - GeoHopper::Holder::iterator _end; - - void incNscanned() { if (ok()) { ++_nscanned; } } - long long _nscanned; - }; - - class GeoCircleBrowse : public GeoBrowse { - public: - GeoCircleBrowse(const Geo2dType * g, const BSONObj& circle, BSONObj filter = BSONObj(), - const string& type = "$center", bool uniqueDocs = true) - : GeoBrowse(g, "circle", filter, uniqueDocs) { - - uassert(13060, "$center needs 2 fields (middle,max distance)", circle.nFields() == 2); - - BSONObjIterator i(circle); - BSONElement center = i.next(); - - uassert(13656, "the first field of $center object must be a location object", - center.isABSONObj()); - - // Get geohash and exact center point - // TODO: For wrapping search, may be useful to allow center points outside-of-bounds - // here. Calculating the nearest point as a hash start inside the region would then be - // required. - _start = g->getConverter().hash(center); - _startPt = Point(center); - - _maxDistance = i.next().numberDouble(); - uassert(13061, "need a max distance >= 0 ", _maxDistance >= 0); - - if (type == "$center") { - // Look in box with bounds of maxDistance in either direction - _type = GEO_PLAIN; - xScanDistance = _maxDistance + _g->getConverter().getError(); - yScanDistance = _maxDistance + _g->getConverter().getError(); - } - else if (type == "$centerSphere") { - // Same, but compute maxDistance using spherical transform - uassert(13461, "Spherical MaxDistance > PI. Are you sure you are using radians?", - _maxDistance < M_PI); - checkEarthBounds(_startPt); - - _type = GEO_SPHERE; - // should this be sphere error? - yScanDistance = rad2deg(_maxDistance) + _g->getConverter().getError(); - xScanDistance = computeXScanDistance(_startPt.y, yScanDistance); - - uassert(13462, "Spherical distance would require (unimplemented) wrapping", - (_startPt.x + xScanDistance < 180) && - (_startPt.x - xScanDistance > -180) && - (_startPt.y + yScanDistance < 90) && - (_startPt.y - yScanDistance > -90)); - } - else { - uassert(13460, "invalid $center query type: " + type, false); - } - - // Bounding box includes fudge factor. - // TODO: Is this correct, since fudge factor may be spherically transformed? - _bBox._min = Point(_startPt.x - xScanDistance, _startPt.y - yScanDistance); - _bBox._max = Point(_startPt.x + xScanDistance, _startPt.y + yScanDistance); - - GEODEBUG("Bounding box for circle query : " << _bBox.toString() - << " (max distance : " << _maxDistance << ")" - << " starting from " << _startPt.toString()); - ok(); - } - - virtual GeoHash expandStartHash() { - return _start; - } - - virtual bool fitsInBox(double width) { - return width >= std::max(xScanDistance, yScanDistance); - } - - virtual double intersectsBox(Box& cur) { - return cur.intersects(_bBox); - } - - virtual KeyResult approxKeyCheck(const Point& p, double& d) { - // Inexact hash distance checks. - double error = 0; - switch (_type) { - case GEO_PLAIN: - d = distance(_startPt, p); - error = _g->getConverter().getError(); - break; - case GEO_SPHERE: { - checkEarthBounds(p); - d = spheredist_deg(_startPt, p); - error = _g->getConverter().getErrorSphere(); - break; - } - default: verify(false); - } - - // If our distance is in the error bounds... - if(d >= _maxDistance - error && d <= _maxDistance + error) return BORDER; - return d > _maxDistance ? BAD : GOOD; - } - - virtual bool exactDocCheck(const Point& p, double& d){ - switch (_type) { - case GEO_PLAIN: { - if(distanceWithin(_startPt, p, _maxDistance)) return true; - break; - } - case GEO_SPHERE: - checkEarthBounds(p); - if(spheredist_deg(_startPt, p) <= _maxDistance) return true; - break; - default: verify(false); - } - - return false; - } - - GeoDistType _type; - GeoHash _start; - Point _startPt; - double _maxDistance; // user input - double xScanDistance; // effected by GeoDistType - double yScanDistance; // effected by GeoDistType - Box _bBox; - - }; - - class GeoBoxBrowse : public GeoBrowse { - public: - GeoBoxBrowse(const Geo2dType * g, const BSONObj& box, BSONObj filter = BSONObj(), - bool uniqueDocs = true) - : GeoBrowse(g, "box", filter, uniqueDocs) { - - uassert(13063, "$box needs 2 fields (bottomLeft,topRight)", box.nFields() == 2); - - // Initialize an *exact* box from the given obj. - BSONObjIterator i(box); - _want._min = Point(i.next()); - _want._max = Point(i.next()); - - _wantRegion = _want; - // Need to make sure we're checking regions within error bounds of where we want - _wantRegion.fudge(g->getConverter().getError()); - fixBox(g, _wantRegion); - fixBox(g, _want); - - // XXX: why do we use _g and g, i think they're the same... - - uassert(13064, "need an area > 0 ", _want.area() > 0); - - Point center = _want.center(); - _start = _g->getConverter().hash(center.x, center.y); - - GEODEBUG("center : " << center.toString() << "\t" << _prefix); - - _fudge = _g->getConverter().getError(); - _wantLen = _fudge + - std::max((_want._max.x - _want._min.x), - (_want._max.y - _want._min.y)) / 2; - - ok(); - } - - void fixBox(const Geo2dType* g, Box& box) { - if(box._min.x > box._max.x) - swap(box._min.x, box._max.x); - if(box._min.y > box._max.y) - swap(box._min.y, box._max.y); - - double gMin = g->getConverter().getMin(); - double gMax = g->getConverter().getMax(); - - if(box._min.x < gMin) box._min.x = gMin; - if(box._min.y < gMin) box._min.y = gMin; - if(box._max.x > gMax) box._max.x = gMax; - if(box._max.y > gMax) box._max.y = gMax; - } - - void swap(double& a, double& b) { - double swap = a; - a = b; - b = swap; - } - - virtual GeoHash expandStartHash() { - return _start; - } - - virtual bool fitsInBox(double width) { - return width >= _wantLen; - } - - virtual double intersectsBox(Box& cur) { - return cur.intersects(_wantRegion); - } - - virtual KeyResult approxKeyCheck(const Point& p, double& d) { - if(_want.onBoundary(p, _fudge)) return BORDER; - else return _want.inside(p, _fudge) ? GOOD : BAD; - - } - - virtual bool exactDocCheck(const Point& p, double& d){ - return _want.inside(p); - } - - Box _want; - Box _wantRegion; - double _wantLen; - double _fudge; - GeoHash _start; - }; - - class GeoPolygonBrowse : public GeoBrowse { - public: - GeoPolygonBrowse(const Geo2dType* g, const BSONObj& polyPoints, - BSONObj filter = BSONObj(), bool uniqueDocs = true) - : GeoBrowse(g, "polygon", filter, uniqueDocs) { - - GEODEBUG("In Polygon") - - BSONObjIterator i(polyPoints); - BSONElement first = i.next(); - _poly.add(Point(first)); - - while (i.more()) { - _poly.add(Point(i.next())); - } - - uassert(14030, "polygon must be defined by three points or more", _poly.size() >= 3); - - _bounds = _poly.bounds(); - // We need to check regions within the error bounds of these bounds - _bounds.fudge(g->getConverter().getError()); - // We don't need to look anywhere outside the space - _bounds.truncate(g->getConverter().getMin(), g->getConverter().getMax()); - _maxDim = g->getConverter().getError() + _bounds.maxDim() / 2; - - ok(); - } - - // The initial geo hash box for our first expansion - virtual GeoHash expandStartHash() { - return _g->getConverter().hash(_bounds.center()); - } - - // Whether the current box width is big enough for our search area - virtual bool fitsInBox(double width) { - return _maxDim <= width; - } - - // Whether the current box overlaps our search area - virtual double intersectsBox(Box& cur) { - return cur.intersects(_bounds); - } - - virtual KeyResult approxKeyCheck(const Point& p, double& d) { - int in = _poly.contains(p, _g->getConverter().getError()); - if(in == 0) return BORDER; - else return in > 0 ? GOOD : BAD; - } - - virtual bool exactDocCheck(const Point& p, double& d){ - return _poly.contains(p); - } - - private: - Polygon _poly; - Box _bounds; - double _maxDim; - GeoHash _start; - }; - - shared_ptr Geo2dType::newCursor(const BSONObj& query, const BSONObj& order, - int numWanted) const { - if (numWanted < 0) - numWanted = numWanted * -1; - else if (numWanted == 0) - numWanted = 100; - - // false means we want to filter OUT geoFieldsToNuke, not filter to include only that. - BSONObj filteredQuery = query.filterFieldsUndotted(BSON(_geo << ""), false); - - BSONObjIterator i(query); - while (i.more()) { - BSONElement e = i.next(); - - if (_geo != e.fieldName()) - continue; - - if (e.type() == Array) { - // If we get an array query, assume it is a location, and do a $within { $center : - // [[x, y], 0] } search - BSONObj circle = BSON("0" << e.embeddedObjectUserCheck() << "1" << 0); - shared_ptr c(new GeoCircleBrowse(this, circle, filteredQuery, "$center", true)); - return c; - } - else if (e.type() == Object) { - // TODO: Filter out _geo : { $special... } field so it doesn't get matched - // accidentally, if matcher changes - - switch (e.embeddedObject().firstElement().getGtLtOp()) { - case BSONObj::opNEAR: { - BSONObj n = e.embeddedObject(); - e = n.firstElement(); - - const char* suffix = e.fieldName() + 5; // strlen("$near") == 5; - GeoDistType type; - if (suffix[0] == '\0') { - type = GEO_PLAIN; - } - else if (strcmp(suffix, "Sphere") == 0) { - type = GEO_SPHERE; - } - else { - uassert(13464, string("invalid $near search type: ") + e.fieldName(), false); - type = GEO_PLAIN; // prevents uninitialized warning - } - - double maxDistance = numeric_limits::max(); - if (e.isABSONObj() && e.embeddedObject().nFields() > 2) { - BSONObjIterator i(e.embeddedObject()); - i.next(); - i.next(); - BSONElement e = i.next(); - if (e.isNumber()) - maxDistance = e.numberDouble(); - } - { - BSONElement e = n["$maxDistance"]; - if (e.isNumber()) - maxDistance = e.numberDouble(); - } - - bool uniqueDocs = false; - if(! n["$uniqueDocs"].eoo()) uniqueDocs = n["$uniqueDocs"].trueValue(); - - shared_ptr s(new GeoSearch(this, Point(e), numWanted, filteredQuery, - maxDistance, type, uniqueDocs)); - s->exec(); - shared_ptr c; - c.reset(new GeoSearchCursor(s)); - return c; - } - case BSONObj::opWITHIN: { - - e = e.embeddedObject().firstElement(); - uassert(13057, "$within has to take an object or array", e.isABSONObj()); - - BSONObj context = e.embeddedObject(); - e = e.embeddedObject().firstElement(); - string type = e.fieldName(); - - bool uniqueDocs = true; - if (!context["$uniqueDocs"].eoo()) - uniqueDocs = context["$uniqueDocs"].trueValue(); - - if (startsWith(type, "$center")) { - uassert(13059, "$center has to take an object or array", e.isABSONObj()); - shared_ptr c(new GeoCircleBrowse(this, e.embeddedObjectUserCheck(), - filteredQuery, type, uniqueDocs)); - return c; - } - else if (type == "$box") { - uassert(13065, "$box has to take an object or array", e.isABSONObj()); - shared_ptr c(new GeoBoxBrowse(this, e.embeddedObjectUserCheck(), - filteredQuery, uniqueDocs)); - return c; - } - else if (startsWith(type, "$poly")) { - uassert(14029, "$polygon has to take an object or array", e.isABSONObj()); - shared_ptr c(new GeoPolygonBrowse(this, e.embeddedObjectUserCheck(), - filteredQuery, uniqueDocs)); - return c; - } - throw UserException(13058, str::stream() << "unknown $within information : " - << context - << ", a shape must be specified."); - } - default: - // Otherwise... assume the object defines a point, and we want to do a - // zero-radius $within $center - - shared_ptr c(new GeoCircleBrowse(this, BSON("0" << e.embeddedObjectUserCheck() << "1" << 0), filteredQuery)); - - return c; - } - } - } - - throw UserException(13042, (string)"missing geo field (" + _geo + ") in : " + query.toString()); - } - - // ------ - // commands - // ------ - bool run2DGeoNear(const IndexDetails &id, const BSONObj& cmdObj, - const GeoNearArguments &parsedArgs, string& errmsg, BSONObjBuilder& result) { - Geo2dType * g = (Geo2dType*)id.getSpec().getType(); - verify(&id == g->getDetails()); - - uassert(13046, "'near' param missing/invalid", !cmdObj["near"].eoo()); - const Point n(cmdObj["near"]); - result.append("near", g->getConverter().hash(cmdObj["near"]).toString()); - - double maxDistance = numeric_limits::max(); - if (cmdObj["maxDistance"].isNumber()) - maxDistance = cmdObj["maxDistance"].number(); - - GeoDistType type = parsedArgs.isSpherical ? GEO_SPHERE : GEO_PLAIN; - - GeoSearch gs(g, n, parsedArgs.numWanted, parsedArgs.query, maxDistance, type, - parsedArgs.uniqueDocs, true); - - if (cmdObj["start"].type() == String) { - GeoHash start ((string) cmdObj["start"].valuestr()); - gs._start = start; - } - - gs.exec(); - - double totalDistance = 0; - - BSONObjBuilder arr(result.subarrayStart("results")); - int x = 0; - for (GeoHopper::Holder::iterator i=gs._points.begin(); i!=gs._points.end(); i++) { - - const GeoPoint& p = *i; - double dis = parsedArgs.distanceMultiplier * p.distance(); - totalDistance += dis; - - BSONObjBuilder bb(arr.subobjStart(BSONObjBuilder::numStr(x++))); - bb.append("dis", dis); - if (parsedArgs.includeLocs) { - if(p._pt.couldBeArray()) bb.append("loc", BSONArray(p._pt)); - else bb.append("loc", p._pt); - } - bb.append("obj", p._o); - bb.done(); - - if (arr.len() > BSONObjMaxUserSize) { - warning() << "Too many results to fit in single document. Truncating..." << endl; - break; - } - } - arr.done(); - - BSONObjBuilder stats(result.subobjStart("stats")); - stats.append("time", cc().curop()->elapsedMillis()); - stats.appendNumber("btreelocs", gs._nscanned); - stats.appendNumber("nscanned", gs._lookedAt); - stats.appendNumber("objectsLoaded", gs._objectsLoaded); - stats.append("avgDistance", totalDistance / x); - stats.append("maxDistance", gs.farthest()); - stats.done(); - - return true; - } - - class GeoWalkCmd : public Command { - public: - GeoWalkCmd() : Command("geoWalk") {} - virtual LockType locktype() const { return READ; } - bool slaveOk() const { return true; } - bool slaveOverrideOk() const { return true; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - string ns = dbname + "." + cmdObj.firstElement().valuestr(); - - NamespaceDetails * d = nsdetails(ns); - if (! d) { - errmsg = "can't find ns"; - return false; - } - - int geoIdx = -1; - { - NamespaceDetails::IndexIterator ii = d->ii(); - while (ii.more()) { - IndexDetails& id = ii.next(); - if (id.getSpec().getTypeName() == GEO2DNAME) { - if (geoIdx >= 0) { - errmsg = "2 geo indexes :("; - return false; - } - geoIdx = ii.pos() - 1; - } - } - } - - if (geoIdx < 0) { - errmsg = "no geo index :("; - return false; - } - - - IndexDetails& id = d->idx(geoIdx); - Geo2dType * g = (Geo2dType*)id.getSpec().getType(); - verify(&id == g->getDetails()); - - int max = 100000; - - auto_ptr bc(BtreeCursor::make(d, id, BSONObj(), BSONObj(), true, 1)); - BtreeCursor &c = *bc; - while (c.ok() && max--) { - GeoHash h(c.currKey().firstElement()); - int len; - cout << "\t" << h.toString() - << "\t" << c.current()[g->_geo] - << "\t" << hex << h.getHash() - << "\t" << hex << ((long long*)c.currKey().firstElement().binData(len))[0] - << "\t" << c.current()["_id"] - << endl; - c.advance(); - } - - return true; - } - } geoWalkCmd; - - struct GeoUnitTest : public StartupTest { - int round(double d) { - return (int)(.5 + (d * 1000)); - } - -#define GEOHEQ(a,b) if (a.toString() != b){ cout << "[" << a.toString() << "] != [" << b << "]" << endl; verify(a == GeoHash(b)); } - - void run() { - verify(!GeoHash::isBitSet(0, 0)); - verify(!GeoHash::isBitSet(0, 31)); - verify(GeoHash::isBitSet(1, 31)); - - IndexSpec i(BSON("loc" << "2d")); - Geo2dType g(&geo2dplugin, &i); - const GeoHashConverter &conv = g.getConverter(); - - { - double x = 73.01212; - double y = 41.352964; - BSONObj in = BSON("x" << x << "y" << y); - GeoHash h = conv.hash(in); - BSONObj out = conv.unhashToBSONObj(h); - verify(round(x) == round(out["x"].number())); - verify(round(y) == round(out["y"].number())); - verify(round(in["x"].number()) == round(out["x"].number())); - verify(round(in["y"].number()) == round(out["y"].number())); - } - { - double x = -73.01212; - double y = 41.352964; - BSONObj in = BSON("x" << x << "y" << y); - GeoHash h = conv.hash(in); - BSONObj out = conv.unhashToBSONObj(h); - verify(round(x) == round(out["x"].number())); - verify(round(y) == round(out["y"].number())); - verify(round(in["x"].number()) == round(out["x"].number())); - verify(round(in["y"].number()) == round(out["y"].number())); - } - { - GeoHash h("0000"); - h.move(0, 1); - GEOHEQ(h, "0001"); - h.move(0, -1); - GEOHEQ(h, "0000"); - - h = GeoHash("0001"); - h.move(0, 1); - GEOHEQ(h, "0100"); - h.move(0, -1); - GEOHEQ(h, "0001"); - - h = GeoHash("0000"); - h.move(1, 0); - GEOHEQ(h, "0010"); - } - { - Box b(5, 5, 2); - verify("(5,5) -->> (7,7)" == b.toString()); - } - { - GeoHash a = conv.hash(1, 1); - GeoHash b = conv.hash(4, 5); - verify(5 == (int)(conv.distanceBetweenHashes(a, b))); - a = conv.hash(50, 50); - b = conv.hash(42, 44); - verify(round(10) == round(conv.distanceBetweenHashes(a, b))); - } - { - GeoHash x("0000"); - verify(0 == x.getHash()); - x = GeoHash(0, 1, 32); - GEOHEQ(x, "0000000000000000000000000000000000000000000000000000000000000001") - - verify(GeoHash("1100").hasPrefix(GeoHash("11"))); - verify(!GeoHash("1000").hasPrefix(GeoHash("11"))); - } - { - GeoHash x("1010"); - GEOHEQ(x, "1010"); - GeoHash y = x + "01"; - GEOHEQ(y, "101001"); - } - { - GeoHash a = conv.hash(5, 5); - GeoHash b = conv.hash(5, 7); - GeoHash c = conv.hash(100, 100); - BSONObj oa = a.wrap(); - BSONObj ob = b.wrap(); - BSONObj oc = c.wrap(); - verify(oa.woCompare(ob) < 0); - verify(oa.woCompare(oc) < 0); - } - { - GeoHash x("000000"); - x.move(-1, 0); - GEOHEQ(x, "101010"); - x.move(1, -1); - GEOHEQ(x, "010101"); - x.move(0, 1); - GEOHEQ(x, "000000"); - } - { - GeoHash prefix("110011000000"); - GeoHash entry( "1100110000011100000111000001110000011100000111000001000000000000"); - verify(!entry.hasPrefix(prefix)); - entry = GeoHash("1100110000001100000111000001110000011100000111000001000000000000"); - verify(entry.toString().find(prefix.toString()) == 0); - verify(entry.hasPrefix(GeoHash("1100"))); - verify(entry.hasPrefix(prefix)); - } - { - GeoHash a = conv.hash(50, 50); - GeoHash b = conv.hash(48, 54); - verify(round(4.47214) == round(conv.distanceBetweenHashes(a, b))); - } - { - Box b(Point(29.762283, -95.364271), Point(29.764283000000002, -95.36227099999999)); - verify(b.inside(29.763, -95.363)); - verify(! b.inside(32.9570255, -96.1082497)); - verify(! b.inside(32.9570255, -96.1082497, .01)); - } - { - GeoHash a("11001111"); - verify(GeoHash("11") == a.commonPrefix(GeoHash("11"))); - verify(GeoHash("11") == a.commonPrefix(GeoHash("11110000"))); - } - { - int N = 10000; -#if 0 // XXX: we want to make sure the two unhash versions both work, but private. - { - Timer t; - for (int i = 0; i < N; i++) { - unsigned x = (unsigned)rand(); - unsigned y = (unsigned)rand(); - GeoHash h(x, y); - unsigned a, b; - h.unhash(&a, &b); - verify(a == x); - verify(b == y); - } - //cout << "slow: " << t.millis() << endl; - } -#endif - { - Timer t; - for (int i=0; i 2469 && dist < 2470); - } - { - Point BNA (-86.67, 36.12); - Point LAX (-118.40, 33.94); - Point JFK (-73.77694444, 40.63861111); - verify(spheredist_deg(BNA, BNA) < 1e-6); - verify(spheredist_deg(LAX, LAX) < 1e-6); - verify(spheredist_deg(JFK, JFK) < 1e-6); - - Point zero (0, 0); - Point antizero (0,-180); - - // these were known to cause NaN - verify(spheredist_deg(zero, zero) < 1e-6); - verify(fabs(M_PI-spheredist_deg(zero, antizero)) < 1e-6); - verify(fabs(M_PI-spheredist_deg(antizero, zero)) < 1e-6); - } - } - } - } geoUnitTest; -} diff --git a/src/mongo/db/geo/2d.h b/src/mongo/db/geo/2d.h deleted file mode 100644 index d54e1f766c6..00000000000 --- a/src/mongo/db/geo/2d.h +++ /dev/null @@ -1,23 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -namespace mongo { - // We need cmdObj and parsedArgs so we can print a useful error msg - // and pull other args out. - bool run2DGeoNear(const IndexDetails &id, const BSONObj& cmdObj, - const GeoNearArguments &parsedArgs, string& errmsg, - BSONObjBuilder& result); -} // namespace mongo diff --git a/src/mongo/db/geo/core.h b/src/mongo/db/geo/core.h index 0fc46a53609..ca4b7138be7 100644 --- a/src/mongo/db/geo/core.h +++ b/src/mongo/db/geo/core.h @@ -1,73 +1,49 @@ -// core.h - /** -* Copyright (C) 2008-2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #pragma once -#include "mongo/pch.h" -#include "mongo/db/jsobj.h" -#include "mongo/util/mongoutils/str.h" - #include #ifndef M_PI # define M_PI 3.14159265358979323846 #endif -#if 0 -# define CDEBUG -1 -#else -# define CDEBUG 10 -#endif - -#if 0 -# define GEODEBUGGING -# define GEODEBUG(x) cout << x << endl; -# define GEODEBUGPRINT(x) PRINT(x) - inline void PREFIXDEBUG(GeoHash prefix, const GeoConvert* g) { - if (!prefix.constrains()) { - cout << "\t empty prefix" << endl; - return ; - } - - Point ll (g, prefix); // lower left - prefix.move(1,1); - Point tr (g, prefix); // top right +namespace mongo { - Point center ((ll._x+tr._x)/2, (ll._y+tr._y)/2); - double radius = fabs(ll._x - tr._x) / 2; + inline double deg2rad(const double deg) { return deg * (M_PI / 180.0); } - cout << "\t ll: " << ll.toString() << " tr: " << tr.toString() - << " center: " << center.toString() << " radius: " << radius << endl; + inline double rad2deg(const double rad) { return rad * (180.0 / M_PI); } + inline double computeXScanDistance(double y, double maxDistDegrees) { + // TODO: this overestimates for large maxDistDegrees far from the equator + return maxDistDegrees / min(cos(deg2rad(min(+89.0, y + maxDistDegrees))), + cos(deg2rad(max(-89.0, y - maxDistDegrees)))); } -#else -# define GEODEBUG(x) -# define GEODEBUGPRINT(x) -# define PREFIXDEBUG(x, y) -#endif - -// Used by haystack.cpp. XXX: change to something else/only have one of these geo things/nuke em -// all? -#define GEOQUADDEBUG(x) -//#define GEOQUADDEBUG(x) cout << x << endl -// XXX: move elsewhere? -namespace mongo { - inline double deg2rad(const double deg) { return deg * (M_PI / 180.0); } - inline double rad2deg(const double rad) { return rad * (180.0 / M_PI); } } diff --git a/src/mongo/db/geo/geoconstants.h b/src/mongo/db/geo/geoconstants.h new file mode 100644 index 00000000000..d7df948150a --- /dev/null +++ b/src/mongo/db/geo/geoconstants.h @@ -0,0 +1,36 @@ +/** +* Copyright (C) 2013 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +namespace mongo { + + // Thanks, Wikipedia. + const double kRadiusOfEarthInMeters = (6378.1 * 1000); + +} // namespace mongo diff --git a/src/mongo/db/geo/geonear.cpp b/src/mongo/db/geo/geonear.cpp deleted file mode 100644 index 118c75d9499..00000000000 --- a/src/mongo/db/geo/geonear.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include - -#include "mongo/db/geo/geonear.h" - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/commands.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/namespace_details.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/geo/2d.h" -#include "mongo/db/geo/s2common.h" -#include "mongo/db/geo/s2index.h" -#include "mongo/db/geo/s2nearcursor.h" - -namespace mongo { - GeoNearArguments::GeoNearArguments(const BSONObj &cmdObj) { - const char* limitName = cmdObj["num"].isNumber() ? "num" : "limit"; - if (cmdObj[limitName].isNumber()) { - numWanted = cmdObj[limitName].numberInt(); - } else { - numWanted = 100; - } - - if (!cmdObj["uniqueDocs"].eoo()) { - uniqueDocs = cmdObj["uniqueDocs"].trueValue(); - } else { - uniqueDocs = false; - } - - if (!cmdObj["includeLocs"].eoo()) { - includeLocs = cmdObj["includeLocs"].trueValue(); - } else { - includeLocs = false; - } - - if (cmdObj["query"].isABSONObj()) { - query = cmdObj["query"].embeddedObject(); - } - - if (cmdObj["distanceMultiplier"].isNumber()) { - distanceMultiplier = cmdObj["distanceMultiplier"].number(); - } else { - distanceMultiplier = 1.0; - } - - isSpherical = cmdObj["spherical"].trueValue(); - } - - class Geo2dFindNearCmd : public Command { - public: - Geo2dFindNearCmd() : Command("geoNear") {} - - virtual LockType locktype() const { return READ; } - bool slaveOk() const { return true; } - bool slaveOverrideOk() const { return true; } - - void help(stringstream& h) const { - h << "http://dochub.mongodb.org/core/geo#GeospatialIndexing-geoNearCommand"; - } - - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - - bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - string ns = dbname + "." + cmdObj.firstElement().valuestr(); - NamespaceDetails *d = nsdetails(ns); - - if (NULL == d) { - errmsg = "can't find ns"; - return false; - } - - GeoNearArguments commonArgs(cmdObj); - if (commonArgs.numWanted < 0) { - errmsg = "numWanted must be >= 0"; - return false; - } - - vector idxs; - - d->findIndexByType("2d", idxs); - if (idxs.size() > 1) { - errmsg = "more than one 2d index, not sure which to run geoNear on"; - return false; - } - - if (1 == idxs.size()) { - result.append("ns", ns); - return run2DGeoNear(d->idx(idxs[0]), cmdObj, commonArgs, errmsg, result); - } - - d->findIndexByType("2dsphere", idxs); - if (idxs.size() > 1) { - errmsg = "more than one 2dsphere index, not sure which to run geoNear on"; - return false; - } - - if (1 == idxs.size()) { - result.append("ns", ns); - return run2DSphereGeoNear(d->idx(idxs[0]), cmdObj, commonArgs, errmsg, result); - } - - errmsg = "no geo indices for geoNear"; - return false; - } - private: - } geo2dFindNearCmd; -} // namespace mongo diff --git a/src/mongo/db/geo/geonear.h b/src/mongo/db/geo/geonear.h deleted file mode 100644 index 1a36449856f..00000000000 --- a/src/mongo/db/geo/geonear.h +++ /dev/null @@ -1,33 +0,0 @@ -/** -* Copyright (C) 2013 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/jsobj.h" - -namespace mongo { - // Arguments in common between 2d and 2dsphere geoNear. - class GeoNearArguments { - public: - GeoNearArguments(const BSONObj& cmdObj); - int numWanted; - bool uniqueDocs; - bool includeLocs; - BSONObj query; - double distanceMultiplier; - bool isSpherical; - private: - GeoNearArguments() { } - }; -} // namespace mongo diff --git a/src/mongo/db/geo/geoparser.cpp b/src/mongo/db/geo/geoparser.cpp index 3de520be0be..a553e6f38ee 100644 --- a/src/mongo/db/geo/geoparser.cpp +++ b/src/mongo/db/geo/geoparser.cpp @@ -1,57 +1,116 @@ /** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/geo/geoparser.h" #include #include + +#include "mongo/db/geo/shapes.h" #include "mongo/db/jsobj.h" -#include "mongo/db/geo/geoparser.h" #include "mongo/util/mongoutils/str.h" -#include "third_party/s2/s2.h" -#include "third_party/s2/s2cap.h" -#include "third_party/s2/s2cell.h" -#include "third_party/s2/s2latlng.h" -#include "third_party/s2/s2loop.h" -#include "third_party/s2/s2polygon.h" #include "third_party/s2/s2polygonbuilder.h" -#include "third_party/s2/s2polyline.h" namespace mongo { + // This field must be present, and... static const string GEOJSON_TYPE = "type"; - // Have one of these three values: + // Have one of these values: static const string GEOJSON_TYPE_POINT = "Point"; static const string GEOJSON_TYPE_LINESTRING = "LineString"; static const string GEOJSON_TYPE_POLYGON = "Polygon"; + static const string GEOJSON_TYPE_MULTI_POINT = "MultiPoint"; + static const string GEOJSON_TYPE_MULTI_LINESTRING = "MultiLineString"; + static const string GEOJSON_TYPE_MULTI_POLYGON = "MultiPolygon"; + static const string GEOJSON_TYPE_GEOMETRY_COLLECTION = "GeometryCollection"; // This field must also be present. The value depends on the type. static const string GEOJSON_COORDINATES = "coordinates"; + static const string GEOJSON_GEOMETRIES = "geometries"; - //// Utility functions used by GeoParser functions below. - static S2Point coordToPoint(double p0, double p1) { - return S2LatLng::FromDegrees(p1, p0).Normalized().ToPoint(); + bool isValidLngLat(double lng, double lat) { + return lat >= -90 && lat <= 90 && lng >= -180 && lng <= 180; } - static S2Point coordsToPoint(const vector& coordElt) { - return coordToPoint(coordElt[0].Number(), coordElt[1].Number()); + static bool isGeoJSONPoint(const BSONObj& obj) { + BSONElement type = obj.getFieldDotted(GEOJSON_TYPE); + if (type.eoo() || (String != type.type())) { return false; } + if (GEOJSON_TYPE_POINT != type.String()) { return false; } + + if (!GeoParser::crsIsOK(obj)) { + warning() << "Invalid CRS: " << obj.toString() << endl; + return false; + } + + BSONElement coordElt = obj.getFieldDotted(GEOJSON_COORDINATES); + if (coordElt.eoo() || (Array != coordElt.type())) { return false; } + + const vector& coordinates = coordElt.Array(); + if (coordinates.size() != 2) { return false; } + if (!coordinates[0].isNumber() || !coordinates[1].isNumber()) { return false; } + double lat = coordinates[1].Number(); + double lng = coordinates[0].Number(); + return isValidLngLat(lng, lat); } - static void parsePoints(const vector& coordElt, vector* out) { - for (size_t i = 0; i < coordElt.size(); ++i) { - const vector& pointElt = coordElt[i].Array(); - if (pointElt.empty()) { continue; } - out->push_back(coordsToPoint(pointElt)); + static bool isLegacyPoint(const BSONObj &obj) { + BSONObjIterator it(obj); + if (!it.more()) { return false; } + BSONElement x = it.next(); + if (!x.isNumber()) { return false; } + if (!it.more()) { return false; } + BSONElement y = it.next(); + if (!y.isNumber()) { return false; } + if (it.more()) { return false; } + return true; + } + + static S2Point coordToPoint(double lng, double lat) { + // We don't rely on drem to clean up non-sane points. We just don't let them become + // spherical. + verify(isValidLngLat(lng, lat)); + // Note that it's (lat, lng) for S2 but (lng, lat) for MongoDB. + S2LatLng ll = S2LatLng::FromDegrees(lat, lng).Normalized(); + // This shouldn't happen since we should only have valid lng/lats. + if (!ll.is_valid()) { + stringstream ss; + ss << "coords invalid after normalization, lng = " << lng << " lat = " << lat << endl; + uasserted(17125, ss.str()); + } + return ll.ToPoint(); + } + + static void eraseDuplicatePoints(vector* vertices) { + for (size_t i = 1; i < vertices->size(); ++i) { + if ((*vertices)[i - 1] == (*vertices)[i]) { + vertices->erase(vertices->begin() + i); + // We could have > 2 adjacent identical vertices, and must examine i again. + --i; + } } } @@ -66,15 +125,99 @@ namespace mongo { for (size_t j = 0; j < thisCoord.size(); ++j) { if (!thisCoord[j].isNumber()) { return false; } } - // ...where the latitude is valid + // ...where the longitude, latitude is valid double lat = thisCoord[1].Number(); double lng = thisCoord[0].Number(); - if (lat < -90 || lat > 90) { return false; } - if (lng < -180 || lng > 180) { return false; } + if (!isValidLngLat(lng, lat)) { return false; } } return true; } + static bool parsePoints(const vector& coordElt, vector* out) { + for (size_t i = 0; i < coordElt.size(); ++i) { + const vector& pointElt = coordElt[i].Array(); + if (pointElt.empty()) { continue; } + if (!isValidLngLat(pointElt[0].Number(), pointElt[1].Number())) { + return false; + } + out->push_back(coordToPoint(pointElt[0].Number(), pointElt[1].Number())); + } + + return true; + } + + static bool isValidLineString(const vector& coordinateArray) { + if (coordinateArray.size() < 2) { return false; } + if (!isArrayOfCoordinates(coordinateArray)) { return false; } + vector vertices; + if (!parsePoints(coordinateArray, &vertices)) { return false; } + eraseDuplicatePoints(&vertices); + return S2Polyline::IsValid(vertices); + } + + static bool parseGeoJSONPolygonCoordinates(const vector& coordinates, + const BSONObj &sourceObject, S2Polygon *out) { + const vector& exteriorRing = coordinates[0].Array(); + vector exteriorVertices; + if (!parsePoints(exteriorRing, &exteriorVertices)) { return false; } + eraseDuplicatePoints(&exteriorVertices); + // The last point is duplicated. We drop it, since S2Loop expects no + // duplicate points + exteriorVertices.resize(exteriorVertices.size() - 1); + // S2 Polygon loops must have 3 vertices + if (exteriorVertices.size() < 3) { return false; } + + S2PolygonBuilderOptions polyOptions; + polyOptions.set_validate(true); + // Don't silently eliminate duplicate edges. + polyOptions.set_xor_edges(false); + S2PolygonBuilder polyBuilder(polyOptions); + S2Loop exteriorLoop(exteriorVertices); + exteriorLoop.Normalize(); + if (exteriorLoop.is_hole()) { + exteriorLoop.Invert(); + } + if (!exteriorLoop.IsValid()) { return false; } + polyBuilder.AddLoop(&exteriorLoop); + + // Subsequent arrays of coordinates are interior rings/holes. + for (size_t i = 1; i < coordinates.size(); ++i) { + vector holePoints; + if (!parsePoints(coordinates[i].Array(), &holePoints)) { return false; } + eraseDuplicatePoints(&holePoints); + // Drop the duplicated last point. + holePoints.resize(holePoints.size() - 1); + // S2 Polygon loops must have 3 vertices + if (holePoints.size() < 3) { return false; } + // Interior rings are clockwise. + S2Loop holeLoop(holePoints); + holeLoop.Normalize(); + if (!holeLoop.IsValid()) { return false; } + if (!holeLoop.is_hole()) { + if (!exteriorLoop.Contains(&holeLoop)) { return false; } + holeLoop.Invert(); + } else { + // It's already clockwise; we need to invert once to check that it's contained in + // the shell, then invert again. + holeLoop.Invert(); + if (!exteriorLoop.Contains(&holeLoop)) { return false; } + holeLoop.Invert(); + } + polyBuilder.AddLoop(&holeLoop); + } + + return polyBuilder.AssemblePolygon(out, NULL); + } + + static bool parseLegacyPoint(const BSONObj &obj, Point *out) { + BSONObjIterator it(obj); + BSONElement x = it.next(); + BSONElement y = it.next(); + out->x = x.number(); + out->y = y.number(); + return true; + } + // Coordinates looks like [[0,0],[5,0],[5,5],[0,5],[0,0]] static bool isLoopClosed(const vector& coordinates) { double x1, y1, x2, y2; @@ -85,13 +228,27 @@ namespace mongo { return (fabs(x1 - x2) < 1e-6) && fabs(y1 - y2) < 1e-6; } - //// What we publicly export - bool GeoParser::isGeoJSONPoint(const BSONObj& obj) { + static bool isGeoJSONPolygonCoordinates(const vector& coordinates) { + // Must be at least one element, the outer shell + if (coordinates.empty()) { return false; } + // Verify that the shell is a bunch'a coordinates. + for (size_t i = 0; i < coordinates.size(); ++i) { + if (Array != coordinates[i].type()) { return false; } + const vector& thisLoop = coordinates[i].Array(); + // A triangle is the simplest 2d shape, and we repeat a vertex, so, 4. + if (thisLoop.size() < 4) { return false; } + if (!isArrayOfCoordinates(thisLoop)) { return false; } + if (!isLoopClosed(thisLoop)) { return false; } + } + return true; + } + + static bool isGeoJSONPolygon(const BSONObj& obj) { BSONElement type = obj.getFieldDotted(GEOJSON_TYPE); if (type.eoo() || (String != type.type())) { return false; } - if (GEOJSON_TYPE_POINT != type.String()) { return false; } + if (GEOJSON_TYPE_POLYGON != type.String()) { return false; } - if (!crsIsOK(obj)) { + if (!GeoParser::crsIsOK(obj)) { warning() << "Invalid CRS: " << obj.toString() << endl; return false; } @@ -99,41 +256,95 @@ namespace mongo { BSONElement coordElt = obj.getFieldDotted(GEOJSON_COORDINATES); if (coordElt.eoo() || (Array != coordElt.type())) { return false; } - const vector& coordinates = coordElt.Array(); - if (coordinates.size() != 2) { return false; } - if (!coordinates[0].isNumber() || !coordinates[1].isNumber()) { return false; } - double lat = coordinates[1].Number(); - double lng = coordinates[0].Number(); - return lat >= -90 && lat <= 90 && lng >= -180 && lng <= 180; + return isGeoJSONPolygonCoordinates(coordElt.Array()); } - void GeoParser::parseGeoJSONPoint(const BSONObj& obj, S2Cell* out) { - S2Point point = coordsToPoint(obj.getFieldDotted(GEOJSON_COORDINATES).Array()); - *out = S2Cell(point); + static bool isLegacyPolygon(const BSONObj &obj) { + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + if (!type.isABSONObj()) { return false; } + if (!mongoutils::str::equals(type.fieldName(), "$polygon")) { return false; } + BSONObjIterator coordIt(type.embeddedObject()); + int vertices = 0; + while (coordIt.more()) { + BSONElement coord = coordIt.next(); + if (!coord.isABSONObj()) { return false; } + if (!isLegacyPoint(coord.Obj())) { return false; } + ++vertices; + } + if (vertices < 3) { return false; } + return true; + } + + static bool isLegacyCenter(const BSONObj &obj) { + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + if (!type.isABSONObj()) { return false; } + bool isCenter = mongoutils::str::equals(type.fieldName(), "$center"); + if (!isCenter) { return false; } + BSONObjIterator objIt(type.embeddedObject()); + BSONElement center = objIt.next(); + if (!center.isABSONObj()) { return false; } + if (!isLegacyPoint(center.Obj())) { return false; } + if (!objIt.more()) { return false; } + BSONElement radius = objIt.next(); + if (!radius.isNumber()) { return false; } + return true; } - void GeoParser::parseGeoJSONPoint(const BSONObj& obj, Point* out) { - const vector& coords = obj.getFieldDotted(GEOJSON_COORDINATES).Array(); - out->x = coords[0].Number(); - out->y = coords[1].Number(); + static bool isLegacyCenterSphere(const BSONObj &obj) { + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + if (!type.isABSONObj()) { return false; } + bool isCenterSphere = mongoutils::str::equals(type.fieldName(), "$centerSphere"); + if (!isCenterSphere) { return false; } + BSONObjIterator objIt(type.embeddedObject()); + BSONElement center = objIt.next(); + if (!center.isABSONObj()) { return false; } + if (!isLegacyPoint(center.Obj())) { return false; } + // Check to make sure the points are valid lng/lat. + BSONObjIterator coordIt(center.Obj()); + BSONElement lng = coordIt.next(); + BSONElement lat = coordIt.next(); + if (!isValidLngLat(lng.Number(), lat.Number())) { return false; } + if (!objIt.more()) { return false; } + BSONElement radius = objIt.next(); + if (!radius.isNumber()) { return false; } + return true; } - void GeoParser::parseGeoJSONPoint(const BSONObj& obj, S2Point* out) { - const vector& coords = obj.getFieldDotted(GEOJSON_COORDINATES).Array(); - *out = coordsToPoint(coords); + /** exported **/ + + bool GeoParser::isPoint(const BSONObj &obj) { + return isGeoJSONPoint(obj) || isLegacyPoint(obj); } - void eraseDuplicatePoints(vector* vertices) { - for (size_t i = 1; i < vertices->size(); ++i) { - if ((*vertices)[i - 1] == (*vertices)[i]) { - vertices->erase(vertices->begin() + i); - // We could have > 2 adjacent identical vertices, and must examine i again. - --i; + bool GeoParser::parsePoint(const BSONObj &obj, PointWithCRS *out) { + if (isGeoJSONPoint(obj)) { + const vector& coords = obj.getFieldDotted(GEOJSON_COORDINATES).Array(); + out->point = coordToPoint(coords[0].Number(), coords[1].Number()); + out->cell = S2Cell(out->point); + out->oldPoint.x = coords[0].Number(); + out->oldPoint.y = coords[1].Number(); + out->crs = SPHERE; + } else if (isLegacyPoint(obj)) { + BSONObjIterator it(obj); + BSONElement x = it.next(); + BSONElement y = it.next(); + if (isValidLngLat(x.Number(), y.Number())) { + out->flatUpgradedToSphere = true; + out->point = coordToPoint(x.Number(), y.Number()); + out->cell = S2Cell(out->point); } + out->oldPoint.x = x.Number(); + out->oldPoint.y = y.Number(); + out->crs = FLAT; } + + return true; } - bool GeoParser::isGeoJSONLineString(const BSONObj& obj) { + bool GeoParser::isLine(const BSONObj& obj) { BSONElement type = obj.getFieldDotted(GEOJSON_TYPE); if (type.eoo() || (String != type.type())) { return false; } if (GEOJSON_TYPE_LINESTRING != type.String()) { return false; } @@ -146,26 +357,74 @@ namespace mongo { BSONElement coordElt = obj.getFieldDotted(GEOJSON_COORDINATES); if (coordElt.eoo() || (Array != coordElt.type())) { return false; } - const vector& coordinateArray = coordElt.Array(); - if (coordinateArray.size() < 2) { return false; } - if (!isArrayOfCoordinates(coordinateArray)) { return false; } - vector vertices; - parsePoints(obj.getFieldDotted(GEOJSON_COORDINATES).Array(), &vertices); - eraseDuplicatePoints(&vertices); - return S2Polyline::IsValid(vertices); + return isValidLineString(coordElt.Array()); } - void GeoParser::parseGeoJSONLineString(const BSONObj& obj, S2Polyline* out) { + bool GeoParser::parseLine(const BSONObj& obj, LineWithCRS* out) { vector vertices; - parsePoints(obj.getFieldDotted(GEOJSON_COORDINATES).Array(), &vertices); + if (!parsePoints(obj.getFieldDotted(GEOJSON_COORDINATES).Array(), &vertices)) { + return false; + } eraseDuplicatePoints(&vertices); - out->Init(vertices); + out->line.Init(vertices); + out->crs = SPHERE; + return true; + } + + bool GeoParser::isBox(const BSONObj &obj) { + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + if (!type.isABSONObj()) { return false; } + if (!mongoutils::str::equals(type.fieldName(), "$box")) { return false; } + BSONObjIterator coordIt(type.embeddedObject()); + BSONElement minE = coordIt.next(); + if (!minE.isABSONObj()) { return false; } + if (!isLegacyPoint(minE.Obj())) { return false; } + if (!coordIt.more()) { return false; } + BSONElement maxE = coordIt.next(); + if (!maxE.isABSONObj()) { return false; } + if (!isLegacyPoint(maxE.Obj())) { return false; } + // XXX: VERIFY AREA >= 0 + return true; } - bool GeoParser::isGeoJSONPolygon(const BSONObj& obj) { + bool GeoParser::parseBox(const BSONObj &obj, BoxWithCRS *out) { + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + BSONObjIterator coordIt(type.embeddedObject()); + BSONElement minE = coordIt.next(); + BSONElement maxE = coordIt.next(); + if (!parseLegacyPoint(minE.Obj(), &out->box._min) || + !parseLegacyPoint(maxE.Obj(), &out->box._max)) { return false; } + out->crs = FLAT; + return true; + } + + bool GeoParser::parsePolygon(const BSONObj &obj, PolygonWithCRS *out) { + if (isGeoJSONPolygon(obj)) { + const vector& coordinates = obj.getFieldDotted(GEOJSON_COORDINATES).Array(); + if (!parseGeoJSONPolygonCoordinates(coordinates, obj, &out->polygon)) { return false; } + out->crs = SPHERE; + } else { + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + BSONObjIterator coordIt(type.embeddedObject()); + vector points; + while (coordIt.more()) { + Point p; + if (!parseLegacyPoint(coordIt.next().Obj(), &p)) { return false; } + points.push_back(p); + } + out->oldPolygon = Polygon(points); + out->crs = FLAT; + } + return true; + } + + bool GeoParser::isMultiPoint(const BSONObj &obj) { BSONElement type = obj.getFieldDotted(GEOJSON_TYPE); if (type.eoo() || (String != type.type())) { return false; } - if (GEOJSON_TYPE_POLYGON != type.String()) { return false; } + if (GEOJSON_TYPE_MULTI_POINT != type.String()) { return false; } if (!crsIsOK(obj)) { warning() << "Invalid CRS: " << obj.toString() << endl; @@ -176,157 +435,124 @@ namespace mongo { if (coordElt.eoo() || (Array != coordElt.type())) { return false; } const vector& coordinates = coordElt.Array(); - // Must be at least one element, the outer shell - if (coordinates.empty()) { return false; } - // Verify that the shell is a bunch'a coordinates. + if (0 == coordinates.size()) { return false; } + return isArrayOfCoordinates(coordinates); + } + + bool GeoParser::parseMultiPoint(const BSONObj &obj, MultiPointWithCRS *out) { + out->points.clear(); + BSONElement coordElt = obj.getFieldDotted(GEOJSON_COORDINATES); + const vector& coordinates = coordElt.Array(); + out->points.resize(coordinates.size()); + out->cells.resize(coordinates.size()); for (size_t i = 0; i < coordinates.size(); ++i) { - if (Array != coordinates[i].type()) { return false; } - const vector& thisLoop = coordinates[i].Array(); - // A triangle is the simplest 2d shape, and we repeat a vertex, so, 4. - if (thisLoop.size() < 4) { return false; } - if (!isArrayOfCoordinates(thisLoop)) { return false; } - if (!isLoopClosed(thisLoop)) { return false; } + const vector& thisCoord = coordinates[i].Array(); + out->points[i] = coordToPoint(thisCoord[0].Number(), thisCoord[1].Number()); + out->cells[i] = S2Cell(out->points[i]); } + return true; } - void GeoParser::parseGeoJSONPolygon(const BSONObj& obj, S2Polygon* out) { - const vector& coordinates = - obj.getFieldDotted(GEOJSON_COORDINATES).Array(); - - const vector& exteriorRing = coordinates[0].Array(); - vector exteriorVertices; - parsePoints(exteriorRing, &exteriorVertices); - // The last point is duplicated. We drop it, since S2Loop expects no - // duplicate points - exteriorVertices.resize(exteriorVertices.size() - 1); + bool GeoParser::isMultiLine(const BSONObj &obj) { + BSONElement type = obj.getFieldDotted(GEOJSON_TYPE); + if (type.eoo() || (String != type.type())) { return false; } + if (GEOJSON_TYPE_MULTI_LINESTRING != type.String()) { return false; } - S2PolygonBuilderOptions polyOptions; - polyOptions.set_validate(true); - // Don't silently eliminate duplicate edges. - polyOptions.set_xor_edges(false); - S2PolygonBuilder polyBuilder(polyOptions); - S2Loop exteriorLoop(exteriorVertices); - exteriorLoop.Normalize(); - if (exteriorLoop.is_hole()) { - exteriorLoop.Invert(); + if (!crsIsOK(obj)) { + warning() << "Invalid CRS: " << obj.toString() << endl; + return false; } - uassert(16693, "Exterior shell of polygon is invalid: " + obj.toString(), - exteriorLoop.IsValid()); - polyBuilder.AddLoop(&exteriorLoop); - // Subsequent arrays of coordinates are interior rings/holes. - for (size_t i = 1; i < coordinates.size(); ++i) { - vector holePoints; - parsePoints(coordinates[i].Array(), &holePoints); - // Drop the duplicated last point. - holePoints.resize(holePoints.size() - 1); - // Interior rings are clockwise. - S2Loop holeLoop(holePoints); - holeLoop.Normalize(); - uassert(16694, "Interior hole of polygon is invalid: " + obj.toString(), - holeLoop.IsValid()); - if (!holeLoop.is_hole()) { - holeLoop.Invert(); - } - polyBuilder.AddLoop(&holeLoop); - } + BSONElement coordElt = obj.getFieldDotted(GEOJSON_COORDINATES); + if (coordElt.eoo() || (Array != coordElt.type())) { return false; } - uassert(16695, "Couldn't assemble polygon: " + obj.toString(), - polyBuilder.AssemblePolygon(out, NULL)); - } + const vector& coordinates = coordElt.Array(); + if (0 == coordinates.size()) { return false; } - bool GeoParser::parsePoint(const BSONObj &obj, Point *out) { - if (isGeoJSONPoint(obj)) { - parseGeoJSONPoint(obj, out); - return true; - } else if (isLegacyPoint(obj)) { - parseLegacyPoint(obj, out); - return true; + for (size_t i = 0; i < coordinates.size(); ++i) { + if (coordinates[i].eoo() || (Array != coordinates[i].type())) { return false; } + if (!isValidLineString(coordinates[i].Array())) { return false; } } - return false; - } - bool GeoParser::parsePoint(const BSONObj &obj, S2Point *out) { - if (isGeoJSONPoint(obj)) { - parseGeoJSONPoint(obj, out); - return true; - } else if (isLegacyPoint(obj)) { - BSONObjIterator it(obj); - BSONElement x = it.next(); - BSONElement y = it.next(); - *out = coordToPoint(x.number(), y.number()); - return true; - } - return false; + return true; } - bool GeoParser::parsePoint(const BSONObj &obj, S2Cell *out) { - S2Point point; - if (parsePoint(obj, &point)) { - *out = S2Cell(point); - return true; + bool GeoParser::parseMultiLine(const BSONObj &obj, MultiLineWithCRS *out) { + vector coordElt = obj.getFieldDotted(GEOJSON_COORDINATES).Array(); + out->lines.clear(); + out->lines.mutableVector().resize(coordElt.size()); + + for (size_t i = 0; i < coordElt.size(); ++i) { + vector vertices; + if (!parsePoints(coordElt[i].Array(), &vertices)) { return false; } + out->lines.mutableVector()[i] = new S2Polyline(); + out->lines.mutableVector()[i]->Init(vertices); } - return false; - } - bool GeoParser::parseLineString(const BSONObj &obj, S2Polyline *out) { - if (!isGeoJSONLineString(obj)) { return false; } - parseGeoJSONLineString(obj, out); return true; } - void GeoParser::parseLegacyPoint(const BSONObj &obj, S2Point *out) { - BSONObjIterator it(obj); - BSONElement x = it.next(); - BSONElement y = it.next(); - *out = coordToPoint(x.number(), y.number()); - } + bool GeoParser::isMultiPolygon(const BSONObj &obj) { + BSONElement type = obj.getFieldDotted(GEOJSON_TYPE); + if (type.eoo() || (String != type.type())) { return false; } + if (GEOJSON_TYPE_MULTI_POLYGON != type.String()) { return false; } - bool GeoParser::parsePolygon(const BSONObj &obj, S2Polygon *out) { - if (isGeoJSONPolygon(obj)) { - parseGeoJSONPolygon(obj, out); - return true; - } else { + if (!crsIsOK(obj)) { + warning() << "Invalid CRS: " << obj.toString() << endl; return false; } - } - bool GeoParser::isLegacyPoint(const BSONObj &obj) { - BSONObjIterator it(obj); - if (!it.more()) { return false; } - BSONElement x = it.next(); - if (!x.isNumber()) { return false; } - if (!it.more()) { return false; } - BSONElement y = it.next(); - if (!y.isNumber()) { return false; } - if (it.more()) { return false; } + BSONElement coordElt = obj.getFieldDotted(GEOJSON_COORDINATES); + if (coordElt.eoo() || (Array != coordElt.type())) { return false; } + + const vector& coordinates = coordElt.Array(); + if (0 == coordinates.size()) { return false; } + for (size_t i = 0; i < coordinates.size(); ++i) { + if (coordinates[i].eoo() || (Array != coordinates[i].type())) { return false; } + if (!isGeoJSONPolygonCoordinates(coordinates[i].Array())) { return false; } + } + return true; } - bool GeoParser::isLegacyPolygon(const BSONObj &obj) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - if (!type.isABSONObj()) { return false; } - if (!mongoutils::str::equals(type.fieldName(), "$polygon")) { return false; } - BSONObjIterator coordIt(type.embeddedObject()); - int vertices = 0; - while (coordIt.more()) { - BSONElement coord = coordIt.next(); - if (!coord.isABSONObj()) { return false; } - if (!isLegacyPoint(coord.Obj())) { return false; } - ++vertices; + bool GeoParser::parseMultiPolygon(const BSONObj &obj, MultiPolygonWithCRS *out) { + vector coordElt = obj.getFieldDotted(GEOJSON_COORDINATES).Array(); + out->polygons.clear(); + out->polygons.mutableVector().resize(coordElt.size()); + + for (size_t i = 0; i < coordElt.size(); ++i) { + out->polygons.mutableVector()[i] = new S2Polygon(); + if (!parseGeoJSONPolygonCoordinates( + coordElt[i].Array(), obj, out->polygons.vector()[i])) { + return false; + } } - if (vertices < 3) { return false; } + return true; } - bool GeoParser::isPoint(const BSONObj &obj) { - return isGeoJSONPoint(obj) || isLegacyPoint(obj); - } + bool GeoParser::isGeometryCollection(const BSONObj &obj) { + BSONElement type = obj.getFieldDotted(GEOJSON_TYPE); + if (type.eoo() || (String != type.type())) { return false; } + if (GEOJSON_TYPE_GEOMETRY_COLLECTION != type.String()) { return false; } - bool GeoParser::isLineString(const BSONObj &obj) { - return isGeoJSONLineString(obj); + BSONElement coordElt = obj.getFieldDotted(GEOJSON_GEOMETRIES); + if (coordElt.eoo() || (Array != coordElt.type())) { return false; } + + const vector& coordinates = coordElt.Array(); + if (0 == coordinates.size()) { return false; } + + for (size_t i = 0; i < coordinates.size(); ++i) { + if (coordinates[i].eoo() || (Object != coordinates[i].type())) { return false; } + BSONObj obj = coordinates[i].Obj(); + if (!isGeoJSONPoint(obj) && !isLine(obj) && !isGeoJSONPolygon(obj) + && !isMultiPoint(obj) && !isMultiPolygon(obj) && !isMultiLine(obj)) { + return false; + } + } + + return true; } bool GeoParser::isPolygon(const BSONObj &obj) { @@ -356,110 +582,113 @@ namespace mongo { return ("urn:ogc:def:crs:OGC:1.3:CRS84" == name) || ("EPSG:4326" == name); } - void GeoParser::parseLegacyPoint(const BSONObj &obj, Point *out) { - BSONObjIterator it(obj); - BSONElement x = it.next(); - BSONElement y = it.next(); - out->x = x.number(); - out->y = y.number(); + bool GeoParser::isCap(const BSONObj &obj) { + return isLegacyCenter(obj) || isLegacyCenterSphere(obj); } - bool GeoParser::isLegacyBox(const BSONObj &obj) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - if (!type.isABSONObj()) { return false; } - if (!mongoutils::str::equals(type.fieldName(), "$box")) { return false; } - BSONObjIterator coordIt(type.embeddedObject()); - BSONElement minE = coordIt.next(); - if (!minE.isABSONObj()) { return false; } - if (!isLegacyPoint(minE.Obj())) { return false; } - if (!coordIt.more()) { return false; } - BSONElement maxE = coordIt.next(); - if (!maxE.isABSONObj()) { return false; } - if (!isLegacyPoint(maxE.Obj())) { return false; } + bool GeoParser::parseCap(const BSONObj& obj, CapWithCRS *out) { + if (isLegacyCenter(obj)) { + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + BSONObjIterator objIt(type.embeddedObject()); + BSONElement center = objIt.next(); + if (!parseLegacyPoint(center.Obj(), &out->circle.center)) { return false; } + BSONElement radius = objIt.next(); + out->circle.radius = radius.number(); + // radius >= 0 and is not NaN + if (!(out->circle.radius >= 0)) + return false; + out->crs = FLAT; + } else { + verify(isLegacyCenterSphere(obj)); + BSONObjIterator typeIt(obj); + BSONElement type = typeIt.next(); + BSONObjIterator objIt(type.embeddedObject()); + BSONObj centerObj = objIt.next().Obj(); + + S2Point centerPoint; + BSONObjIterator it(centerObj); + BSONElement x = it.next(); + BSONElement y = it.next(); + centerPoint = coordToPoint(x.Number(), y.Number()); + BSONElement radiusElt = objIt.next(); + double radius = radiusElt.number(); + // radius >= 0 and is not NaN + if (!(radius >= 0)) + return false; + out->cap = S2Cap::FromAxisAngle(centerPoint, S1Angle::Radians(radius)); + out->circle.radius = radius; + out->circle.center = Point(x.Number(), y.Number()); + out->crs = SPHERE; + } return true; } - void GeoParser::parseLegacyBox(const BSONObj &obj, Box *out) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - BSONObjIterator coordIt(type.embeddedObject()); - BSONElement minE = coordIt.next(); - BSONElement maxE = coordIt.next(); - parseLegacyPoint(minE.Obj(), &out->_min); - parseLegacyPoint(maxE.Obj(), &out->_max); - } + bool GeoParser::parseGeometryCollection(const BSONObj &obj, GeometryCollection *out) { + BSONElement coordElt = obj.getFieldDotted(GEOJSON_GEOMETRIES); + const vector& geometries = coordElt.Array(); + + for (size_t i = 0; i < geometries.size(); ++i) { + const BSONObj& geoObj = geometries[i].Obj(); + + if (isGeoJSONPoint(geoObj)) { + PointWithCRS point; + if (!parsePoint(geoObj, &point)) { return false; } + out->points.push_back(point); + } else if (isLine(geoObj)) { + out->lines.mutableVector().push_back(new LineWithCRS()); + if (!parseLine(geoObj, out->lines.vector().back())) { return false; } + } else if (isGeoJSONPolygon(geoObj)) { + out->polygons.mutableVector().push_back(new PolygonWithCRS()); + if (!parsePolygon(geoObj, out->polygons.vector().back())) { return false; } + } else if (isMultiPoint(geoObj)) { + out->multiPoints.mutableVector().push_back(new MultiPointWithCRS()); + if (!parseMultiPoint(geoObj, out->multiPoints.mutableVector().back())) { + return false; + } + } else if (isMultiPolygon(geoObj)) { + out->multiPolygons.mutableVector().push_back(new MultiPolygonWithCRS()); + if (!parseMultiPolygon(geoObj, out->multiPolygons.mutableVector().back())) { + return false; + } + } else { + verify(isMultiLine(geoObj)); + out->multiLines.mutableVector().push_back(new MultiLineWithCRS()); + if (!parseMultiLine(geoObj, out->multiLines.mutableVector().back())) { + return false; + } + } + } - bool GeoParser::isLegacyCenter(const BSONObj &obj) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - if (!type.isABSONObj()) { return false; } - bool isCenter = mongoutils::str::equals(type.fieldName(), "$center"); - if (!isCenter) { return false; } - BSONObjIterator objIt(type.embeddedObject()); - BSONElement center = objIt.next(); - if (!center.isABSONObj()) { return false; } - if (!isLegacyPoint(center.Obj())) { return false; } - if (!objIt.more()) { return false; } - BSONElement radius = objIt.next(); - if (!radius.isNumber()) { return false; } return true; } - void GeoParser::parseLegacyCenter(const BSONObj &obj, Circle *out) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - BSONObjIterator objIt(type.embeddedObject()); - BSONElement center = objIt.next(); - parseLegacyPoint(center.Obj(), &out->center); - BSONElement radius = objIt.next(); - out->radius = radius.number(); - } + bool GeoParser::parsePointWithMaxDistance(const BSONObj& obj, PointWithCRS* out, double* maxOut) { + BSONObjIterator it(obj); + if (!it.more()) { return false; } - bool GeoParser::isLegacyCenterSphere(const BSONObj &obj) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - if (!type.isABSONObj()) { return false; } - bool isCenterSphere = mongoutils::str::equals(type.fieldName(), "$centerSphere"); - if (!isCenterSphere) { return false; } - BSONObjIterator objIt(type.embeddedObject()); - BSONElement center = objIt.next(); - if (!center.isABSONObj()) { return false; } - if (!isLegacyPoint(center.Obj())) { return false; } - if (!objIt.more()) { return false; } - BSONElement radius = objIt.next(); - if (!radius.isNumber()) { return false; } - return true; - } + BSONElement lng = it.next(); + if (!lng.isNumber()) { return false; } + if (!it.more()) { return false; } - void GeoParser::parseLegacyCenterSphere(const BSONObj &obj, S2Cap *out) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - BSONObjIterator objIt(type.embeddedObject()); - BSONElement center = objIt.next(); - S2Point centerPoint; - parseLegacyPoint(center.Obj(), ¢erPoint); - BSONElement radiusElt = objIt.next(); - double radius = radiusElt.number(); - *out = S2Cap::FromAxisAngle(centerPoint, S1Angle::Radians(radius)); - } + BSONElement lat = it.next(); + if (!lat.isNumber()) { return false; } + if (!it.more()) { return false; } - void GeoParser::parseLegacyPolygon(const BSONObj &obj, Polygon *out) { - BSONObjIterator typeIt(obj); - BSONElement type = typeIt.next(); - BSONObjIterator coordIt(type.embeddedObject()); - vector points; - while (coordIt.more()) { - Point p; - parseLegacyPoint(coordIt.next().Obj(), &p); - points.push_back(p); - } - *out = Polygon(points); - } + BSONElement dist = it.next(); + if (!dist.isNumber()) { return false; } + if (it.more()) { return false; } - bool GeoParser::parsePolygon(const BSONObj &obj, Polygon *out) { - if (!isLegacyPolygon(obj)) { return false; } - parseLegacyPolygon(obj, out); + out->crs = FLAT; + out->oldPoint.x = lng.number(); + out->oldPoint.y = lat.number(); + *maxOut = dist.number(); + if (isValidLngLat(lng.Number(), lat.Number())) { + out->flatUpgradedToSphere = true; + out->point = coordToPoint(lng.Number(), lat.Number()); + out->cell = S2Cell(out->point); + } return true; } + } // namespace mongo diff --git a/src/mongo/db/geo/geoparser.h b/src/mongo/db/geo/geoparser.h index 72d998267ff..a133f8a3989 100644 --- a/src/mongo/db/geo/geoparser.h +++ b/src/mongo/db/geo/geoparser.h @@ -1,5 +1,5 @@ /** -* Copyright (C) 2008-2012 10gen Inc. +* Copyright (C) 2013 10gen Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -12,73 +12,66 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "mongo/db/jsobj.h" -#include -#include "third_party/s2/s2.h" #include "mongo/db/geo/shapes.h" - -class S2Cap; -class S2Cell; -class S2Polyline; -class S2Polygon; +#include "mongo/db/jsobj.h" namespace mongo { + // This class parses geographic data. // It parses a subset of GeoJSON and creates S2 shapes from it. // See http://geojson.org/geojson-spec.html for the spec. // // This class also parses the ad-hoc geo formats that MongoDB introduced. // - // The parseFoo methods that return a bool internally call isFoo and return true - // if the foo is parsed correctly. - // The parseFoo methods that do not return a bool assume isFoo is true. - // - // We assume that if you're trying to parse something, you know it's valid. + // parse* methods may do some more validation than the is* methods; they return false if they + // encounter invalid geometry and true if the geometry is parsed successfully. class GeoParser { public: - // Try to parse GeoJSON, then try legacy format, return true if either succeed. - // These call the various isPoint and parsePoint methods below. - // You can just use these bool parsePoint(...) methods. - static bool parsePoint(const BSONObj &obj, S2Point *out); - static bool parsePoint(const BSONObj &obj, S2Cell *out); - static bool parsePoint(const BSONObj &obj, Point *out); - // Check to see if it's GeoJSON or if it's legacy geo. static bool isPoint(const BSONObj &obj); + static bool parsePoint(const BSONObj &obj, PointWithCRS *out); - static bool isGeoJSONPoint(const BSONObj &obj); - static void parseGeoJSONPoint(const BSONObj &obj, S2Point *out); - static void parseGeoJSONPoint(const BSONObj &obj, S2Cell *out); - static void parseGeoJSONPoint(const BSONObj &obj, Point *out); + static bool isLine(const BSONObj &obj); + static bool parseLine(const BSONObj &obj, LineWithCRS *out); - static bool isLegacyPoint(const BSONObj &obj); - static void parseLegacyPoint(const BSONObj &obj, S2Point *out); - static void parseLegacyPoint(const BSONObj &obj, Point *out); + static bool isBox(const BSONObj &obj); + static bool parseBox(const BSONObj &obj, BoxWithCRS *out); - static bool parseLineString(const BSONObj &obj, S2Polyline *out); - static bool isLineString(const BSONObj &obj); - static bool isGeoJSONLineString(const BSONObj &obj); - static void parseGeoJSONLineString(const BSONObj &obj, S2Polyline *out); - - static bool parsePolygon(const BSONObj &obj, S2Polygon *out); - static bool parsePolygon(const BSONObj &obj, Polygon *out); static bool isPolygon(const BSONObj &obj); - static bool isGeoJSONPolygon(const BSONObj &obj); - static bool isLegacyPolygon(const BSONObj &obj); - static void parseGeoJSONPolygon(const BSONObj &obj, S2Polygon *out); - static void parseLegacyPolygon(const BSONObj &obj, Polygon *out); + static bool parsePolygon(const BSONObj &obj, PolygonWithCRS *out); + + // AKA $center or $centerSphere + static bool isCap(const BSONObj &obj); + static bool parseCap(const BSONObj &obj, CapWithCRS *out); + + static bool isMultiPoint(const BSONObj &obj); + static bool parseMultiPoint(const BSONObj &obj, MultiPointWithCRS *out); - static bool isLegacyBox(const BSONObj &obj); - static void parseLegacyBox(const BSONObj &obj, Box *out); + static bool isMultiLine(const BSONObj &obj); + static bool parseMultiLine(const BSONObj &obj, MultiLineWithCRS *out); - static bool isLegacyCenter(const BSONObj &obj); - static void parseLegacyCenter(const BSONObj &obj, Circle *out); + static bool isMultiPolygon(const BSONObj &obj); + static bool parseMultiPolygon(const BSONObj &obj, MultiPolygonWithCRS *out); - static bool isLegacyCenterSphere(const BSONObj &obj); - static void parseLegacyCenterSphere(const BSONObj &obj, S2Cap *out); + static bool isGeometryCollection(const BSONObj &obj); + static bool parseGeometryCollection(const BSONObj &obj, GeometryCollection *out); + + static bool parsePointWithMaxDistance(const BSONObj& obj, PointWithCRS* out, double* maxOut); // Return true if the CRS field is 1. missing, or 2. is well-formed and // has a datum we accept. Otherwise, return false. @@ -87,4 +80,5 @@ namespace mongo { // needed. static bool crsIsOK(const BSONObj& obj); }; + } // namespace mongo diff --git a/src/mongo/db/geo/geoparser_test.cpp b/src/mongo/db/geo/geoparser_test.cpp index cb3af1a119e..eeac3725336 100644 --- a/src/mongo/db/geo/geoparser_test.cpp +++ b/src/mongo/db/geo/geoparser_test.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ /** @@ -22,20 +34,13 @@ #include #include "mongo/db/geo/geoparser.h" +#include "mongo/db/geo/shapes.h" #include "mongo/db/json.h" #include "mongo/db/jsobj.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" -#include "mongo/db/geo/shapes.h" - -#include "third_party/s2/s2.h" -#include "third_party/s2/s2polygon.h" -#include "third_party/s2/s2polyline.h" -using mongo::BSONObj; -using mongo::fromjson; -using mongo::GeoParser; -using mongo::Polygon; +using namespace mongo; namespace { @@ -63,27 +68,27 @@ namespace { } TEST(GeoParser, isValidLineString) { - ASSERT_TRUE(GeoParser::isLineString( + ASSERT_TRUE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[1,2], [3,4]]}"))); - ASSERT_TRUE(GeoParser::isLineString( + ASSERT_TRUE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[0,-90], [0,90]]}"))); - ASSERT_TRUE(GeoParser::isLineString( + ASSERT_TRUE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[180,-90], [-180,90]]}"))); - ASSERT_FALSE(GeoParser::isLineString( + ASSERT_FALSE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[180.1,-90], [-180.1,90]]}"))); - ASSERT_FALSE(GeoParser::isLineString( + ASSERT_FALSE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[0,-91], [0,90]]}"))); - ASSERT_FALSE(GeoParser::isLineString( + ASSERT_FALSE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[0,-90], [0,91]]}"))); - ASSERT_TRUE(GeoParser::isLineString( + ASSERT_TRUE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[1,2], [3,4], [5,6]]}"))); - ASSERT_FALSE(GeoParser::isLineString( + ASSERT_FALSE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[1,2]]}"))); - ASSERT_FALSE(GeoParser::isLineString( + ASSERT_FALSE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[['chicken','little']]}"))); - ASSERT_FALSE(GeoParser::isLineString( + ASSERT_FALSE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[1,2, 3, 4]}"))); - ASSERT_FALSE(GeoParser::isLineString( + ASSERT_FALSE(GeoParser::isLine( fromjson("{'type':'LineString', 'coordinates':[[1,2, 3], [3,4, 5], [5,6]]}"))); } @@ -111,71 +116,79 @@ namespace { } TEST(GeoParser, parsePoint) { - S2Point point; - ASSERT_TRUE(GeoParser::parsePoint(fromjson("{'type':'Point', 'coordinates': [40, 5]}"), - &point)); - ASSERT_TRUE(GeoParser::parsePoint(fromjson("{'type':'Point', 'coordinates': [-4.3, -5.0]}"), - &point)); + PointWithCRS point; + GeoParser::parsePoint(fromjson("{'type':'Point', 'coordinates': [40, 5]}"), &point); + GeoParser::parsePoint(fromjson("{'type':'Point', 'coordinates': [-4.3, -5.0]}"), &point); } - TEST(GeoParser, parseLineString) { - S2Polyline polyline; - GeoParser::parseLineString( + TEST(GeoParser, parseLine) { + LineWithCRS polyline; + GeoParser::parseLine( fromjson("{'type':'LineString', 'coordinates':[[1,2],[3,4]]}"), &polyline); - GeoParser::parseLineString( + GeoParser::parseLine( fromjson("{'type':'LineString', 'coordinates':[[1,2], [3,4], [5,6]]}"), &polyline); - GeoParser::parseLineString( + GeoParser::parseLine( fromjson("{'type':'LineString', 'coordinates':[[1,2], [3,4], [5,6]]}"), &polyline); } TEST(GeoParser, parsePolygon) { - S2Point point; + PointWithCRS point; GeoParser::parsePoint(fromjson("{'type':'Point', 'coordinates': [2, 2]}"), &point); - S2Polygon polygonA; + PolygonWithCRS polygonA; GeoParser::parsePolygon( fromjson("{'type':'Polygon', 'coordinates':[ [[0,0],[5,0],[5,5],[0,5],[0,0]] ]}"), &polygonA); - ASSERT_TRUE(polygonA.Contains(point)); + ASSERT_TRUE(polygonA.polygon.Contains(point.point)); - S2Polygon polygonB; + PolygonWithCRS polygonB; GeoParser::parsePolygon( fromjson("{'type':'Polygon', 'coordinates':[ [[0,0],[5,0],[5,5],[0,5],[0,0]]," " [[1,1],[1,4],[4,4],[4,1],[1,1]] ]}"), &polygonB); // We removed this in the hole. - ASSERT_FALSE(polygonB.Contains(point)); + ASSERT_FALSE(polygonB.polygon.Contains(point.point)); // Now we reverse the orientations and verify that the code fixes it up // (outer loop must be CCW, inner CW). - S2Polygon polygonC; + PolygonWithCRS polygonC; GeoParser::parsePolygon( fromjson("{'type':'Polygon', 'coordinates':[ [[0,0],[0,5],[5,5],[5,0],[0,0]] ]}"), &polygonC); - ASSERT_TRUE(polygonC.Contains(point)); + ASSERT_TRUE(polygonC.polygon.Contains(point.point)); - S2Polygon polygonD; + PolygonWithCRS polygonD; GeoParser::parsePolygon( fromjson("{'type':'Polygon', 'coordinates':[ [[0,0],[0,5],[5,5],[5,0],[0,0]]," " [[1,1],[1,4],[4,4],[4,1],[1,1]] ]}"), &polygonD); // Also removed in the loop. - ASSERT_FALSE(polygonD.Contains(point)); + ASSERT_FALSE(polygonD.polygon.Contains(point.point)); + + // + // Bad polygon examples + // + + // Polygon with not enough points, because some are duplicated + PolygonWithCRS polygonBad; + ASSERT_FALSE(GeoParser::parsePolygon( + fromjson("{'type':'Polygon', 'coordinates':[[ [0,0], [0,0], [5,5], [5,5], [0,0] ]]}"), + &polygonBad)); } - TEST(GeoParser, parseLegacyPoint) { - S2Point point; - ASSERT(GeoParser::parsePoint(BSON_ARRAY(0 << 1), &point)); - ASSERT_FALSE(GeoParser::parsePoint(BSON_ARRAY(0), &point)); - ASSERT_FALSE(GeoParser::parsePoint(BSON_ARRAY(0 << 1 << 2), &point)); - ASSERT(GeoParser::parsePoint(fromjson("{x: 50, y:40}"), &point)); - ASSERT_FALSE(GeoParser::parsePoint(fromjson("{x: '50', y:40}"), &point)); - ASSERT_FALSE(GeoParser::parsePoint(fromjson("{x: 5, y:40, z:50}"), &point)); - ASSERT_FALSE(GeoParser::parsePoint(fromjson("{x: 5}"), &point)); + TEST(GeoParser, legacyPoint) { + PointWithCRS point; + ASSERT(GeoParser::isPoint(BSON_ARRAY(0 << 1))); + ASSERT_FALSE(GeoParser::isPoint(BSON_ARRAY(0))); + ASSERT_FALSE(GeoParser::isPoint(BSON_ARRAY(0 << 1 << 2))); + ASSERT(GeoParser::isPoint(fromjson("{x: 50, y:40}"))); + ASSERT_FALSE(GeoParser::isPoint(fromjson("{x: '50', y:40}"))); + ASSERT_FALSE(GeoParser::isPoint(fromjson("{x: 5, y:40, z:50}"))); + ASSERT_FALSE(GeoParser::isPoint(fromjson("{x: 5}"))); } TEST(GeoParser, verifyCRS) { @@ -186,43 +199,204 @@ namespace { BSONObj point1 = fromjson("{'type':'Point', 'coordinates': [40, 5], " + goodCRS1 + "}"); BSONObj point2 = fromjson("{'type':'Point', 'coordinates': [40, 5], " + goodCRS2 + "}"); - ASSERT(GeoParser::isGeoJSONPoint(point1)); + ASSERT(GeoParser::isPoint(point1)); ASSERT(GeoParser::crsIsOK(point1)); - ASSERT(GeoParser::isGeoJSONPoint(point2)); + ASSERT(GeoParser::isPoint(point2)); ASSERT(GeoParser::crsIsOK(point2)); BSONObj point3 = fromjson("{'type':'Point', 'coordinates': [40, 5], " + badCRS1 + "}"); BSONObj point4 = fromjson("{'type':'Point', 'coordinates': [40, 5], " + badCRS2 + "}"); - ASSERT_FALSE(GeoParser::isGeoJSONPoint(point3)); + ASSERT_FALSE(GeoParser::isPoint(point3)); ASSERT_FALSE(GeoParser::crsIsOK(point3)); - ASSERT_FALSE(GeoParser::isGeoJSONPoint(point4)); + ASSERT_FALSE(GeoParser::isPoint(point4)); ASSERT_FALSE(GeoParser::crsIsOK(point4)); BSONObj polygon1 = fromjson("{'type':'Polygon', 'coordinates':[ [[0,0],[5,0],[5,5],[0,5],[0,0]]," " [[1,1],[1,4],[4,4],[4,1],[1,1]] ]," + goodCRS1 + "}"); - ASSERT(GeoParser::isGeoJSONPolygon(polygon1)); + ASSERT(GeoParser::isPolygon(polygon1)); ASSERT(GeoParser::crsIsOK(polygon1)); BSONObj polygon2 = fromjson("{'type':'Polygon', 'coordinates':[ [[0,0],[5,0],[5,5],[0,5],[0,0]]," " [[1,1],[1,4],[4,4],[4,1],[1,1]] ]," + badCRS2 + "}"); - ASSERT_FALSE(GeoParser::isGeoJSONPolygon(polygon2)); + ASSERT_FALSE(GeoParser::isPolygon(polygon2)); ASSERT_FALSE(GeoParser::crsIsOK(polygon2)); BSONObj line1 = fromjson("{'type':'LineString', 'coordinates':[[1,2], [3,4], [5,6]]," + goodCRS2 + "}"); - ASSERT(GeoParser::isGeoJSONLineString(line1)); + ASSERT(GeoParser::isLine(line1)); ASSERT(GeoParser::crsIsOK(line1)); BSONObj line2 = fromjson("{'type':'LineString', 'coordinates':[[1,2], [3,4], [5,6]]," + badCRS1 + "}"); - ASSERT_FALSE(GeoParser::isGeoJSONLineString(line2)); + ASSERT_FALSE(GeoParser::isLine(line2)); ASSERT_FALSE(GeoParser::crsIsOK(line2)); } - TEST(GeoParser, parseLegacyPolygon) { - mongo::Polygon polygon; - ASSERT(GeoParser::parsePolygon(fromjson("{$polygon: [[10,20],[10,40],[30,40],[30,20]]}"), - &polygon)); - ASSERT(GeoParser::parsePolygon(fromjson("{$polygon: [[10,20], [10,40], [30,40]]}"), &polygon)); - ASSERT_FALSE(GeoParser::parsePolygon(fromjson("{$polygon: [[10,20],[10,40]]}"), &polygon)); - ASSERT_FALSE(GeoParser::parsePolygon(fromjson("{$polygon: [['10', 20],[10,40],[30,40],[30,20]]}"), &polygon)); - ASSERT_FALSE(GeoParser::parsePolygon(fromjson("{$polygon: [[10,20,30],[10,40],[30,40],[30,20]]}"), &polygon)); - ASSERT(GeoParser::parsePolygon( - fromjson("{$polygon: {a:{x:40,y:5},b:{x:40,y:6},c:{x:41,y:6},d:{x:41,y:5}}}"), &polygon)); + TEST(GeoParser, legacyPolygon) { + PolygonWithCRS polygon; + GeoParser::parsePolygon(fromjson("{$polygon: [[10,20],[10,40],[30,40],[30,20]]}"), + &polygon); + ASSERT(polygon.crs == FLAT); + + GeoParser::parsePolygon(fromjson("{$polygon: [[10,20], [10,40], [30,40]]}"), &polygon); + ASSERT(polygon.crs == FLAT); + + ASSERT_FALSE(GeoParser::isPolygon(fromjson("{$polygon: [[10,20],[10,40]]}"))); + ASSERT_FALSE(GeoParser::isPolygon(fromjson("{$polygon: [['10',20],[10,40],[30,40],[30,20]]}"))); + ASSERT_FALSE(GeoParser::isPolygon(fromjson("{$polygon: [[10,20,30],[10,40],[30,40],[30,20]]}"))); + ASSERT(GeoParser::isPolygon(fromjson("{$polygon: {a:{x:40,y:5},b:{x:40,y:6},c:{x:41,y:6},d:{x:41,y:5}}}"))); + } + + TEST(GeoParser, multiPoint) { + ASSERT(GeoParser::isMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[[1,2],[3,4]]}"))); + ASSERT(GeoParser::isMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[[3,4]]}"))); + ASSERT(GeoParser::isMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[[1,2],[3,4],[5,6],[7,8]]}"))); + + ASSERT_FALSE(GeoParser::isMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[]}"))); + ASSERT_FALSE(GeoParser::isMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[[181,2],[3,4]]}"))); + ASSERT_FALSE(GeoParser::isMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[[1,-91],[3,4]]}"))); + ASSERT_FALSE(GeoParser::isMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[[181,2],[3,'chicken']]}"))); + } + + TEST(GeoParser, parseMultiPoint) { + mongo::MultiPointWithCRS mp; + GeoParser::parseMultiPoint(fromjson("{'type':'MultiPoint','coordinates':[[1,2],[3,4]]}"), + &mp); + GeoParser::parseMultiPoint(fromjson("{'type':'MultiPoint','coordinates':[[3,4]]}"), + &mp); + GeoParser::parseMultiPoint( + fromjson("{'type':'MultiPoint','coordinates':[[1,2],[3,4],[5,6],[7,8]]}"), &mp); + } + + TEST(GeoParser, multiLineString) { + ASSERT(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[1,1],[2,2],[3,3]]," + "[[4,5],[6,7]]]}"))); + ASSERT(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[1,1],[2,2]]," + "[[4,5],[6,7]]]}"))); + ASSERT(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[1,1],[2,2]]]}"))); + + ASSERT(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[1,1],[2,2]]," + "[[2,2],[1,1]]]}"))); + ASSERT_FALSE(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[1,1]]]}"))); + ASSERT_FALSE(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[1,1]],[[1,2],[3,4]]]}"))); + ASSERT_FALSE(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[181,1],[2,2]]]}"))); + ASSERT_FALSE(GeoParser::isMultiLine( + fromjson("{'type':'MultiLineString','coordinates':[ [[181,1],[2,-91]]]}"))); + } + + TEST(GeoParser, parseMultiLine) { + mongo::MultiLineWithCRS mls; + + GeoParser::parseMultiLine( + fromjson("{'type':'MultiLine','coordinates':[ [[1,1],[2,2],[3,3]]," + "[[4,5],[6,7]]]}"), + &mls); + + GeoParser::parseMultiLine( + fromjson("{'type':'MultiLine','coordinates':[ [[1,1],[2,2]]," + "[[4,5],[6,7]]]}"), + &mls); + + GeoParser::parseMultiLine( + fromjson("{'type':'MultiLine','coordinates':[ [[1,1],[2,2]]]}"), + &mls); + + GeoParser::parseMultiLine( + fromjson("{'type':'MultiLine','coordinates':[ [[1,1],[2,2]]," + "[[2,2],[1,1]]]}"), + &mls); + } + + TEST(GeoParser, multiPolygon) { + ASSERT(GeoParser::isMultiPolygon( + fromjson("{'type':'MultiPolygon','coordinates':[" + "[[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]]," + "[[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]," + "[[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]" + "]}"))); + ASSERT(GeoParser::isMultiPolygon( + fromjson("{'type':'MultiPolygon','coordinates':[" + "[[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]," + "[[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]" + "]}"))); + } + + TEST(GeoParser, parseMultiPolygon) { + mongo::MultiPolygonWithCRS mp; + GeoParser::parseMultiPolygon( + fromjson("{'type':'MultiPolygon','coordinates':[" + "[[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]]," + "[[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]," + "[[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]" + "]}"), &mp); + } + + TEST(GeoParser, parseGeometryCollection) { + { + mongo::GeometryCollection gc; + BSONObj obj = fromjson( + "{ 'type': 'GeometryCollection', 'geometries': [" + "{ 'type': 'Point','coordinates': [100.0,0.0]}," + "{ 'type': 'LineString', 'coordinates': [ [101.0, 0.0], [102.0, 1.0] ]}" + "]}"); + ASSERT(GeoParser::isGeometryCollection(obj)); + GeoParser::parseGeometryCollection(obj, &gc); + ASSERT_FALSE(gc.supportsContains()); + } + + { + BSONObj obj = fromjson( + "{ 'type': 'GeometryCollection', 'geometries': [" + "{'type':'MultiPolygon','coordinates':[" + "[[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]]," + "[[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]," + "[[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]" + "]}" + "]}"); + + ASSERT(GeoParser::isGeometryCollection(obj)); + mongo::GeometryCollection gc; + GeoParser::parseGeometryCollection(obj, &gc); + ASSERT_TRUE(gc.supportsContains()); + } + { + BSONObj obj = fromjson( + "{ 'type': 'GeometryCollection', 'geometries': [" + "{'type':'Polygon', 'coordinates':[ [[0,0],[0,91],[5,5],[5,0],[0,0]] ]}," + "{'type':'MultiPolygon','coordinates':[" + "[[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]]," + "[[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]," + "[[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]" + "]}" + "]}"); + + ASSERT_FALSE(GeoParser::isGeometryCollection(obj)); + } + + { + BSONObj obj = fromjson( + "{ 'type': 'GeometryCollection', 'geometries': [" + "{'type':'Polygon', 'coordinates':[ [[0,0],[0,5],[5,5],[5,0],[0,0]] ]}," + "{'type':'MultiPolygon','coordinates':[" + "[[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]]," + "[[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]," + "[[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]" + "]}" + "]}"); + + ASSERT(GeoParser::isGeometryCollection(obj)); + mongo::GeometryCollection gc; + GeoParser::parseGeometryCollection(obj, &gc); + ASSERT_TRUE(gc.supportsContains()); + } } } diff --git a/src/mongo/db/geo/geoquery.cpp b/src/mongo/db/geo/geoquery.cpp index 4a5cc6f0127..a25c7f0cb0e 100644 --- a/src/mongo/db/geo/geoquery.cpp +++ b/src/mongo/db/geo/geoquery.cpp @@ -1,131 +1,194 @@ /** -* Copyright (C) 2013 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #include "mongo/db/geo/geoquery.h" +#include "mongo/db/geo/geoconstants.h" + namespace mongo { - bool NearQuery::parseFromGeoNear(const BSONObj &obj, double radius) { - if (obj["near"].eoo()) { return false; } - BSONObj nearObj = obj["near"].embeddedObject(); - // The CRS for the legacy points dictates that distances are in radians. - fromRadians = GeoParser::isLegacyPoint(nearObj); - if (!GeoParser::parsePoint(nearObj, ¢roid)) { return false; } - - if (!obj["maxDistance"].eoo()) { - if (obj["maxDistance"].isNumber()) { - double distArg = obj["maxDistance"].number(); - if (fromRadians) { - maxDistance = distArg * radius; - } else { - maxDistance = distArg; - } - } else { - return false; - } - } - return true; - } + using mongoutils::str::equals; - bool NearQuery::parseFrom(const BSONObj &obj, double radius) { + bool NearQuery::parseLegacyQuery(const BSONObj &obj) { bool hasGeometry = false; - // First, try legacy near. - // Legacy near parsing: t.find({ loc : { $nearSphere: [0,0], $maxDistance: 3 }}) - // Legacy near parsing: t.find({ loc : { $nearSphere: [0,0] }}) - // Legacy near parsing: t.find({ loc : { $near: { someGeoJSONPoint}}) + // First, try legacy near, e.g.: + // t.find({ loc : { $nearSphere: [0,0], $minDistance: 1, $maxDistance: 3 }}) + // t.find({ loc : { $nearSphere: [0,0] }}) + // t.find({ loc : { $near: { someGeoJSONPoint}}) + // t.find({ loc : { $geoNear: { someGeoJSONPoint}}) BSONObjIterator it(obj); while (it.more()) { BSONElement e = it.next(); - bool isNearSphere = mongoutils::str::equals(e.fieldName(), "$nearSphere"); - bool isMaxDistance = mongoutils::str::equals(e.fieldName(), "$maxDistance"); - bool isNear = mongoutils::str::equals(e.fieldName(), "$near") - || mongoutils::str::equals(e.fieldName(), "$geoNear"); - if (isNearSphere || isNear) { + if (equals(e.fieldName(), "$near") || equals(e.fieldName(), "$geoNear") + || equals(e.fieldName(), "$nearSphere")) { if (!e.isABSONObj()) { return false; } BSONObj embeddedObj = e.embeddedObject(); - if (isNearSphere && GeoParser::isPoint(embeddedObj)) { - fromRadians = GeoParser::isLegacyPoint(embeddedObj); - GeoParser::parsePoint(embeddedObj, ¢roid); - hasGeometry = true; - } else if (isNear && GeoParser::isGeoJSONPoint(embeddedObj)) { - GeoParser::parseGeoJSONPoint(embeddedObj, ¢roid); + + if ((GeoParser::isPoint(embeddedObj) && GeoParser::parsePoint(embeddedObj, ¢roid)) + || GeoParser::parsePointWithMaxDistance(embeddedObj, ¢roid, &maxDistance)) { + uassert(18522, "max distance must be non-negative", maxDistance >= 0.0); hasGeometry = true; + isNearSphere = equals(e.fieldName(), "$nearSphere"); } - } else if (isMaxDistance) { + } else if (equals(e.fieldName(), "$minDistance")) { + uassert(16893, "$minDistance must be a number", e.isNumber()); + minDistance = e.Number(); + uassert(16894, "$minDistance must be non-negative", minDistance >= 0.0); + } else if (equals(e.fieldName(), "$maxDistance")) { + uassert(16895, "$maxDistance must be a number", e.isNumber()); maxDistance = e.Number(); + uassert(16896, "$maxDistance must be non-negative", maxDistance >= 0.0); + } else if (equals(e.fieldName(), "$uniqueDocs")) { + warning() << "ignoring deprecated option $uniqueDocs"; } } - if (fromRadians) { - maxDistance *= radius; + // The user-provided point can be flat. We need to make sure that it's in bounds. + if (isNearSphere) { + uassert(17444, + "Legacy point is out of bounds for spherical query", + centroid.flatUpgradedToSphere || (SPHERE == centroid.crs)); } - if (hasGeometry) { return true; } + return hasGeometry; + } - // Next, try "new" near - // New near: t.find({ "geo" : { "$near" : { "$geometry" : pointA, $maxDistance : 20 }}}) - BSONElement e = obj.firstElement(); - if (!e.isABSONObj()) { return false; } + Status NearQuery::parseNewQuery(const BSONObj &obj) { + bool hasGeometry = false; + + BSONObjIterator objIt(obj); + if (!objIt.more()) { + return Status(ErrorCodes::BadValue, "empty geo near query object"); + } + BSONElement e = objIt.next(); + // Just one arg. to $geoNear. + if (objIt.more()) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "geo near accepts just one argument when querying for a GeoJSON " << + "point. Extra field found: " << objIt.next()); + } + + // Parse "new" near: + // t.find({"geo" : {"$near" : {"$geometry": pointA, $minDistance: 1, $maxDistance: 3}}}) + // t.find({"geo" : {"$geoNear" : {"$geometry": pointA, $minDistance: 1, $maxDistance: 3}}}) + if (!e.isABSONObj()) { + return Status(ErrorCodes::BadValue, "geo near query argument is not an object"); + } BSONObj::MatchType matchType = static_cast(e.getGtLtOp()); - if (BSONObj::opNEAR != matchType) { return false; } + if (BSONObj::opNEAR != matchType) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "invalid geo near query operator: " << e.fieldName()); + } - // Restart it. - it = BSONObjIterator(e.embeddedObject()); + // Iterate over the argument. + BSONObjIterator it(e.embeddedObject()); while (it.more()) { BSONElement e = it.next(); - if (mongoutils::str::equals(e.fieldName(), "$geometry")) { + if (equals(e.fieldName(), "$geometry")) { if (e.isABSONObj()) { BSONObj embeddedObj = e.embeddedObject(); + uassert(16885, "$near requires a point, given " + embeddedObj.toString(), + GeoParser::isPoint(embeddedObj)); + if (!GeoParser::parsePoint(embeddedObj, ¢roid)) { + return Status(ErrorCodes::BadValue, mongoutils::str::stream() << + "invalid point in geo near query $geometry argument: " << + embeddedObj); + } uassert(16681, "$near requires geojson point, given " + embeddedObj.toString(), - GeoParser::isGeoJSONPoint(embeddedObj)); - GeoParser::parseGeoJSONPoint(embeddedObj, ¢roid); + (SPHERE == centroid.crs)); hasGeometry = true; } - } else if (mongoutils::str::equals(e.fieldName(), "$maxDistance")) { - if (e.isNumber()) { - maxDistance = e.Number(); - } + } else if (equals(e.fieldName(), "$minDistance")) { + uassert(16897, "$minDistance must be a number", e.isNumber()); + minDistance = e.Number(); + uassert(16898, "$minDistance must be non-negative", minDistance >= 0.0); + } else if (equals(e.fieldName(), "$maxDistance")) { + uassert(16899, "$maxDistance must be a number", e.isNumber()); + maxDistance = e.Number(); + uassert(16900, "$maxDistance must be non-negative", maxDistance >= 0.0); } } - return hasGeometry; + + if (!hasGeometry) { + return Status(ErrorCodes::BadValue, "$geometry is required for geo near query"); + } + + return Status::OK(); + } + + + Status NearQuery::parseFrom(const BSONObj &obj) { + if (parseLegacyQuery(obj)) { + return Status::OK(); + } + // Clear out any half-baked data. + minDistance = 0; + isNearSphere = false; + maxDistance = std::numeric_limits::max(); + centroid = PointWithCRS(); + // And try parsing new format. + return parseNewQuery(obj); } bool GeoQuery::parseLegacyQuery(const BSONObj &obj) { - // Legacy within parsing #1: t.find({ loc : [0,0] }) This is should be - // point-only. We tag it as intersect and limit $within to - // space-containing geometry. - if (GeoParser::isPoint(obj) && geoContainer.parseFrom(obj)) { - predicate = GeoQuery::INTERSECT; - return true; + // The only legacy syntax is {$within: {.....}} + BSONObjIterator outerIt(obj); + if (!outerIt.more()) { return false; } + BSONElement withinElt = outerIt.next(); + if (outerIt.more()) { return false; } + if (!withinElt.isABSONObj()) { return false; } + if (!equals(withinElt.fieldName(), "$within") && !equals(withinElt.fieldName(), "$geoWithin")) { + return false; } + BSONObj withinObj = withinElt.embeddedObject(); - BSONObjIterator it(obj); - if (!it.more()) { return false; } - BSONElement e = it.next(); - if (!e.isABSONObj()) { return false; } - BSONObj embeddedObj = e.embeddedObject(); - // Legacy within #2 : t.find({ loc : { $within : { $box/etc : ... - bool contains = (BSONObj::opWITHIN == static_cast(e.getGtLtOp())); - if (contains && geoContainer.parseFrom(embeddedObj)) { - predicate = GeoQuery::WITHIN; - return true; + bool hasGeometry = false; + + BSONObjIterator withinIt(withinObj); + while (withinIt.more()) { + BSONElement elt = withinIt.next(); + if (equals(elt.fieldName(), "$uniqueDocs")) { + warning() << "deprecated $uniqueDocs option: " << obj.toString() << endl; + // return false; + } + else if (elt.isABSONObj()) { + hasGeometry = geoContainer.parseFrom(elt.wrap()); + } + else { + warning() << "bad geo query: " << obj.toString() << endl; + return false; + } } - return false; + predicate = GeoQuery::WITHIN; + + return hasGeometry; } bool GeoQuery::parseNewQuery(const BSONObj &obj) { @@ -188,6 +251,40 @@ namespace mongo { return geoContainer.getRegion(); } + bool GeoQuery::hasS2Region() const { + return geoContainer.hasS2Region(); + } + + bool GeometryContainer::isSimpleContainer() const { + return NULL != _point || NULL != _line || NULL != _polygon; + } + + bool GeometryContainer::supportsContains() const { + return NULL != _polygon + || NULL != _cap + || NULL != _multiPolygon + || (NULL != _geometryCollection + && (_geometryCollection->polygons.vector().size() > 0 + || _geometryCollection->multiPolygons.vector().size() > 0)); + } + + bool GeometryContainer::hasS2Region() const { + return (NULL != _point && (_point->crs == SPHERE || _point->flatUpgradedToSphere)) + || NULL != _line + || (NULL != _polygon && _polygon->crs == SPHERE) + || (NULL != _cap && _cap->crs == SPHERE) + || NULL != _multiPoint + || NULL != _multiLine + || NULL != _multiPolygon + || NULL != _geometryCollection; + } + + bool GeometryContainer::hasFlatRegion() const { + return (NULL != _polygon && _polygon->crs == FLAT) + || (NULL != _cap && _cap->crs == FLAT) + || NULL != _box; + } + bool GeoQuery::satisfiesPredicate(const GeometryContainer &otherContainer) const { verify(predicate == WITHIN || predicate == INTERSECT); @@ -199,154 +296,650 @@ namespace mongo { } bool GeometryContainer::contains(const GeometryContainer& otherContainer) const { - if (NULL != _oldPolygon) { - if (NULL == otherContainer._oldPoint) { return false; } - return _oldPolygon->contains(*otherContainer._oldPoint); + // First let's deal with the case where we are FLAT. + if (NULL != _polygon && (FLAT == _polygon->crs)) { + if (NULL == otherContainer._point) { return false; } + return _polygon->oldPolygon.contains(otherContainer._point->oldPoint); } - if (NULL != _oldBox) { - if (NULL == otherContainer._oldPoint) { return false; } - return _oldBox->inside(*otherContainer._oldPoint); - } - if (NULL != _cap) { - if (NULL == otherContainer._cell) { return false; } - return _cap->MayIntersect(*otherContainer._cell); + + if (NULL != _box) { + verify(FLAT == _box->crs); + if (NULL == otherContainer._point) { return false; } + return _box->box.inside(otherContainer._point->oldPoint); } - if (NULL != _oldCircle) { - if (NULL == otherContainer._oldPoint) { return false; } + + if (NULL != _cap && (FLAT == _cap->crs)) { + if (NULL == otherContainer._point) { return false; } // Let's be as consistent epsilon-wise as we can with the '2d' indextype. - return distanceWithin(_oldCircle->center, *otherContainer._oldPoint, - _oldCircle->radius); - } - if (NULL != _polygon) { - if (NULL != otherContainer._cell) { - // Intersecting a point is containing a point. Hooray! - return _polygon->MayIntersect(*otherContainer._cell); - } else if (NULL != otherContainer._line) { - // Kind of a mess. We get a function for clipping the line to the - // polygon. We do this and make sure the line is the same as the - // line we're clipping against. - vector clipped; - _polygon->IntersectWithPolyline(otherContainer._line.get(), &clipped); - if (1 != clipped.size()) { return false; } - // If the line is entirely contained within the polygon, we should be - // getting it back verbatim, so really there should be no error. - bool ret = clipped[0]->NearlyCoversPolyline(*otherContainer._line, - S1Angle::Degrees(1e-10)); - for (size_t i = 0; i < clipped.size(); ++i) delete clipped[i]; - return ret; - } else if (NULL != otherContainer._polygon) { - return _polygon->Contains(otherContainer._polygon.get()); - } else { return false; } - } - // Containment only works for polygons/boxes/circles. + return distanceWithin(_cap->circle.center, otherContainer._point->oldPoint, + _cap->circle.radius); + } + + // Now we deal with all the SPHERE stuff. + + // Iterate over the other thing and see if we contain it all. + if (NULL != otherContainer._point) { + // The point must be valid lng, lat if it was old-style. + if (FLAT == otherContainer._point->crs + && !otherContainer._point->flatUpgradedToSphere) { + return false; + } + return contains(otherContainer._point->cell, otherContainer._point->point); + } + + if (NULL != otherContainer._line) { + return contains(otherContainer._line->line); + } + + if (NULL != otherContainer._polygon) { + return contains(otherContainer._polygon->polygon); + } + + if (NULL != otherContainer._multiPoint) { + for (size_t i = 0; i < otherContainer._multiPoint->points.size(); ++i) { + if (!contains(otherContainer._multiPoint->cells[i], + otherContainer._multiPoint->points[i])) { + return false; + } + } + return true; + } + + if (NULL != otherContainer._multiLine) { + const vector& lines = otherContainer._multiLine->lines.vector(); + for (size_t i = 0; i < lines.size(); ++i) { + if (!contains(*lines[i])) { return false; } + } + return true; + } + + if (NULL != otherContainer._multiPolygon) { + const vector& polys = otherContainer._multiPolygon->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (!contains(*polys[i])) { return false; } + } + return true; + } + + if (NULL != otherContainer._geometryCollection) { + GeometryCollection& c = *otherContainer._geometryCollection; + + for (size_t i = 0; i < c.points.size(); ++i) { + if (!contains(c.points[i].cell, c.points[i].point)) { + return false; + } + } + + const vector& lines = c.lines.vector(); + for (size_t i = 0; i < lines.size(); ++i) { + if (!contains(lines[i]->line)) { return false; } + } + + const vector& polys = c.polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (!contains(polys[i]->polygon)) { return false; } + } + + const vector& multipoints = c.multiPoints.vector(); + for (size_t i = 0; i < multipoints.size(); ++i) { + MultiPointWithCRS* mp = multipoints[i]; + for (size_t j = 0; j < mp->points.size(); ++j) { + if (!contains(mp->cells[j], mp->points[j])) { return false; } + } + } + + const vector& multilines = c.multiLines.vector(); + for (size_t i = 0; i < multilines.size(); ++i) { + const vector& lines = multilines[i]->lines.vector(); + for (size_t j = 0; j < lines.size(); ++j) { + if (!contains(*lines[j])) { return false; } + } + } + + const vector& multipolys = c.multiPolygons.vector(); + for (size_t i = 0; i < multipolys.size(); ++i) { + const vector& polys = multipolys[i]->polygons.vector(); + for (size_t j = 0; j < polys.size(); ++j) { + if (!contains(*polys[j])) { return false; } + } + } + + return true; + } + + return false; + } + + bool containsPoint(const S2Polygon& poly, const S2Cell& otherCell, const S2Point& otherPoint) { + // This is much faster for actual containment checking. + if (poly.Contains(otherPoint)) { return true; } + // This is slower but contains edges/vertices. + return poly.MayIntersect(otherCell); + } + + bool GeometryContainer::contains(const S2Cell& otherCell, const S2Point& otherPoint) const { + if (NULL != _polygon && (_polygon->crs == SPHERE)) { + return containsPoint(_polygon->polygon, otherCell, otherPoint); + } + + if (NULL != _cap && (_cap->crs == SPHERE)) { + return _cap->cap.MayIntersect(otherCell); + } + + if (NULL != _multiPolygon) { + const vector& polys = _multiPolygon->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (containsPoint(*polys[i], otherCell, otherPoint)) { return true; } + } + } + + if (NULL != _geometryCollection) { + const vector& polys = _geometryCollection->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (containsPoint(polys[i]->polygon, otherCell, otherPoint)) { return true; } + } + + const vector& multipolys =_geometryCollection->multiPolygons.vector(); + for (size_t i = 0; i < multipolys.size(); ++i) { + const vector& innerpolys = multipolys[i]->polygons.vector(); + for (size_t j = 0; j < innerpolys.size(); ++j) { + if (containsPoint(*innerpolys[j], otherCell, otherPoint)) { return true; } + } + } + } + + return false; + } + + bool containsLine(const S2Polygon& poly, const S2Polyline& otherLine) { + // Kind of a mess. We get a function for clipping the line to the + // polygon. We do this and make sure the line is the same as the + // line we're clipping against. + OwnedPointerVector clippedOwned; + vector& clipped = clippedOwned.mutableVector(); + + poly.IntersectWithPolyline(&otherLine, &clipped); + if (1 != clipped.size()) { return false; } + + // If the line is entirely contained within the polygon, we should be + // getting it back verbatim, so really there should be no error. + bool ret = clipped[0]->NearlyCoversPolyline(otherLine, + S1Angle::Degrees(1e-10)); + + return ret; + } + + bool GeometryContainer::contains(const S2Polyline& otherLine) const { + if (NULL != _polygon && (_polygon->crs == SPHERE)) { + return containsLine(_polygon->polygon, otherLine); + } + + if (NULL != _multiPolygon) { + const vector& polys = _multiPolygon->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (containsLine(*polys[i], otherLine)) { return true; } + } + } + + if (NULL != _geometryCollection) { + const vector& polys = _geometryCollection->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (containsLine(polys[i]->polygon, otherLine)) { return true; } + } + + const vector& multipolys =_geometryCollection->multiPolygons.vector(); + for (size_t i = 0; i < multipolys.size(); ++i) { + const vector& innerpolys = multipolys[i]->polygons.vector(); + for (size_t j = 0; j < innerpolys.size(); ++j) { + if (containsLine(*innerpolys[j], otherLine)) { return true; } + } + } + } + + return false; + } + + bool containsPolygon(const S2Polygon& poly, const S2Polygon& otherPoly) { + return poly.Contains(&otherPoly); + } + + bool GeometryContainer::contains(const S2Polygon& otherPolygon) const { + if (NULL != _polygon && (_polygon->crs == SPHERE)) { + return containsPolygon(_polygon->polygon, otherPolygon); + } + + if (NULL != _multiPolygon) { + const vector& polys = _multiPolygon->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (containsPolygon(*polys[i], otherPolygon)) { return true; } + } + } + + if (NULL != _geometryCollection) { + const vector& polys = _geometryCollection->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (containsPolygon(polys[i]->polygon, otherPolygon)) { return true; } + } + + const vector& multipolys =_geometryCollection->multiPolygons.vector(); + for (size_t i = 0; i < multipolys.size(); ++i) { + const vector& innerpolys = multipolys[i]->polygons.vector(); + for (size_t j = 0; j < innerpolys.size(); ++j) { + if (containsPolygon(*innerpolys[j], otherPolygon)) { return true; } + } + } + } + return false; } bool GeometryContainer::intersects(const GeometryContainer& otherContainer) const { - if (NULL != otherContainer._cell) { - return intersects(*otherContainer._cell); + if (NULL != otherContainer._point) { + // The point must be valid lng, lat if it was old-style. + if (FLAT == otherContainer._point->crs + && !otherContainer._point->flatUpgradedToSphere) { + return false; + } + return intersects(otherContainer._point->cell); } else if (NULL != otherContainer._line) { - return intersects(*otherContainer._line); + return intersects(otherContainer._line->line); } else if (NULL != otherContainer._polygon) { - return intersects(*otherContainer._polygon); - } else { - return false; + if (SPHERE != otherContainer._polygon->crs) { return false; } + return intersects(otherContainer._polygon->polygon); + } else if (NULL != otherContainer._multiPoint) { + return intersects(*otherContainer._multiPoint); + } else if (NULL != otherContainer._multiLine) { + return intersects(*otherContainer._multiLine); + } else if (NULL != otherContainer._multiPolygon) { + return intersects(*otherContainer._multiPolygon); + } else if (NULL != otherContainer._geometryCollection) { + const GeometryCollection& c = *otherContainer._geometryCollection; + + for (size_t i = 0; i < c.points.size(); ++i) { + if (intersects(c.points[i].cell)) { return true; } + } + + for (size_t i = 0; i < c.polygons.vector().size(); ++i) { + if (intersects(c.polygons.vector()[i]->polygon)) { return true; } + } + + for (size_t i = 0; i < c.lines.vector().size(); ++i) { + if (intersects(c.lines.vector()[i]->line)) { return true; } + } + + for (size_t i = 0; i < c.multiPolygons.vector().size(); ++i) { + if (intersects(*c.multiPolygons.vector()[i])) { return true; } + } + + for (size_t i = 0; i < c.multiLines.vector().size(); ++i) { + if (intersects(*c.multiLines.vector()[i])) { return true; } + } + + for (size_t i = 0; i < c.multiPoints.vector().size(); ++i) { + if (intersects(*c.multiPoints.vector()[i])) { return true; } + } + } + + return false; + } + + bool GeometryContainer::intersects(const MultiPointWithCRS& otherMultiPoint) const { + for (size_t i = 0; i < otherMultiPoint.cells.size(); ++i) { + if (intersects(otherMultiPoint.cells[i])) { return true; } + } + return false; + } + + bool GeometryContainer::intersects(const MultiLineWithCRS& otherMultiLine) const { + for (size_t i = 0; i < otherMultiLine.lines.vector().size(); ++i) { + if (intersects(*otherMultiLine.lines.vector()[i])) { return true; } } + return false; + } + + bool GeometryContainer::intersects(const MultiPolygonWithCRS& otherMultiPolygon) const { + for (size_t i = 0; i < otherMultiPolygon.polygons.vector().size(); ++i) { + if (intersects(*otherMultiPolygon.polygons.vector()[i])) { return true; } + } + return false; } // Does this (GeometryContainer) intersect the provided data? bool GeometryContainer::intersects(const S2Cell &otherPoint) const { - if (NULL != _cell) { - return _cell->MayIntersect(otherPoint); + if (NULL != _point) { + // The point must be valid lng, lat if it was old-style. + if (FLAT == _point->crs && !_point->flatUpgradedToSphere) { + return false; + } + return _point->cell.MayIntersect(otherPoint); } else if (NULL != _line) { - return _line->MayIntersect(otherPoint); - } else { - verify(NULL != _polygon); - return _polygon->MayIntersect(otherPoint); + return _line->line.MayIntersect(otherPoint); + } else if (NULL != _polygon) { + return _polygon->polygon.MayIntersect(otherPoint); + } else if (NULL != _multiPoint) { + const vector& cells = _multiPoint->cells; + for (size_t i = 0; i < cells.size(); ++i) { + if (cells[i].MayIntersect(otherPoint)) { return true; } + } + } else if (NULL != _multiLine) { + const vector& lines = _multiLine->lines.vector(); + for (size_t i = 0; i < lines.size(); ++i) { + if (lines[i]->MayIntersect(otherPoint)) { return true; } + } + } else if (NULL != _multiPolygon) { + const vector& polys = _multiPolygon->polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + if (polys[i]->MayIntersect(otherPoint)) { return true; } + } + } else if (NULL != _geometryCollection) { + const GeometryCollection& c = *_geometryCollection; + + for (size_t i = 0; i < c.points.size(); ++i) { + if (c.points[i].cell.MayIntersect(otherPoint)) { return true; } + } + + for (size_t i = 0; i < c.polygons.vector().size(); ++i) { + if (c.polygons.vector()[i]->polygon.MayIntersect(otherPoint)) { return true; } + } + + for (size_t i = 0; i < c.lines.vector().size(); ++i) { + if (c.lines.vector()[i]->line.MayIntersect(otherPoint)) { return true; } + } + + for (size_t i = 0; i < c.multiPolygons.vector().size(); ++i) { + const vector& innerPolys = + c.multiPolygons.vector()[i]->polygons.vector(); + for (size_t j = 0; j < innerPolys.size(); ++j) { + if (innerPolys[j]->MayIntersect(otherPoint)) { return true; } + } + } + + for (size_t i = 0; i < c.multiLines.vector().size(); ++i) { + const vector& innerLines = + c.multiLines.vector()[i]->lines.vector(); + for (size_t j = 0; j < innerLines.size(); ++j) { + if (innerLines[j]->MayIntersect(otherPoint)) { return true; } + } + } + + for (size_t i = 0; i < c.multiPoints.vector().size(); ++i) { + const vector& innerCells = c.multiPoints.vector()[i]->cells; + for (size_t j = 0; j < innerCells.size(); ++j) { + if (innerCells[j].MayIntersect(otherPoint)) { return true; } + } + } } + + return false; + } + + bool polygonLineIntersection(const S2Polyline& line, const S2Polygon& poly) { + // TODO(hk): modify s2 library to just let us know if it intersected + // rather than returning all this. + vector clipped; + poly.IntersectWithPolyline(&line, &clipped); + bool ret = clipped.size() > 0; + for (size_t i = 0; i < clipped.size(); ++i) delete clipped[i]; + return ret; } bool GeometryContainer::intersects(const S2Polyline& otherLine) const { - if (NULL != _cell) { - return otherLine.MayIntersect(*_cell); + if (NULL != _point) { + // The point must be valid lng, lat if it was old-style. + if (FLAT == _point->crs && !_point->flatUpgradedToSphere) { + return false; + } + return otherLine.MayIntersect(_point->cell); } else if (NULL != _line) { - return otherLine.Intersects(_line.get()); - } else { - verify(NULL != _polygon); - // TODO(hk): modify s2 library to just let us know if it intersected - // rather than returning all this. - vector clipped; - _polygon->IntersectWithPolyline(&otherLine, &clipped); - bool ret = clipped.size() > 0; - for (size_t i = 0; i < clipped.size(); ++i) delete clipped[i]; - return ret; + return otherLine.Intersects(&_line->line); + } else if (NULL != _polygon && (_polygon->crs == SPHERE)) { + return polygonLineIntersection(otherLine, _polygon->polygon); + } else if (NULL != _multiPoint) { + for (size_t i = 0; i < _multiPoint->cells.size(); ++i) { + if (otherLine.MayIntersect(_multiPoint->cells[i])) { return true; } + } + } else if (NULL != _multiLine) { + for (size_t i = 0; i < _multiLine->lines.vector().size(); ++i) { + if (otherLine.Intersects(_multiLine->lines.vector()[i])) { + return true; + } + } + } else if (NULL != _multiPolygon) { + for (size_t i = 0; i < _multiPolygon->polygons.vector().size(); ++i) { + if (polygonLineIntersection(otherLine, *_multiPolygon->polygons.vector()[i])) { + return true; + } + } + } else if (NULL != _geometryCollection) { + const GeometryCollection& c = *_geometryCollection; + + for (size_t i = 0; i < c.points.size(); ++i) { + if (otherLine.MayIntersect(c.points[i].cell)) { return true; } + } + + for (size_t i = 0; i < c.polygons.vector().size(); ++i) { + if (polygonLineIntersection(otherLine, c.polygons.vector()[i]->polygon)) { + return true; + } + } + + for (size_t i = 0; i < c.lines.vector().size(); ++i) { + if (c.lines.vector()[i]->line.Intersects(&otherLine)) { return true; } + } + + for (size_t i = 0; i < c.multiPolygons.vector().size(); ++i) { + const vector& innerPolys = + c.multiPolygons.vector()[i]->polygons.vector(); + for (size_t j = 0; j < innerPolys.size(); ++j) { + if (polygonLineIntersection(otherLine, *innerPolys[j])) { + return true; + } + } + } + + for (size_t i = 0; i < c.multiLines.vector().size(); ++i) { + const vector& innerLines = + c.multiLines.vector()[i]->lines.vector(); + for (size_t j = 0; j < innerLines.size(); ++j) { + if (innerLines[j]->Intersects(&otherLine)) { return true; } + } + } + + for (size_t i = 0; i < c.multiPoints.vector().size(); ++i) { + const vector& innerCells = c.multiPoints.vector()[i]->cells; + for (size_t j = 0; j < innerCells.size(); ++j) { + if (otherLine.MayIntersect(innerCells[j])) { return true; } + } + } } + + return false; } + // Does 'this' intersect with the provided polygon? bool GeometryContainer::intersects(const S2Polygon& otherPolygon) const { - if (NULL != _cell) { - return otherPolygon.MayIntersect(*_cell); + if (NULL != _point) { + // The point must be valid lng, lat if it was old-style. + if (FLAT == _point->crs && !_point->flatUpgradedToSphere) { + return false; + } + return otherPolygon.MayIntersect(_point->cell); } else if (NULL != _line) { - // TODO(hk): modify s2 library to just let us know if it intersected - // rather than returning all this. - vector clipped; - otherPolygon.IntersectWithPolyline(_line.get(), &clipped); - bool ret = clipped.size() > 0; - for (size_t i = 0; i < clipped.size(); ++i) delete clipped[i]; - return ret; - } else { - verify(NULL != _polygon); - return otherPolygon.Intersects(_polygon.get()); + return polygonLineIntersection(_line->line, otherPolygon); + } else if (NULL != _polygon) { + return otherPolygon.Intersects(&_polygon->polygon); + } else if (NULL != _multiPoint) { + for (size_t i = 0; i < _multiPoint->cells.size(); ++i) { + if (otherPolygon.MayIntersect(_multiPoint->cells[i])) { return true; } + } + } else if (NULL != _multiLine) { + for (size_t i = 0; i < _multiLine->lines.vector().size(); ++i) { + if (polygonLineIntersection(*_multiLine->lines.vector()[i], otherPolygon)) { + return true; + } + } + } else if (NULL != _multiPolygon) { + for (size_t i = 0; i < _multiPolygon->polygons.vector().size(); ++i) { + if (otherPolygon.Intersects(_multiPolygon->polygons.vector()[i])) { + return true; + } + } + } else if (NULL != _geometryCollection) { + const GeometryCollection& c = *_geometryCollection; + + for (size_t i = 0; i < c.points.size(); ++i) { + if (otherPolygon.MayIntersect(c.points[i].cell)) { return true; } + } + + for (size_t i = 0; i < c.polygons.vector().size(); ++i) { + if (otherPolygon.Intersects(&c.polygons.vector()[i]->polygon)) { + return true; + } + } + + for (size_t i = 0; i < c.lines.vector().size(); ++i) { + if (polygonLineIntersection(c.lines.vector()[i]->line, otherPolygon)) { + return true; + } + } + + for (size_t i = 0; i < c.multiPolygons.vector().size(); ++i) { + const vector& innerPolys = + c.multiPolygons.vector()[i]->polygons.vector(); + for (size_t j = 0; j < innerPolys.size(); ++j) { + if (otherPolygon.Intersects(innerPolys[j])) { + return true; + } + } + } + + for (size_t i = 0; i < c.multiLines.vector().size(); ++i) { + const vector& innerLines = + c.multiLines.vector()[i]->lines.vector(); + for (size_t j = 0; j < innerLines.size(); ++j) { + if (polygonLineIntersection(*innerLines[j], otherPolygon)) { + return true; + } + } + } + + for (size_t i = 0; i < c.multiPoints.vector().size(); ++i) { + const vector& innerCells = c.multiPoints.vector()[i]->cells; + for (size_t j = 0; j < innerCells.size(); ++j) { + if (otherPolygon.MayIntersect(innerCells[j])) { + return true; + } + } + } } + + return false; } bool GeometryContainer::parseFrom(const BSONObj& obj) { - // Free up any pointers we might have left over from previous parses. *this = GeometryContainer(); - if (GeoParser::isGeoJSONPolygon(obj)) { + + if (GeoParser::isPolygon(obj)) { // We can't really pass these things around willy-nilly except by ptr. - _polygon.reset(new S2Polygon()); - GeoParser::parseGeoJSONPolygon(obj, _polygon.get()); + _polygon.reset(new PolygonWithCRS()); + if (!GeoParser::parsePolygon(obj, _polygon.get())) { return false; } } else if (GeoParser::isPoint(obj)) { - _cell.reset(new S2Cell()); - GeoParser::parsePoint(obj, _cell.get()); - _oldPoint.reset(new Point()); - GeoParser::parsePoint(obj, _oldPoint.get()); - } else if (GeoParser::isLineString(obj)) { - _line.reset(new S2Polyline()); - GeoParser::parseLineString(obj, _line.get()); - } else if (GeoParser::isLegacyBox(obj)) { - _oldBox.reset(new Box()); - GeoParser::parseLegacyBox(obj, _oldBox.get()); - } else if (GeoParser::isLegacyPolygon(obj)) { - _oldPolygon.reset(new Polygon()); - GeoParser::parseLegacyPolygon(obj, _oldPolygon.get()); - } else if (GeoParser::isLegacyCenter(obj)) { - _oldCircle.reset(new Circle()); - GeoParser::parseLegacyCenter(obj, _oldCircle.get()); - } else if (GeoParser::isLegacyCenterSphere(obj)) { - _cap.reset(new S2Cap()); - GeoParser::parseLegacyCenterSphere(obj, _cap.get()); + _point.reset(new PointWithCRS()); + if (!GeoParser::parsePoint(obj, _point.get())) { return false; } + } else if (GeoParser::isLine(obj)) { + _line.reset(new LineWithCRS()); + if (!GeoParser::parseLine(obj, _line.get())) { return false; } + } else if (GeoParser::isBox(obj)) { + _box.reset(new BoxWithCRS()); + if (!GeoParser::parseBox(obj, _box.get())) { return false; } + } else if (GeoParser::isCap(obj)) { + _cap.reset(new CapWithCRS()); + if (!GeoParser::parseCap(obj, _cap.get())) { return false; } + } else if (GeoParser::isMultiPoint(obj)) { + _multiPoint.reset(new MultiPointWithCRS()); + if (!GeoParser::parseMultiPoint(obj, _multiPoint.get())) { return false; } + _region.reset(new S2RegionUnion()); + for (size_t i = 0; i < _multiPoint->cells.size(); ++i) { + _region->Add(&_multiPoint->cells[i]); + } + } else if (GeoParser::isMultiLine(obj)) { + _multiLine.reset(new MultiLineWithCRS()); + if (!GeoParser::parseMultiLine(obj, _multiLine.get())) { return false; } + _region.reset(new S2RegionUnion()); + for (size_t i = 0; i < _multiLine->lines.vector().size(); ++i) { + _region->Add(_multiLine->lines.vector()[i]); + } + } else if (GeoParser::isMultiPolygon(obj)) { + _multiPolygon.reset(new MultiPolygonWithCRS()); + if (!GeoParser::parseMultiPolygon(obj, _multiPolygon.get())) { return false; } + _region.reset(new S2RegionUnion()); + for (size_t i = 0; i < _multiPolygon->polygons.vector().size(); ++i) { + _region->Add(_multiPolygon->polygons.vector()[i]); + } + } else if (GeoParser::isGeometryCollection(obj)) { + _geometryCollection.reset(new GeometryCollection()); + if (!GeoParser::parseGeometryCollection(obj, _geometryCollection.get())) { + return false; + } + _region.reset(new S2RegionUnion()); + for (size_t i = 0; i < _geometryCollection->points.size(); ++i) { + _region->Add(&_geometryCollection->points[i].cell); + } + for (size_t i = 0; i < _geometryCollection->lines.vector().size(); ++i) { + _region->Add(&_geometryCollection->lines.vector()[i]->line); + } + for (size_t i = 0; i < _geometryCollection->polygons.vector().size(); ++i) { + _region->Add(&_geometryCollection->polygons.vector()[i]->polygon); + } + for (size_t i = 0; i < _geometryCollection->multiPoints.vector().size(); ++i) { + MultiPointWithCRS* multiPoint = _geometryCollection->multiPoints.vector()[i]; + for (size_t j = 0; j < multiPoint->cells.size(); ++j) { + _region->Add(&multiPoint->cells[j]); + } + } + for (size_t i = 0; i < _geometryCollection->multiLines.vector().size(); ++i) { + const MultiLineWithCRS* multiLine = _geometryCollection->multiLines.vector()[i]; + for (size_t j = 0; j < multiLine->lines.vector().size(); ++j) { + _region->Add(multiLine->lines.vector()[j]); + } + } + for (size_t i = 0; i < _geometryCollection->multiPolygons.vector().size(); ++i) { + const MultiPolygonWithCRS* multiPolygon = + _geometryCollection->multiPolygons.vector()[i]; + for (size_t j = 0; j < multiPolygon->polygons.vector().size(); ++j) { + _region->Add(multiPolygon->polygons.vector()[j]); + } + } } else { return false; } - return true; - } - bool GeoQuery::hasS2Region() const { - return geoContainer.hasS2Region(); + return true; } const S2Region& GeometryContainer::getRegion() const { - if (NULL != _cell) { - return *_cell; + if (NULL != _point) { + // _point->crs might be FLAT but we "upgrade" it for free if it was in bounds. + if (FLAT == _point->crs) { + verify(_point->flatUpgradedToSphere); + } + return _point->cell; } else if (NULL != _line) { - return *_line; - } else if (NULL != _cap) { - return *_cap; + return _line->line; + } else if (NULL != _cap && SPHERE == _cap->crs) { + return _cap->cap; + } else if (NULL != _multiPoint) { + return *_region; + } else if (NULL != _multiLine) { + return *_region; + } else if (NULL != _multiPolygon) { + return *_region; + } else if (NULL != _geometryCollection) { + return *_region; } else { verify(NULL != _polygon); - return *_polygon; + verify(SPHERE == _polygon->crs); + return _polygon->polygon; } } } // namespace mongo diff --git a/src/mongo/db/geo/geoquery.h b/src/mongo/db/geo/geoquery.h index 2933cb18bba..79a7c321673 100644 --- a/src/mongo/db/geo/geoquery.h +++ b/src/mongo/db/geo/geoquery.h @@ -12,80 +12,141 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ +#pragma once + #include "mongo/db/geo/geoparser.h" +#include "mongo/db/geo/shapes.h" #include "mongo/util/mongoutils/str.h" -#include "third_party/s2/s2.h" -#include "third_party/s2/s2cap.h" -#include "third_party/s2/s2regioncoverer.h" -#include "third_party/s2/s2cell.h" -#include "third_party/s2/s2polyline.h" -#include "third_party/s2/s2polygon.h" -#include "third_party/s2/s2regioncoverer.h" - -#pragma once +#include "third_party/s2/s2regionunion.h" namespace mongo { + class GeometryContainer { public: bool parseFrom(const BSONObj &obj); - // Does we intersect the provided data? Sadly there is no common good - // way to check this, so we do different things for all pairs of - // geometry_of(query,data). + /** + * Is the geometry any of {Point, Line, Polygon}? + */ + bool isSimpleContainer() const; + + /** + * To check intersection, we iterate over the otherContainer's geometries, checking each + * geometry to see if we intersect it. If we intersect one geometry, we intersect the + * entire other container. + */ bool intersects(const GeometryContainer& otherContainer) const; - bool intersects(const S2Cell& otherPoint) const; - bool intersects(const S2Polyline& otherLine) const; - bool intersects(const S2Polygon& otherPolygon) const; - // And, within. + + /** + * To check containment, we iterate over the otherContainer's geometries. If we don't + * contain any sub-geometry of the otherContainer, the otherContainer is not contained + * within us. If each sub-geometry of the otherContainer is contained within us, we contain + * the entire otherContainer. + */ bool contains(const GeometryContainer& otherContainer) const; - bool supportsContains() const { - return NULL != _polygon.get() - || NULL != _cap.get() - || NULL != _oldPolygon.get() - || NULL != _oldCircle.get(); - } + /** + * Only polygons (and aggregate types thereof) support contains. + */ + bool supportsContains() const; - bool hasS2Region() const { - return NULL != _cell - || NULL != _line - || NULL != _polygon - || NULL != _cap; - } + bool hasS2Region() const; + bool hasFlatRegion() const; // Used by s2cursor only to generate a covering of the query object. // One region is not NULL and this returns it. const S2Region& getRegion() const; - private: + // XXX FIXME + // private: + // Does 'this' intersect with the provided type? + bool intersects(const S2Cell& otherPoint) const; + bool intersects(const S2Polyline& otherLine) const; + bool intersects(const S2Polygon& otherPolygon) const; + // These three just iterate over the geometries and call the 3 methods above. + bool intersects(const MultiPointWithCRS& otherMultiPoint) const; + bool intersects(const MultiLineWithCRS& otherMultiLine) const; + bool intersects(const MultiPolygonWithCRS& otherMultiPolygon) const; + + // Used when 'this' has a polygon somewhere, either in _polygon or _multiPolygon or + // _geometryCollection. + bool contains(const S2Cell& otherCell, const S2Point& otherPoint) const; + bool contains(const S2Polyline& otherLine) const; + bool contains(const S2Polygon& otherPolygon) const; + // Only one of these shared_ptrs should be non-NULL. S2Region is a // superclass but it only supports testing against S2Cells. We need // the most specific class we can get. - shared_ptr _cell; - shared_ptr _line; - shared_ptr _polygon; - shared_ptr _cap; - // Legacy shapes. - shared_ptr _oldPolygon; - shared_ptr _oldBox; - shared_ptr _oldCircle; - shared_ptr _oldPoint; + shared_ptr _point; + shared_ptr _line; + shared_ptr _polygon; + shared_ptr _cap; + shared_ptr _multiPoint; + shared_ptr _multiLine; + shared_ptr _multiPolygon; + shared_ptr _geometryCollection; + shared_ptr _box; + + shared_ptr _region; }; + // TODO: Make a struct, turn parse stuff into something like + // static Status parseNearQuery(const BSONObj& obj, NearQuery** out); class NearQuery { public: - NearQuery() : maxDistance(std::numeric_limits::max()), fromRadians(false) {} - NearQuery(const string& f) : field(f), maxDistance(std::numeric_limits::max()), - fromRadians(false) {} - bool parseFrom(const BSONObj &obj, double radius); - bool parseFromGeoNear(const BSONObj &obj, double radius); + NearQuery() + : minDistance(0), + maxDistance(std::numeric_limits::max()), + isNearSphere(false) { } + + NearQuery(const string& f) + : field(f), + minDistance(0), + maxDistance(std::numeric_limits::max()), + isNearSphere(false) { } + + Status parseFrom(const BSONObj &obj); + + // The name of the field that contains the geometry. string field; - S2Point centroid; - // Distance IN METERS that we're willing to search. + + // The starting point of the near search. + PointWithCRS centroid; + + // Min and max distance from centroid that we're willing to search. + // Distance is in whatever units the centroid's CRS implies. + // If centroid.crs == FLAT these are radians. + // If centroid.crs == SPHERE these are meters. + double minDistance; double maxDistance; - // Did we convert to this distance from radians? (If so, we output distances in radians.) - bool fromRadians; + + // It's either $near or $nearSphere. + bool isNearSphere; + + string toString() const { + stringstream ss; + ss << " field=" << field; + ss << " maxdist=" << maxDistance; + ss << " isNearSphere=" << isNearSphere; + return ss.str(); + } + + private: + bool parseLegacyQuery(const BSONObj &obj); + Status parseNewQuery(const BSONObj &obj); }; // This represents either a $within or a $geoIntersects. @@ -106,6 +167,10 @@ namespace mongo { bool hasS2Region() const; const S2Region& getRegion() const; string getField() const { return field; } + + Predicate getPred() const { return predicate; } + const GeometryContainer& getGeometry() const { return geoContainer; } + private: // Try to parse the provided object into the right place. bool parseLegacyQuery(const BSONObj &obj); diff --git a/src/mongo/db/geo/hash.cpp b/src/mongo/db/geo/hash.cpp index b1841cfae76..be16bd0037f 100644 --- a/src/mongo/db/geo/hash.cpp +++ b/src/mongo/db/geo/hash.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/pch.h" @@ -490,7 +502,7 @@ namespace mongo { BSONElement y = i.next(); uassert(13026, - str::stream() << "geo values have to be numbers" + str::stream() << "geo values must be 'legacy coordinate pairs' for 2d indexes" << causedBy(src ? (*src).toString() : BSON_ARRAY(x << y).toString()), x.isNumber() && y.isNumber()); @@ -554,6 +566,14 @@ namespace mongo { *y = convertFromHashScale(b); } + Box GeoHashConverter::unhashToBox(const GeoHash &h) const { + double sizeEdgeBox = sizeEdge(h); + Point min(unhashToPoint(h)); + Point max(min.x + sizeEdgeBox, min.y + sizeEdgeBox); + Box box(min, max); + return box; + } + double GeoHashConverter::sizeOfDiag(const GeoHash& a) const { GeoHash b = a; b.move(1, 1); diff --git a/src/mongo/db/geo/hash.h b/src/mongo/db/geo/hash.h index ba6a9b3c586..b65dfbb78cb 100644 --- a/src/mongo/db/geo/hash.h +++ b/src/mongo/db/geo/hash.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -23,6 +35,7 @@ namespace mongo { class GeoHash; + class Box; struct Point; std::ostream& operator<<(std::ostream &s, const GeoHash &h); @@ -141,6 +154,12 @@ namespace mongo { GeoHashConverter(const Parameters ¶ms); + /** + * Return converter parameterss which can be used to + * construct an copy of this converter. + */ + const Parameters& getParams() const { return _params; } + int getBits() const { return _params.bits; } double getError() const { return _error; } double getErrorSphere() const { return _errorSphere ;} @@ -168,6 +187,7 @@ namespace mongo { * Convert from a hash to the following types: * double, double * Point + * Box * BSONObj */ // XXX: these should have consistent naming @@ -176,6 +196,14 @@ namespace mongo { BSONObj unhashToBSONObj(const GeoHash& h) const; void unhash(const GeoHash &h, double *x, double *y) const; + /** + * Generates bounding box from geo hash using converter. + * Used in GeoBrowse::fillStack and db/query/explain_plan.cpp + * to generate index bounds from + * geo hashes in plan stats. + */ + Box unhashToBox(const GeoHash &h) const; + double sizeOfDiag(const GeoHash& a) const; // XXX: understand/clean this. double sizeEdge(const GeoHash& a) const; diff --git a/src/mongo/db/geo/hash_test.cpp b/src/mongo/db/geo/hash_test.cpp index 65a700fae76..acb196faaca 100644 --- a/src/mongo/db/geo/hash_test.cpp +++ b/src/mongo/db/geo/hash_test.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ /** @@ -24,7 +36,7 @@ #include "mongo/db/geo/hash.h" #include "mongo/platform/random.h" #include "mongo/unittest/unittest.h" -#include "util/assert_util.h" +#include "mongo/util/assert_util.h" using mongo::GeoHash; using std::string; diff --git a/src/mongo/db/geo/haystack.cpp b/src/mongo/db/geo/haystack.cpp index 30c3abc1fd6..27a78ba1511 100644 --- a/src/mongo/db/geo/haystack.cpp +++ b/src/mongo/db/geo/haystack.cpp @@ -1,7 +1,5 @@ -// db/geo/haystack.cpp - /** - * Copyright (C) 2008-2012 10gen Inc. + * Copyright (C) 2008-2013 10gen Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -14,299 +12,46 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" - #include #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" -#include "mongo/db/namespace-inl.h" +#include "mongo/db/curop.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/index/haystack_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index_names.h" #include "mongo/db/jsobj.h" -#include "mongo/db/index.h" -#include "mongo/db/commands.h" +#include "mongo/db/structure/catalog/namespace_details-inl.h" #include "mongo/db/pdfile.h" -#include "mongo/db/btreecursor.h" -#include "mongo/db/curop-inl.h" -#include "mongo/db/matcher.h" -#include "mongo/db/geo/core.h" -#include "mongo/db/geo/hash.h" -#include "mongo/db/geo/shapes.h" -#include "mongo/util/timer.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/commands.h" /** - * Provides the geoHaystack index type and the command "geoSearch." * Examines all documents in a given radius of a given point. * Returns all documents that match a given search restriction. * See http://dochub.mongodb.org/core/haystackindexes * * Use when you want to look for restaurants within 25 miles with a certain name. - * Don't use when you want to find the closest open restaurants; see 2d.cpp for that. + * Don't use when you want to find the closest open restaurants. */ namespace mongo { - static const string GEOSEARCHNAME = "geoHaystack"; - - class GeoHaystackSearchHopper { - public: - /** - * Constructed with a point, a max distance from that point, and a max number of - * matched points to store. - * @param n The centroid that we're searching - * @param maxDistance The maximum distance to consider from that point - * @param limit The maximum number of results to return - * @param geoField Which field in the provided DiskLoc has the point to test. - */ - GeoHaystackSearchHopper(const BSONObj& nearObj, double maxDistance, unsigned limit, - const string& geoField) - : _near(nearObj), _maxDistance(maxDistance), _limit(limit), _geoField(geoField) { } - - // Consider the point in loc, and keep it if it's within _maxDistance (and we have space for - // it) - void consider(const DiskLoc& loc) { - if (limitReached()) return; - Point p(loc.obj().getFieldDotted(_geoField)); - if (distance(_near, p) > _maxDistance) - return; - _locs.push_back(loc); - } - - int appendResultsTo(BSONArrayBuilder* b) { - for (unsigned i = 0; i <_locs.size(); i++) - b->append(_locs[i].obj()); - return _locs.size(); - } - - // Have we stored as many points as we can? - const bool limitReached() const { - return _locs.size() >= _limit; - } - private: - Point _near; - double _maxDistance; - unsigned _limit; - const string _geoField; - vector _locs; - }; - - /** - * Provides the IndexType for geoSearch. - * Maps (lat, lng) to the bucketSize-sided square bucket that contains it. - * Usage: - * db.foo.ensureIndex({ pos : "geoHaystack", type : 1 }, { bucketSize : 1 }) - * pos is the name of the field to be indexed that has lat/lng data in an array. - * type is the name of the secondary field to be indexed. - * bucketSize specifies the dimension of the square bucket for the data in pos. - * ALL fields are mandatory. - */ - class GeoHaystackSearchIndex : public IndexType { - public: - GeoHaystackSearchIndex(const IndexPlugin* plugin, const IndexSpec* spec) - : IndexType(plugin, spec) { - - BSONElement e = spec->info["bucketSize"]; - uassert(13321, "need bucketSize", e.isNumber()); - _bucketSize = e.numberDouble(); - uassert(16455, "bucketSize cannot be zero", _bucketSize != 0.0); - - // Example: - // db.foo.ensureIndex({ pos : "geoHaystack", type : 1 }, { bucketSize : 1 }) - BSONObjIterator i(spec->keyPattern); - while (i.more()) { - BSONElement e = i.next(); - if (e.type() == String && GEOSEARCHNAME == e.valuestr()) { - uassert(13314, "can't have more than one geo field", _geoField.size() == 0); - uassert(13315, "the geo field has to be first in index", - _otherFields.size() == 0); - _geoField = e.fieldName(); - } else { - // TODO(hk): Do we want to do any checking on e.type and e.valuestr? - uassert(13326, "geoSearch can only have 1 non-geo field for now", - _otherFields.size() == 0); - _otherFields.push_back(e.fieldName()); - } - } - - uassert(13316, "no geo field specified", _geoField.size()); - // XXX: Fix documentation that says the other field is optional; code says it's mandatory. - uassert(13317, "no non-geo fields specified", _otherFields.size()); - } - - void getKeys(const BSONObj &obj, BSONObjSet &keys) const { - BSONElement loc = obj.getFieldDotted(_geoField); - if (loc.eoo()) - return; - - uassert(13323, "latlng not an array", loc.isABSONObj()); - string root; - { - BSONObjIterator i(loc.Obj()); - BSONElement x = i.next(); - BSONElement y = i.next(); - root = makeString(hash(x), hash(y)); - } - - verify(_otherFields.size() == 1); - - BSONElementSet all; - - // This is getFieldsDotted (plural not singular) since the object we're indexing - // may be an array. - obj.getFieldsDotted(_otherFields[0], all); - - if (all.size() == 0) { - // We're indexing a document that doesn't have the secondary non-geo field present. - // XXX: do we want to add this even if all.size() > 0? result:empty search terms - // match everything instead of only things w/empty search terms) - addKey(root, BSONElement(), keys); - } else { - // Ex:If our secondary field is type: "foo" or type: {a:"foo", b:"bar"}, - // all.size()==1. We can query on the complete field. - // Ex: If our secondary field is type: ["A", "B"] all.size()==2 and all has values - // "A" and "B". The query looks for any of the fields in the array. - for (BSONElementSet::iterator i = all.begin(); i != all.end(); ++i) { - addKey(root, *i, keys); - } - } - } - - // XXX: Who could call this and how do they know not to actually do so? - shared_ptr newCursor(const BSONObj& query, const BSONObj& order, - int numWanted) const { - shared_ptr c; - verify(0); - return c; - } - - void searchCommand(NamespaceDetails* nsd, - const BSONObj& n /*near*/, double maxDistance, const BSONObj& search, - BSONObjBuilder& result, unsigned limit) { - Timer t; - - LOG(1) << "SEARCH near:" << n << " maxDistance:" << maxDistance - << " search: " << search << endl; - int x, y; - { - BSONObjIterator i(n); - x = hash(i.next()); - y = hash(i.next()); - } - int scale = static_cast(ceil(maxDistance / _bucketSize)); - - GeoHaystackSearchHopper hopper(n, maxDistance, limit, _geoField); - - long long btreeMatches = 0; - - // TODO(hk): Consider starting with a (or b)=0, then going to a=+-1, then a=+-2, etc. - // Would want a HaystackKeyIterator or similar for this, but it'd be a nice - // encapsulation allowing us to S2-ify this trivially/abstract the key details. - for (int a = -scale; a <= scale && !hopper.limitReached(); ++a) { - for (int b = -scale; b <= scale && !hopper.limitReached(); ++b) { - BSONObjBuilder bb; - bb.append("", makeString(x + a, y + b)); - - for (unsigned i = 0; i < _otherFields.size(); i++) { - // See if the non-geo field we're indexing on is in the provided search term. - BSONElement e = search.getFieldDotted(_otherFields[i]); - if (e.eoo()) - bb.appendNull(""); - else - bb.appendAs(e, ""); - } - - BSONObj key = bb.obj(); - - GEOQUADDEBUG("KEY: " << key); - - // TODO(hk): this keeps a set of all DiskLoc seen in this pass so that we don't - // consider the element twice. Do we want to instead store a hash of the set? - // Is this often big? - set thisPass; - - // Lookup from key to key, inclusive. - scoped_ptr cursor(BtreeCursor::make(nsd, - *getDetails(), - key, - key, - true, - 1)); - while (cursor->ok() && !hopper.limitReached()) { - pair::iterator, bool> p = thisPass.insert(cursor->currLoc()); - // If a new element was inserted (haven't seen the DiskLoc before), p.second - // is true. - if (p.second) { - hopper.consider(cursor->currLoc()); - GEOQUADDEBUG("\t" << cursor->current()); - btreeMatches++; - } - cursor->advance(); - } - } - } - - BSONArrayBuilder arr(result.subarrayStart("results")); - int num = hopper.appendResultsTo(&arr); - arr.done(); - - { - BSONObjBuilder b(result.subobjStart("stats")); - b.append("time", t.millis()); - b.appendNumber("btreeMatches", btreeMatches); - b.append("n", num); - b.done(); - } - } - - const IndexDetails* getDetails() const { - return _spec->getDetails(); - } - private: - // TODO(hk): consider moving hash/unhash/makeString out - int hash(const BSONElement& e) const { - uassert(13322, "geo field is not a number", e.isNumber()); - return hash(e.numberDouble()); - } - - int hash(double d) const { - d += 180; - d /= _bucketSize; - return static_cast(d); - } - - string makeString(int hashedX, int hashedY) const { - stringstream ss; - ss << hashedX << "_" << hashedY; - return ss.str(); - } - - // Build a new BSONObj with root in it. If e is non-empty, append that to the key. Insert - // the BSONObj into keys. - void addKey(const string& root, const BSONElement& e, BSONObjSet& keys) const { - BSONObjBuilder buf; - buf.append("", root); - - if (e.eoo()) - buf.appendNull(""); - else - buf.appendAs(e, ""); - - keys.insert(buf.obj()); - } - - string _geoField; - vector _otherFields; - double _bucketSize; - }; - - class GeoHaystackSearchIndexPlugin : public IndexPlugin { - public: - GeoHaystackSearchIndexPlugin() : IndexPlugin(GEOSEARCHNAME) { } - - virtual IndexType* generate(const IndexSpec* spec) const { - return new GeoHaystackSearchIndex(this, spec); - } - } nameIndexPlugin; class GeoHaystackSearchCommand : public Command { public: @@ -315,25 +60,33 @@ namespace mongo { virtual LockType locktype() const { return READ; } bool slaveOk() const { return true; } bool slaveOverrideOk() const { return true; } + virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { ActionSet actions; actions.addAction(ActionType::find); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); + out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } + bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { string ns = dbname + "." + cmdObj.firstElement().valuestr(); - NamespaceDetails *nsd = nsdetails(ns); - if (NULL == nsd) { + Database* db = cc().database(); + if ( !db ) { errmsg = "can't find ns"; return false; } - vector idxs; - nsd->findIndexByType(GEOSEARCHNAME, idxs); + Collection* collection = db->getCollection( ns ); + if ( !collection ) { + errmsg = "can't find ns"; + return false; + } + + vector idxs; + collection->getIndexCatalog()->findIndexByType(IndexNames::GEO_HAYSTACK, idxs); if (idxs.size() == 0) { errmsg = "no geoSearch index"; return false; @@ -343,13 +96,6 @@ namespace mongo { return false; } - int idxNum = idxs[0]; - - IndexDetails& id = nsd->idx(idxNum); - GeoHaystackSearchIndex *si = - static_cast(id.getSpec().getType()); - verify(&id == si->getDetails()); - BSONElement nearElt = cmdObj["near"]; BSONElement maxDistance = cmdObj["maxDistance"]; BSONElement search = cmdObj["search"]; @@ -362,9 +108,13 @@ namespace mongo { if (cmdObj["limit"].isNumber()) limit = static_cast(cmdObj["limit"].numberInt()); - si->searchCommand(nsd, nearElt.Obj(), maxDistance.numberDouble(), search.Obj(), - result, limit); + IndexDescriptor* desc = idxs[0]; + HaystackAccessMethod* ham = + static_cast( collection->getIndexCatalog()->getIndex(desc) ); + ham->searchCommand(nearElt.Obj(), maxDistance.numberDouble(), search.Obj(), + &result, limit); return 1; } } nameSearchCommand; -} + +} // namespace mongo diff --git a/src/mongo/db/geo/s2.h b/src/mongo/db/geo/s2.h new file mode 100644 index 00000000000..426e9f0e5b5 --- /dev/null +++ b/src/mongo/db/geo/s2.h @@ -0,0 +1,45 @@ +/** +* Copyright (C) 2008-2012 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +/* + * This file's purpose is to confine the suppression of the Clang warning for mismatched-tags (struct vs class) + * in only the s2.h file + */ + +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wmismatched-tags" +#endif + +#include "third_party/s2/s2.h" + +#ifdef __clang__ +#pragma GCC diagnostic pop +#endif + diff --git a/src/mongo/db/geo/s2common.cpp b/src/mongo/db/geo/s2common.cpp index 8c3687969e4..3f2ab4bbd10 100644 --- a/src/mongo/db/geo/s2common.cpp +++ b/src/mongo/db/geo/s2common.cpp @@ -12,13 +12,30 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/geo/s2common.h" +#include "mongo/db/geo/geoconstants.h" +#include "mongo/db/geo/geoparser.h" +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/geo/s2.h" +#include "third_party/s2/s2cell.h" +#include "third_party/s2/s2regioncoverer.h" + namespace mongo { - // Thanks, Wikipedia. - const double S2IndexingParams::kRadiusOfEarthInMeters = (6378.1 * 1000); static string myitoa(int d) { stringstream ss; @@ -26,6 +43,127 @@ namespace mongo { return ss.str(); } + double dist(const S2Point& a, const S2Point& b) { + S1Angle angle(a, b); + return angle.radians(); + } + + double dist(const S2Point& a, const MultiPointWithCRS& b) { + double minDist = numeric_limits::max(); + for (size_t i = 0; i < b.points.size(); ++i) { + minDist = min(minDist, dist(a, b.points[i])); + } + return minDist; + } + + double dist(const S2Point& a, const S2Polyline& b) { + int tmp; + S1Angle angle(a, b.Project(a, &tmp)); + return angle.radians(); + } + + double dist(const S2Point& a, const MultiLineWithCRS& b) { + double minDist = numeric_limits::max(); + for (size_t i = 0; i < b.lines.vector().size(); ++i) { + minDist = min(minDist, dist(a, *b.lines.vector()[i])); + } + return minDist; + } + + double dist(const S2Point& a, const S2Polygon& b) { + S1Angle angle(a, b.Project(a)); + return angle.radians(); + } + + double dist(const S2Point& a, const MultiPolygonWithCRS& b) { + double minDist = numeric_limits::max(); + for (size_t i = 0; i < b.polygons.vector().size(); ++i) { + minDist = min(minDist, dist(a, *b.polygons.vector()[i])); + } + return minDist; + } + + bool S2SearchUtil::distanceBetween(const S2Point& us, const BSONObj& them, double *out) { + if (GeoParser::isGeometryCollection(them)) { + GeometryCollection c; + if (!GeoParser::parseGeometryCollection(them, &c)) { return false; } + double minDist = numeric_limits::max(); + + for (size_t i = 0; i < c.points.size(); ++i) { + minDist = min(minDist, dist(us, c.points[i].point)); + } + + const vector& lines = c.lines.vector(); + for (size_t i = 0; i < lines.size(); ++i) { + minDist = min(minDist, dist(us, lines[i]->line)); + } + + const vector& polys = c.polygons.vector(); + for (size_t i = 0; i < polys.size(); ++i) { + minDist = min(minDist, dist(us, polys[i]->polygon)); + } + + const vector& multipoints = c.multiPoints.vector(); + for (size_t i = 0; i < multipoints.size(); ++i) { + MultiPointWithCRS* mp = multipoints[i]; + for (size_t j = 0; j < mp->points.size(); ++j) { + minDist = min(minDist, dist(us, mp->points[i])); + } + } + + const vector& multilines = c.multiLines.vector(); + for (size_t i = 0; i < multilines.size(); ++i) { + const vector& lines = multilines[i]->lines.vector(); + for (size_t j = 0; j < lines.size(); ++j) { + minDist = min(minDist, dist(us, *lines[j])); + } + } + + const vector& multipolys = c.multiPolygons.vector(); + for (size_t i = 0; i < multipolys.size(); ++i) { + const vector& polys = multipolys[i]->polygons.vector(); + for (size_t j = 0; j < polys.size(); ++j) { + minDist = min(minDist, dist(us, *polys[j])); + } + } + + *out = kRadiusOfEarthInMeters * minDist; + return true; + } else if (GeoParser::isMultiPoint(them)) { + MultiPointWithCRS multiPoint; + if (!GeoParser::parseMultiPoint(them, &multiPoint)) { return false; } + *out = dist(us, multiPoint) * kRadiusOfEarthInMeters; + return true; + } else if (GeoParser::isMultiLine(them)) { + MultiLineWithCRS multiLine; + if (!GeoParser::parseMultiLine(them, &multiLine)) { return false; } + *out = dist(us, multiLine) * kRadiusOfEarthInMeters; + return true; + } else if (GeoParser::isMultiPolygon(them)) { + MultiPolygonWithCRS multiPolygon; + if (!GeoParser::parseMultiPolygon(them, &multiPolygon)) { return false; } + *out = dist(us, multiPolygon) * kRadiusOfEarthInMeters; + return true; + } else if (GeoParser::isPolygon(them)) { + PolygonWithCRS poly; + if (!GeoParser::parsePolygon(them, &poly)) { return false; } + *out = dist(us, poly.polygon) * kRadiusOfEarthInMeters; + return true; + } else if (GeoParser::isLine(them)) { + LineWithCRS line; + if (!GeoParser::parseLine(them, &line)) { return false; } + *out = dist(us, line.line) * kRadiusOfEarthInMeters; + return true; + } else if (GeoParser::isPoint(them)) { + PointWithCRS point; + if (!GeoParser::parsePoint(them, &point)) { return false; } + *out = dist(us, point.point) * kRadiusOfEarthInMeters; + return true; + } else { + return false; + } + } + void S2SearchUtil::setCoverLimitsBasedOnArea(double area, S2RegionCoverer *coverer, int coarsestIndexedLevel) { area = sqrt(area); diff --git a/src/mongo/db/geo/s2common.h b/src/mongo/db/geo/s2common.h index 60e836eb3d3..6cf07aa48b3 100644 --- a/src/mongo/db/geo/s2common.h +++ b/src/mongo/db/geo/s2common.h @@ -12,11 +12,23 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "mongo/db/diskloc.h" #include "mongo/db/geo/geoparser.h" -#include "third_party/s2/s2.h" +#include "mongo/db/geo/geoconstants.h" +#include "mongo/db/geo/s2.h" #include "third_party/s2/s2regioncoverer.h" #include "third_party/s2/s2cell.h" #include "third_party/s2/s2polyline.h" @@ -26,19 +38,20 @@ #pragma once namespace mongo { - // This is used by both s2cursor and s2nearcursor. - class S2SearchUtil { - public: - // Given a coverer, region, and field name, generate a BSONObj that we can pass to a - // FieldRangeSet so that we only examine the keys that the provided region may intersect. - static BSONObj coverAsBSON(const vector &cover, const string& field, - const int coarsestIndexedLevel); - static void setCoverLimitsBasedOnArea(double area, S2RegionCoverer *coverer, int coarsestIndexedLevel); + + // An enum describing the version of an S2 index. + enum S2IndexVersion { + // The first version of the S2 index, introduced in MongoDB 2.4.0. Compatible with MongoDB + // 2.4.0 and later. Supports the following GeoJSON objects: Point, LineString, Polygon. + S2_INDEX_VERSION_1 = 1, + + // The current version of the S2 index, introduced in MongoDB 2.6.0. Compatible with + // MongoDB 2.6.0 and later. Introduced support for the following GeoJSON objects: + // MultiPoint, MultiLineString, MultiPolygon, GeometryCollection. + S2_INDEX_VERSION_2 = 2 }; struct S2IndexingParams { - static const double kRadiusOfEarthInMeters; - // Since we take the cartesian product when we generate keys for an insert, // we need a cap. size_t maxKeysPerInsert; @@ -51,6 +64,8 @@ namespace mongo { // And, what's the coarsest? When we search in larger coverings we know we // can stop here -- we index nothing coarser than this. int coarsestIndexedLevel; + // Version of this index (specific to the index type). + S2IndexVersion indexVersion; double radius; @@ -60,6 +75,7 @@ namespace mongo { ss << "maxCellsInCovering: " << maxCellsInCovering << endl; ss << "finestIndexedLevel: " << finestIndexedLevel << endl; ss << "coarsestIndexedLevel: " << coarsestIndexedLevel << endl; + ss << "indexVersion: " << indexVersion << endl; return ss.str(); } @@ -70,4 +86,15 @@ namespace mongo { coverer->set_max_cells(maxCellsInCovering); } }; + + class S2SearchUtil { + public: + // Given a coverer, region, and field name, generate a BSONObj that we can pass to a + // FieldRangeSet so that we only examine the keys that the provided region may intersect. + static BSONObj coverAsBSON(const vector &cover, const string& field, + const int coarsestIndexedLevel); + static void setCoverLimitsBasedOnArea(double area, S2RegionCoverer *coverer, int coarsestIndexedLevel); + static bool distanceBetween(const S2Point& us, const BSONObj& them, double *out); + }; + } // namespace mongo diff --git a/src/mongo/db/geo/s2cursor.cpp b/src/mongo/db/geo/s2cursor.cpp deleted file mode 100644 index a0e07f56d8e..00000000000 --- a/src/mongo/db/geo/s2cursor.cpp +++ /dev/null @@ -1,184 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/geo/s2cursor.h" - -#include "mongo/db/btreecursor.h" -#include "mongo/db/index.h" -#include "mongo/db/matcher.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/geo/s2common.h" - -namespace mongo { - S2Cursor::S2Cursor(const BSONObj &keyPattern, const IndexDetails *details, - const BSONObj &query, const vector &fields, - const S2IndexingParams ¶ms) - : _details(details), _fields(fields), _params(params), _keyPattern(keyPattern), - _nscanned(0), _matchTested(0), _geoTested(0) { - - BSONObjBuilder geoFieldsToNuke; - for (size_t i = 0; i < _fields.size(); ++i) { - geoFieldsToNuke.append(_fields[i].getField(), ""); - } - // false means we want to filter OUT geoFieldsToNuke, not filter to include only that. - _filteredQuery = query.filterFieldsUndotted(geoFieldsToNuke.obj(), false); - _matcher.reset(new CoveredIndexMatcher(_filteredQuery, keyPattern)); - } - - S2Cursor::~S2Cursor() { } - - CoveredIndexMatcher* S2Cursor::matcher() const { return _matcher.get(); } - - bool S2Cursor::ok() { - if (NULL == _btreeCursor.get()) { - // FieldRangeVector needs an IndexSpec so we make it one. - BSONObjBuilder specBuilder; - BSONObjIterator i(_keyPattern); - while (i.more()) { - BSONElement e = i.next(); - // Checked in AccessMethod already, so we know this spec has only numbers and 2dsphere - if ( e.type() == String ) { - specBuilder.append( e.fieldName(), 1 ); - } - else { - specBuilder.append( e.fieldName(), e.numberInt() ); - } - } - BSONObj spec = specBuilder.obj(); - IndexSpec specForFRV(spec); - // All the magic is in makeUnifiedFRS. See below. - // A lot of these arguments are opaque. - BSONObj frsObj; - if (!makeFRSObject(&frsObj)) { return false; } - FieldRangeSet frs(_details->parentNS().c_str(), frsObj, false, false); - shared_ptr frv(new FieldRangeVector(frs, specForFRV, 1)); - _btreeCursor.reset(BtreeCursor::make(nsdetails(_details->parentNS()), - *_details, frv, 0, 1)); - return advance(); - } - return _btreeCursor->ok(); - } - - // Make the FieldRangeSet of keys we look for. Uses coverAsBSON to go from - // a region to a covering to a set of keys for that covering. - // Returns false if the FRS object would be empty. - bool S2Cursor::makeFRSObject(BSONObj *out) { - BSONObjBuilder frsObjBuilder; - frsObjBuilder.appendElements(_filteredQuery); - - S2RegionCoverer coverer; - - for (size_t i = 0; i < _fields.size(); ++i) { - vector cover; - double area = _fields[i].getRegion().GetRectBound().Area(); - S2SearchUtil::setCoverLimitsBasedOnArea(area, &coverer, _params.coarsestIndexedLevel); - coverer.GetCovering(_fields[i].getRegion(), &cover); - if (0 == cover.size()) { return false; } - _cellsInCover = cover.size(); - BSONObj fieldRange = S2SearchUtil::coverAsBSON(cover, _fields[i].getField(), - _params.coarsestIndexedLevel); - frsObjBuilder.appendElements(fieldRange); - } - - *out = frsObjBuilder.obj(); - return true; - } - - Record* S2Cursor::_current() { return _btreeCursor->currLoc().rec(); } - BSONObj S2Cursor::current() { return _btreeCursor->currLoc().obj(); } - DiskLoc S2Cursor::currLoc() { return _btreeCursor->currLoc(); } - BSONObj S2Cursor::currKey() const { return _btreeCursor->currKey(); } - DiskLoc S2Cursor::refLoc() { return DiskLoc(); } - long long S2Cursor::nscanned() { return _nscanned; } - bool S2Cursor::getsetdup(DiskLoc loc) { return _btreeCursor->getsetdup(loc); } - void S2Cursor::aboutToDeleteBucket(const DiskLoc& b) { - if (NULL != _btreeCursor) { - _btreeCursor->aboutToDeleteBucket(b); - } - } - - // This is the actual search. - bool S2Cursor::advance() { - for (; _btreeCursor->ok(); _btreeCursor->advance()) { - ++_nscanned; - if (_seen.end() != _seen.find(_btreeCursor->currLoc())) { continue; } - _seen.insert(_btreeCursor->currLoc()); - - ++_matchTested; - MatchDetails details; - bool matched = _matcher->matchesCurrent(_btreeCursor.get(), &details); - if (!matched) { continue; } - - const BSONObj &indexedObj = _btreeCursor->currLoc().obj(); - - ++_geoTested; - size_t geoFieldsMatched = 0; - // OK, cool, non-geo match satisfied. See if the object actually overlaps w/the geo - // query fields. - for (size_t i = 0; i < _fields.size(); ++i) { - BSONElementSet geoFieldElements; - indexedObj.getFieldsDotted(_fields[i].getField(), geoFieldElements, false); - if (geoFieldElements.empty()) { continue; } - - bool match = false; - - for (BSONElementSet::iterator oi = geoFieldElements.begin(); - !match && (oi != geoFieldElements.end()); ++oi) { - if (!oi->isABSONObj()) { continue; } - const BSONObj &geoObj = oi->Obj(); - GeometryContainer geoContainer; - uassert(16698, "malformed geometry: " + geoObj.toString(), - geoContainer.parseFrom(geoObj)); - match = _fields[i].satisfiesPredicate(geoContainer); - } - - if (match) { ++geoFieldsMatched; } - } - - if (geoFieldsMatched == _fields.size()) { - // We have a winner! And we point at it. - return true; - } - } - return false; - } - - // This is called when we're supposed to yield. - void S2Cursor::noteLocation() { - _btreeCursor->noteLocation(); - _seen.clear(); - } - - // Called when we're un-yielding. - void S2Cursor::checkLocation() { - _btreeCursor->checkLocation(); - // We are pointing at a valid btree location now, but it may not be a valid result. - // This ensures that we're pointing at a valid result that satisfies the query. - - // There is something subtle here: Say we point at something valid, and note the location - // (yield), then checkLocation (unyield), when we call advance, we don't go past the object - // that we were/are pointing at since we only do that if we've seen it before (that is, it's - // in _seen, which we clear when we yield). - advance(); - } - - void S2Cursor::explainDetails(BSONObjBuilder& b) { - b << "nscanned" << _nscanned; - b << "matchTested" << _matchTested; - b << "geoTested" << _geoTested; - b << "cellsInCover" << _cellsInCover; - } -} // namespace mongo diff --git a/src/mongo/db/geo/s2cursor.h b/src/mongo/db/geo/s2cursor.h deleted file mode 100644 index d3cfa9e21b1..00000000000 --- a/src/mongo/db/geo/s2cursor.h +++ /dev/null @@ -1,92 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include -#include "mongo/db/jsobj.h" -#include "mongo/db/commands.h" -#include "mongo/db/cursor.h" -#include "mongo/db/diskloc.h" -#include "mongo/db/matcher.h" -#include "mongo/db/queryutil.h" -#include "mongo/db/geo/geoquery.h" -#include "mongo/db/geo/s2common.h" - -namespace mongo { - class BtreeCursor; - - class S2Cursor : public Cursor { - public: - S2Cursor(const BSONObj &keyPattern, const IndexDetails* details, const BSONObj &query, - const vector ®ions, const S2IndexingParams ¶ms); - virtual ~S2Cursor(); - virtual CoveredIndexMatcher *matcher() const; - - virtual bool supportYields() { return true; } - virtual bool supportGetMore() { return true; } - virtual bool isMultiKey() const { return true; } - virtual bool autoDedup() const { return false; } - virtual bool modifiedKeys() const { return true; } - virtual bool getsetdup(DiskLoc loc); - virtual void aboutToDeleteBucket(const DiskLoc& b); - virtual string toString() { return "S2Cursor"; } - BSONObj indexKeyPattern() { return _keyPattern; } - virtual bool ok(); - virtual Record* _current(); - virtual BSONObj current(); - virtual DiskLoc currLoc(); - virtual bool advance(); - virtual BSONObj currKey() const; - virtual DiskLoc refLoc(); - virtual void noteLocation(); - virtual void checkLocation(); - virtual long long nscanned(); - virtual void explainDetails(BSONObjBuilder& b); - private: - // Make an object that describes the restrictions on all possible valid keys. - // It's kind of a monstrous object. Thanks, FieldRangeSet, for doing all the work - // for us. - // Returns false if the FRS object would be empty. - bool makeFRSObject(BSONObj *out); - - // Need this to make a FieldRangeSet. - const IndexDetails *_details; - // The query with the geo stuff taken out. We use this with a matcher. - BSONObj _filteredQuery; - // What geo regions are we looking for? - vector _fields; - // We use this for matching non-GEO stuff. - shared_ptr _matcher; - // How were the keys created? We need this to search for the right stuff. - S2IndexingParams _params; - // We have to pass this to the FieldRangeVector ctor (in modified form). - BSONObj _keyPattern; - - // What have we checked so we don't repeat it and waste time? - set _seen; - // This really does all the work/points into the btree. - scoped_ptr _btreeCursor; - - // Stat counters/debug information goes below. - // How many items did we look at in the btree? - long long _nscanned; - // How many did we try to match? - long long _matchTested; - // How many did we geo-test? - long long _geoTested; - // How many cells were in our cover? - long long _cellsInCover; - }; -} // namespace mongo diff --git a/src/mongo/db/geo/s2index.cpp b/src/mongo/db/geo/s2index.cpp deleted file mode 100644 index d52f0ad2991..00000000000 --- a/src/mongo/db/geo/s2index.cpp +++ /dev/null @@ -1,467 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/namespace-inl.h" -#include "mongo/db/client.h" -#include "mongo/db/curop.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/index.h" -#include "mongo/db/queryutil.h" -#include "mongo/db/geo/geonear.h" -#include "mongo/db/geo/geoparser.h" -#include "mongo/db/geo/geoquery.h" -#include "mongo/db/geo/s2common.h" -#include "mongo/db/geo/s2cursor.h" -#include "mongo/db/geo/s2nearcursor.h" -#include "third_party/s2/s2.h" -#include "third_party/s2/s2cell.h" -#include "third_party/s2/s2polygon.h" -#include "third_party/s2/s2polyline.h" -#include "third_party/s2/s2regioncoverer.h" - -namespace { - // Used in a handful of places in GeoSphere2DType below. - static void keysFromRegion(S2RegionCoverer *coverer, const S2Region ®ion, - vector *out) { - vector covering; - coverer->GetCovering(region, &covering); - for (size_t i = 0; i < covering.size(); ++i) { - out->push_back(covering[i].toString()); - } - } -} // namespace - -namespace mongo { - class S2IndexType : public IndexType { - public: - // We keep track of what fields we've indexed and if they're geo or not. - struct IndexedField { - enum Type { - GEO, - LITERAL - }; - - Type type; - string name; - IndexedField(Type t, const string& n) : type(t), name(n) { } - }; - - S2IndexType(const string& geoIdxName, const IndexPlugin *plugin, const IndexSpec *spec, - const S2IndexingParams ¶ms) : IndexType(plugin, spec), _params(params) { - int geoFields = 0; - // Categorize the fields we're indexing and make sure we have a geo field. - BSONObjIterator i(spec->keyPattern); - while (i.more()) { - BSONElement e = i.next(); - if (e.type() == String && geoIdxName == e.valuestr()) { - _fields.push_back(IndexedField(IndexedField::GEO, e.fieldName())); - ++geoFields; - } else { - _fields.push_back(IndexedField(IndexedField::LITERAL, e.fieldName())); - } - } - uassert(16450, "Expect at least one geo field, spec=" + spec->keyPattern.toString(), - geoFields >= 1); - } - - virtual ~S2IndexType() { } - - void getKeys(const BSONObj& obj, BSONObjSet& keys) const { - verify(_fields.size() >= 1); - - BSONObjSet keysToAdd; - // We output keys in the same order as the fields we index. - for (size_t i = 0; i < _fields.size(); ++i) { - const IndexedField &field = _fields[i]; - - // First, we get the keys that this field adds. Either they're added literally from - // the value of the field, or they're transformed if the field is geo. - BSONElementSet fieldElements; - // false means Don't expand the last array, duh. - obj.getFieldsDotted(field.name, fieldElements, false); - - BSONObjSet keysForThisField; - if (IndexedField::GEO == field.type) { - getGeoKeys(fieldElements, &keysForThisField); - } else if (IndexedField::LITERAL == field.type) { - getLiteralKeys(fieldElements, &keysForThisField); - } else { - verify(0); - } - - // We expect there to be _spec->_missingField() present in the keys if data is - // missing. So, this should be non-empty. - verify(!keysForThisField.empty()); - - // We take the Cartesian product of all of the keys. This requires that we have - // some keys to take the Cartesian product with. If keysToAdd.empty(), we - // initialize it. - if (keysToAdd.empty()) { - keysToAdd = keysForThisField; - continue; - } - - BSONObjSet updatedKeysToAdd; - for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); - ++it) { - for (BSONObjSet::const_iterator newIt = keysForThisField.begin(); - newIt!= keysForThisField.end(); ++newIt) { - BSONObjBuilder b; - b.appendElements(*it); - b.append(newIt->firstElement()); - updatedKeysToAdd.insert(b.obj()); - } - } - keysToAdd = updatedKeysToAdd; - } - - if (keysToAdd.size() > _params.maxKeysPerInsert) { - warning() << "insert of geo object generated lots of keys (" << keysToAdd.size() - << ") consider creating larger buckets. obj=" - << obj; - } - - for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); ++it) { - keys.insert(*it); - } - } - - // Entry point for a search. - virtual shared_ptr newCursor(const BSONObj& query, const BSONObj& order, - int numWanted) const { - vector regions; - bool isNearQuery = false; - NearQuery nearQuery; - - // Go through the fields that we index, and for each geo one, make - // a GeoQuery object for the S2*Cursor class to do intersection - // testing/cover generating with. - for (size_t i = 0; i < _fields.size(); ++i) { - const IndexedField &field = _fields[i]; - if (IndexedField::GEO != field.type) { continue; } - - BSONElement e = query.getFieldDotted(field.name); - if (e.eoo()) { continue; } - if (!e.isABSONObj()) { continue; } - BSONObj obj = e.Obj(); - - if (nearQuery.parseFrom(obj, _params.radius)) { - uassert(16685, "Only one $near clause allowed: " + query.toString(), - !isNearQuery); - isNearQuery = true; - nearQuery.field = field.name; - continue; - } - - GeoQuery geoQueryField(field.name); - if (!geoQueryField.parseFrom(obj)) { - uasserted(16535, "can't parse query (2dsphere): " + obj.toString()); - } - uassert(16684, "Geometry unsupported: " + obj.toString(), - geoQueryField.hasS2Region()); - regions.push_back(geoQueryField); - } - - // Remove all the indexed geo regions from the query. The s2*cursor will - // instead create a covering for that key to speed up the search. - // - // One thing to note is that we create coverings for indexed geo keys during - // a near search to speed it up further. - BSONObjBuilder geoFieldsToNuke; - if (isNearQuery) { - geoFieldsToNuke.append(nearQuery.field, ""); - } - for (size_t i = 0; i < regions.size(); ++i) { - geoFieldsToNuke.append(regions[i].getField(), ""); - } - - // false means we want to filter OUT geoFieldsToNuke, not filter to include only that. - BSONObj filteredQuery = query.filterFieldsUndotted(geoFieldsToNuke.obj(), false); - - if (isNearQuery) { - S2NearCursor *cursor = new S2NearCursor(keyPattern(), getDetails(), filteredQuery, - nearQuery, regions, _params); - return shared_ptr(cursor); - } else { - S2Cursor *cursor = new S2Cursor(keyPattern(), getDetails(), filteredQuery, regions, - _params); - return shared_ptr(cursor); - } - } - - virtual IndexSuitability suitability(const FieldRangeSet& queryConstraints, - const BSONObj& order) const { - BSONObj query = queryConstraints.originalQuery(); - - for (size_t i = 0; i < _fields.size(); ++i) { - const IndexedField &field = _fields[i]; - if (IndexedField::GEO != field.type) { continue; } - - BSONElement e = query.getFieldDotted(field.name); - // Some locations are given to us as arrays. Sigh. - if (Array == e.type()) { return HELPFUL; } - if (Object != e.type()) { continue; } - // getGtLtOp is horribly misnamed and really means get the operation. - switch (e.embeddedObject().firstElement().getGtLtOp()) { - case BSONObj::opNEAR: - return OPTIMAL; - case BSONObj::opWITHIN: { - BSONElement elt = e.embeddedObject().firstElement(); - if (Object != elt.type()) { continue; } - const char* fname = elt.embeddedObject().firstElement().fieldName(); - if (mongoutils::str::equals("$geometry", fname) - || mongoutils::str::equals("$centerSphere", fname)) { - return OPTIMAL; - } else { - return USELESS; - } - } - case BSONObj::opGEO_INTERSECTS: - return OPTIMAL; - default: - return USELESS; - } - } - return USELESS; - } - - const IndexDetails* getDetails() const { return _spec->getDetails(); } - - // These are used by the geoNear command. geoNear constructs its own cursor. - const S2IndexingParams& getParams() const { return _params; } - void getGeoFieldNames(vector *out) const { - for (size_t i = 0; i < _fields.size(); ++i) { - if (IndexedField::GEO == _fields[i].type) { - out->push_back(_fields[i].name); - } - } - } - private: - // Get the index keys for elements that are GeoJSON. - void getGeoKeys(const BSONElementSet &elements, BSONObjSet *out) const { - S2RegionCoverer coverer; - _params.configureCoverer(&coverer); - - // See here for GeoJSON format: geojson.org/geojson-spec.html - for (BSONElementSet::iterator i = elements.begin(); i != elements.end(); ++i) { - uassert(16700, "Can't parse geometry from element: " + i->toString(), - i->isABSONObj()); - const BSONObj &obj = i->Obj(); - - vector cells; - S2Polyline line; - S2Cell point; - // We only support GeoJSON polygons. Why?: - // 1. we don't automagically do WGS84/flat -> WGS84, and - // 2. the old polygon format must die. - if (GeoParser::isGeoJSONPolygon(obj)) { - S2Polygon polygon; - GeoParser::parseGeoJSONPolygon(obj, &polygon); - keysFromRegion(&coverer, polygon, &cells); - } else if (GeoParser::parseLineString(obj, &line)) { - keysFromRegion(&coverer, line, &cells); - } else if (GeoParser::parsePoint(obj, &point)) { - S2CellId parent(point.id().parent(_params.finestIndexedLevel)); - cells.push_back(parent.toString()); - } else { - uasserted(16572, "Can't extract geo keys from object, malformed geometry?:" - + obj.toString()); - } - uassert(16673, "Unable to generate keys for (likely malformed) geometry: " - + obj.toString(), - cells.size() > 0); - - for (vector::const_iterator it = cells.begin(); it != cells.end(); ++it) { - BSONObjBuilder b; - b.append("", *it); - out->insert(b.obj()); - } - } - - if (0 == out->size()) { - BSONObjBuilder b; - b.appendNull(""); - out->insert(b.obj()); - } - } - - void getLiteralKeysArray(BSONObj obj, BSONObjSet *out) const { - BSONObjIterator objIt(obj); - if (!objIt.more()) { - // Empty arrays are indexed as undefined. - BSONObjBuilder b; - b.appendUndefined(""); - out->insert(b.obj()); - } else { - // Non-empty arrays are exploded. - while (objIt.more()) { - BSONObjBuilder b; - b.appendAs(objIt.next(), ""); - out->insert(b.obj()); - } - } - } - - void getOneLiteralKey(BSONElement elt, BSONObjSet *out) const { - if (Array == elt.type()) { - getLiteralKeysArray(elt.Obj(), out); - } else { - // One thing, not an array, index as-is. - BSONObjBuilder b; - b.appendAs(elt, ""); - out->insert(b.obj()); - } - } - - // elements is a non-geo field. Add the values literally, expanding arrays. - void getLiteralKeys(const BSONElementSet &elements, BSONObjSet *out) const { - if (0 == elements.size()) { - // Missing fields are indexed as null. - BSONObjBuilder b; - b.appendNull(""); - out->insert(b.obj()); - } else { - for (BSONElementSet::iterator i = elements.begin(); i != elements.end(); ++i) { - getOneLiteralKey(*i, out); - } - } - } - - vector _fields; - S2IndexingParams _params; - }; - - static const string SPHERE_2D_NAME = "2dsphere"; - class S2IndexPlugin : public IndexPlugin { - public: - S2IndexPlugin() : IndexPlugin(SPHERE_2D_NAME) { } - - virtual IndexType* generate(const IndexSpec* spec) const { - S2IndexingParams params; - params.maxKeysPerInsert = 200; - // This is advisory. - params.maxCellsInCovering = 50; - // Near distances are specified in meters...sometimes. - params.radius = S2IndexingParams::kRadiusOfEarthInMeters; - // These are not advisory. - params.finestIndexedLevel = configValueWithDefault(spec, "finestIndexedLevel", - S2::kAvgEdge.GetClosestLevel(500.0 / params.radius)); - params.coarsestIndexedLevel = configValueWithDefault(spec, "coarsestIndexedLevel", - S2::kAvgEdge.GetClosestLevel(100 * 1000.0 / params.radius)); - uassert(16687, "coarsestIndexedLevel must be >= 0", params.coarsestIndexedLevel >= 0); - uassert(16688, "finestIndexedLevel must be <= 30", params.finestIndexedLevel <= 30); - uassert(16689, "finestIndexedLevel must be >= coarsestIndexedLevel", - params.finestIndexedLevel >= params.coarsestIndexedLevel); - - // Categorize the fields we're indexing and make sure we have a geo field. - int geoFields = 0; - BSONObjIterator i( spec->keyPattern ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( e.type() == String && SPHERE_2D_NAME == e.String() ) { - ++geoFields; - } - else { - // We check for numeric in 2d, so that's the check here - uassert( 16823, (string)"Cannot use " + SPHERE_2D_NAME + - " index with other special index types: " + e.toString(), - e.isNumber() ); - } - } - uassert(16750, (string)"Expect at least one geo field, spec=" - + spec->keyPattern.toString(), - geoFields >= 1); - - return new S2IndexType(SPHERE_2D_NAME, this, spec, params); - } - - int configValueWithDefault(const IndexSpec* spec, const string& name, int def) const { - BSONElement e = spec->info[name]; - if (e.isNumber()) { return e.numberInt(); } - return def; - } - } S2IndexPluginS2D; - - bool run2DSphereGeoNear(const IndexDetails &id, BSONObj& cmdObj, - const GeoNearArguments &parsedArgs, string& errmsg, - BSONObjBuilder& result) { - S2IndexType *idxType = static_cast(id.getSpec().getType()); - verify(&id == idxType->getDetails()); - - vector geoFieldNames; - idxType->getGeoFieldNames(&geoFieldNames); - - // NOTE(hk): If we add a new argument to geoNear, we could have a - // 2dsphere index with multiple indexed geo fields, and the geoNear - // could pick the one to run over. Right now, we just require one. - uassert(16552, "geoNear requires exactly one indexed geo field", 1 == geoFieldNames.size()); - NearQuery nearQuery(geoFieldNames[0]); - uassert(16679, "Invalid geometry given as arguments to geoNear: " + cmdObj.toString(), - nearQuery.parseFromGeoNear(cmdObj, idxType->getParams().radius)); - uassert(16683, "geoNear on 2dsphere index requires spherical", - parsedArgs.isSpherical); - - // NOTE(hk): For a speedup, we could look through the query to see if - // we've geo-indexed any of the fields in it. - vector regions; - - scoped_ptr cursor(new S2NearCursor(idxType->keyPattern(), - idxType->getDetails(), parsedArgs.query, nearQuery, regions, idxType->getParams())); - - double totalDistance = 0; - BSONObjBuilder resultBuilder(result.subarrayStart("results")); - double farthestDist = 0; - - int results; - for (results = 0; results < parsedArgs.numWanted && cursor->ok(); ++results) { - double dist = cursor->currentDistance(); - // If we got the distance in radians, output it in radians too. - if (nearQuery.fromRadians) { dist /= idxType->getParams().radius; } - dist *= parsedArgs.distanceMultiplier; - totalDistance += dist; - if (dist > farthestDist) { farthestDist = dist; } - - BSONObjBuilder oneResultBuilder( - resultBuilder.subobjStart(BSONObjBuilder::numStr(results))); - oneResultBuilder.append("dis", dist); - if (parsedArgs.includeLocs) { - BSONElementSet geoFieldElements; - cursor->current().getFieldsDotted(geoFieldNames[0], geoFieldElements, false); - for (BSONElementSet::iterator oi = geoFieldElements.begin(); - oi != geoFieldElements.end(); ++oi) { - if (oi->isABSONObj()) { - oneResultBuilder.appendAs(*oi, "loc"); - } - } - } - - oneResultBuilder.append("obj", cursor->current()); - oneResultBuilder.done(); - cursor->advance(); - } - - resultBuilder.done(); - - BSONObjBuilder stats(result.subobjStart("stats")); - stats.append("time", cc().curop()->elapsedMillis()); - stats.appendNumber("nscanned", cursor->nscanned()); - stats.append("avgDistance", totalDistance / results); - stats.append("maxDistance", farthestDist); - stats.done(); - - return true; - } -} // namespace mongo diff --git a/src/mongo/db/geo/s2index.h b/src/mongo/db/geo/s2index.h deleted file mode 100644 index 5a2f6be6056..00000000000 --- a/src/mongo/db/geo/s2index.h +++ /dev/null @@ -1,22 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -namespace mongo { - // We need cmdObj and parsedArgs so we can print a useful error msg. - bool run2DSphereGeoNear(const IndexDetails &id, BSONObj& cmdObj, - const GeoNearArguments &parsedArgs, string& errmsg, - BSONObjBuilder& result); -} // namespace mongo diff --git a/src/mongo/db/geo/s2nearcursor.cpp b/src/mongo/db/geo/s2nearcursor.cpp deleted file mode 100644 index e4ef547c5d8..00000000000 --- a/src/mongo/db/geo/s2nearcursor.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/btree.h" -#include "mongo/db/index.h" -#include "mongo/db/matcher.h" -#include "mongo/db/pdfile.h" -#include "third_party/s2/s2cap.h" -#include "third_party/s2/s2regionintersection.h" -#include "mongo/db/geo/s2common.h" -#include "mongo/db/geo/s2nearcursor.h" - -namespace mongo { - S2NearCursor::S2NearCursor(const BSONObj &keyPattern, const IndexDetails *details, - const BSONObj &query, const NearQuery &nearQuery, - const vector &indexedGeoFields, - const S2IndexingParams ¶ms) - : _details(details), _nearQuery(nearQuery), _indexedGeoFields(indexedGeoFields), - _params(params), _keyPattern(keyPattern), _nearFieldIndex(0), _returnedDistance(0) { - - BSONObjBuilder geoFieldsToNuke; - for (size_t i = 0; i < _indexedGeoFields.size(); ++i) { - geoFieldsToNuke.append(_indexedGeoFields[i].getField(), ""); - } - // false means we want to filter OUT geoFieldsToNuke, not filter to include only that. - _filteredQuery = query.filterFieldsUndotted(geoFieldsToNuke.obj(), false); - _matcher.reset(new CoveredIndexMatcher(_filteredQuery, keyPattern)); - - // More indexing machinery. - BSONObjBuilder specBuilder; - BSONObjIterator specIt(_keyPattern); - while (specIt.more()) { - BSONElement e = specIt.next(); - // Checked in AccessMethod already, so we know this spec has only numbers and 2dsphere - if ( e.type() == String ) { - specBuilder.append( e.fieldName(), 1 ); - } - else { - specBuilder.append( e.fieldName(), e.numberInt() ); - } - } - BSONObj spec = specBuilder.obj(); - _specForFRV = IndexSpec(spec); - - specIt = BSONObjIterator(_keyPattern); - while (specIt.more()) { - if (specIt.next().fieldName() == _nearQuery.field) { break; } - ++_nearFieldIndex; - } - - // _outerRadius can't be greater than (pi * r) or we wrap around the opposite - // side of the world. - _maxDistance = min(M_PI * _params.radius, _nearQuery.maxDistance); - - // Start with a conservative _radiusIncrement. - _radiusIncrement = 5 * S2::kAvgEdge.GetValue(_params.finestIndexedLevel) * _params.radius; - _innerRadius = _outerRadius = 0; - // We might want to adjust the sizes of our coverings if our search - // isn't local to the start point. - // Set up _outerRadius with proper checks (maybe maxDistance is really small?) - nextAnnulus(); - } - - S2NearCursor::~S2NearCursor() { - // Annulus takes ownership of the pointers we pass in. - // Those are actually pointers to the member variables _innerCap and _outerCap. - _annulus.Release(NULL); - } - - CoveredIndexMatcher* S2NearCursor::matcher() const { return _matcher.get(); } - - Record* S2NearCursor::_current() { return _results.top().loc.rec(); } - BSONObj S2NearCursor::current() { return _results.top().loc.obj(); } - DiskLoc S2NearCursor::currLoc() { return _results.top().loc; } - BSONObj S2NearCursor::currKey() const { return _results.top().key; } - DiskLoc S2NearCursor::refLoc() { return DiskLoc(); } - long long S2NearCursor::nscanned() { return _stats._nscanned; } - - double S2NearCursor::currentDistance() const { return _results.top().distance; } - - // This is called when we're about to yield. - void S2NearCursor::noteLocation() { - LOG(1) << "yielding, tossing " << _results.size() << " results" << endl; - _results = priority_queue(); - } - - // Called when we're un-yielding. - // Note that this is (apparently) a valid call sequence: - // 1. noteLocation() - // 2. ok() - // 3. checkLocation() - // As such we might have results and only want to fill the result queue if it's empty. - void S2NearCursor::checkLocation() { - LOG(1) << "unyielding, have " << _results.size() << " results in queue"; - if (_results.empty()) { - LOG(1) << ", filling..." << endl; - fillResults(); - LOG(1) << "now have " << _results.size() << " results in queue"; - } - LOG(1) << endl; - } - - void S2NearCursor::explainDetails(BSONObjBuilder& b) { - b << "nscanned" << _stats._nscanned; - b << "matchTested" << _stats._matchTested; - b << "geoMatchTested" << _stats._geoMatchTested; - b << "numShells" << _stats._numShells; - b << "keyGeoSkip" << _stats._keyGeoSkip; - b << "returnSkip" << _stats._returnSkip; - b << "btreeDups" << _stats._btreeDups; - b << "inAnnulusTested" << _stats._inAnnulusTested; - } - - bool S2NearCursor::ok() { - if (_innerRadius > _maxDistance) { - LOG(1) << "not OK, exhausted search bounds" << endl; - return false; - } - if (_results.empty()) { - LOG(1) << "results empty in OK, filling" << endl; - fillResults(); - } - // If fillResults can't find anything, we're outta results. - return !_results.empty(); - } - - void S2NearCursor::nextAnnulus() { - LOG(1) << "growing annulus from (" << _innerRadius << ", " << _outerRadius; - _innerRadius = _outerRadius; - _outerRadius += _radiusIncrement; - _outerRadius = min(_outerRadius, _maxDistance); - verify(_innerRadius <= _outerRadius); - LOG(1) << ") to (" << _innerRadius << ", " << _outerRadius << ")" << endl; - ++_stats._numShells; - } - - bool S2NearCursor::advance() { - if (_innerRadius > _maxDistance) { - LOG(2) << "advancing but exhausted search distance" << endl; - return false; - } - - if (!_results.empty()) { - _returnedDistance = _results.top().distance; - _returned.insert(_results.top().loc); - _results.pop(); - ++_stats._numReturned; - // Safe to grow the radius as we've returned everything in our shell. We don't do this - // check outside of !_results.empty() because we could have results, yield, dump them - // (_results would be empty), then need to recreate them w/the same radii. In that case - // we'd grow when we shouldn't. - if (_results.empty()) { nextAnnulus(); } - } - - if (_results.empty()) { fillResults(); } - - // The only reason _results should be empty now is if there are no more possible results. - return !_results.empty(); - } - - BSONObj S2NearCursor::makeFRSObject() { - BSONObjBuilder frsObjBuilder; - frsObjBuilder.appendElements(_filteredQuery); - - S2RegionCoverer coverer; - // Step 1: Make the BSON'd covering for our search annulus. - BSONObj inExpr; - // Caps are inclusive and inverting a cap includes the border. This means that our - // initial _innerRadius of 0 is OK -- we'll still find a point that is exactly at - // the start of our search. - _innerCap = S2Cap::FromAxisAngle(_nearQuery.centroid, - S1Angle::Radians(_innerRadius / _params.radius)); - _outerCap = S2Cap::FromAxisAngle(_nearQuery.centroid, - S1Angle::Radians(_outerRadius / _params.radius)); - double area = _outerCap.area() - _innerCap.area(); - _innerCap = _innerCap.Complement(); - vector regions; - regions.push_back(&_innerCap); - regions.push_back(&_outerCap); - _annulus.Release(NULL); - _annulus.Init(®ions); - vector cover; - S2SearchUtil::setCoverLimitsBasedOnArea(area, &coverer, _params.coarsestIndexedLevel); - coverer.GetCovering(_annulus, &cover); - LOG(2) << "annulus cover size is " << cover.size() - << ", params (" << coverer.min_level() << ", " << coverer.max_level() << ")" - << endl; - inExpr = S2SearchUtil::coverAsBSON(cover, _nearQuery.field, - _params.coarsestIndexedLevel); - frsObjBuilder.appendElements(inExpr); - - _params.configureCoverer(&coverer); - // Cover the indexed geo components of the query. - for (size_t i = 0; i < _indexedGeoFields.size(); ++i) { - vector cover; - coverer.GetCovering(_indexedGeoFields[i].getRegion(), &cover); - uassert(16682, "Couldn't generate index keys for geo field " - + _indexedGeoFields[i].getField(), - cover.size() > 0); - BSONObj fieldRange = S2SearchUtil::coverAsBSON(cover, _indexedGeoFields[i].getField(), - _params.coarsestIndexedLevel); - frsObjBuilder.appendElements(fieldRange); - } - - return frsObjBuilder.obj(); - } - - // Fill _results with the next shell of results. We may have to search several times to do - // this. If _results.empty() after calling fillResults, there are no more possible results. - void S2NearCursor::fillResults() { - verify(_results.empty()); - if (_innerRadius >= _outerRadius) { return; } - if (_innerRadius > _maxDistance) { return; } - - // We iterate until 1. our search radius is too big or 2. we find results. - do { - // Some of these arguments are opaque, look at the definitions of the involved classes. - FieldRangeSet frs(_details->parentNS().c_str(), makeFRSObject(), false, false); - shared_ptr frv(new FieldRangeVector(frs, _specForFRV, 1)); - scoped_ptr cursor(BtreeCursor::make(nsdetails(_details->parentNS()), - *_details, frv, 0, 1)); - - // The cursor may return the same obj more than once for a given - // FRS, so we make sure to only consider it once in any given annulus. - // - // We don't want this outside of the 'do' loop because the covering - // for an annulus may return an object whose distance to the query - // point is actually contained in a subsequent annulus. If we - // didn't consider every object in a given annulus we might miss - // the point. - // - // We don't use a global 'seen' because we get that by requiring - // the distance from the query point to the indexed geo to be - // within our 'current' annulus, and I want to dodge all yield - // issues if possible. - unordered_set seen; - - LOG(1) << "looking at annulus from " << _innerRadius << " to " << _outerRadius << endl; - LOG(1) << "Total # returned: " << _stats._numReturned << endl; - // Do the actual search through this annulus. - for (; cursor->ok(); cursor->advance()) { - // Don't bother to look at anything we've returned. - if (_returned.end() != _returned.find(cursor->currLoc())) { - ++_stats._returnSkip; - continue; - } - - ++_stats._nscanned; - if (seen.end() != seen.find(cursor->currLoc())) { - ++_stats._btreeDups; - continue; - } - - // Get distance interval from our query point to the cell. - // If it doesn't overlap with our current shell, toss. - BSONObj currKey(cursor->currKey()); - BSONObjIterator it(currKey); - BSONElement geoKey; - for (int i = 0; i <= _nearFieldIndex; ++i) { - geoKey = it.next(); - } - - S2Cell keyCell = S2Cell(S2CellId::FromString(geoKey.String())); - if (!_annulus.MayIntersect(keyCell)) { - ++_stats._keyGeoSkip; - continue; - } - - // Match against non-indexed fields. - ++_stats._matchTested; - MatchDetails details; - if (!_matcher->matchesCurrent(cursor.get(), &details)) { - continue; - } - - // We have to add this document to seen *AFTER* the key intersection test. - // A geometry may have several keys, one of which may be in our search shell and one - // of which may be outside of it. We don't want to ignore a document just because - // one of its covers isn't inside this annulus. - seen.insert(cursor->currLoc()); - - // At this point forward, we will not examine the document again in this annulus. - - const BSONObj& indexedObj = cursor->currLoc().obj(); - - // Match against indexed geo fields. - ++_stats._geoMatchTested; - size_t geoFieldsMatched = 0; - // See if the object actually overlaps w/the geo query fields. - for (size_t i = 0; i < _indexedGeoFields.size(); ++i) { - BSONElementSet geoFieldElements; - indexedObj.getFieldsDotted(_indexedGeoFields[i].getField(), geoFieldElements, - false); - if (geoFieldElements.empty()) { continue; } - - bool match = false; - - for (BSONElementSet::iterator oi = geoFieldElements.begin(); - !match && (oi != geoFieldElements.end()); ++oi) { - if (!oi->isABSONObj()) { continue; } - const BSONObj &geoObj = oi->Obj(); - GeometryContainer geoContainer; - uassert(16699, "ill-formed geometry: " + geoObj.toString(), - geoContainer.parseFrom(geoObj)); - match = _indexedGeoFields[i].satisfiesPredicate(geoContainer); - } - - if (match) { ++geoFieldsMatched; } - } - - if (geoFieldsMatched != _indexedGeoFields.size()) { - continue; - } - - // Get all the fields with that name from the document. - BSONElementSet geoFieldElements; - indexedObj.getFieldsDotted(_nearQuery.field, geoFieldElements, false); - if (geoFieldElements.empty()) { continue; } - - ++_stats._inAnnulusTested; - double minDistance = 1e20; - // Look at each field in the document and take the min. distance. - for (BSONElementSet::iterator oi = geoFieldElements.begin(); - oi != geoFieldElements.end(); ++oi) { - if (!oi->isABSONObj()) { continue; } - double dist = distanceTo(oi->Obj()); - minDistance = min(dist, minDistance); - } - - // We could be in an annulus, yield, add new points closer to - // query point than the last point we returned, then unyield. - // This would return points out of order. - if (minDistance < _returnedDistance) { continue; } - - // If the min. distance satisfies our distance criteria - if (minDistance >= _innerRadius && minDistance < _outerRadius) { - // The result is valid. We have to de-dup ourselves here. - if (_returned.end() == _returned.find(cursor->currLoc())) { - _results.push(Result(cursor->currLoc(), cursor->currKey(), - minDistance)); - } - } - } - - if (_results.empty()) { - LOG(1) << "results empty!\n"; - _radiusIncrement *= 2; - nextAnnulus(); - } else if (_results.size() < 300) { - _radiusIncrement *= 2; - } else if (_results.size() > 600) { - _radiusIncrement /= 2; - } - } while (_results.empty() - && _innerRadius < _maxDistance - && _innerRadius < _outerRadius); - LOG(1) << "Filled shell with " << _results.size() << " results" << endl; - } - - double S2NearCursor::distanceTo(const BSONObj &obj) { - const S2Point &us = _nearQuery.centroid; - S2Point them; - - S2Polygon polygon; - S2Polyline line; - S2Cell point; - if (GeoParser::parsePolygon(obj, &polygon)) { - them = polygon.Project(us); - } else if (GeoParser::parseLineString(obj, &line)) { - int tmp; - them = line.Project(us, &tmp); - } else if (GeoParser::parsePoint(obj, &point)) { - them = point.GetCenter(); - } else { - warning() << "unknown geometry: " << obj.toString(); - return numeric_limits::max(); - } - S1Angle angle(us, them); - return angle.radians() * _params.radius; - } -} // namespace mongo diff --git a/src/mongo/db/geo/s2nearcursor.h b/src/mongo/db/geo/s2nearcursor.h deleted file mode 100644 index 1d7040f5769..00000000000 --- a/src/mongo/db/geo/s2nearcursor.h +++ /dev/null @@ -1,157 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include -#include "mongo/db/jsobj.h" -#include "mongo/db/commands.h" -#include "mongo/db/btreecursor.h" -#include "mongo/db/cursor.h" -#include "mongo/db/diskloc.h" -#include "mongo/db/matcher.h" -#include "mongo/db/queryutil.h" -#include "mongo/db/geo/s2common.h" -#include "mongo/db/geo/geoquery.h" -#include "mongo/platform/unordered_set.h" -#include "third_party/s2/s2cap.h" -#include "third_party/s2/s2regionintersection.h" - -namespace mongo { - class S2NearCursor : public Cursor { - public: - S2NearCursor(const BSONObj &keyPattern, const IndexDetails* details, const BSONObj &query, - const NearQuery &nearQuery, const vector &indexedGeoRegions, - const S2IndexingParams ¶ms); - virtual ~S2NearCursor(); - virtual CoveredIndexMatcher *matcher() const; - - virtual bool supportYields() { return true; } - virtual bool supportGetMore() { return true; } - virtual bool isMultiKey() const { return true; } - virtual bool autoDedup() const { return false; } - virtual bool modifiedKeys() const { return true; } - virtual bool getsetdup(DiskLoc loc) { return false; } - virtual string toString() { return "S2NearCursor"; } - BSONObj indexKeyPattern() { return _keyPattern; } - virtual bool ok(); - virtual Record* _current(); - virtual BSONObj current(); - virtual DiskLoc currLoc(); - virtual bool advance(); - virtual BSONObj currKey() const; - virtual DiskLoc refLoc(); - virtual void noteLocation(); - virtual void checkLocation(); - virtual long long nscanned(); - virtual void explainDetails(BSONObjBuilder& b); - - double currentDistance() const; - private: - // We use this to cache results of the search. Results are sorted to have decreasing - // distance, and callers are interested in loc and key. - struct Result { - Result(const DiskLoc &dl, const BSONObj &ck, double dist) : loc(dl), key(ck), - distance(dist) { } - bool operator<(const Result& other) const { - // We want increasing distance, not decreasing, so we reverse the <. - return distance > other.distance; - } - DiskLoc loc; - BSONObj key; - double distance; - }; - - // Make the object that describes all keys that are within our current search annulus. - BSONObj makeFRSObject(); - // Fill _results with all of the results in the annulus defined by _innerRadius and - // _outerRadius. If no results are found, grow the annulus and repeat until success (or - // until the edge of the world). - void fillResults(); - // Grow _innerRadius and _outerRadius by _radiusIncrement, capping _outerRadius at halfway - // around the world (pi * _params.radius). - void nextAnnulus(); - double distanceTo(const BSONObj &obj); - - // Need this to make a FieldRangeSet. - const IndexDetails *_details; - - // How we need/use the query: - // Matcher: Can have geo fields in it, but only with $within. - // This only really happens (right now) from geoNear command. - // We assume the caller takes care of this in the right way. - // FRS: No geo fields allowed! - // So, on that note: the query with the geo stuff taken out, used by makeFRSObject(). - BSONObj _filteredQuery; - // The GeoQuery for the point we're doing near searching from. - NearQuery _nearQuery; - // What geo regions are we looking for? - vector _indexedGeoFields; - // We use this for matching non-GEO stuff. - shared_ptr _matcher; - // How were the keys created? We need this to search for the right stuff. - S2IndexingParams _params; - // We have to pass this to the FieldRangeVector ctor (in modified form). - BSONObj _keyPattern; - // We also pass this to the FieldRangeVector ctor. - IndexSpec _specForFRV; - - // Geo-related variables. - // What's the max distance (arc length) we're willing to look for results? - double _maxDistance; - // We compute an annulus of results and cache it here. - priority_queue _results; - // These radii define the annulus we're currently looking at. - double _innerRadius; - double _outerRadius; - // When we search the next annulus, what to adjust our radius by? Grows when we search an - // annulus and find no results. - double _radiusIncrement; - // What have we returned already? - unordered_set _returned; - - struct Stats { - Stats() : _nscanned(0), _matchTested(0), _geoMatchTested(0), _numShells(0), - _keyGeoSkip(0), _returnSkip(0), _btreeDups(0), _inAnnulusTested(0), - _numReturned(0) {} - // Stat counters/debug information goes below. - // How many items did we look at in the btree? - long long _nscanned; - // How many did we try to match? - long long _matchTested; - // How many did we geo-test? - long long _geoMatchTested; - // How many search shells did we use? - long long _numShells; - // How many did we skip due to key-geo check? - long long _keyGeoSkip; - long long _returnSkip; - long long _btreeDups; - long long _inAnnulusTested; - long long _numReturned; - }; - - Stats _stats; - - // The S2 machinery that represents the search annulus - S2Cap _innerCap; - S2Cap _outerCap; - S2RegionIntersection _annulus; - // This is the "array index" of the key field that is the near field. We use this to do - // cheap is-this-doc-in-the-annulus testing. - int _nearFieldIndex; - // The max distance we've returned so far. - double _returnedDistance; - }; -} // namespace mongo diff --git a/src/mongo/db/geo/shapes.cpp b/src/mongo/db/geo/shapes.cpp index 136ab1cf979..2be932344c3 100644 --- a/src/mongo/db/geo/shapes.cpp +++ b/src/mongo/db/geo/shapes.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -194,9 +206,6 @@ namespace mongo { // XXX: why is there a mod here? Point p2 = _points[i % size()]; - GEODEBUG("Doing intersection check of " << fudgeBox.toString() - << " with seg " << p1.toString() << " to " << p2.toString()); - // We need to check whether or not this segment intersects our error box if (fudge > 0 && // Points not too far below box @@ -208,16 +217,13 @@ namespace mongo { // Points not too far to right of box fudgeBox._max.x >= std::min(p1.x, p2.x)) { - GEODEBUG("Doing detailed check"); // If our box contains one or more of these points, we need to do an exact // check. if (fudgeBox.inside(p1)) { - GEODEBUG("Point 1 inside"); return 0; } if (fudgeBox.inside(p2)) { - GEODEBUG("Point 2 inside"); return 0; } @@ -227,13 +233,11 @@ namespace mongo { double xintersT = (fudgeBox._max.y - p1.y) * invSlope + p1.x; if (fudgeBox._min.x <= xintersT && fudgeBox._max.x >= xintersT) { - GEODEBUG("Top intersection @ " << xintersT); return 0; } double xintersB = (fudgeBox._min.y - p1.y) * invSlope + p1.x; if (fudgeBox._min.x <= xintersB && fudgeBox._max.x >= xintersB) { - GEODEBUG("Bottom intersection @ " << xintersB); return 0; } } @@ -244,13 +248,11 @@ namespace mongo { double yintersR = (p1.x - fudgeBox._max.x) * slope + p1.y; if (fudgeBox._min.y <= yintersR && fudgeBox._max.y >= yintersR) { - GEODEBUG("Right intersection @ " << yintersR); return 0; } double yintersL = (p1.x - fudgeBox._min.x) * slope + p1.y; if (fudgeBox._min.y <= yintersL && fudgeBox._max.y >= yintersL) { - GEODEBUG("Left intersection @ " << yintersL); return 0; } } @@ -459,4 +461,5 @@ namespace mongo { return sqrt((a * a) + (b * b)); } + } // namespace mongo diff --git a/src/mongo/db/geo/shapes.h b/src/mongo/db/geo/shapes.h index 7ec31bf23da..2f18f6731c2 100644 --- a/src/mongo/db/geo/shapes.h +++ b/src/mongo/db/geo/shapes.h @@ -12,14 +12,36 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "mongo/pch.h" +#include +#include + +#include "mongo/base/owned_pointer_vector.h" #include "mongo/db/jsobj.h" +#include "mongo/db/geo/s2.h" +#include "third_party/s2/s2cap.h" +#include "third_party/s2/s2cell.h" +#include "third_party/s2/s2latlng.h" +#include "third_party/s2/s2polygon.h" +#include "third_party/s2/s2polyline.h" namespace mongo { + struct Point; double distance(const Point& p1, const Point &p2); bool distanceWithin(const Point &p1, const Point &p2, double radius); @@ -103,4 +125,77 @@ namespace mongo { bool _boundsCalculated; vector _points; }; + + // Clearly this isn't right but currently it's sufficient. + enum CRS { + FLAT, + SPHERE + }; + + struct PointWithCRS { + PointWithCRS() : flatUpgradedToSphere(false) { } + S2Point point; + S2Cell cell; + Point oldPoint; + CRS crs; + // If crs is FLAT, we might be able to upgrade the point to SPHERE if it's a valid SPHERE + // point (lng/lat in bounds). In this case, we can use FLAT data with SPHERE predicates. + bool flatUpgradedToSphere; + }; + + struct LineWithCRS { + S2Polyline line; + CRS crs; + }; + + struct CapWithCRS { + S2Cap cap; + Circle circle; + CRS crs; + }; + + struct BoxWithCRS { + Box box; + CRS crs; + }; + + struct PolygonWithCRS { + S2Polygon polygon; + Polygon oldPolygon; + CRS crs; + }; + + struct MultiPointWithCRS { + vector points; + vector cells; + CRS crs; + }; + + struct MultiLineWithCRS { + OwnedPointerVector lines; + CRS crs; + }; + + struct MultiPolygonWithCRS { + OwnedPointerVector polygons; + CRS crs; + }; + + struct GeometryCollection { + vector points; + + // The amount of indirection here is painful but we can't operator= scoped_ptr or + // OwnedPointerVector. + OwnedPointerVector lines; + OwnedPointerVector polygons; + OwnedPointerVector multiPoints; + OwnedPointerVector multiLines; + OwnedPointerVector multiPolygons; + + bool supportsContains() { + // Only polygons (and multiPolygons) support containment. + return (polygons.vector().size() > 0 || multiPolygons.vector().size() > 0); + } + }; + } // namespace mongo diff --git a/src/mongo/db/hasher.cpp b/src/mongo/db/hasher.cpp index ae51d8707da..4b41b163156 100644 --- a/src/mongo/db/hasher.cpp +++ b/src/mongo/db/hasher.cpp @@ -18,6 +18,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/hasher.h" diff --git a/src/mongo/db/hasher.h b/src/mongo/db/hasher.h index ae3e9567cee..2410306038d 100644 --- a/src/mongo/db/hasher.h +++ b/src/mongo/db/hasher.h @@ -18,6 +18,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -68,6 +80,9 @@ namespace mongo { /* The hash function we use can be given a seed, to effectively randomize it * by choosing from among a family of hash functions. When it is not specified, * use this. + * + * WARNING: do not change the hash see value. Hash-based sharding clusters will + * expect that value to be zero. */ static const int DEFAULT_HASH_SEED = 0; @@ -85,9 +100,6 @@ namespace mongo { */ static long long int hash64( const BSONElement& e , HashSeed seed ); - private: - BSONElementHasher(); - /* This incrementally computes the hash of BSONElement "e" * using hash function "h". If "includeFieldName" is true, * then the name of the field is hashed in between the type of @@ -98,6 +110,9 @@ namespace mongo { */ static void recursiveHash( Hasher* h , const BSONElement& e , bool includeFieldName ); + private: + BSONElementHasher(); + }; } diff --git a/src/mongo/db/hasher_test.cpp b/src/mongo/db/hasher_test.cpp new file mode 100644 index 00000000000..a11113d13a9 --- /dev/null +++ b/src/mongo/db/hasher_test.cpp @@ -0,0 +1,385 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** Unit tests for BSONElementHasher. */ + +#include "mongo/db/hasher.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/bson/bsontypes.h" + +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + + // Helper methods + long long hashIt( const BSONObj& object, int seed ) { + return BSONElementHasher::hash64( object.firstElement(), seed ); + } + long long hashIt( const BSONObj& object ) { + int seed = 0; + return hashIt( object, seed ); + } + + // Test different oids hash to different things + TEST( BSONElementHasher, DifferentOidsAreDifferentHashes ) { + int seed = 0; + + long long int oidHash = BSONElementHasher::hash64( + BSONObjBuilder().genOID().obj().firstElement() , seed ); + long long int oidHash2 = BSONElementHasher::hash64( + BSONObjBuilder().genOID().obj().firstElement() , seed ); + long long int oidHash3 = BSONElementHasher::hash64( + BSONObjBuilder().genOID().obj().firstElement() , seed ); + + ASSERT_NOT_EQUALS( oidHash , oidHash2 ); + ASSERT_NOT_EQUALS( oidHash , oidHash3 ); + ASSERT_NOT_EQUALS( oidHash3 , oidHash2 ); + } + + // Test 32-bit ints, 64-bit ints, doubles hash to same thing + TEST( BSONElementHasher, ConsistentHashOfIntLongAndDouble ) { + int i = 3; + BSONObj p1 = BSON("a" << i); + long long int intHash = hashIt( p1 ); + + long long int ilong = 3; + BSONObj p2 = BSON("a" << ilong); + long long int longHash = hashIt( p2 ); + + double d = 3.1; + BSONObj p3 = BSON("a" << d); + long long int doubleHash = hashIt( p3 ); + + ASSERT_EQUALS( intHash, longHash ); + ASSERT_EQUALS( doubleHash, longHash ); + } + + // Test different ints don't hash to same thing + TEST( BSONElementHasher, DifferentIntHashesDiffer ) { + ASSERT_NOT_EQUALS( + hashIt( BSON("a" << 3) ) , + hashIt( BSON("a" << 4) ) + ); + } + + // Test seed makes a difference + TEST( BSONElementHasher, SeedMatters ) { + ASSERT_NOT_EQUALS( + hashIt( BSON("a" << 4), 0 ) , + hashIt( BSON("a" << 4), 1 ) + ); + } + + // Test strings hash to different things + TEST( BSONElementHasher, IntAndStringHashesDiffer ) { + ASSERT_NOT_EQUALS( + hashIt( BSON("a" << 3) ) , + hashIt( BSON("a" << "3") ) + ); + } + + // Test regexps and strings hash to different things + TEST( BSONElementHasher, RegexAndStringHashesDiffer ) { + BSONObjBuilder builder; + + ASSERT_NOT_EQUALS( + hashIt( BSON("a" << "3") ) , + hashIt( builder.appendRegex("a","3").obj() ) + ); + } + + // Test arrays and subobject hash to different things + TEST( BSONElementHasher, ArrayAndSubobjectHashesDiffer ) { + ASSERT_NOT_EQUALS( + hashIt( fromjson("{a : {'0' : 0 , '1' : 1}}") ) , + hashIt( fromjson("{a : [0,1]}") ) + ); + } + + // Testing sub-document grouping + TEST( BSONElementHasher, SubDocumentGroupingHashesDiffer ) { + ASSERT_NOT_EQUALS( + hashIt( fromjson("{x : {a : {}, b : 1}}") ) , + hashIt( fromjson("{x : {a : {b : 1}}}") ) + ); + } + + // Testing codeWscope scope squashing + TEST( BSONElementHasher, CodeWithScopeSquashesScopeIntsAndDoubles ) { + int seed = 0; + + BSONObjBuilder b1; + b1.appendCodeWScope("a","print('this is some stupid code')", BSON("a" << 3)); + BSONObj p10 = b1.obj(); + + BSONObjBuilder b2; + b2.appendCodeWScope("a","print('this is some stupid code')", BSON("a" << 3.1)); + + BSONObjBuilder b3; + b3.appendCodeWScope("a","print('this is \nsome stupider code')", BSON("a" << 3)); + ASSERT_EQUALS( + BSONElementHasher::hash64( p10.firstElement() , seed ) , + BSONElementHasher::hash64( b2.obj().firstElement() , seed ) + ); + ASSERT_NOT_EQUALS( + BSONElementHasher::hash64( p10.firstElement() , seed ) , + BSONElementHasher::hash64( b3.obj().firstElement() , seed ) + ); + } + + // Test some recursive squashing + TEST( BSONElementHasher, RecursiveSquashingIntsAndDoubles ) { + ASSERT_EQUALS( + hashIt( fromjson("{x : {a : 3 , b : [ 3.1, {c : 3 }]}}") ) , + hashIt( fromjson("{x : {a : 3.1 , b : [ 3, {c : 3.0}]}}") ) + ); + } + + // Test minkey and maxkey don't hash to same thing + TEST( BSONElementHasher, MinKeyMaxKeyHashesDiffer ) { + ASSERT_NOT_EQUALS( + hashIt( BSON("a" << MAXKEY) ) , + hashIt( BSON("a" << MINKEY) ) + ); + } + + // Test squashing very large doubles and very small doubles + TEST( BSONElementHasher, VeryLargeAndSmallDoubles ) { + long long maxInt = std::numeric_limits::max(); + double smallerDouble = maxInt/2; + double biggerDouble = ( (double)maxInt )*( (double)maxInt ); + ASSERT_NOT_EQUALS( + hashIt( BSON("a" << maxInt ) ) , + hashIt( BSON("a" << smallerDouble ) ) + ); + ASSERT_EQUALS( + hashIt( BSON("a" << maxInt ) ) , + hashIt( BSON("a" << biggerDouble ) ) + ); + + long long minInt = std::numeric_limits::min(); + double negativeDouble = -( (double)maxInt )*( (double)maxInt ); + ASSERT_EQUALS( + hashIt( BSON("a" << minInt ) ) , + hashIt( BSON("a" << negativeDouble ) ) + ); + } + + // Remaining tests are hard-coded checks to ensure the hash function is + // consistent across platforms and server versions + // + // All of the values in the remaining tests have been determined experimentally. + TEST( BSONElementHasher, HashIntOrLongOrDouble ) { + BSONObj o = BSON( "check" << 42 ); + ASSERT_EQUALS( hashIt( o ), -944302157085130861LL ); + o = BSON( "check" << 42.123 ); + ASSERT_EQUALS( hashIt( o ), -944302157085130861LL ); + o = BSON( "check" << (long long) 42 ); + ASSERT_EQUALS( hashIt( o ), -944302157085130861LL ); + + o = BSON( "check" << 0 ); + ASSERT_EQUALS( hashIt( o ), 4854801880128277513LL ); + o = BSON( "check" << 0.456 ); + ASSERT_EQUALS( hashIt( o ), 4854801880128277513LL ); + o = BSON( "check" << (long long) 0 ); + ASSERT_EQUALS( hashIt( o ), 4854801880128277513LL ); + // NAN is treated as zero. + o = BSON( "check" << std::numeric_limits::signaling_NaN() ); + ASSERT_EQUALS( hashIt( o ), 4854801880128277513LL ); + o = BSON( "check" << std::numeric_limits::quiet_NaN() ); + ASSERT_EQUALS( hashIt( o ), 4854801880128277513LL ); + + o = BSON( "check" << 1 ); + ASSERT_EQUALS( hashIt( o ), 5902408780260971510LL ); + o = BSON( "check" << 1.987 ); + ASSERT_EQUALS( hashIt( o ), 5902408780260971510LL ); + o = BSON( "check" << (long long) 1 ); + ASSERT_EQUALS( hashIt( o ), 5902408780260971510LL ); + + o = BSON( "check" << -1 ); + ASSERT_EQUALS( hashIt( o ), 1140205862565771219LL ); + o = BSON( "check" << -1.789 ); + ASSERT_EQUALS( hashIt( o ), 1140205862565771219LL ); + o = BSON( "check" << (long long) -1 ); + ASSERT_EQUALS( hashIt( o ), 1140205862565771219LL ); + + o = BSON( "check" << std::numeric_limits::min() ); + ASSERT_EQUALS( hashIt( o ), 6165898260261354870LL ); + o = BSON( "check" << (double) std::numeric_limits::min() ); + ASSERT_EQUALS( hashIt( o ), 6165898260261354870LL ); + o = BSON( "check" << (long long) std::numeric_limits::min() ); + ASSERT_EQUALS( hashIt( o ), 6165898260261354870LL ); + + o = BSON( "check" << std::numeric_limits::max() ); + ASSERT_EQUALS( hashIt( o ), 1143184177162245883LL ); + o = BSON( "check" << (double) std::numeric_limits::max() ); + ASSERT_EQUALS( hashIt( o ), 1143184177162245883LL ); + o = BSON( "check" << (long long) std::numeric_limits::max() ); + ASSERT_EQUALS( hashIt( o ), 1143184177162245883LL ); + + // Large/small double values. + ASSERT( std::numeric_limits::max() < std::numeric_limits::max() ); + o = BSON( "check" << std::numeric_limits::max() ); + ASSERT_EQUALS( hashIt( o ), 921523596458303250LL ); + o = BSON( "check" << std::numeric_limits::max() ); // 9223372036854775807 + ASSERT_EQUALS( hashIt( o ), 921523596458303250LL ); + + // Have to create our own small double. + // std::numeric_limits::lowest() - Not available until C++11 + // std::numeric_limits::min() - Closest positive value to zero, not most negative. + double smallDouble = - std::numeric_limits::max(); + ASSERT( smallDouble < static_cast( std::numeric_limits::min() ) ); + o = BSON( "check" << smallDouble ); + ASSERT_EQUALS( hashIt( o ), 4532067210535695462LL ); + o = BSON( "check" << std::numeric_limits::min() ); // -9223372036854775808 + ASSERT_EQUALS( hashIt( o ), 4532067210535695462LL ); + } + + TEST( BSONElementHasher, HashMinKey ) { + BSONObj o = BSON( "check" << MINKEY ); + ASSERT_EQUALS( hashIt( o ), 7961148599568647290LL ); + } + + TEST( BSONElementHasher, HashMaxKey ) { + BSONObj o = BSON( "check" << MAXKEY ); + ASSERT_EQUALS( hashIt( o ), 5504842513779440750LL ); + } + + TEST( BSONElementHasher, HashUndefined ) { + BSONObj o = BSON( "check" << BSONUndefined ); + ASSERT_EQUALS( hashIt( o ), 40158834000849533LL ); + } + + TEST( BSONElementHasher, HashNull ) { + BSONObj o = BSON( "check" << BSONNULL ); + ASSERT_EQUALS( hashIt( o ), 2338878944348059895LL ); + } + + TEST( BSONElementHasher, HashString ) { + BSONObj o = BSON( "check" << "abc" ); + ASSERT_EQUALS( hashIt( o ), 8478485326885698097LL ); + o = BSON( "check" << BSONSymbol( "abc" ) ); + ASSERT_EQUALS( hashIt( o ), 8478485326885698097LL ); + + o = BSON( "check" << "" ); + ASSERT_EQUALS( hashIt( o ), 2049396243249673340LL ); + o = BSON( "check" << BSONSymbol( "" ) ); + ASSERT_EQUALS( hashIt( o ), 2049396243249673340LL ); + } + + TEST( BSONElementHasher, HashObject ) { + BSONObj o = BSON( "check" << BSON( "a" << "abc" << "b" << 123LL ) ); + ASSERT_EQUALS( hashIt( o ), 4771603801758380216LL ); + + o = BSON( "check" << BSONObj() ); + ASSERT_EQUALS( hashIt( o ), 7980500913326740417LL ); + } + + TEST( BSONElementHasher, HashArray ) { + BSONObj o = BSON( "check" << BSON_ARRAY( "bar" << "baz" << "qux" ) ); + ASSERT_EQUALS( hashIt( o ), -2938911267422831539LL ); + + o = BSON( "check" << BSONArray() ); + ASSERT_EQUALS( hashIt( o ), 8849948234993459283LL ); + } + + TEST( BSONElementHasher, HashBinary ) { + uint8_t bytes[] = { 0, 1, 2, 3, 4, 6 }; + BSONObj o = BSON( "check" << BSONBinData( bytes, 6, BinDataGeneral ) ); + ASSERT_EQUALS( hashIt( o ), 7252465090394235301LL ); + + o = BSON( "check" << BSONBinData( bytes, 6, bdtCustom ) ); + ASSERT_EQUALS( hashIt( o ), 5736670452907618262LL ); + + uint8_t uuidBytes[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; + o = BSON( "check" << BSONBinData( uuidBytes, 16, newUUID ) ); + ASSERT_EQUALS( hashIt( o ), 6084661258071355978LL ); + } + + TEST( BSONElementHasher, HashObjectId ) { + BSONObj o = BSON( "check" << OID( "010203040506070809101112" ) ); + ASSERT_EQUALS( hashIt( o ), -5588663249627035708LL ); + + o = BSON( "check" << OID( "000000000000000000000000" ) ); + ASSERT_EQUALS( hashIt( o ), -4293118519463489418LL ); + } + + TEST( BSONElementHasher, HashBoolean ) { + BSONObj o = BSON( "check" << true ); + ASSERT_EQUALS( hashIt( o ), 6405873908747105701LL ); + + o = BSON( "check" << false ); + ASSERT_EQUALS( hashIt( o ), 6289544573401934092LL ); + } + + TEST( BSONElementHasher, HashTimeStamp ) { + BSONObjBuilder builder1; + BSONObjBuilder builder2; + + BSONObj o = BSON( "check" << Date_t( 0x5566778811223344LL ) ); + ASSERT_EQUALS( hashIt( o ), 4476222765095560467LL ); + o = builder1.appendTimestamp( "check", 0x55667788LL * 1000LL, 0x11223344LL ).obj(); + ASSERT_EQUALS( hashIt( o ), 4476222765095560467LL ); + + o = BSON( "check" << Date_t( 0 ) ); + ASSERT_EQUALS( hashIt( o ), -1178696894582842035LL ); + o = builder2.appendTimestamp( "check", 0 ).obj(); + ASSERT_EQUALS( hashIt( o ), -1178696894582842035LL ); + } + + TEST( BSONElementHasher, HashRegEx ) { + BSONObj o = BSON( "check" << BSONRegEx( "mongodb" ) ); + ASSERT_EQUALS( hashIt( o ), -7275792090268217043LL ); + + o = BSON( "check" << BSONRegEx( ".*", "i" ) ); + ASSERT_EQUALS( hashIt( o ), 7095855029187981886LL ); + } + + TEST( BSONElementHasher, HashDBRef ) { + BSONObj o = BSON( "check" << BSONDBRef( "c", OID( "010203040506070809101112" ) ) ); + ASSERT_EQUALS( hashIt( o ), 940175826736461384LL ); + + o = BSON( "check" << BSONDBRef( "db.c", OID( "010203040506070809101112" ) ) ); + ASSERT_EQUALS( hashIt( o ), 2426768198104018194LL ); + } + + TEST( BSONElementHasher, HashCode ) { + BSONObj o = BSON( "check" << BSONCode( "func f() { return 1; }" ) ); + ASSERT_EQUALS( hashIt( o ), 6861638109178014270LL ); + } + + TEST( BSONElementHasher, HashCodeWScope ) { + BSONObj o = BSON( "check" << + BSONCodeWScope( "func f() { return 1; }", BSON( "c" << true ) ) ); + ASSERT_EQUALS( hashIt( o ), 501342939894575968LL ); + } + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/hashindex.cpp b/src/mongo/db/hashindex.cpp deleted file mode 100644 index a769dc73a98..00000000000 --- a/src/mongo/db/hashindex.cpp +++ /dev/null @@ -1,170 +0,0 @@ -// mongo/db/hashindex.cpp - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/hashindex.h" - -#include "mongo/db/btreecursor.h" -#include "mongo/db/json.h" -#include "mongo/db/queryutil.h" - -namespace mongo { - - const string HashedIndexType::HASHED_INDEX_TYPE_IDENTIFIER = "hashed"; - - HashedIndexType::HashedIndexType( const IndexPlugin* plugin , const IndexSpec* spec ) : - IndexType( plugin , spec ) , _keyPattern( spec->keyPattern ) { - - //change these if single-field limitation lifted later - uassert( 16241 , "Currently only single field hashed index supported." , - _keyPattern.toBSON().nFields() == 1 ); - uassert( 16242 , "Currently hashed indexes cannot guarantee uniqueness. Use a regular index." , - ! (spec->info).getField("unique").booleanSafe() ); - - //Default _seed to 0 if "seed" is not included in the index spec - //or if the value of "seed" is not a number - _seed = (spec->info).getField("seed").numberInt(); - - //Default _isSparse to false if "sparse" is not included in the index spec - //or if the value of "sparse" is not a boolean - _isSparse = (spec->info).getField("sparse").booleanSafe(); - - //In case we have hashed indexes based on other hash functions in - //the future, we store a hashVersion number. If hashVersion changes, - // "makeSingleKey" will need to change accordingly. - //Defaults to 0 if "hashVersion" is not included in the index spec - //or if the value of "hashversion" is not a number - _hashVersion = (spec->info).getField("hashVersion").numberInt(); - - //Get the hashfield name - BSONElement firstElt = spec->keyPattern.firstElement(); - massert( 16243 , "error: no hashed index field" , - firstElt.str().compare( HASHED_INDEX_TYPE_IDENTIFIER ) == 0 ); - _hashedField = firstElt.fieldName(); - - // Explicit null valued fields and missing fields are both represented in hashed indexes - // using the hash value of the null BSONElement. This is partly for historical reasons - // (hash of null was used in the initial release of hashed indexes and changing would alter - // the data format). Additionally, in certain places the hashed index code and the index - // bound calculation code assume null and missing are indexed identically. - BSONObj nullObj = BSON( "" << BSONNULL ); - _missingKey = BSON( "" << makeSingleKey( nullObj.firstElement(), _seed, _hashVersion ) ); - } - - HashedIndexType::~HashedIndexType() { } - - IndexSuitability HashedIndexType::suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const { - if ( queryConstraints.isPointIntervalSet( _hashedField ) ) - return HELPFUL; - return USELESS; - } - - void HashedIndexType::getKeys( const BSONObj &obj, BSONObjSet &keys ) const { - string hashedFieldCopy = string( _hashedField ); - const char* hashedFieldCopyPtr = hashedFieldCopy.c_str(); - BSONElement fieldVal = obj.getFieldDottedOrArray( hashedFieldCopyPtr ); - - uassert( 16244 , "Error: hashed indexes do not currently support array values" , fieldVal.type() != Array ); - - if ( ! fieldVal.eoo() ) { - BSONObj key = BSON( "" << makeSingleKey( fieldVal , _seed , _hashVersion ) ); - keys.insert( key ); - } - else if (! _isSparse ) { - keys.insert( _missingKey.copy() ); - } - } - - shared_ptr HashedIndexType::newCursor( const BSONObj& query , - const BSONObj& order , int numWanted ) const { - - //Use FieldRangeSet to parse the query into a vector of intervals - //These should be point-intervals if this cursor is ever used - //So the FieldInterval vector will be, e.g. <[1,1], [3,3], [6,6]> - FieldRangeSet frs( "" , query , true, true ); - const vector& intervals = frs.range( _hashedField.c_str() ).intervals(); - - //Force a match of the query against the actual document by giving - //the cursor a matcher with an empty indexKeyPattern. This ensures the - //index is not used as a covered index. - //NOTE: this forcing is necessary due to potential hash collisions - const shared_ptr< CoveredIndexMatcher > forceDocMatcher( - new CoveredIndexMatcher( query , BSONObj() ) ); - - //Construct a new query based on the hashes of the previous point-intervals - //e.g. {a : {$in : [ hash(1) , hash(3) , hash(6) ]}} - BSONObjBuilder newQueryBuilder; - BSONObjBuilder inObj( newQueryBuilder.subobjStart( _hashedField ) ); - BSONArrayBuilder inArray( inObj.subarrayStart("$in") ); - vector::const_iterator i; - for( i = intervals.begin(); i != intervals.end(); ++i ){ - if ( ! i->equality() ){ - const shared_ptr< BtreeCursor > exhaustiveCursor( - BtreeCursor::make( nsdetails( _spec->getDetails()->parentNS()), - *( _spec->getDetails() ), - BSON( "" << MINKEY ) , - BSON( "" << MAXKEY ) , - true , - 1 ) ); - exhaustiveCursor->setMatcher( forceDocMatcher ); - return exhaustiveCursor; - } - inArray.append( makeSingleKey( i->_lower._bound , _seed , _hashVersion ) ); - } - inArray.done(); - inObj.done(); - BSONObj newQuery = newQueryBuilder.obj(); - - //Use the point-intervals of the new query to create a Btree cursor - FieldRangeSet newfrs( "" , newQuery , true, true ); - shared_ptr newVector( - new FieldRangeVector( newfrs , *_spec , 1 ) ); - - const shared_ptr< BtreeCursor > cursor( - BtreeCursor::make( nsdetails( _spec->getDetails()->parentNS()), - *( _spec->getDetails() ), newVector, 0, 1 ) ); - cursor->setMatcher( forceDocMatcher ); - return cursor; - } - - - /* This class registers HASHED_INDEX_NAME in a global map of special index types - * Using this pattern, any index with the pattern, {fieldname : HASHED_INDEX_NAME} - * will be recognized as a HashedIndexType and the associated methods will be used. - */ - class HashedIndexPlugin : public IndexPlugin { - public: - - HashedIndexPlugin() : IndexPlugin( HashedIndexType::HASHED_INDEX_TYPE_IDENTIFIER ) {} - - virtual IndexType* generate( const IndexSpec* spec ) const { - return new HashedIndexType( this , spec ); - } - - } hashedIndexPlugin; - - - long long int HashedIndexType::makeSingleKey( const BSONElement& e , - HashSeed seed , - HashVersion v ) { - massert( 16245 , "Only HashVersion 0 has been defined" , v == 0 ); - return BSONElementHasher::hash64( e , seed ); - } - -} - diff --git a/src/mongo/db/hashindex.h b/src/mongo/db/hashindex.h deleted file mode 100644 index 8c020acf8c0..00000000000 --- a/src/mongo/db/hashindex.h +++ /dev/null @@ -1,125 +0,0 @@ -// hashindex.h - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/db/hasher.h" -#include "mongo/db/index.h" -#include "mongo/db/keypattern.h" -#include "mongo/db/matcher.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/pdfile.h" - -namespace mongo { - - /* This is an index where the keys are hashes of a given field. - * - * Optional arguments: - * "seed" : int (default = 0, a seed for the hash function) - * "hashVersion : int (default = 0, determines which hash function to use) - * - * Example use in the mongo shell: - * > db.foo.ensureIndex({a : "hashed"}, {seed : 3, hashVersion : 0}) - * - * LIMITATION: Only works with a single field. The HashedIndexType - * constructor uses uassert to ensure that the spec has the form - * { : "hashed"}, and not, for example, - * { a : "hashed" , b : 1} - * - * LIMITATION: Cannot be used as a unique index. - * The HashedIndexType constructor uses uassert to ensure that - * the spec does not contain {"unique" : true} - * - * LIMITATION: Cannot be used to index arrays. - * The getKeys function uasserts that value being inserted - * is not an array. This index will not be built if any - * array values of the hashed field exist. - * - */ - class HashedIndexType : public IndexType{ - public: - - static const string HASHED_INDEX_TYPE_IDENTIFIER; - typedef int HashVersion; - - /* Creates a new HashedIndex around a HashedIndexPlugin - * and an IndexSpec. New HashedIndexTypes are created via - * a factory method in the HashedIndexPlugin class. - */ - HashedIndexType( const IndexPlugin* plugin , const IndexSpec* spec ); - virtual ~HashedIndexType(); - - /* This index is only considered "HELPFUL" for a query - * if it's the union of at least one equality constraint on the - * hashed field. Otherwise it's considered USELESS. - * Example queries (supposing the indexKey is {a : "hashed"}): - * {a : 3} HELPFUL - * {a : 3 , b : 3} HELPFUL - * {a : {$in : [3,4]}} HELPFUL - * {a : {$gte : 3, $lte : 3}} HELPFUL - * {} USELESS - * {b : 3} USELESS - * {a : {$gt : 3}} USELESS - */ - IndexSuitability suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const; - - /* The input is "obj" which should have a field corresponding to the hashedfield. - * The output is a BSONObj with a single BSONElement whose value is the hash - * Eg if this is an index on "a" we have - * obj is {a : 45} --> key becomes {"" : hash(45) } - * - * Limitation: arrays values are not currently supported. This function uasserts - * that the value is not an array, and errors out in that case. - */ - void getKeys( const BSONObj &obj, BSONObjSet &keys ) const; - - /* A field missing from a document is represented by the hash value of a null BSONElement. - */ - BSONElement missingField() const { return _missingKey.firstElement(); } - - /* The newCursor method works for suitable queries by generating a BtreeCursor - * using the hash of point-intervals parsed by FieldRangeSet. - * For unsuitable queries it just instantiates a btree cursor over the whole tree - */ - shared_ptr newCursor( const BSONObj& query , - const BSONObj& order , int numWanted ) const; - - /* Takes a BSONElement, seed and hashVersion, and outputs the - * 64-bit hash used for this index - * E.g. if the element is {a : 3} this outputs v1-hash(3) - * */ - static long long int makeSingleKey( const BSONElement& e , - HashSeed seed , - HashVersion v = 0 ); - - /* Since the keys for this index are hashes, documents are not stored in order, - * thus we will need to perform scanAndOrder whenever the "order" is non-empty. - */ - bool scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const { - return ! order.isEmpty(); - } - - private: - string _hashedField; - KeyPattern _keyPattern; - HashSeed _seed; //defaults to zero if not in the IndexSpec - HashVersion _hashVersion; //defaults to zero if not in the IndexSpec - bool _isSparse; - BSONObj _missingKey; - }; - -} diff --git a/src/mongo/db/index.cpp b/src/mongo/db/index.cpp deleted file mode 100644 index f7346978313..00000000000 --- a/src/mongo/db/index.cpp +++ /dev/null @@ -1,492 +0,0 @@ -/** @file index.cpp */ - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/index.h" - -#include - -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/background.h" -#include "mongo/db/btree.h" -#include "mongo/db/index_update.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/ops/delete.h" -#include "mongo/db/repl/rs.h" -#include "mongo/util/scopeguard.h" -#include "mongo/util/mongoutils/str.h" - -namespace mongo { - - IndexInterface::IndexInserter::IndexInserter() {} - IndexInterface::IndexInserter::~IndexInserter() { - for (size_t i = 0; i < _continuations.size(); ++i) - delete _continuations[i]; - } - - void IndexInterface::IndexInserter::addInsertionContinuation(IndexInsertionContinuation *c) { - _continuations.push_back(c); - } - - void IndexInterface::IndexInserter::finishAllInsertions() { - for (size_t i = 0; i < _continuations.size(); ++i) { - _continuations[i]->doIndexInsertionWrites(); - } - } - - IndexInterface& IndexInterface::defaultVersion() { - return *IndexDetails::iis[ DefaultIndexVersionNumber ]; - } - - - template< class V > - class IndexInterfaceImpl : public IndexInterface { - public: - typedef typename V::KeyOwned KeyOwned; - virtual int keyCompare(const BSONObj& l,const BSONObj& r, const Ordering &ordering); - - public: - IndexInsertionContinuation *beginInsertIntoIndex( - int idxNo, IndexDetails &_idx, - DiskLoc _recordLoc, const BSONObj &_key, - const Ordering& _order, bool dupsAllowed) { - - IndexInsertionContinuationImpl *continuation = new IndexInsertionContinuationImpl( - _idx.head, _recordLoc, _key, _order, _idx); - ScopeGuard allocGuard = MakeGuard(boost::checked_delete, - continuation); - _idx.head.btree()->twoStepInsert(_idx.head, *continuation, dupsAllowed); - allocGuard.Dismiss(); - return continuation; - } - - virtual long long fullValidate(const DiskLoc& thisLoc, const BSONObj &order) { - return thisLoc.btree()->fullValidate(thisLoc, order); - } - virtual DiskLoc findSingle(const IndexDetails &indexdetails , const DiskLoc& thisLoc, const BSONObj& key) const { - return thisLoc.btree()->findSingle(indexdetails,thisLoc,key); - } - virtual bool unindex(const DiskLoc thisLoc, IndexDetails& id, const BSONObj& key, const DiskLoc recordLoc) const { - return thisLoc.btree()->unindex(thisLoc, id, key, recordLoc); - } - virtual int bt_insert(const DiskLoc thisLoc, const DiskLoc recordLoc, - const BSONObj& key, const Ordering &order, bool dupsAllowed, - IndexDetails& idx, bool toplevel = true) const { - return thisLoc.btree()->bt_insert(thisLoc, recordLoc, key, order, dupsAllowed, idx, toplevel); - } - virtual DiskLoc addBucket(const IndexDetails& id) { - return BtreeBucket::addBucket(id); - } - virtual void uassertIfDups(IndexDetails& idx, vector& addedKeys, DiskLoc head, DiskLoc self, const Ordering& ordering) { - const BtreeBucket *h = head.btree(); - for( vector::iterator i = addedKeys.begin(); i != addedKeys.end(); i++ ) { - KeyOwned k(**i); - bool dup = h->wouldCreateDup(idx, head, k, ordering, self); - uassert( 11001 , h->dupKeyError( idx , k ) , !dup); - } - } - - // for geo: - virtual bool isUsed(DiskLoc thisLoc, int pos) { return thisLoc.btree()->isUsed(pos); } - virtual void keyAt(DiskLoc thisLoc, int pos, BSONObj& key, DiskLoc& recordLoc) { - recordLoc = DiskLoc(); - const BtreeBucket* bucket = thisLoc.btree(); - int n = bucket->nKeys(); - - if( pos < 0 || pos >= n || n == 0xffff /* bucket deleted */ || ! bucket->isUsed( pos ) ){ - // log() << "Pos: " << pos << " n " << n << endl; - return; - } - - typename BtreeBucket::KeyNode kn = bucket->keyNode(pos); - key = kn.key.toBson(); - recordLoc = kn.recordLoc; - } - virtual BSONObj keyAt(DiskLoc thisLoc, int pos) { - return thisLoc.btree()->keyAt(pos).toBson(); - } - virtual DiskLoc locate(const IndexDetails &idx , const DiskLoc& thisLoc, const BSONObj& key, const Ordering &order, - int& pos, bool& found, const DiskLoc &recordLoc, int direction=1) { - return thisLoc.btree()->locate(idx, thisLoc, key, order, pos, found, recordLoc, direction); - } - virtual DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) { - return thisLoc.btree()->advance(thisLoc,keyOfs,direction,caller); - } - }; - - int oldCompare(const BSONObj& l,const BSONObj& r, const Ordering &o); // key.cpp - - template <> - int IndexInterfaceImpl< V0 >::keyCompare(const BSONObj& l, const BSONObj& r, const Ordering &ordering) { - return oldCompare(l, r, ordering); - } - - template <> - int IndexInterfaceImpl< V1 >::keyCompare(const BSONObj& l, const BSONObj& r, const Ordering &ordering) { - return l.woCompare(r, ordering, /*considerfieldname*/false); - } - - IndexInterfaceImpl iii_v0; - IndexInterfaceImpl iii_v1; - - IndexInterface *IndexDetails::iis[] = { &iii_v0, &iii_v1 }; - - int removeFromSysIndexes(const char *ns, const char *idxName) { - string system_indexes = cc().database()->name + ".system.indexes"; - BSONObjBuilder b; - b.append("ns", ns); - b.append("name", idxName); // e.g.: { name: "ts_1", ns: "foo.coll" } - BSONObj cond = b.done(); - return (int) deleteObjects(system_indexes.c_str(), cond, false, false, true); - } - - /* this is just an attempt to clean up old orphaned stuff on a delete all indexes - call. repair database is the clean solution, but this gives one a lighter weight - partial option. see dropIndexes() - */ - void assureSysIndexesEmptied(const char *ns, IndexDetails *idIndex) { - string system_indexes = cc().database()->name + ".system.indexes"; - BSONObjBuilder b; - b.append("ns", ns); - if( idIndex ) { - b.append("name", BSON( "$ne" << idIndex->indexName().c_str() )); - } - BSONObj cond = b.done(); - int n = (int) deleteObjects(system_indexes.c_str(), cond, false, false, true); - if( n ) { - log() << "info: assureSysIndexesEmptied cleaned up " << n << " entries" << endl; - } - } - - int IndexDetails::keyPatternOffset( const string& key ) const { - BSONObjIterator i( keyPattern() ); - int n = 0; - while ( i.more() ) { - BSONElement e = i.next(); - if ( key == e.fieldName() ) - return n; - n++; - } - return -1; - } - - const IndexSpec& IndexDetails::getSpec() const { - SimpleMutex::scoped_lock lk(NamespaceDetailsTransient::_qcMutex); - return NamespaceDetailsTransient::get_inlock( info.obj()["ns"].valuestr() ).getIndexSpec( this ); - } - - /* delete this index. does NOT clean up the system catalog - (system.indexes or system.namespaces) -- only NamespaceIndex. - */ - void IndexDetails::kill_idx() { - string ns = indexNamespace(); // e.g. foo.coll.$ts_1 - try { - - string pns = parentNS(); // note we need a copy, as parentNS() won't work after the drop() below - - // clean up parent namespace index cache - NamespaceDetailsTransient::get( pns.c_str() ).deletedIndex(); - - string name = indexName(); - - /* important to catch exception here so we can finish cleanup below. */ - try { - dropNS(ns.c_str()); - } - catch(DBException& ) { - LOG(2) << "IndexDetails::kill(): couldn't drop ns " << ns << endl; - } - head.setInvalid(); - info.setInvalid(); - - // clean up in system.indexes. we do this last on purpose. - int n = removeFromSysIndexes(pns.c_str(), name.c_str()); - wassert( n == 1 ); - - } - catch ( DBException &e ) { - log() << "exception in kill_idx: " << e << ", ns: " << ns << endl; - } - } - - void IndexDetails::getKeysFromObject( const BSONObj& obj, BSONObjSet& keys) const { - getSpec().getKeys( obj, keys ); - } - - void setDifference(BSONObjSet &l, BSONObjSet &r, vector &diff) { - // l and r must use the same ordering spec. - verify( l.key_comp().order() == r.key_comp().order() ); - BSONObjSet::iterator i = l.begin(); - BSONObjSet::iterator j = r.begin(); - while ( 1 ) { - if ( i == l.end() ) - break; - while ( j != r.end() && j->woCompare( *i ) < 0 ) - j++; - if ( j == r.end() || i->woCompare(*j) != 0 ) { - const BSONObj *jo = &*i; - diff.push_back( (BSONObj *) jo ); - } - i++; - } - } - - void getIndexChanges(vector& v, const char *ns, NamespaceDetails& d, - BSONObj newObj, BSONObj oldObj, bool &changedId) { - int z = d.getTotalIndexCount(); - v.resize(z); - for( int i = 0; i < z; i++ ) { - IndexDetails& idx = d.idx(i); - BSONObj idxKey = idx.info.obj().getObjectField("key"); // eg { ts : 1 } - IndexChanges& ch = v[i]; - idx.getKeysFromObject(oldObj, ch.oldkeys); - idx.getKeysFromObject(newObj, ch.newkeys); - if( ch.newkeys.size() > 1 ) - d.setIndexIsMultikey(ns, i); - setDifference(ch.oldkeys, ch.newkeys, ch.removed); - setDifference(ch.newkeys, ch.oldkeys, ch.added); - if ( ch.removed.size() > 0 && ch.added.size() > 0 && idx.isIdIndex() ) { - changedId = true; - } - } - } - - void dupCheck(vector& v, NamespaceDetails& d, DiskLoc curObjLoc) { - int z = d.getTotalIndexCount(); - for( int i = 0; i < z; i++ ) { - IndexDetails& idx = d.idx(i); - v[i].dupCheck(idx, curObjLoc); - } - } - - // should be { : , .keyp.. } - static bool validKeyPattern(BSONObj kp) { - BSONObjIterator i(kp); - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if( e.type() == Object || e.type() == Array ) - return false; - } - return true; - } - - static bool needToUpgradeMinorVersion(const string& newPluginName) { - if (IndexPlugin::existedBefore24(newPluginName)) - return false; - - DataFileHeader* dfh = cc().database()->getFile(0)->getHeader(); - if (dfh->versionMinor == PDFILE_VERSION_MINOR_24_AND_NEWER) - return false; // these checks have already been done - - fassert(16737, dfh->versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER); - - return true; - } - - static void upgradeMinorVersionOrAssert(const string& newPluginName) { - const string systemIndexes = cc().database()->name + ".system.indexes"; - shared_ptr cursor(theDataFileMgr.findAll(systemIndexes)); - for ( ; cursor && cursor->ok(); cursor->advance()) { - const BSONObj index = cursor->current(); - const BSONObj key = index.getObjectField("key"); - const string plugin = IndexPlugin::findPluginName(key); - if (IndexPlugin::existedBefore24(plugin)) - continue; - - const string errmsg = str::stream() - << "Found pre-existing index " << index << " with invalid type '" << plugin << "'. " - << "Disallowing creation of new index type '" << newPluginName << "'. See " - << "http://dochub.mongodb.org/core/index-type-changes" - ; - - error() << errmsg << endl; - uasserted(16738, errmsg); - } - - DataFileHeader* dfh = cc().database()->getFile(0)->getHeader(); - getDur().writingInt(dfh->versionMinor) = PDFILE_VERSION_MINOR_24_AND_NEWER; - } - - bool prepareToBuildIndex(const BSONObj& io, - bool mayInterrupt, - bool god, - string& sourceNS, - NamespaceDetails*& sourceCollection, - BSONObj& fixedIndexObject) { - sourceCollection = 0; - - // the collection for which we are building an index - sourceNS = io.getStringField("ns"); - uassert(10096, "invalid ns to index", sourceNS.find( '.' ) != string::npos); - massert(10097, str::stream() << "bad table to index name on add index attempt current db: " << cc().database()->name << " source: " << sourceNS , - cc().database()->name == nsToDatabase(sourceNS)); - - // logical name of the index. todo: get rid of the name, we don't need it! - const char *name = io.getStringField("name"); - uassert(12523, "no index name specified", *name); - - BSONObj key = io.getObjectField("key"); - uassert(12524, "index key pattern too large", key.objsize() <= 2048); - if( !validKeyPattern(key) ) { - string s = string("bad index key pattern ") + key.toString(); - uasserted(10098 , s.c_str()); - } - - if ( sourceNS.empty() || key.isEmpty() ) { - LOG(2) << "bad add index attempt name:" << (name?name:"") << "\n ns:" << - sourceNS << "\n idxobj:" << io.toString() << endl; - string s = "bad add index attempt " + sourceNS + " key:" + key.toString(); - uasserted(12504, s); - } - - sourceCollection = nsdetails(sourceNS); - if( sourceCollection == 0 ) { - // try to create it - string err; - if ( !userCreateNS(sourceNS.c_str(), BSONObj(), err, false) ) { - problem() << "ERROR: failed to create collection while adding its index. " << sourceNS << endl; - return false; - } - sourceCollection = nsdetails(sourceNS); - tlog() << "info: creating collection " << sourceNS << " on add index" << endl; - verify( sourceCollection ); - } - - // Check both existing and in-progress indexes (2nd param = true) - if ( sourceCollection->findIndexByName(name, true) >= 0 ) { - // index already exists. - return false; - } - - // Check both existing and in-progress indexes (2nd param = true) - if( sourceCollection->findIndexByKeyPattern(key, true) >= 0 ) { - LOG(2) << "index already exists with diff name " << name << ' ' << key.toString() << endl; - return false; - } - - if ( sourceCollection->nIndexes >= NamespaceDetails::NIndexesMax ) { - stringstream ss; - ss << "add index fails, too many indexes for " << sourceNS << " key:" << key.toString(); - string s = ss.str(); - log() << s << endl; - uasserted(12505,s); - } - - /* this is because we want key patterns like { _id : 1 } and { _id : } to - all be treated as the same pattern. - */ - if ( IndexDetails::isIdIndexPattern(key) ) { - if( !god ) { - ensureHaveIdIndex( sourceNS.c_str(), mayInterrupt ); - return false; - } - } - else { - /* is buildIndexes:false set for this replica set member? - if so we don't build any indexes except _id - */ - if( theReplSet && !theReplSet->buildIndexes() ) - return false; - } - - string pluginName = IndexPlugin::findPluginName( key ); - IndexPlugin * plugin = NULL; - if (pluginName.size()) { - plugin = IndexPlugin::get(pluginName); - uassert(16734, str::stream() << "Unknown index plugin '" << pluginName << "' " - << "in index "<< key - , plugin); - - if (needToUpgradeMinorVersion(pluginName)) - upgradeMinorVersionOrAssert(pluginName); - } - - { - BSONObj o = io; - if ( plugin ) { - o = plugin->adjustIndexSpec(o); - } - BSONObjBuilder b; - int v = DefaultIndexVersionNumber; - if( !o["v"].eoo() ) { - double vv = o["v"].Number(); - // note (one day) we may be able to fresh build less versions than we can use - // isASupportedIndexVersionNumber() is what we can use - uassert(14803, str::stream() << "this version of mongod cannot build new indexes of version number " << vv, - vv == 0 || vv == 1); - v = (int) vv; - } - // idea is to put things we use a lot earlier - b.append("v", v); - b.append(o["key"]); - if( o["unique"].trueValue() ) - b.appendBool("unique", true); // normalize to bool true in case was int 1 or something... - b.append(o["ns"]); - - { - // stripping _id - BSONObjIterator i(o); - while ( i.more() ) { - BSONElement e = i.next(); - string s = e.fieldName(); - if( s != "_id" && s != "v" && s != "ns" && s != "unique" && s != "key" ) - b.append(e); - } - } - - fixedIndexObject = b.obj(); - } - - return true; - } - - void IndexSpec::reset(const IndexDetails * details) { - const DataFileHeader* dfh = cc().database()->getFile(0)->getHeader(); - IndexSpec::PluginRules rules = dfh->versionMinor == PDFILE_VERSION_MINOR_24_AND_NEWER - ? IndexSpec::RulesFor24 - : IndexSpec::RulesFor22 - ; - - _details = details; - reset(details->info, rules); - } - - void IndexSpec::reset(const BSONObj& _info, PluginRules rules) { - info = _info; - keyPattern = info["key"].embeddedObjectUserCheck(); - if ( keyPattern.objsize() == 0 ) { - out() << info.toString() << endl; - verify(false); - } - _init(rules); - } - - void IndexChanges::dupCheck(IndexDetails& idx, DiskLoc curObjLoc) { - if (added.empty() || - !idx.unique() || - ignoreUniqueIndex(idx)) { - return; - } - const Ordering ordering = Ordering::make(idx.keyPattern()); - - // "E11001 duplicate key on update" - idx.idxInterface().uassertIfDups(idx, added, idx.head, curObjLoc, ordering); - } -} diff --git a/src/mongo/db/index.h b/src/mongo/db/index.h deleted file mode 100644 index a522d58a906..00000000000 --- a/src/mongo/db/index.h +++ /dev/null @@ -1,287 +0,0 @@ -// index.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/pch.h" - -#include - -#include "mongo/db/diskloc.h" -#include "mongo/db/index_insertion_continuation.h" -#include "mongo/db/indexkey.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/key.h" -#include "mongo/db/namespace.h" - -namespace mongo { - - class IndexInterface { - protected: - virtual ~IndexInterface() { } - public: - class IndexInserter : private boost::noncopyable { - public: - IndexInserter(); - ~IndexInserter(); - - void addInsertionContinuation(IndexInsertionContinuation *c); - void finishAllInsertions(); - - private: - std::vector _continuations; - }; - - virtual IndexInsertionContinuation *beginInsertIntoIndex( - int idxNo, - IndexDetails &_idx, DiskLoc _recordLoc, const BSONObj &_key, - const Ordering& _order, bool dupsAllowed) = 0; - - virtual int keyCompare(const BSONObj& l,const BSONObj& r, const Ordering &ordering) = 0; - virtual long long fullValidate(const DiskLoc& thisLoc, const BSONObj &order) = 0; - virtual DiskLoc findSingle(const IndexDetails &indexdetails , const DiskLoc& thisLoc, const BSONObj& key) const = 0; - virtual bool unindex(const DiskLoc thisLoc, IndexDetails& id, const BSONObj& key, const DiskLoc recordLoc) const = 0; - virtual int bt_insert(const DiskLoc thisLoc, const DiskLoc recordLoc, - const BSONObj& key, const Ordering &order, bool dupsAllowed, - IndexDetails& idx, bool toplevel = true) const = 0; - virtual DiskLoc addBucket(const IndexDetails&) = 0; - virtual void uassertIfDups(IndexDetails& idx, vector& addedKeys, DiskLoc head, - DiskLoc self, const Ordering& ordering) = 0; - - // these are for geo - virtual bool isUsed(DiskLoc thisLoc, int pos) = 0; - virtual void keyAt(DiskLoc thisLoc, int pos, BSONObj&, DiskLoc& recordLoc) = 0; - virtual BSONObj keyAt(DiskLoc thisLoc, int pos) = 0; - virtual DiskLoc locate(const IndexDetails &idx , const DiskLoc& thisLoc, const BSONObj& key, const Ordering &order, - int& pos, bool& found, const DiskLoc &recordLoc, int direction=1) = 0; - virtual DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) = 0; - - /** - * @return a static IndexInterface consistent with index version DefaultIndexVersionNumber. - * An IndexInterface should generally not be retrieved via this function, but from the - * IndexDetails for an existing index. - */ - static IndexInterface& defaultVersion(); - }; - - /* Details about a particular index. There is one of these effectively for each object in - system.namespaces (although this also includes the head pointer, which is not in that - collection). - - ** MemoryMapped Record ** (i.e., this is on disk data) - */ - class IndexDetails { - public: - /** - * btree head disk location - * TODO We should make this variable private, since btree operations - * may change its value and we don't want clients to rely on an old - * value. If we create a btree class, we can provide a btree object - * to clients instead of 'head'. - */ - DiskLoc head; - - /* Location of index info object. Format: - - { name:"nameofindex", ns:"parentnsname", key: {keypattobject} - [, unique: , background: , v:] - } - - This object is in the system.indexes collection. Note that since we - have a pointer to the object here, the object in system.indexes MUST NEVER MOVE. - */ - DiskLoc info; - - /* extract key value from the query object - e.g., if key() == { x : 1 }, - { x : 70, y : 3 } -> { x : 70 } - */ - BSONObj getKeyFromQuery(const BSONObj& query) const { - BSONObj k = keyPattern(); - BSONObj res = query.extractFieldsUnDotted(k); - return res; - } - - /* pull out the relevant key objects from obj, so we - can index them. Note that the set is multiple elements - only when it's a "multikey" array. - keys will be left empty if key not found in the object. - */ - void getKeysFromObject( const BSONObj& obj, BSONObjSet& keys) const; - - /* get the key pattern for this object. - e.g., { lastname:1, firstname:1 } - */ - BSONObj keyPattern() const { - return info.obj().getObjectField("key"); - } - - /** - * @return offset into keyPattern for key - -1 if doesn't exist - */ - int keyPatternOffset( const string& key ) const; - bool inKeyPattern( const string& key ) const { return keyPatternOffset( key ) >= 0; } - - /* true if the specified key is in the index */ - bool hasKey(const BSONObj& key); - - // returns name of this index's storage area - // database.table.$index - string indexNamespace() const { - BSONObj io = info.obj(); - string s; - s.reserve(Namespace::MaxNsLen); - s = io.getStringField("ns"); - verify( !s.empty() ); - s += ".$"; - s += io.getStringField("name"); - return s; - } - - string indexName() const { // e.g. "ts_1" - BSONObj io = info.obj(); - return io.getStringField("name"); - } - - static bool isIdIndexPattern( const BSONObj &pattern ) { - BSONObjIterator i(pattern); - BSONElement e = i.next(); - //_id index must have form exactly {_id : 1} or {_id : -1}. - //Allows an index of form {_id : "hashed"} to exist but - //do not consider it to be the primary _id index - if(! ( strcmp(e.fieldName(), "_id") == 0 - && (e.numberInt() == 1 || e.numberInt() == -1))) - return false; - return i.next().eoo(); - } - - /* returns true if this is the _id index. */ - bool isIdIndex() const { - return isIdIndexPattern( keyPattern() ); - } - - /* gets not our namespace name (indexNamespace for that), - but the collection we index, its name. - */ - string parentNS() const { - BSONObj io = info.obj(); - return io.getStringField("ns"); - } - - static int versionForIndexObj( const BSONObj &obj ) { - BSONElement e = obj["v"]; - if( e.type() == NumberInt ) - return e._numberInt(); - // should normally be an int. this is for backward compatibility - int v = e.numberInt(); - uassert(14802, "index v field should be Integer type", v == 0); - return v; - } - - int version() const { - return versionForIndexObj( info.obj() ); - } - - /** @return true if index has unique constraint */ - bool unique() const { - BSONObj io = info.obj(); - return io["unique"].trueValue() || - /* temp: can we juse make unique:true always be there for _id and get rid of this? */ - isIdIndex(); - } - - /** return true if dropDups was set when building index (if any duplicates, dropdups drops the duplicating objects) */ - bool dropDups() const { - return info.obj().getBoolField( "dropDups" ); - } - - /** delete this index. does NOT clean up the system catalog - (system.indexes or system.namespaces) -- only NamespaceIndex. - */ - void kill_idx(); - - const IndexSpec& getSpec() const; - - string toString() const { - return info.obj().toString(); - } - - /** @return true if supported. supported means we can use the index, including adding new keys. - it may not mean we can build the index version in question: we may not maintain building - of indexes in old formats in the future. - */ - static bool isASupportedIndexVersionNumber(int v) { return (v&1)==v; } // v == 0 || v == 1 - - /** @return the interface for this interface, which varies with the index version. - used for backward compatibility of index versions/formats. - */ - IndexInterface& idxInterface() const { - int v = version(); - dassert( isASupportedIndexVersionNumber(v) ); - return *iis[v&1]; - } - - static IndexInterface *iis[]; - }; - - struct IndexChanges { /*on an update*/ - BSONObjSet oldkeys; - BSONObjSet newkeys; - vector removed; // these keys were removed as part of the change - vector added; // these keys were added as part of the change - - /** @curObjLoc - the object we want to add's location. if it is already in the - index, that is allowed here (for bg indexing case). - */ - void dupCheck(IndexDetails& idx, DiskLoc curObjLoc); - }; - - class NamespaceDetails; - // changedId should be initialized to false - void getIndexChanges(vector& v, const char *ns, NamespaceDetails& d, - BSONObj newObj, BSONObj oldObj, bool &cangedId); - void dupCheck(vector& v, NamespaceDetails& d, DiskLoc curObjLoc); - - void assureSysIndexesEmptied(const char *ns, IndexDetails *exceptForIdIndex); - int removeFromSysIndexes(const char *ns, const char *idxName); - - /** - * Prepare to build an index. Does not actually build it (except for a special _id case). - * - We validate that the params are good - * - That the index does not already exist - * - Creates the source collection if it DNE - * - * example of 'io': - * { ns : 'test.foo', name : 'z', key : { z : 1 } } - * - * @throws DBException - * - * @param mayInterrupt - When true, killop may interrupt the function call. - * @param sourceNS - source NS we are indexing - * @param sourceCollection - its details ptr - * @return true if ok to continue. when false we stop/fail silently (index already exists) - */ - bool prepareToBuildIndex(const BSONObj& io, - bool mayInterrupt, - bool god, - string& sourceNS, - NamespaceDetails*& sourceCollection, - BSONObj& fixedIndexObject); - -} // namespace mongo diff --git a/src/mongo/db/index/2d_access_method.cpp b/src/mongo/db/index/2d_access_method.cpp new file mode 100644 index 00000000000..5e1680cf10a --- /dev/null +++ b/src/mongo/db/index/2d_access_method.cpp @@ -0,0 +1,63 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/2d_access_method.h" + +#include +#include + +#include "mongo/db/geo/core.h" +#include "mongo/db/index_names.h" +#include "mongo/db/index/2d_common.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pdfile.h" + +namespace mongo { + + TwoDAccessMethod::TwoDAccessMethod(IndexCatalogEntry* btreeState) + : BtreeBasedAccessMethod(btreeState) { + + const IndexDescriptor* descriptor = btreeState->descriptor(); + + ExpressionParams::parseTwoDParams(descriptor->infoObj(), &_params); + + _keyGenerator.reset( new TwoDKeyGenerator( _params ) ); + } + + /** Finds the key objects to put in an index */ + void TwoDAccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) { + _keyGenerator->getKeys( obj, keys ); + } + + /** Finds all locations in a geo-indexed object */ + void TwoDAccessMethod::getKeys(const BSONObj& obj, vector& locs) const { + _keyGenerator->getKeys( obj, NULL, &locs ); + } + +} // namespace mongo diff --git a/src/mongo/db/index/2d_access_method.h b/src/mongo/db/index/2d_access_method.h new file mode 100644 index 00000000000..a9894cbf05e --- /dev/null +++ b/src/mongo/db/index/2d_access_method.h @@ -0,0 +1,113 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/index/2d_common.h" +#include "mongo/db/index/2d_key_generator.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class IndexCatalogEntry; + class IndexCursor; + class IndexDescriptor; + struct TwoDIndexingParams; + + namespace twod_exec { + class GeoPoint; + class GeoAccumulator; + class GeoBrowse; + class GeoHopper; + class GeoSearch; + class GeoCircleBrowse; + class GeoBoxBrowse; + class GeoPolygonBrowse; + class TwoDGeoNearRunner; + } + + namespace twod_internal { + class GeoPoint; + class GeoAccumulator; + class GeoBrowse; + class GeoHopper; + class GeoSearch; + class GeoCircleBrowse; + class GeoBoxBrowse; + class GeoPolygonBrowse; + class TwoDGeoNearRunner; + } + + class TwoDAccessMethod : public BtreeBasedAccessMethod { + public: + using BtreeBasedAccessMethod::_descriptor; + using BtreeBasedAccessMethod::_interface; + + TwoDAccessMethod(IndexCatalogEntry* btreeState); + virtual ~TwoDAccessMethod() { } + + virtual shared_ptr getKeyGenerator() const { return _keyGenerator; } + private: + friend class TwoDIndexCursor; + friend class twod_internal::GeoPoint; + friend class twod_internal::GeoAccumulator; + friend class twod_internal::GeoBrowse; + friend class twod_internal::GeoHopper; + friend class twod_internal::GeoSearch; + friend class twod_internal::GeoCircleBrowse; + friend class twod_internal::GeoBoxBrowse; + friend class twod_internal::GeoPolygonBrowse; + + friend class twod_exec::GeoPoint; + friend class twod_exec::GeoAccumulator; + friend class twod_exec::GeoBrowse; + friend class twod_exec::GeoHopper; + friend class twod_exec::GeoSearch; + friend class twod_exec::GeoCircleBrowse; + friend class twod_exec::GeoBoxBrowse; + friend class twod_exec::GeoPolygonBrowse; + + friend class twod_internal::TwoDGeoNearRunner; + + BtreeInterface* getInterface() { return _interface; } + const IndexDescriptor* getDescriptor() { return _descriptor; } + TwoDIndexingParams& getParams() { return _params; } + + // This really gets the 'locs' from the provided obj. + void getKeys(const BSONObj& obj, vector& locs) const; + + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys); + + TwoDIndexingParams _params; + + shared_ptr _keyGenerator; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/2d_common.h b/src/mongo/db/index/2d_common.h new file mode 100644 index 00000000000..1027a73c507 --- /dev/null +++ b/src/mongo/db/index/2d_common.h @@ -0,0 +1,44 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include + +#include "mongo/db/geo/hash.h" + +namespace mongo { + + struct TwoDIndexingParams { + string geo; + vector > other; + shared_ptr geoHashConverter; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/2d_key_generator.cpp b/src/mongo/db/index/2d_key_generator.cpp new file mode 100644 index 00000000000..dcbaccd68e9 --- /dev/null +++ b/src/mongo/db/index/2d_key_generator.cpp @@ -0,0 +1,149 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/2d_access_method.h" + +#include +#include + +#include "mongo/db/geo/core.h" +#include "mongo/db/index_names.h" +#include "mongo/db/index/2d_common.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pdfile.h" + +namespace mongo { + + + TwoDKeyGenerator::TwoDKeyGenerator( const TwoDIndexingParams& params ) + : _params( params ) { + } + + void TwoDKeyGenerator::getKeys(const BSONObj& obj, BSONObjSet* keys) const { + getKeys( obj, keys, NULL ); + } + + void TwoDKeyGenerator::getKeys(const BSONObj& obj, + BSONObjSet* keys, + std::vector* locs ) const { + BSONElementMSet bSet; + + // Get all the nested location fields, but don't return individual elements from + // the last array, if it exists. + obj.getFieldsDotted(_params.geo.c_str(), bSet, false); + + if (bSet.empty()) + return; + + for (BSONElementMSet::iterator setI = bSet.begin(); setI != bSet.end(); ++setI) { + BSONElement geo = *setI; + + if (geo.eoo() || !geo.isABSONObj()) + continue; + + // + // Grammar for location lookup: + // locs ::= [loc,loc,...,loc]|{:loc,:loc,...,:loc}|loc + // loc ::= { : #, : # }|[#, #]|{} + // + // Empty locations are ignored, preserving single-location semantics + // + + BSONObj embed = geo.embeddedObject(); + if (embed.isEmpty()) + continue; + + // Differentiate between location arrays and locations + // by seeing if the first element value is a number + bool singleElement = embed.firstElement().isNumber(); + + BSONObjIterator oi(embed); + + while (oi.more()) { + BSONObj locObj; + + if (singleElement) { + locObj = embed; + } else { + BSONElement locElement = oi.next(); + + uassert(16804, mongoutils::str::stream() << + "location object expected, location array not in correct format", + locElement.isABSONObj()); + + locObj = locElement.embeddedObject(); + if(locObj.isEmpty()) + continue; + } + + BSONObjBuilder b(64); + + // Remember the actual location object if needed + if (locs) + locs->push_back(locObj); + + // Stop if we don't need to get anything but location objects + if (!keys) { + if (singleElement) break; + else continue; + } + + _params.geoHashConverter->hash(locObj, &obj).appendToBuilder(&b, ""); + + // Go through all the other index keys + for (vector >::const_iterator i = _params.other.begin(); + i != _params.other.end(); ++i) { + // Get *all* fields for the index key + BSONElementSet eSet; + obj.getFieldsDotted(i->first, eSet); + + if (eSet.size() == 0) + b.appendNull(""); + else if (eSet.size() == 1) + b.appendAs(*(eSet.begin()), ""); + else { + // If we have more than one key, store as an array of the objects + BSONArrayBuilder aBuilder; + + for (BSONElementSet::iterator ei = eSet.begin(); ei != eSet.end(); + ++ei) { + aBuilder.append(*ei); + } + + b.append("", aBuilder.arr()); + } + } + keys->insert(b.obj()); + if(singleElement) break; + } + } + } + + +} // namespace mongo diff --git a/src/mongo/db/index/2d_key_generator.h b/src/mongo/db/index/2d_key_generator.h new file mode 100644 index 00000000000..b612819a42b --- /dev/null +++ b/src/mongo/db/index/2d_key_generator.h @@ -0,0 +1,59 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/index/2d_common.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class IndexCatalogEntry; + class IndexCursor; + class IndexDescriptor; + struct TwoDIndexingParams; + + class TwoDKeyGenerator : public KeyGenerator { + public: + TwoDKeyGenerator( const TwoDIndexingParams& params ); + virtual ~TwoDKeyGenerator() {} + + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys) const; + + virtual void getKeys(const BSONObj& obj, + BSONObjSet* keys, + std::vector* locs ) const; + + private: + TwoDIndexingParams _params; + }; + + +} // namespace mongo diff --git a/src/mongo/db/index/SConscript b/src/mongo/db/index/SConscript new file mode 100644 index 00000000000..d651a7c7be4 --- /dev/null +++ b/src/mongo/db/index/SConscript @@ -0,0 +1,36 @@ +# -*- mode: python -*- + +Import("env") + +env.Library( + target='key_generator', + source=[ + '2d_key_generator.cpp', + 'btree_key_generator.cpp', + 'fts_key_generator.cpp', + 'hash_key_generator.cpp', + 'haystack_key_generator.cpp', + 'key_generator.cpp', + 's2_key_generator.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/bson', + '$BUILD_DIR/mongo/db/fts/base', + '$BUILD_DIR/mongo/geometry', + '$BUILD_DIR/mongo/geoparser', + '$BUILD_DIR/mongo/geoquery', + '$BUILD_DIR/mongo/index_names', + '$BUILD_DIR/third_party/s2/s2', + ], +) + +env.Library( + target='external_key_generator', + source=[ + 'external_key_generator.cpp', + ], + LIBDEPS=[ + 'key_generator', + '$BUILD_DIR/mongo/bson', + ], +) diff --git a/src/mongo/db/index/btree_access_method.cpp b/src/mongo/db/index/btree_access_method.cpp new file mode 100644 index 00000000000..37558a40bdf --- /dev/null +++ b/src/mongo/db/index/btree_access_method.cpp @@ -0,0 +1,73 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/btree_access_method.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/db/index/btree_index_cursor.h" +#include "mongo/db/index/btree_interface.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/keypattern.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/pdfile_private.h" + +namespace mongo { + + // Standard Btree implementation below. + BtreeAccessMethod::BtreeAccessMethod(IndexCatalogEntry* btreeState) + : BtreeBasedAccessMethod(btreeState) { + + // The key generation wants these values. + vector fieldNames; + vector fixed; + + BSONObjIterator it(_descriptor->keyPattern()); + while (it.more()) { + BSONElement elt = it.next(); + fieldNames.push_back(elt.fieldName()); + fixed.push_back(BSONElement()); + } + + if (0 == _descriptor->version()) { + _keyGenerator.reset(new BtreeKeyGeneratorV0(fieldNames, fixed, + _descriptor->isSparse())); + } else if (1 == _descriptor->version()) { + _keyGenerator.reset(new BtreeKeyGeneratorV1(fieldNames, fixed, + _descriptor->isSparse())); + } else { + massert(16745, "Invalid index version for key generation.", false ); + } + } + + void BtreeAccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) { + _keyGenerator->getKeys(obj, keys); + } + +} // namespace mongo diff --git a/src/mongo/db/index/btree_access_method.h b/src/mongo/db/index/btree_access_method.h new file mode 100644 index 00000000000..779dfe80b56 --- /dev/null +++ b/src/mongo/db/index/btree_access_method.h @@ -0,0 +1,69 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/db/structure/btree/btree.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index/btree_key_generator.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class BtreeInterface; + class IndexCursor; + class IndexDescriptor; + + /** + * The IndexAccessMethod for a Btree index. + * Any index created with {field: 1} or {field: -1} uses this. + */ + class BtreeAccessMethod : public BtreeBasedAccessMethod { + public: + // Every Btree-based index needs these. We put them in the BtreeBasedAccessMethod + // superclass and subclasses (like this) can use them. + using BtreeBasedAccessMethod::_descriptor; + using BtreeBasedAccessMethod::_interface; + + BtreeAccessMethod(IndexCatalogEntry* btreeState ); + virtual ~BtreeAccessMethod() { } + + virtual shared_ptr getKeyGenerator() const { return _keyGenerator; } + private: + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys); + + // Our keys differ for V0 and V1. + // this is shared so that we can pass it off + shared_ptr _keyGenerator; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/btree_based_access_method.cpp b/src/mongo/db/index/btree_based_access_method.cpp new file mode 100644 index 00000000000..a46a69e3d47 --- /dev/null +++ b/src/mongo/db/index/btree_based_access_method.cpp @@ -0,0 +1,595 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/btree_access_method.h" + +#include + +#include "mongo/base/error_codes.h" +#include "mongo/base/status.h" +#include "mongo/db/curop.h" +#include "mongo/db/extsort.h" +#include "mongo/db/index/btree_index_cursor.h" +#include "mongo/db/index/btree_interface.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/keypattern.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/pdfile_private.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/sort_phase_one.h" +#include "mongo/db/structure/btree/btreebuilder.h" +#include "mongo/util/progress_meter.h" + +namespace mongo { + + BtreeBasedAccessMethod::BtreeBasedAccessMethod(IndexCatalogEntry* btreeState) + : _btreeState(btreeState), _descriptor(btreeState->descriptor()) { + + verify(0 == _descriptor->version() || 1 == _descriptor->version()); + _interface = BtreeInterface::interfaces[_descriptor->version()]; + } + + // Find the keys for obj, put them in the tree pointing to loc + Status BtreeBasedAccessMethod::insert(const BSONObj& obj, + const DiskLoc& loc, + const InsertDeleteOptions& options, + int64_t* numInserted, + const PregeneratedKeysOnIndex* prepared ) { + + const BSONObjSet* keysToUse = NULL; + + BSONObjSet myOwnedKeys; + + if ( prepared && prepared->generator->getId() == getKeyGenerator()->getId() ) { + keysToUse = &prepared->keys; + } + else { + getKeys(obj, &myOwnedKeys); + keysToUse = &myOwnedKeys; + } + + *numInserted = 0; + + for (BSONObjSet::const_iterator i = keysToUse->begin(); i != keysToUse->end(); ++i) { + try { + _interface->bt_insert(_btreeState, + _btreeState->head(), + loc, + *i, + options.dupsAllowed, + true); + ++*numInserted; + } + catch (AssertionException& e) { + const int code = e.getCode(); + if (10287 == code && !_btreeState->isReady()) { + // This is the duplicate key exception. We ignore it for some reason in BG + // indexing. + DEV log() << "info: key already in index during bg indexing (ok)\n"; + } + else if (options.ignoreKeyTooLong && (code == 17280 || code == 17281)) { + // Behave as-if failIndexKeyTooLong was false and bt_insert silently didn't + // insert the record. + continue; + } + else if (!options.dupsAllowed) { + // Assuming it's a duplicate key exception. Clean up any inserted keys. + for (BSONObjSet::const_iterator j = keysToUse->begin(); j != i; ++j) { + removeOneKey(*j, loc); + } + *numInserted = 0; + return Status(ErrorCodes::DuplicateKey, e.what(), e.getCode()); + } + else { + problem() << " caught assertion addKeysToIndex " + << _descriptor->indexNamespace() + << obj["_id"] << endl; + return Status(ErrorCodes::InternalError, e.what(), e.getCode()); + } + } + } + + if (*numInserted > 1) { + _btreeState->setMultikey(); + } + + return Status::OK(); + } + + bool BtreeBasedAccessMethod::removeOneKey(const BSONObj& key, const DiskLoc& loc) { + bool ret = false; + + try { + ret = _interface->unindex(_btreeState, + _btreeState->head(), + key, + loc); + } catch (AssertionException& e) { + problem() << "Assertion failure: _unindex failed " + << _descriptor->indexNamespace() << endl; + out() << "Assertion failure: _unindex failed: " << e.what() << '\n'; + out() << " obj:" << loc.obj().toString() << '\n'; + out() << " key:" << key.toString() << '\n'; + out() << " dl:" << loc.toString() << endl; + logContext(); + } + + return ret; + } + + Status BtreeBasedAccessMethod::newCursor(IndexCursor **out) const { + *out = new BtreeIndexCursor(_btreeState, _btreeState->head(), _interface); + return Status::OK(); + } + + // Remove the provided doc from the index. + Status BtreeBasedAccessMethod::remove(const BSONObj &obj, const DiskLoc& loc, + const InsertDeleteOptions &options, int64_t* numDeleted) { + + BSONObjSet keys; + getKeys(obj, &keys); + *numDeleted = 0; + + for (BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i) { + bool thisKeyOK = removeOneKey(*i, loc); + + if (thisKeyOK) { + ++*numDeleted; + } else if (options.logIfError) { + log() << "unindex failed (key too big?) " << _descriptor->indexNamespace() + << " key: " << *i << " " << loc.obj()["_id"] << endl; + } + } + + return Status::OK(); + } + + // Return keys in l that are not in r. + // Lifted basically verbatim from elsewhere. + static void setDifference(const BSONObjSet &l, const BSONObjSet &r, vector *diff) { + // l and r must use the same ordering spec. + verify(l.key_comp().order() == r.key_comp().order()); + BSONObjSet::const_iterator i = l.begin(); + BSONObjSet::const_iterator j = r.begin(); + while ( 1 ) { + if ( i == l.end() ) + break; + while ( j != r.end() && j->woCompare( *i ) < 0 ) + j++; + if ( j == r.end() || i->woCompare(*j) != 0 ) { + const BSONObj *jo = &*i; + diff->push_back( (BSONObj *) jo ); + } + i++; + } + } + + Status BtreeBasedAccessMethod::initializeAsEmpty() { + if ( !_btreeState->head().isNull() ) + return Status( ErrorCodes::InternalError, "index already initialized" ); + + DiskLoc newHead; + if ( 0 == _descriptor->version() ) { + newHead = BtreeBucket::addBucket( _btreeState ); + } + else if ( 1 == _descriptor->version() ) { + newHead = BtreeBucket::addBucket( _btreeState ); + } + else { + return Status( ErrorCodes::InternalError, "invalid index number" ); + } + _btreeState->setHead( newHead ); + + return Status::OK(); + } + + Status BtreeBasedAccessMethod::touch(const BSONObj& obj) { + BSONObjSet keys; + getKeys(obj, &keys); + return touch( keys ); + } + + Status BtreeBasedAccessMethod::touch(const BSONObjSet& keys) { + for (BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i) { + int unusedPos; + bool unusedFound; + DiskLoc unusedDiskLoc; + _interface->locate(_btreeState, + _btreeState->head(), + *i, + unusedPos, + unusedFound, + unusedDiskLoc, + 1); + } + + return Status::OK(); + } + + DiskLoc BtreeBasedAccessMethod::findSingle( const BSONObj& key ) const { + DiskLoc head = _btreeState->head(); + Record* record = _btreeState->recordStore()->recordFor( head ); + + if ( 0 == _descriptor->version() ) { + return BtreeBucket::asVersion( record )->findSingle( _btreeState, + _btreeState->head(), + key ); + } + if ( 1 == _descriptor->version() ) { + return BtreeBucket::asVersion( record )->findSingle( _btreeState, + _btreeState->head(), + key ); + } + verify( 0 ); + } + + + Status BtreeBasedAccessMethod::validate(int64_t* numKeys) { + *numKeys = _interface->fullValidate(_btreeState, + _btreeState->head(), + _descriptor->keyPattern()); + return Status::OK(); + } + + Status BtreeBasedAccessMethod::validateUpdate( + const BSONObj &from, const BSONObj &to, const DiskLoc &record, + const InsertDeleteOptions &options, UpdateTicket* status) { + + BtreeBasedPrivateUpdateData *data = new BtreeBasedPrivateUpdateData(); + status->_indexSpecificUpdateData.reset(data); + + getKeys(from, &data->oldKeys); + getKeys(to, &data->newKeys); + data->loc = record; + data->dupsAllowed = options.dupsAllowed; + + setDifference(data->oldKeys, data->newKeys, &data->removed); + setDifference(data->newKeys, data->oldKeys, &data->added); + + bool checkForDups = !data->added.empty() + && (KeyPattern::isIdKeyPattern(_descriptor->keyPattern()) || _descriptor->unique()) + && !options.dupsAllowed; + + for (vector::iterator i = data->added.begin(); i != data->added.end(); i++) { + if (checkForDups) { + if (_interface->wouldCreateDup(_btreeState, + _btreeState->head(), + **i, record)) { + status->_isValid = false; + return Status(ErrorCodes::DuplicateKey, + _interface->dupKeyError(_btreeState, + _btreeState->head(), + **i)); + } + } + + _interface->assertIfKeyTooLongAndNotIgnored(_btreeState, _btreeState->head(), **i); + } + + status->_isValid = true; + + return Status::OK(); + } + + Status BtreeBasedAccessMethod::update(const UpdateTicket& ticket, int64_t* numUpdated) { + if (!ticket._isValid) { + return Status(ErrorCodes::InternalError, "Invalid updateticket in update"); + } + + BtreeBasedPrivateUpdateData* data = + static_cast(ticket._indexSpecificUpdateData.get()); + + if (data->oldKeys.size() + data->added.size() - data->removed.size() > 1) { + _btreeState->setMultikey(); + } + + for (size_t i = 0; i < data->added.size(); ++i) { + _interface->bt_insert(_btreeState, + _btreeState->head(), + data->loc, + *data->added[i], + data->dupsAllowed, + true); + } + + for (size_t i = 0; i < data->removed.size(); ++i) { + _interface->unindex(_btreeState, + _btreeState->head(), + *data->removed[i], + data->loc); + } + + *numUpdated = data->added.size(); + + return Status::OK(); + } + + // ------- + + class BtreeBulk : public IndexAccessMethod { + public: + BtreeBulk( BtreeBasedAccessMethod* real ) { + _real = real; + } + + ~BtreeBulk() {} + + virtual shared_ptr getKeyGenerator() const { + invariant( false ); + } + + virtual Status insert(const BSONObj& obj, + const DiskLoc& loc, + const InsertDeleteOptions& options, + int64_t* numInserted, + const PregeneratedKeysOnIndex* pregen ) { + BSONObjSet keys; + _real->getKeys(obj, &keys); + _phase1.addKeys(keys, loc, false); + if ( numInserted ) + *numInserted += keys.size(); + return Status::OK(); + } + + virtual Status remove(const BSONObj& obj, + const DiskLoc& loc, + const InsertDeleteOptions& options, + int64_t* numDeleted) { + return _notAllowed(); + } + + virtual Status validateUpdate(const BSONObj& from, + const BSONObj& to, + const DiskLoc& loc, + const InsertDeleteOptions& options, + UpdateTicket* ticket) { + return _notAllowed(); + } + + virtual Status update(const UpdateTicket& ticket, int64_t* numUpdated) { + return _notAllowed(); + } + + virtual Status newCursor(IndexCursor **out) const { + return _notAllowed(); + } + + virtual Status initializeAsEmpty() { + return _notAllowed(); + } + + virtual IndexAccessMethod* initiateBulk() { + return NULL; + } + + virtual Status commitBulk( IndexAccessMethod* bulk, + bool mayInterrupt, + std::set* dups ) { + verify( this == bulk ); + return Status::OK(); + } + + virtual Status touch(const BSONObj& obj) { + return _notAllowed(); + } + + virtual Status touch(const BSONObjSet& obj) { + return _notAllowed(); + } + + virtual Status validate(int64_t* numKeys) { + return _notAllowed(); + } + + // ------- + + template< class V > + void commit( set* dupsToDrop, + CurOp* op, + bool mayInterrupt ) { + + Timer timer; + + IndexCatalogEntry* entry = _real->_btreeState; + + bool dupsAllowed = !entry->descriptor()->unique() || + ignoreUniqueIndex(entry->descriptor()); + bool dropDups = entry->descriptor()->dropDups() || inDBRepair; + + BtreeBuilder btBuilder(dupsAllowed, entry); + + BSONObj keyLast; + scoped_ptr i( _phase1.sorter->iterator() ); + + // verifies that pm and op refer to the same ProgressMeter + ProgressMeter& pm = op->setMessage("Index Bulk Build: (2/3) btree bottom up", + "Index: (2/3) BTree Bottom Up Progress", + _phase1.nkeys, + 10); + + while( i->more() ) { + RARELY if ( mayInterrupt ) killCurrentOp.checkForInterrupt(); + ExternalSortDatum d = i->next(); + + try { + if ( !dupsAllowed && dropDups ) { + LastError::Disabled led( lastError.get() ); + btBuilder.addKey(d.first, d.second); + } + else { + btBuilder.addKey(d.first, d.second); + } + } + catch( AssertionException& e ) { + if ( dupsAllowed ) { + // unknown exception?? + throw; + } + + if (ErrorCodes::isInterruption( + DBException::convertExceptionCode(e.getCode()))) { + killCurrentOp.checkForInterrupt(); + } + + if ( ! dropDups ) + throw; + + /* we could queue these on disk, but normally there are very few dups, + * so instead we keep in ram and have a limit. + */ + if ( dupsToDrop ) { + dupsToDrop->insert(d.second); + uassert( 10092, + "too may dups on index build with dropDups=true", + dupsToDrop->size() < 1000000 ); + } + } + pm.hit(); + } + pm.finished(); + op->setMessage("Index Bulk Build: (3/3) btree-middle", + "Index: (3/3) BTree Middle Progress"); + LOG(timer.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit"; + btBuilder.commit( mayInterrupt ); + if ( btBuilder.getn() != _phase1.nkeys && ! dropDups ) { + warning() << "not all entries were added to the index, probably some " + << "keys were too large" << endl; + } + } + + // ------- + + Status _notAllowed() const { + return Status( ErrorCodes::InternalError, "cannot use bulk for this yet" ); + } + + BtreeBasedAccessMethod* _real; // now owned here + SortPhaseOne _phase1; + }; + + int oldCompare(const BSONObj& l,const BSONObj& r, const Ordering &o); // key.cpp + + class BtreeExternalSortComparisonV0 : public ExternalSortComparison { + public: + BtreeExternalSortComparisonV0(const BSONObj& ordering) + : _ordering(Ordering::make(ordering)){ + } + + virtual ~BtreeExternalSortComparisonV0() { } + + virtual int compare(const ExternalSortDatum& l, const ExternalSortDatum& r) const { + int x = oldCompare(l.first, r.first, _ordering); + if (x) { return x; } + return l.second.compare(r.second); + } + private: + const Ordering _ordering; + }; + + class BtreeExternalSortComparisonV1 : public ExternalSortComparison { + public: + BtreeExternalSortComparisonV1(const BSONObj& ordering) + : _ordering(Ordering::make(ordering)) { + } + + virtual ~BtreeExternalSortComparisonV1() { } + + virtual int compare(const ExternalSortDatum& l, const ExternalSortDatum& r) const { + int x = l.first.woCompare(r.first, _ordering, /*considerfieldname*/false); + if (x) { return x; } + return l.second.compare(r.second); + } + private: + const Ordering _ordering; + }; + + ExternalSortComparison* BtreeBasedAccessMethod::getComparison(int version, + const BSONObj& keyPattern) { + + if ( 0 == version ) { + return new BtreeExternalSortComparisonV0( keyPattern ); + } + else if ( 1 == version ) { + return new BtreeExternalSortComparisonV1( keyPattern ); + } + verify( 0 ); + return NULL; + } + + IndexAccessMethod* BtreeBasedAccessMethod::initiateBulk() { + + if ( _interface->nKeys( _btreeState, + _btreeState->head() ) > 0 ) + return NULL; + + auto_ptr bulk( new BtreeBulk( this ) ); + bulk->_phase1.sortCmp.reset( getComparison( _descriptor->version(), + _descriptor->keyPattern() ) ); + + bulk->_phase1.sorter.reset( new BSONObjExternalSorter(bulk->_phase1.sortCmp.get()) ); + bulk->_phase1.sorter->hintNumObjects( _btreeState->collection()->numRecords() ); + + return bulk.release(); + } + + Status BtreeBasedAccessMethod::commitBulk( IndexAccessMethod* bulkRaw, + bool mayInterrupt, + set* dupsToDrop ) { + + if ( _interface->nKeys( _btreeState, + _btreeState->head() ) > 0 ) { + return Status( ErrorCodes::InternalError, "trying to commit, but has data already" ); + } + + { + DiskLoc oldHead = _btreeState->head(); + _btreeState->setHead( DiskLoc() ); + _btreeState->recordStore()->deleteRecord( oldHead ); + } + + string ns = _btreeState->collection()->ns().ns(); + + BtreeBulk* bulk = static_cast( bulkRaw ); + if ( bulk->_phase1.multi ) + _btreeState->setMultikey(); + + bulk->_phase1.sorter->sort( false ); + + if ( _descriptor->version() == 0 ) + bulk->commit( dupsToDrop, cc().curop(), mayInterrupt ); + else if ( _descriptor->version() == 1 ) + bulk->commit( dupsToDrop, cc().curop(), mayInterrupt ); + else + return Status( ErrorCodes::InternalError, "bad btree version" ); + + return Status::OK(); + } + + +} // namespace mongo diff --git a/src/mongo/db/index/btree_based_access_method.h b/src/mongo/db/index/btree_based_access_method.h new file mode 100644 index 00000000000..714b33f6a59 --- /dev/null +++ b/src/mongo/db/index/btree_based_access_method.h @@ -0,0 +1,142 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/index/btree_interface.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/index/index_cursor.h" +#include "mongo/db/index/index_descriptor.h" + +namespace mongo { + + class BtreeBulk; + class ExternalSortComparison; + + /** + * Any access method that is Btree based subclasses from this. + * + * Subclassers must: + * 1. Call the constructor for this class from their constructors, and + * 2. override getKeys. + * + * XXX: Should really think of the sub-class as providing an expression mapping of the input, + * don't need so many AMs, just really precomputing some data and mapping doc for getKeys(?). + * See SERVER-12397 for tracking. + */ + class BtreeBasedAccessMethod : public IndexAccessMethod { + MONGO_DISALLOW_COPYING( BtreeBasedAccessMethod ); + public: + BtreeBasedAccessMethod( IndexCatalogEntry* btreeState ); + + virtual ~BtreeBasedAccessMethod() { } + + virtual Status insert(const BSONObj& obj, + const DiskLoc& loc, + const InsertDeleteOptions& options, + int64_t* numInserted, + const PregeneratedKeysOnIndex* prepared = NULL ) ; + + virtual Status remove(const BSONObj& obj, + const DiskLoc& loc, + const InsertDeleteOptions& options, + int64_t* numDeleted); + + virtual Status validateUpdate(const BSONObj& from, + const BSONObj& to, + const DiskLoc& loc, + const InsertDeleteOptions& options, + UpdateTicket* ticket); + + virtual Status update(const UpdateTicket& ticket, int64_t* numUpdated); + + virtual Status newCursor(IndexCursor **out) const; + + virtual Status initializeAsEmpty(); + + virtual IndexAccessMethod* initiateBulk() ; + + virtual Status commitBulk( IndexAccessMethod* bulk, + bool mayInterrupt, + std::set* dups ); + + virtual Status touch(const BSONObj& obj); + + virtual Status touch( const BSONObjSet& keys ); + + virtual Status validate(int64_t* numKeys); + + // XXX: consider migrating callers to use IndexCursor instead + virtual DiskLoc findSingle( const BSONObj& key ) const; + + // exposed for testing, used for bulk commit + static ExternalSortComparison* getComparison(int version, + const BSONObj& keyPattern); + + protected: + // Friends who need getKeys. + friend class BtreeBulk; + + // See below for body. + class BtreeBasedPrivateUpdateData; + + virtual void getKeys(const BSONObj &obj, BSONObjSet *keys) = 0; + + IndexCatalogEntry* _btreeState; // owned by IndexCatalogEntry + const IndexDescriptor* _descriptor; + + // There are 2 types of Btree disk formats. We put them both behind one interface. + BtreeInterface* _interface; + + private: + bool removeOneKey(const BSONObj& key, const DiskLoc& loc); + }; + + /** + * What data do we need to perform an update? + */ + class BtreeBasedAccessMethod::BtreeBasedPrivateUpdateData + : public UpdateTicket::PrivateUpdateData { + public: + virtual ~BtreeBasedPrivateUpdateData() { } + + BSONObjSet oldKeys, newKeys; + + // These point into the sets oldKeys and newKeys. + vector removed, added; + + DiskLoc loc; + bool dupsAllowed; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/btree_index_cursor.cpp b/src/mongo/db/index/btree_index_cursor.cpp new file mode 100644 index 00000000000..528a0a5ff4d --- /dev/null +++ b/src/mongo/db/index/btree_index_cursor.cpp @@ -0,0 +1,282 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/btree_index_cursor.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/index/index_cursor.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/platform/unordered_set.h" + +namespace mongo { + + unordered_set BtreeIndexCursor::_activeCursors; + SimpleMutex BtreeIndexCursor::_activeCursorsMutex("active_btree_index_cursors"); + + // Go forward by default. + BtreeIndexCursor::BtreeIndexCursor(const IndexCatalogEntry* btreeState, + const DiskLoc head, + BtreeInterface *interface) + : _direction(1), + _btreeState(btreeState), + _interface(interface), + _bucket(head), + _keyOffset(0) { + + SimpleMutex::scoped_lock lock(_activeCursorsMutex); + _activeCursors.insert(this); + } + + BtreeIndexCursor::~BtreeIndexCursor() { + SimpleMutex::scoped_lock lock(_activeCursorsMutex); + _activeCursors.erase(this); + } + + bool BtreeIndexCursor::isEOF() const { return _bucket.isNull(); } + + void BtreeIndexCursor::aboutToDeleteBucket(const IndexCatalogEntry* index, + const DiskLoc& bucket) { + SimpleMutex::scoped_lock lock(_activeCursorsMutex); + for (unordered_set::iterator i = _activeCursors.begin(); + i != _activeCursors.end(); ++i) { + + BtreeIndexCursor* ic = *i; + if (bucket == ic->_bucket && ic->_btreeState == index) { + ic->_keyOffset = -1; + } + } + } + + Status BtreeIndexCursor::setOptions(const CursorOptions& options) { + if (CursorOptions::DECREASING == options.direction) { + _direction = -1; + } else { + _direction = 1; + } + return Status::OK(); + } + + Status BtreeIndexCursor::seek(const BSONObj& position) { + _keyOffset = 0; + + // Unused out parameter. + bool found; + + _bucket = _interface->locate( _btreeState, + _btreeState->head(), + position, + _keyOffset, + found, + 1 == _direction ? minDiskLoc : maxDiskLoc, + _direction); + + skipUnusedKeys(); + + return Status::OK(); + } + + void BtreeIndexCursor::seek(const BSONObj& position, bool afterKey) { + _keyOffset = 0; + + // Unused out parameter. + bool found; + + // Find our key. + _bucket = _interface->locate(_btreeState, + _btreeState->head(), + position, + _keyOffset, + found, + afterKey ? maxDiskLoc : minDiskLoc, + 1); + skipUnusedKeys(); + } + + bool BtreeIndexCursor::pointsAt(const BtreeIndexCursor& other) { + // XXX: do we need this + if (isEOF()) { + return other.isEOF(); + } + + return _bucket == other._bucket && _keyOffset == other._keyOffset; + } + + Status BtreeIndexCursor::seek(const vector& position, + const vector& inclusive) { + pair ignored; + + // Bucket is modified by customLocate. Seeks start @ the root, so we set _bucket to the + // root here. + _bucket = _btreeState->head(); + _keyOffset = 0; + + _interface->customLocate(_btreeState, + _bucket, + _keyOffset, + _emptyObj, + 0, false, + position, + inclusive, + (int)_direction, + ignored); + + skipUnusedKeys(); + + return Status::OK(); + } + + Status BtreeIndexCursor::skip(const BSONObj &keyBegin, int keyBeginLen, bool afterKey, + const vector& keyEnd, + const vector& keyEndInclusive) { + _interface->advanceTo(_btreeState, + _bucket, + _keyOffset, + keyBegin, + keyBeginLen, + afterKey, + keyEnd, + keyEndInclusive, + (int)_direction); + + skipUnusedKeys(); + return Status::OK(); + } + + BSONObj BtreeIndexCursor::getKey() const { + verify(!_bucket.isNull()); + return _interface->keyAt(_btreeState, _bucket, _keyOffset); + } + + DiskLoc BtreeIndexCursor::getValue() const { + verify(!_bucket.isNull()); + return _interface->recordAt(_btreeState, _bucket, _keyOffset); + } + + void BtreeIndexCursor::next() { advance("BtreeIndexCursor::next"); skipUnusedKeys(); } + + Status BtreeIndexCursor::savePosition() { + if (!isEOF()) { + _savedKey = getKey().getOwned(); + _savedLoc = getValue(); + return Status::OK(); + } else { + return Status(ErrorCodes::IllegalOperation, "Can't save position when EOF"); + } + } + + Status BtreeIndexCursor::restorePosition() { + // _keyOffset could be -1 if the bucket was deleted. When buckets are deleted, the + // Btree calls a clientcursor function that calls down to all BTree buckets. Really, + // this deletion thing should be kept BTree-internal. + if (_keyOffset >= 0) { + verify(!_savedKey.isEmpty()); + + try { + if (isSavedPositionValid()) { return Status::OK(); } + if (_keyOffset > 0) { + --_keyOffset; + // "we check one key earlier too, in case a key was just deleted. this is + // important so that multi updates are reasonably fast." -- btreecursor.cpp + if (isSavedPositionValid()) { return Status::OK(); } + } + // Object isn't at the saved position. Fall through to calling seek. + } catch (UserException& e) { + // deletedBucketCode is what keyAt throws if the bucket was deleted. Not a + // problem... + if (BtreeInterface::deletedBucketCode != e.getCode()) { + return e.toStatus(); + } + // Our bucket was deleted, so we look for the saved key. + DEV log() << "debug info: bucket was deleted" << endl; + } + } + + // Our old position was invalidated (keyOfs was set to -1) or our saved position + // is no longer valid, so we must re-find. + RARELY log() << "key seems to have moved in the index, refinding. " + << _bucket.toString() << endl; + + bool found; + + // Why don't we just call seek? Because we want to pass _savedLoc. + _bucket = _interface->locate(_btreeState, + _btreeState->head(), + _savedKey, + _keyOffset, + found, + _savedLoc, + _direction); + + skipUnusedKeys(); + + return Status::OK(); + } + + string BtreeIndexCursor::toString() { return "I AM A BTREE INDEX CURSOR!\n"; } + + void BtreeIndexCursor::skipUnusedKeys() { + int skipped = 0; + + while (!isEOF() && !_interface->keyIsUsed(_btreeState, _bucket, _keyOffset)) { + advance("BtreeIndexCursor::skipUnusedKeys"); + ++skipped; + } + + if (skipped > 10) { + OCCASIONALLY log() << "btree unused skipped: " << skipped << endl; + } + } + + bool BtreeIndexCursor::isSavedPositionValid() { + // We saved the key. If it's in the same position we saved it from... + if (_interface->keyAt(_btreeState, _bucket, _keyOffset).binaryEqual(_savedKey)) { + // And the record it points to is the same record... + if (_interface->recordAt(_btreeState, _bucket, _keyOffset) == _savedLoc) { + // Success! We found it. However! + if (!_interface->keyIsUsed(_btreeState, _bucket, _keyOffset)) { + // We could have been deleted but still exist as a "vacant" key, so skip + // over any unused keys. + skipUnusedKeys(); + } + return true; + } + } + + return false; + } + + // Move to the next/prev. key. Used by normal getNext and also skipping unused keys. + void BtreeIndexCursor::advance(const char* caller) { + _bucket = _interface->advance(_btreeState, _bucket, _keyOffset, _direction, caller); + } + +} // namespace mongo diff --git a/src/mongo/db/index/btree_index_cursor.h b/src/mongo/db/index/btree_index_cursor.h new file mode 100644 index 00000000000..8562660738e --- /dev/null +++ b/src/mongo/db/index/btree_index_cursor.h @@ -0,0 +1,137 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/index/btree_interface.h" +#include "mongo/db/index/index_cursor.h" +#include "mongo/db/index/index_descriptor.h" + +namespace mongo { + + class BtreeIndexCursor : public IndexCursor { + public: + virtual ~BtreeIndexCursor(); + + bool isEOF() const; + + /** + * Called from btree.cpp when we're about to delete a Btree bucket. The index descriptor + * is needed as just the DiskLoc of the bucket is not unique across databases, which might + * result in incorrect invalidation of cursors in other unlocked databases. + */ + static void aboutToDeleteBucket(const IndexCatalogEntry* index, + const DiskLoc& bucket); + + virtual Status setOptions(const CursorOptions& options); + + virtual Status seek(const BSONObj& position); + + // Btree-specific seeking functions. + Status seek(const vector& position, + const vector& inclusive); + + /** + * Seek to the key 'position'. If 'afterKey' is true, seeks to the first + * key that is oriented after 'position'. + * + * Btree-specific. + */ + void seek(const BSONObj& position, bool afterKey); + + Status skip(const BSONObj &keyBegin, int keyBeginLen, bool afterKey, + const vector& keyEnd, + const vector& keyEndInclusive); + + virtual BSONObj getKey() const; + virtual DiskLoc getValue() const; + virtual void next(); + + /** + * BtreeIndexCursor-only. + * Returns true if 'this' points at the same exact key as 'other'. + * Returns false otherwise. + */ + bool pointsAt(const BtreeIndexCursor& other); + + virtual Status savePosition(); + + virtual Status restorePosition(); + + virtual string toString(); + + private: + // We keep the constructor private and only allow the AM to create us. + friend class BtreeBasedAccessMethod; + + // For handling bucket deletion. + static unordered_set _activeCursors; + static SimpleMutex _activeCursorsMutex; + + /** + * btreeState is the ICE of the Btree that we're going to traverse. + * head is the head of the Btree. + * interface is an abstraction to hide the fact that we have two types of Btrees. + * + * Go forward by default. + * + * Intentionally private, we're friends with the only class allowed to call it. + */ + BtreeIndexCursor(const IndexCatalogEntry* btreeState, + const DiskLoc head, + BtreeInterface *interface); + + void skipUnusedKeys(); + + bool isSavedPositionValid(); + + // Move to the next/prev. key. Used by normal getNext and also skipping unused keys. + void advance(const char* caller); + + // For saving/restoring position. + BSONObj _savedKey; + DiskLoc _savedLoc; + + BSONObj _emptyObj; + + int _direction; + const IndexCatalogEntry* _btreeState; // not-owned + BtreeInterface* _interface; + + // What are we looking at RIGHT NOW? We look at a bucket. + DiskLoc _bucket; + // And we look at an offset in the bucket. + int _keyOffset; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/btree_interface.cpp b/src/mongo/db/index/btree_interface.cpp new file mode 100644 index 00000000000..ace3d45ada8 --- /dev/null +++ b/src/mongo/db/index/btree_interface.cpp @@ -0,0 +1,235 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/catalog/index_catalog_entry.h" +#include "mongo/db/index/btree_interface.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/structure/btree/btree.h" +#include "mongo/db/structure/record_store.h" + +namespace mongo { + + template + class BtreeInterfaceImpl : public BtreeInterface { + public: + // typedef typename BucketBasics::VersionNode VersionNode; + + virtual ~BtreeInterfaceImpl() { } + + const BtreeBucket* getHeadBucket( const IndexCatalogEntry* entry ) const { + return getBucket( entry->head() ); + } + + const BtreeBucket* getBucket( const IndexCatalogEntry* entry, + const DiskLoc& loc ) const { + Record* record = entry->recordStore()->recordFor( loc ); + return BtreeBucket::asVersion( record ); + } + + virtual int bt_insert(IndexCatalogEntry* btreeState, + const DiskLoc thisLoc, + const DiskLoc recordLoc, + const BSONObj& key, + bool dupsallowed, + bool toplevel) { + // FYI: toplevel has a default value of true in btree.h + return getBucket( btreeState, thisLoc )->bt_insert(btreeState, + thisLoc, + recordLoc, + key, + dupsallowed, + toplevel); + } + + virtual bool unindex(IndexCatalogEntry* btreeState, + const DiskLoc thisLoc, + const BSONObj& key, + const DiskLoc recordLoc) { + return getBucket( btreeState, thisLoc )->unindex(btreeState, + thisLoc, + key, + recordLoc); + } + + virtual DiskLoc locate(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& key, + int& pos, + bool& found, + const DiskLoc& recordLoc, + int direction) const { + // FYI: direction has a default of 1 + return getBucket( btreeState, thisLoc )->locate(btreeState, + thisLoc, + key, + pos, + found, + recordLoc, + direction); + } + + virtual bool wouldCreateDup(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& key, + const DiskLoc& self) const { + typename Version::KeyOwned ownedVersion(key); + return getBucket( btreeState, thisLoc )->wouldCreateDup(btreeState, + thisLoc, + ownedVersion, + self); + } + + virtual void assertIfKeyTooLongAndNotIgnored(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& key) const { + typename Version::KeyOwned ownedVersion(key); + getBucket( btreeState, thisLoc )->assertIfKeyTooLongAndNotIgnored(btreeState, + ownedVersion); + } + + virtual void customLocate(const IndexCatalogEntry* btreeState, + DiskLoc& locInOut, + int& keyOfs, + const BSONObj& keyBegin, + int keyBeginLen, bool afterVersion, + const vector& keyEnd, + const vector& keyEndInclusive, + int direction, + pair& bestParent) const { + return BtreeBucket::customLocate(btreeState, + locInOut, + keyOfs, + keyBegin, + keyBeginLen, afterVersion, + keyEnd, + keyEndInclusive, + direction, + bestParent); + } + + virtual void advanceTo(const IndexCatalogEntry* btreeState, + DiskLoc &thisLoc, + int &keyOfs, + const BSONObj &keyBegin, + int keyBeginLen, + bool afterVersion, + const vector& keyEnd, + const vector& keyEndInclusive, + int direction) const { + return getBucket( btreeState, thisLoc )->advanceTo(btreeState, + thisLoc, + keyOfs, + keyBegin, + keyBeginLen, + afterVersion, + keyEnd, + keyEndInclusive, + direction); + } + + + virtual int nKeys(const IndexCatalogEntry* btreeState, + DiskLoc bucket ) { + return getBucket(btreeState,bucket)->nKeys(); + } + + + virtual bool keyIsUsed(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset) const { + return getBucket(btreeState,bucket)->k(keyOffset).isUsed(); + } + + virtual BSONObj keyAt(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset) const { + verify(!bucket.isNull()); + const BtreeBucket *b = getBucket(btreeState,bucket); + int n = b->getN(); + if (n == b->INVALID_N_SENTINEL) { + throw UserException(deletedBucketCode, "keyAt bucket deleted"); + } + dassert( n >= 0 && n < 10000 ); + return keyOffset >= n ? BSONObj() : b->keyNode(keyOffset).key.toBson(); + } + + virtual DiskLoc recordAt(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset) const { + const BtreeBucket *b = getBucket(btreeState,bucket); + return b->keyNode(keyOffset).recordLoc; + } + + virtual void keyAndRecordAt(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset, BSONObj* keyOut, + DiskLoc* recordOut) const { + verify(!bucket.isNull()); + const BtreeBucket *b = getBucket(btreeState,bucket); + + int n = b->getN(); + + // If n is 0xffff the bucket was deleted. + if (keyOffset < 0 || keyOffset >= n || n == 0xffff || !b->isUsed(keyOffset)) { + return; + } + + if (keyOffset >= n) { + *keyOut = BSONObj(); + *recordOut = DiskLoc(); + } else { + *keyOut = b->keyNode(keyOffset).key.toBson(); + *recordOut = b->keyNode(keyOffset).recordLoc; + } + } + + virtual string dupKeyError(const IndexCatalogEntry* btreeState, + DiskLoc bucket, + const BSONObj& keyObj) const { + typename Version::KeyOwned key(keyObj); + return getBucket( btreeState, bucket )->dupKeyError(btreeState->descriptor(), + key); + } + + virtual DiskLoc advance(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + int& keyOfs, + int direction, + const char* caller) const { + return getBucket( btreeState, thisLoc )->advance(thisLoc, keyOfs, direction, caller); + } + + virtual long long fullValidate(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& keyPattern) { + return getBucket( btreeState, thisLoc )->fullValidate(thisLoc, keyPattern); + } + }; + + BtreeInterfaceImpl interface_v0; + BtreeInterfaceImpl interface_v1; + BtreeInterface* BtreeInterface::interfaces[] = { &interface_v0, &interface_v1 }; + +} // namespace mongo + diff --git a/src/mongo/db/index/btree_interface.h b/src/mongo/db/index/btree_interface.h new file mode 100644 index 00000000000..edeb05dacd5 --- /dev/null +++ b/src/mongo/db/index/btree_interface.h @@ -0,0 +1,156 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class IndexCatalogEntry; + + /** + * We have two Btree on-disk formats which support identical operations. We hide this as much + * as possible by having one implementation of this interface per format. + * + * For documentation on all of the methods here, look at mongo/db/structure/btree/btree.h + */ + class BtreeInterface { + public: + virtual ~BtreeInterface() { } + + static BtreeInterface *interfaces[]; + + // This is the # of the exception that is thrown if we're trying to access a bucket that + // was deleted. Calling code needs to be able to recognize this and possibly ignore it. + static const int deletedBucketCode = 16738; + + virtual int bt_insert(IndexCatalogEntry* btreeState, + const DiskLoc thisLoc, + const DiskLoc recordLoc, + const BSONObj& key, + bool dupsallowed, + bool toplevel = true) = 0; + + virtual bool unindex(IndexCatalogEntry* btreeState, + const DiskLoc thisLoc, + const BSONObj& key, + const DiskLoc recordLoc) = 0; + + virtual DiskLoc locate(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& key, + int& pos, // out + bool& found, // out + const DiskLoc& recordLoc, // out + int direction = 1) const = 0; + + virtual bool wouldCreateDup(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& key, + const DiskLoc& self) const = 0; + + virtual void assertIfKeyTooLongAndNotIgnored(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& key) const = 0; + + virtual void customLocate(const IndexCatalogEntry* btreeState, + DiskLoc& locInOut, + int& keyOfs, + const BSONObj& keyBegin, + int keyBeginLen, bool afterKey, + const vector& keyEnd, + const vector& keyEndInclusive, + int direction, + pair& bestParent) const = 0 ; + + virtual void advanceTo(const IndexCatalogEntry* btreeState, + DiskLoc &thisLoc, + int &keyOfs, + const BSONObj &keyBegin, + int keyBeginLen, + bool afterKey, + const vector& keyEnd, + const vector& keyEndInclusive, + int direction) const = 0; + + virtual string dupKeyError(const IndexCatalogEntry* btreeState, + DiskLoc bucket, + const BSONObj& keyObj) const =0; + + virtual DiskLoc advance(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + int& keyOfs, + int direction, + const char* caller) const = 0; + + virtual long long fullValidate(const IndexCatalogEntry* btreeState, + const DiskLoc& thisLoc, + const BSONObj& keyPattern) = 0; + + /** + * These methods are here so that the BtreeCursor doesn't need to do any templating for the + * two on-disk formats. + */ + + /** + * Returns number of total keys just in provided bucket + * (not recursive) + */ + virtual int nKeys(const IndexCatalogEntry* btreeState, + DiskLoc bucket ) = 0; + + /** + * Is the key at (bucket, keyOffset) being used or not? + * Some keys are marked as not used and skipped. + */ + virtual bool keyIsUsed(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset) const = 0; + + /** + * Get the BSON representation of the key at (bucket, keyOffset). + */ + virtual BSONObj keyAt(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset) const = 0; + + /** + * Get the DiskLoc that the key at (bucket, keyOffset) points at. + */ + virtual DiskLoc recordAt(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset) const = 0; + + /** + * keyAt and recordAt at the same time. + */ + virtual void keyAndRecordAt(const IndexCatalogEntry* btreeState, + DiskLoc bucket, int keyOffset, BSONObj* keyOut, + DiskLoc* recordOut) const = 0; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/btree_key_generator.cpp b/src/mongo/db/index/btree_key_generator.cpp new file mode 100644 index 00000000000..2a88148ef6d --- /dev/null +++ b/src/mongo/db/index/btree_key_generator.cpp @@ -0,0 +1,332 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/btree_key_generator.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + // Used in scanandorder.cpp to inforatively error when we try to sort keys with parallel arrays. + const int BtreeKeyGenerator::ParallelArraysCode = 10088; + + BtreeKeyGenerator::BtreeKeyGenerator(vector fieldNames, vector fixed, + bool isSparse) + : _isSparse(isSparse), _fixed(fixed) { + + for ( size_t i = 0; i < fieldNames.size(); i++ ) + _fieldNames.push_back( fieldNames[i] ); + + BSONObjBuilder nullKeyBuilder; + for (size_t i = 0; i < fieldNames.size(); ++i) { + nullKeyBuilder.appendNull(""); + } + _nullKey = nullKeyBuilder.obj(); + + BSONObjBuilder nullEltBuilder; + nullEltBuilder.appendNull(""); + _nullObj = nullEltBuilder.obj(); + _nullElt = _nullObj.firstElement(); + } + + void BtreeKeyGenerator::getKeys(const BSONObj &obj, BSONObjSet *keys) const { + // These are mutated as part of the getKeys call. :| + vector fieldNames; + for ( size_t i = 0; i < _fieldNames.size(); i++ ) + fieldNames.push_back( _fieldNames[i].c_str() ); + + vector fixed(_fixed); + getKeysImpl(fieldNames, fixed, obj, keys); + if (keys->empty() && ! _isSparse) { + keys->insert(_nullKey); + } + } + + static void assertParallelArrays( const char *first, const char *second ) { + stringstream ss; + ss << "cannot index parallel arrays [" << first << "] [" << second << "]"; + uasserted( BtreeKeyGenerator::ParallelArraysCode , ss.str() ); + } + + BtreeKeyGeneratorV0::BtreeKeyGeneratorV0(vector fieldNames, + vector fixed, bool isSparse) + : BtreeKeyGenerator(fieldNames, fixed, isSparse) { } + + void BtreeKeyGeneratorV0::getKeysImpl(vector fieldNames, vector fixed, + const BSONObj &obj, BSONObjSet *keys) const { + BSONElement arrElt; + unsigned arrIdx = ~0; + unsigned numNotFound = 0; + + for ( unsigned i = 0; i < fieldNames.size(); ++i ) { + if ( *fieldNames[ i ] == '\0' ) + continue; + + BSONElement e = obj.getFieldDottedOrArray( fieldNames[ i ] ); + + if ( e.eoo() ) { + e = _nullElt; // no matching field + numNotFound++; + } + + if ( e.type() != Array ) + fieldNames[ i ] = ""; // no matching field or non-array match + + if ( *fieldNames[ i ] == '\0' ) + // no need for further object expansion (though array expansion still possible) + fixed[ i ] = e; + + if ( e.type() == Array && arrElt.eoo() ) { + // we only expand arrays on a single path -- track the path here + arrIdx = i; + arrElt = e; + } + + // enforce single array path here + if ( e.type() == Array && e.rawdata() != arrElt.rawdata() ) { + assertParallelArrays( e.fieldName(), arrElt.fieldName() ); + } + } + + bool allFound = true; // have we found elements for all field names in the key spec? + for (vector::const_iterator i = fieldNames.begin(); i != fieldNames.end(); + ++i ) { + if ( **i != '\0' ) { + allFound = false; + break; + } + } + + if ( _isSparse && numNotFound == _fieldNames.size()) { + // we didn't find any fields + // so we're not going to index this document + return; + } + + bool insertArrayNull = false; + + if ( allFound ) { + if ( arrElt.eoo() ) { + // no terminal array element to expand + BSONObjBuilder b(_sizeTracker); + for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i ) + b.appendAs( *i, "" ); + keys->insert( b.obj() ); + } + else { + // terminal array element to expand, so generate all keys + BSONObjIterator i( arrElt.embeddedObject() ); + if ( i.more() ) { + while( i.more() ) { + BSONObjBuilder b(_sizeTracker); + for( unsigned j = 0; j < fixed.size(); ++j ) { + if ( j == arrIdx ) + b.appendAs( i.next(), "" ); + else + b.appendAs( fixed[ j ], "" ); + } + keys->insert( b.obj() ); + } + } + else if ( fixed.size() > 1 ) { + insertArrayNull = true; + } + } + } + else { + // nonterminal array element to expand, so recurse + verify( !arrElt.eoo() ); + BSONObjIterator i( arrElt.embeddedObject() ); + if ( i.more() ) { + while( i.more() ) { + BSONElement e = i.next(); + if ( e.type() == Object ) { + getKeysImpl( fieldNames, fixed, e.embeddedObject(), keys ); + } + } + } + else { + insertArrayNull = true; + } + } + + if ( insertArrayNull ) { + // x : [] - need to insert undefined + BSONObjBuilder b(_sizeTracker); + for( unsigned j = 0; j < fixed.size(); ++j ) { + if ( j == arrIdx ) { + b.appendUndefined( "" ); + } + else { + BSONElement e = fixed[j]; + if ( e.eoo() ) + b.appendNull( "" ); + else + b.appendAs( e , "" ); + } + } + keys->insert( b.obj() ); + } + } + + BtreeKeyGeneratorV1::BtreeKeyGeneratorV1(vector fieldNames, + vector fixed, bool isSparse) + : BtreeKeyGenerator(fieldNames, fixed, isSparse) { + + BSONObjBuilder b; + b.appendUndefined( "" ); + _undefinedObj = b.obj(); + _undefinedElt = _undefinedObj.firstElement(); + } + + BSONElement BtreeKeyGeneratorV1::extractNextElement(const BSONObj &obj, const BSONObj &arr, + const char *&field, + bool &arrayNestedArray) const { + string firstField = mongoutils::str::before( field, '.' ); + bool haveObjField = !obj.getField( firstField ).eoo(); + BSONElement arrField = arr.getField( firstField ); + bool haveArrField = !arrField.eoo(); + + // An index component field name cannot exist in both a document + // array and one of that array's children. + uassert(16746, + mongoutils::str::stream() << + "Ambiguous field name found in array (do not use numeric field names in " + "embedded elements in an array), field: '" << arrField.fieldName() << + "' for array: " << arr, + !haveObjField || !haveArrField ); + + arrayNestedArray = false; + if ( haveObjField ) { + return obj.getFieldDottedOrArray( field ); + } + else if ( haveArrField ) { + if ( arrField.type() == Array ) { + arrayNestedArray = true; + } + return arr.getFieldDottedOrArray( field ); + } + return BSONElement(); + } + + void BtreeKeyGeneratorV1::_getKeysArrEltFixed(vector &fieldNames, + vector &fixed, + const BSONElement &arrEntry, BSONObjSet *keys, + unsigned numNotFound, + const BSONElement &arrObjElt, + const set &arrIdxs, + bool mayExpandArrayUnembedded) const { + // set up any terminal array values + for( set::const_iterator j = arrIdxs.begin(); j != arrIdxs.end(); ++j ) { + if ( *fieldNames[ *j ] == '\0' ) { + fixed[ *j ] = mayExpandArrayUnembedded ? arrEntry : arrObjElt; + } + } + // recurse + getKeysImplWithArray(fieldNames, + fixed, + arrEntry.type() == Object ? arrEntry.embeddedObject() : BSONObj(), + keys, + numNotFound, + arrObjElt.embeddedObject()); + } + + void BtreeKeyGeneratorV1::getKeysImpl(vector fieldNames, vector fixed, + const BSONObj &obj, BSONObjSet *keys) const { + getKeysImplWithArray(fieldNames, fixed, obj, keys, 0, BSONObj()); + } + + void BtreeKeyGeneratorV1::getKeysImplWithArray(vector fieldNames, + vector fixed, const BSONObj &obj, + BSONObjSet *keys, unsigned numNotFound, + const BSONObj &array) const { + BSONElement arrElt; + set arrIdxs; + bool mayExpandArrayUnembedded = true; + for( unsigned i = 0; i < fieldNames.size(); ++i ) { + if ( *fieldNames[ i ] == '\0' ) { + continue; + } + + bool arrayNestedArray; + // Extract element matching fieldName[ i ] from object xor array. + BSONElement e = extractNextElement( obj, array, fieldNames[ i ], arrayNestedArray ); + + if ( e.eoo() ) { + // if field not present, set to null + fixed[ i ] = _nullElt; + // done expanding this field name + fieldNames[ i ] = ""; + numNotFound++; + } + else if ( e.type() == Array ) { + arrIdxs.insert( i ); + if ( arrElt.eoo() ) { + // we only expand arrays on a single path -- track the path here + arrElt = e; + } + else if ( e.rawdata() != arrElt.rawdata() ) { + // enforce single array path here + assertParallelArrays( e.fieldName(), arrElt.fieldName() ); + } + if ( arrayNestedArray ) { + mayExpandArrayUnembedded = false; + } + } + else { + // not an array - no need for further expansion + fixed[ i ] = e; + } + } + + if ( arrElt.eoo() ) { + // No array, so generate a single key. + if ( _isSparse && numNotFound == fieldNames.size()) { + return; + } + BSONObjBuilder b(_sizeTracker); + for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i ) { + b.appendAs( *i, "" ); + } + keys->insert( b.obj() ); + } + else if ( arrElt.embeddedObject().firstElement().eoo() ) { + // Empty array, so set matching fields to undefined. + _getKeysArrEltFixed(fieldNames, fixed, _undefinedElt, keys, numNotFound, arrElt, + arrIdxs, true ); + } + else { + // Non empty array that can be expanded, so generate a key for each member. + BSONObj arrObj = arrElt.embeddedObject(); + BSONObjIterator i( arrObj ); + while( i.more() ) { + _getKeysArrEltFixed(fieldNames, fixed, i.next(), keys, numNotFound, arrElt, arrIdxs, + mayExpandArrayUnembedded ); + } + } + } +} // namespace mongo diff --git a/src/mongo/db/index/btree_key_generator.h b/src/mongo/db/index/btree_key_generator.h new file mode 100644 index 00000000000..eba22214b12 --- /dev/null +++ b/src/mongo/db/index/btree_key_generator.h @@ -0,0 +1,115 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include "mongo/db/jsobj.h" +#include "mongo/db/index/key_generator.h" + +namespace mongo { + + /** + * Internal class used by BtreeAccessMethod to generate keys for indexed documents. + * This class is meant to be kept under the index access layer. + */ + class BtreeKeyGenerator : public KeyGenerator { + public: + BtreeKeyGenerator(vector fieldNames, vector fixed, bool isSparse); + virtual ~BtreeKeyGenerator() { } + + void getKeys(const BSONObj &obj, BSONObjSet *keys) const; + + static const int ParallelArraysCode; + + protected: + // These are used by the getKeysImpl(s) below. + vector _fieldNames; + bool _isSparse; + BSONObj _nullKey; // a full key with all fields null + BSONObj _nullObj; // only used for _nullElt + BSONElement _nullElt; // jstNull + BSONSizeTracker _sizeTracker; + private: + // We have V0 and V1. Sigh. + virtual void getKeysImpl(vector fieldNames, vector fixed, + const BSONObj &obj, BSONObjSet *keys) const = 0; + vector _fixed; + }; + + class BtreeKeyGeneratorV0 : public BtreeKeyGenerator { + public: + BtreeKeyGeneratorV0(vector fieldNames, vector fixed, + bool isSparse); + virtual ~BtreeKeyGeneratorV0() { } + + private: + virtual void getKeysImpl(vector fieldNames, vector fixed, + const BSONObj &obj, BSONObjSet *keys) const; + }; + + class BtreeKeyGeneratorV1 : public BtreeKeyGenerator { + public: + BtreeKeyGeneratorV1(vector fieldNames, vector fixed, + bool isSparse); + virtual ~BtreeKeyGeneratorV1() { } + + private: + /** + * @param fieldNames - fields to index, may be postfixes in recursive calls + * @param fixed - values that have already been identified for their index fields + * @param obj - object from which keys should be extracted, based on names in fieldNames + * @param keys - set where index keys are written + * @param numNotFound - number of index fields that have already been identified as missing + * @param array - array from which keys should be extracted, based on names in fieldNames + * If obj and array are both nonempty, obj will be one of the elements of array. + */ + virtual void getKeysImpl(vector fieldNames, vector fixed, + const BSONObj &obj, BSONObjSet *keys) const; + + // These guys are called by getKeysImpl. + void getKeysImplWithArray(vector fieldNames, vector fixed, + const BSONObj &obj, BSONObjSet *keys, unsigned numNotFound, + const BSONObj &array) const; + /** + * @param arrayNestedArray - set if the returned element is an array nested directly + within arr. + */ + BSONElement extractNextElement(const BSONObj &obj, const BSONObj &arr, const char *&field, + bool &arrayNestedArray ) const; + void _getKeysArrEltFixed(vector &fieldNames, vector &fixed, + const BSONElement &arrEntry, BSONObjSet *keys, + unsigned numNotFound, const BSONElement &arrObjElt, + const set &arrIdxs, bool mayExpandArrayUnembedded) const; + + BSONObj _undefinedObj; + BSONElement _undefinedElt; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/expression_index.h b/src/mongo/db/index/expression_index.h new file mode 100644 index 00000000000..672f78457f1 --- /dev/null +++ b/src/mongo/db/index/expression_index.h @@ -0,0 +1,162 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/db/hasher.h" +#include "mongo/db/query/index_bounds_builder.h" + +namespace mongo { + + /** + * Functions that compute expression index mappings. + * + * TODO: I think we could structure this more generally with respect to planning. + */ + class ExpressionMapping { + public: + static BSONObj hash(const BSONElement& value) { + BSONObjBuilder bob; + bob.append("", BSONElementHasher::hash64(value, BSONElementHasher::DEFAULT_HASH_SEED)); + return bob.obj(); + } + + // TODO: what should we really pass in for indexInfoObj? + static void cover2dsphere(const S2Region& region, + const BSONObj& indexInfoObj, + OrderedIntervalList* oilOut) { + + int coarsestIndexedLevel; + BSONElement ce = indexInfoObj["coarsestIndexedLevel"]; + if (ce.isNumber()) { + coarsestIndexedLevel = ce.numberInt(); + } + else { + coarsestIndexedLevel = + S2::kAvgEdge.GetClosestLevel(100 * 1000.0 / kRadiusOfEarthInMeters); + } + + // The min level of our covering is the level whose cells are the closest match to the + // *area* of the region (or the max indexed level, whichever is smaller) The max level + // is 4 sizes larger. + double edgeLen = sqrt(region.GetRectBound().Area()); + S2RegionCoverer coverer; + coverer.set_min_level(min(coarsestIndexedLevel, + 2 + S2::kAvgEdge.GetClosestLevel(edgeLen))); + coverer.set_max_level(4 + coverer.min_level()); + + vector cover; + coverer.GetCovering(region, &cover); + + // Look at the cells we cover and all cells that are within our covering and finer. + // Anything with our cover as a strict prefix is contained within the cover and should + // be intersection tested. + set intervalSet; + set exactSet; + for (size_t i = 0; i < cover.size(); ++i) { + + S2CellId coveredCell = cover[i]; + intervalSet.insert(coveredCell.toString()); + + // Look at the cells that cover us. We want to look at every cell that contains the + // covering we would index on if we were to insert the query geometry. We generate + // the would-index-with-this-covering and find all the cells strictly containing the + // cells in that set, until we hit the coarsest indexed cell. We use equality, not + // a prefix match. Why not prefix? Because we've already looked at everything + // finer or as fine as our initial covering. + // + // Say we have a fine point with cell id 212121, we go up one, get 21212, we don't + // want to look at cells 21212[not-1] because we know they're not going to intersect + // with 212121, but entries inserted with cell value 21212 (no trailing digits) may. + // And we've already looked at points with the cell id 211111 from the regex search + // created above, so we only want things where the value of the last digit is not + // stored (and therefore could be 1). + + while (coveredCell.level() > coarsestIndexedLevel) { + + // Add the parent cell of the currently covered cell since we aren't at the + // coarsest level yet + // NOTE: Be careful not to generate cells strictly less than the + // coarsestIndexedLevel - this can result in S2 failures when level < 0. + + coveredCell = coveredCell.parent(); + exactSet.insert(coveredCell.toString()); + } + } + + // We turned the cell IDs into strings which define point intervals or prefixes of + // strings we want to look for. + set::iterator exactIt = exactSet.begin(); + set::iterator intervalIt = intervalSet.begin(); + while (exactSet.end() != exactIt && intervalSet.end() != intervalIt) { + const string& exact = *exactIt; + const string& ival = *intervalIt; + if (exact < ival) { + // add exact + oilOut->intervals.push_back(IndexBoundsBuilder::makePointInterval(exact)); + exactIt++; + } + else { + string end = ival; + end[end.size() - 1]++; + oilOut->intervals.push_back( + IndexBoundsBuilder::makeRangeInterval(ival, end, true, false)); + intervalIt++; + } + } + + if (exactSet.end() != exactIt) { + verify(intervalSet.end() == intervalIt); + do { + oilOut->intervals.push_back(IndexBoundsBuilder::makePointInterval(*exactIt)); + exactIt++; + } while (exactSet.end() != exactIt); + } + else if (intervalSet.end() != intervalIt) { + verify(exactSet.end() == exactIt); + do { + const string& ival = *intervalIt; + string end = ival; + end[end.size() - 1]++; + oilOut->intervals.push_back( + IndexBoundsBuilder::makeRangeInterval(ival, end, true, false)); + intervalIt++; + } while (intervalSet.end() != intervalIt); + } + + // Make sure that our intervals don't overlap each other and are ordered correctly. + // This perhaps should only be done in debug mode. + if (!oilOut->isValidFor(1)) { + cout << "check your assumptions! OIL = " << oilOut->toString() << endl; + verify(0); + } + } + }; + +} // namespace mongo diff --git a/src/mongo/db/index/expression_params.h b/src/mongo/db/index/expression_params.h new file mode 100644 index 00000000000..845497c90af --- /dev/null +++ b/src/mongo/db/index/expression_params.h @@ -0,0 +1,190 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/geo/s2common.h" +#include "mongo/db/hasher.h" +#include "mongo/db/index/2d_common.h" +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + class ExpressionParams { + public: + static void parseTwoDParams(const BSONObj& infoObj, TwoDIndexingParams* out) { + BSONObjIterator i(infoObj.getObjectField("key")); + + while (i.more()) { + BSONElement e = i.next(); + if (e.type() == String && IndexNames::GEO_2D == e.valuestr()) { + uassert(16800, "can't have 2 geo fields", out->geo.size() == 0); + uassert(16801, "2d has to be first in index", out->other.size() == 0); + out->geo = e.fieldName(); + } else { + int order = 1; + if (e.isNumber()) { + order = static_cast(e.Number()); + } + out->other.push_back(make_pair(e.fieldName(), order)); + } + } + + uassert(16802, "no geo field specified", out->geo.size()); + + double bits = configValueWithDefaultDouble(infoObj, "bits", 26); // for lat/long, ~ 1ft + uassert(16803, "bits in geo index must be between 1 and 32", bits > 0 && bits <= 32); + + GeoHashConverter::Parameters hashParams; + hashParams.bits = static_cast(bits); + hashParams.max = configValueWithDefaultDouble(infoObj, "max", 180.0); + hashParams.min = configValueWithDefaultDouble(infoObj, "min", -180.0); + double numBuckets = (1024 * 1024 * 1024 * 4.0); + hashParams.scaling = numBuckets / (hashParams.max - hashParams.min); + + out->geoHashConverter.reset(new GeoHashConverter(hashParams)); + } + + static void parseHashParams(const BSONObj& infoObj, + HashSeed* seedOut, + int* versionOut, + string* fieldOut) { + + // Default _seed to DEFAULT_HASH_SEED if "seed" is not included in the index spec + // or if the value of "seed" is not a number + + // *** WARNING *** + // Choosing non-default seeds will invalidate hashed sharding + // Changing the seed default will break existing indexes and sharded collections + if (infoObj["seed"].eoo()) { + *seedOut = BSONElementHasher::DEFAULT_HASH_SEED; + } + else { + *seedOut = infoObj["seed"].numberInt(); + } + + // In case we have hashed indexes based on other hash functions in the future, we store + // a hashVersion number. If hashVersion changes, "makeSingleHashKey" will need to change + // accordingly. Defaults to 0 if "hashVersion" is not included in the index spec or if + // the value of "hashversion" is not a number + *versionOut = infoObj["hashVersion"].numberInt(); + + // Get the hashfield name + BSONElement firstElt = infoObj.getObjectField("key").firstElement(); + massert(16765, "error: no hashed index field", + firstElt.str().compare(IndexNames::HASHED) == 0); + *fieldOut = firstElt.fieldName(); + } + + static void parseHaystackParams(const BSONObj& infoObj, + string* geoFieldOut, + vector* otherFieldsOut, + double* bucketSizeOut) { + + BSONElement e = infoObj["bucketSize"]; + uassert(16777, "need bucketSize", e.isNumber()); + *bucketSizeOut = e.numberDouble(); + uassert(16769, "bucketSize cannot be zero", *bucketSizeOut != 0.0); + + // Example: + // db.foo.ensureIndex({ pos : "geoHaystack", type : 1 }, { bucketSize : 1 }) + BSONObjIterator i(infoObj.getObjectField("key")); + while (i.more()) { + BSONElement e = i.next(); + if (e.type() == String && IndexNames::GEO_HAYSTACK == e.valuestr()) { + uassert(16770, "can't have more than one geo field", geoFieldOut->size() == 0); + uassert(16771, "the geo field has to be first in index", + otherFieldsOut->size() == 0); + *geoFieldOut = e.fieldName(); + } else { + uassert(16772, "geoSearch can only have 1 non-geo field for now", + otherFieldsOut->size() == 0); + otherFieldsOut->push_back(e.fieldName()); + } + } + } + + static void parse2dsphereParams(const BSONObj& infoObj, + S2IndexingParams* out) { + // Set up basic params. + out->maxKeysPerInsert = 200; + + // This is advisory. + out->maxCellsInCovering = 50; + + // Near distances are specified in meters...sometimes. + out->radius = kRadiusOfEarthInMeters; + + // These are not advisory. + out->finestIndexedLevel = configValueWithDefaultInt(infoObj, + "finestIndexedLevel", + S2::kAvgEdge.GetClosestLevel(500.0 / out->radius)); + + out->coarsestIndexedLevel = configValueWithDefaultInt(infoObj, + "coarsestIndexedLevel", + S2::kAvgEdge.GetClosestLevel(100 * 1000.0 / out->radius)); + + static const string kIndexVersionFieldName("2dsphereIndexVersion"); + + // Determine which version of this index we're using. If none was set in the descriptor, + // assume S2_INDEX_VERSION_1 (alas, the first version predates the existence of the version + // field). + out->indexVersion = static_cast(configValueWithDefaultInt(infoObj, + kIndexVersionFieldName, + S2_INDEX_VERSION_1)); + + uassert(16747, "coarsestIndexedLevel must be >= 0", out->coarsestIndexedLevel >= 0); + uassert(16748, "finestIndexedLevel must be <= 30", out->finestIndexedLevel <= 30); + uassert(16749, "finestIndexedLevel must be >= coarsestIndexedLevel", + out->finestIndexedLevel >= out->coarsestIndexedLevel); + + massert(17395, + mongoutils::str::stream() << "unsupported geo index version { " << kIndexVersionFieldName + << " : " << out->indexVersion << " }, only support versions: [" + << S2_INDEX_VERSION_1 << "," << S2_INDEX_VERSION_2 << "]", + out->indexVersion == S2_INDEX_VERSION_2 || out->indexVersion == S2_INDEX_VERSION_1); + } + + private: + static double configValueWithDefaultDouble(const BSONObj& infoObj, + const string& name, + double def) { + BSONElement e = infoObj[name]; + if (e.isNumber()) { return e.numberDouble(); } + return def; + } + + static int configValueWithDefaultInt(const BSONObj& infoObj, const string& name, int def) { + BSONElement e = infoObj[name]; + if (e.isNumber()) { return e.numberInt(); } + return def; + } + + }; + +} // namespace mongo diff --git a/src/mongo/db/index/external_key_generator.cpp b/src/mongo/db/index/external_key_generator.cpp new file mode 100644 index 00000000000..89d94ea0bbe --- /dev/null +++ b/src/mongo/db/index/external_key_generator.cpp @@ -0,0 +1,209 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/index/external_key_generator.h" + +#include "mongo/db/index/2d_key_generator.h" +#include "mongo/db/index/btree_key_generator.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/index/fts_key_generator.h" +#include "mongo/db/index/hash_key_generator.h" +#include "mongo/db/index/haystack_key_generator.h" +#include "mongo/db/index/s2_key_generator.h" +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + +namespace { + void getKeysForUpgradeChecking(const BSONObj& infoObj, + const BSONObj& doc, + BSONObjSet* keys) { + + BSONObj keyPattern = infoObj.getObjectField("key"); + + string type = IndexNames::findPluginName(keyPattern); + + if (IndexNames::GEO_2D == type) { + TwoDIndexingParams params; + ExpressionParams::parseTwoDParams(infoObj, ¶ms); + TwoDKeyGenerator gen( params ); + gen.getKeys( doc, keys ); + } + else if (IndexNames::GEO_HAYSTACK == type) { + string geoField; + vector otherFields; + double bucketSize; + ExpressionParams::parseHaystackParams(infoObj, &geoField, &otherFields, &bucketSize); + HaystackKeyGenerator gen( geoField, otherFields, bucketSize ); + gen.getKeys( doc, keys ); + } + else if (IndexNames::GEO_2DSPHERE == type) { + S2IndexingParams params; + ExpressionParams::parse2dsphereParams(infoObj, ¶ms); + S2KeyGenerator gen( keyPattern, params ); + gen.getKeys( doc, keys ); + } + else if (IndexNames::TEXT == type) { + FTSKeyGenerator gen(infoObj); + gen.getKeys( doc, keys ); + } + else if (IndexNames::HASHED == type) { + HashSeed seed; + int version; + string field; + ExpressionParams::parseHashParams(infoObj, &seed, &version, &field); + HashKeyGenerator gen( field, seed, version, infoObj["sparse"].trueValue() ); + gen.getKeys( doc, keys ); + } + else { + invariant(IndexNames::BTREE == type); + + std::vector fieldNames; + std::vector fixed; + BSONObjIterator keyIt(keyPattern); + while (keyIt.more()) { + BSONElement patternElt = keyIt.next(); + fieldNames.push_back(patternElt.fieldName()); + fixed.push_back(BSONElement()); + } + + // XXX: do we care about version + BtreeKeyGeneratorV1 keyGen(fieldNames, fixed, infoObj["sparse"].trueValue()); + + keyGen.getKeys(doc, keys); + } + } + + // cloned from key.cpp to build the below set + const int binDataCodeLengths[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 32}; + const std::set acceptableBinDataLengths( + binDataCodeLengths, + binDataCodeLengths + (sizeof(binDataCodeLengths) / sizeof(int))); + + // modified version of the KeyV1Owned constructor that returns the would-be-key's datasize() + int keyV1Size(const BSONObj& obj) { + BSONObj::iterator i(obj); + int size = 0; + const int traditionalSize = obj.objsize() + 1; + while (i.more()) { + BSONElement e = i.next(); + switch (e.type()) { + case MinKey: + case jstNULL: + case MaxKey: + case Bool: + size += 1; + break; + case jstOID: + size += 1; + size += sizeof(OID); + break; + case BinData: + { + int t = e.binDataType(); + // 0-7 and 0x80 to 0x87 are supported by Key + if( (t & 0x78) == 0 && t != ByteArrayDeprecated ) { + int len; + e.binData(len); + if (acceptableBinDataLengths.count(len)) { + size += 1; + size += 1; + size += len; + break; + } + } + return traditionalSize; + } + case Date: + size += 1; + size += sizeof(Date_t); + break; + case String: + { + size += 1; + // note we do not store the terminating null, to save space. + unsigned x = (unsigned) e.valuestrsize() - 1; + if (x > 255) { + return traditionalSize; + } + size += 1; + size += x; + break; + } + case NumberInt: + size += 1; + size += sizeof(double); + break; + case NumberLong: + { + long long n = e._numberLong(); + long long m = 2LL << 52; + if( n >= m || n <= -m ) { + // can't represent exactly as a double + return traditionalSize; + } + size += 1; + size += sizeof(double); + break; + } + case NumberDouble: + { + double d = e._numberDouble(); + if (isNaN(d)) { + return traditionalSize; + } + size += 1; + size += sizeof(double); + break; + } + default: + // if other types involved, store as traditional BSON + return traditionalSize; + } + } + return size; + } + +} // namespace + + bool isAnyIndexKeyTooLarge(const BSONObj& index, const BSONObj& doc) { + BSONObjSet keys; + getKeysForUpgradeChecking(index, doc, &keys); + + int largestKeySize = 0; + + for (BSONObjSet::const_iterator it = keys.begin(); it != keys.end(); ++it) { + largestKeySize = std::max(largestKeySize, keyV1Size(*it)); + } + + // BtreeData_V1::KeyMax is 1024 + return largestKeySize > 1024; + } + +} // namespace mongo diff --git a/src/mongo/db/index/external_key_generator.h b/src/mongo/db/index/external_key_generator.h new file mode 100644 index 00000000000..a340fd56e0e --- /dev/null +++ b/src/mongo/db/index/external_key_generator.h @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/jsobj.h" + +/** + * Please, please, please do not rely on this existing. If you write code that uses + * it please talk to hk first. + * + * This is visible only for for upgrade checking. + */ +namespace mongo { + + // Returns whether or not the largest key the index will generate for the document is too large. + bool isAnyIndexKeyTooLarge(const BSONObj& index, const BSONObj& doc); + +} // namespace mongo diff --git a/src/mongo/db/index/fts_access_method.cpp b/src/mongo/db/index/fts_access_method.cpp new file mode 100644 index 00000000000..b2cadaeb6f9 --- /dev/null +++ b/src/mongo/db/index/fts_access_method.cpp @@ -0,0 +1,45 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/fts/fts_index_format.h" +#include "mongo/db/index/fts_access_method.h" +#include "mongo/db/index/fts_key_generator.h" + +namespace mongo { + + FTSAccessMethod::FTSAccessMethod(IndexCatalogEntry* btreeState) + : BtreeBasedAccessMethod(btreeState), + _ftsSpec(btreeState->descriptor()->infoObj()), + _keyGenerator( new FTSKeyGenerator(btreeState->descriptor()->infoObj() ) ) { + } + + void FTSAccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) { + _keyGenerator->getKeys( obj, keys ); + } + +} // namespace mongo diff --git a/src/mongo/db/index/fts_access_method.h b/src/mongo/db/index/fts_access_method.h new file mode 100644 index 00000000000..21f66bde2d3 --- /dev/null +++ b/src/mongo/db/index/fts_access_method.h @@ -0,0 +1,55 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class FTSAccessMethod : public BtreeBasedAccessMethod { + public: + FTSAccessMethod(IndexCatalogEntry* btreeState ); + virtual ~FTSAccessMethod() { } + + const fts::FTSSpec& getSpec() const { return _ftsSpec; } + + virtual shared_ptr getKeyGenerator() const { return _keyGenerator; } + private: + // Implemented: + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys); + + fts::FTSSpec _ftsSpec; + shared_ptr _keyGenerator; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/fts_key_generator.cpp b/src/mongo/db/index/fts_key_generator.cpp new file mode 100644 index 00000000000..cbbae831bb6 --- /dev/null +++ b/src/mongo/db/index/fts_key_generator.cpp @@ -0,0 +1,42 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/fts/fts_index_format.h" +#include "mongo/db/index/fts_key_generator.h" + +namespace mongo { + + FTSKeyGenerator::FTSKeyGenerator( const BSONObj& infoObj ) + : _ftsSpec( infoObj ) { + } + + void FTSKeyGenerator::getKeys(const BSONObj& obj, BSONObjSet* keys) const { + return fts::FTSIndexFormat::getKeys( _ftsSpec, obj, keys ); + } + +} // namespace mongo diff --git a/src/mongo/db/index/fts_key_generator.h b/src/mongo/db/index/fts_key_generator.h new file mode 100644 index 00000000000..86c66d916fb --- /dev/null +++ b/src/mongo/db/index/fts_key_generator.h @@ -0,0 +1,50 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class FTSKeyGenerator : public KeyGenerator { + public: + FTSKeyGenerator( const BSONObj& infoObj ); + virtual ~FTSKeyGenerator(){} + + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys) const; + + private: + fts::FTSSpec _ftsSpec; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/hash_access_method.cpp b/src/mongo/db/index/hash_access_method.cpp new file mode 100644 index 00000000000..b9483017191 --- /dev/null +++ b/src/mongo/db/index/hash_access_method.cpp @@ -0,0 +1,64 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/structure/btree/btree.h" +#include "mongo/db/hasher.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/index/hash_access_method.h" +#include "mongo/db/index/hash_key_generator.h" + +namespace mongo { + + HashAccessMethod::HashAccessMethod(IndexCatalogEntry* btreeState) + : BtreeBasedAccessMethod(btreeState) { + + const IndexDescriptor* descriptor = btreeState->descriptor(); + + // We can change these if the single-field limitation is lifted later. + uassert(16763, "Currently only single field hashed index supported.", + 1 == descriptor->getNumFields()); + + uassert(16764, "Currently hashed indexes cannot guarantee uniqueness. Use a regular index.", + !descriptor->unique()); + + ExpressionParams::parseHashParams(descriptor->infoObj(), + &_seed, + &_hashVersion, + &_hashedField); + + _keyGenerator.reset( new HashKeyGenerator( _hashedField, + _seed, + _hashVersion, + _descriptor->isSparse() ) ); + } + + void HashAccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) { + _keyGenerator->getKeys( obj, keys ); + } + +} // namespace mongo diff --git a/src/mongo/db/index/hash_access_method.h b/src/mongo/db/index/hash_access_method.h new file mode 100644 index 00000000000..c68b7ec39dd --- /dev/null +++ b/src/mongo/db/index/hash_access_method.h @@ -0,0 +1,74 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/db/hasher.h" // For HashSeed. +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + /** + * This is the access method for "hashed" indices. + */ + class HashAccessMethod : public BtreeBasedAccessMethod { + public: + using BtreeBasedAccessMethod::_descriptor; + + HashAccessMethod(IndexCatalogEntry* btreeState); + virtual ~HashAccessMethod() { } + + // This is a NO-OP. + virtual Status setOptions(const CursorOptions& options) { + return Status::OK(); + } + + virtual shared_ptr getKeyGenerator() const { return _keyGenerator; } + private: + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys); + + // Only one of our fields is hashed. This is the field name for it. + string _hashedField; + + // _seed defaults to zero. + HashSeed _seed; + + // _hashVersion defaults to zero. + int _hashVersion; + + BSONObj _missingKey; + + shared_ptr _keyGenerator; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/hash_key_generator.cpp b/src/mongo/db/index/hash_key_generator.cpp new file mode 100644 index 00000000000..2e891701fba --- /dev/null +++ b/src/mongo/db/index/hash_key_generator.cpp @@ -0,0 +1,70 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/structure/btree/btree.h" +#include "mongo/db/hasher.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/index/hash_access_method.h" +#include "mongo/db/index/hash_key_generator.h" + +namespace mongo { + + HashKeyGenerator::HashKeyGenerator( const std::string& hashedField, + HashSeed seed, + int hashVersion, + bool isSparse ) + : _hashedField( hashedField ), + _seed( seed ), + _hashVersion( hashVersion ), + _isSparse( isSparse ) { + } + + void HashKeyGenerator::getKeys( const BSONObj& obj, BSONObjSet* keys ) const { + const char* cstr = _hashedField.c_str(); + BSONElement fieldVal = obj.getFieldDottedOrArray(cstr); + uassert(16766, "Error: hashed indexes do not currently support array values", + fieldVal.type() != Array ); + + if (!fieldVal.eoo()) { + BSONObj key = BSON( "" << makeSingleHashKey(fieldVal, _seed, _hashVersion)); + keys->insert(key); + } + else if (!_isSparse) { + BSONObj nullObj = BSON("" << BSONNULL); + keys->insert(BSON("" << makeSingleHashKey(nullObj.firstElement(), _seed, _hashVersion))); + } + + } + + long long int HashKeyGenerator::makeSingleHashKey(const BSONElement& e, HashSeed seed, int v) { + massert(16767, "Only HashVersion 0 has been defined" , v == 0 ); + return BSONElementHasher::hash64(e, seed); + } + + +} // namespace mongo diff --git a/src/mongo/db/index/hash_key_generator.h b/src/mongo/db/index/hash_key_generator.h new file mode 100644 index 00000000000..dd8f851be4f --- /dev/null +++ b/src/mongo/db/index/hash_key_generator.h @@ -0,0 +1,60 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/db/hasher.h" // For HashSeed. +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index/key_generator.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class HashKeyGenerator : public KeyGenerator { + public: + HashKeyGenerator( const std::string& hashedField, + HashSeed seed, + int hashVersion, + bool isSparse ); + virtual ~HashKeyGenerator() {} + + virtual void getKeys( const BSONObj& obj, BSONObjSet* keys ) const; + + static long long int makeSingleHashKey(const BSONElement& e, HashSeed seed, int v); + + private: + std::string _hashedField; + HashSeed _seed; + int _hashVersion; + bool _isSparse; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/haystack_access_method.cpp b/src/mongo/db/index/haystack_access_method.cpp new file mode 100644 index 00000000000..31441d1f114 --- /dev/null +++ b/src/mongo/db/index/haystack_access_method.cpp @@ -0,0 +1,131 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/index/haystack_access_method.h" + +#include "mongo/base/status.h" +#include "mongo/db/geo/hash.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/index/haystack_access_method_internal.h" +#include "mongo/db/index/haystack_key_generator.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/internal_plans.h" + +namespace mongo { + + HaystackAccessMethod::HaystackAccessMethod(IndexCatalogEntry* btreeState) + : BtreeBasedAccessMethod(btreeState) { + + const IndexDescriptor* descriptor = btreeState->descriptor(); + + ExpressionParams::parseHaystackParams(descriptor->infoObj(), + &_geoField, + &_otherFields, + &_bucketSize); + + uassert(16773, "no geo field specified", _geoField.size()); + uassert(16774, "no non-geo fields specified", _otherFields.size()); + + _keyGenerator.reset( new HaystackKeyGenerator( _geoField, _otherFields, _bucketSize ) ); + } + + void HaystackAccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) { + _keyGenerator->getKeys( obj, keys ); + } + + void HaystackAccessMethod::searchCommand(const BSONObj& nearObj, double maxDistance, + const BSONObj& search, BSONObjBuilder* result, + unsigned limit) { + Timer t; + + LOG(1) << "SEARCH near:" << nearObj << " maxDistance:" << maxDistance + << " search: " << search << endl; + int x, y; + { + BSONObjIterator i(nearObj); + x = HaystackKeyGenerator::hashHaystackElement(i.next(), _bucketSize); + y = HaystackKeyGenerator::hashHaystackElement(i.next(), _bucketSize); + } + int scale = static_cast(ceil(maxDistance / _bucketSize)); + + GeoHaystackSearchHopper hopper(nearObj, maxDistance, limit, _geoField); + + long long btreeMatches = 0; + + for (int a = -scale; a <= scale && !hopper.limitReached(); ++a) { + for (int b = -scale; b <= scale && !hopper.limitReached(); ++b) { + BSONObjBuilder bb; + bb.append("", HaystackKeyGenerator::makeHaystackString(x + a, y + b)); + + for (unsigned i = 0; i < _otherFields.size(); i++) { + // See if the non-geo field we're indexing on is in the provided search term. + BSONElement e = search.getFieldDotted(_otherFields[i]); + if (e.eoo()) + bb.appendNull(""); + else + bb.appendAs(e, ""); + } + + BSONObj key = bb.obj(); + + unordered_set thisPass; + + + scoped_ptr runner(InternalPlanner::indexScan(_btreeState->collection(), + _descriptor, key, key, true)); + Runner::RunnerState state; + DiskLoc loc; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &loc))) { + if (hopper.limitReached()) { break; } + pair::iterator, bool> p + = thisPass.insert(loc); + // If a new element was inserted (haven't seen the DiskLoc before), p.second + // is true. + if (p.second) { + hopper.consider(loc); + btreeMatches++; + } + } + } + } + + BSONArrayBuilder arr(result->subarrayStart("results")); + int num = hopper.appendResultsTo(&arr); + arr.done(); + + { + BSONObjBuilder b(result->subobjStart("stats")); + b.append("time", t.millis()); + b.appendNumber("btreeMatches", btreeMatches); + b.append("n", num); + b.done(); + } + } + +} // namespace mongo diff --git a/src/mongo/db/index/haystack_access_method.h b/src/mongo/db/index/haystack_access_method.h new file mode 100644 index 00000000000..dc377083f96 --- /dev/null +++ b/src/mongo/db/index/haystack_access_method.h @@ -0,0 +1,77 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + /** + * Maps (lat, lng) to the bucketSize-sided square bucket that contains it. + * Examines all documents in a given radius of a given point. + * Returns all documents that match a given search restriction. + * See http://dochub.mongodb.org/core/haystackindexes + * + * Use when you want to look for restaurants within 25 miles with a certain name. + * Don't use when you want to find the closest open restaurants; see 2d.cpp for that. + * + * Usage: + * db.foo.ensureIndex({ pos : "geoHaystack", type : 1 }, { bucketSize : 1 }) + * pos is the name of the field to be indexed that has lat/lng data in an array. + * type is the name of the secondary field to be indexed. + * bucketSize specifies the dimension of the square bucket for the data in pos. + * ALL fields are mandatory. + */ + class HaystackAccessMethod : public BtreeBasedAccessMethod { + public: + using BtreeBasedAccessMethod::_descriptor; + + HaystackAccessMethod(IndexCatalogEntry* btreeState); + virtual ~HaystackAccessMethod() { } + + virtual shared_ptr getKeyGenerator() const { return _keyGenerator; } + protected: + friend class GeoHaystackSearchCommand; + void searchCommand(const BSONObj& nearObj, double maxDistance, const BSONObj& search, + BSONObjBuilder* result, unsigned limit); + + private: + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys); + + string _geoField; + vector _otherFields; + double _bucketSize; + + shared_ptr _keyGenerator; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/haystack_access_method_internal.h b/src/mongo/db/index/haystack_access_method_internal.h new file mode 100644 index 00000000000..c563cb3f045 --- /dev/null +++ b/src/mongo/db/index/haystack_access_method_internal.h @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/diskloc.h" +#include "mongo/db/geo/core.h" +#include "mongo/db/geo/shapes.h" + +namespace mongo { + + class GeoHaystackSearchHopper { + public: + /** + * Constructed with a point, a max distance from that point, and a max number of + * matched points to store. + * @param n The centroid that we're searching + * @param maxDistance The maximum distance to consider from that point + * @param limit The maximum number of results to return + * @param geoField Which field in the provided DiskLoc has the point to test. + */ + GeoHaystackSearchHopper(const BSONObj& nearObj, double maxDistance, unsigned limit, + const string& geoField) + : _near(nearObj), _maxDistance(maxDistance), _limit(limit), _geoField(geoField) { } + + // Consider the point in loc, and keep it if it's within _maxDistance (and we have space for + // it) + void consider(const DiskLoc& loc) { + if (limitReached()) return; + Point p(loc.obj().getFieldDotted(_geoField)); + if (distance(_near, p) > _maxDistance) + return; + _locs.push_back(loc); + } + + int appendResultsTo(BSONArrayBuilder* b) { + for (unsigned i = 0; i <_locs.size(); i++) + b->append(_locs[i].obj()); + return _locs.size(); + } + + // Have we stored as many points as we can? + bool limitReached() const { + return _locs.size() >= _limit; + } + private: + Point _near; + double _maxDistance; + unsigned _limit; + const string _geoField; + vector _locs; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/haystack_key_generator.cpp b/src/mongo/db/index/haystack_key_generator.cpp new file mode 100644 index 00000000000..f49f0e33f79 --- /dev/null +++ b/src/mongo/db/index/haystack_key_generator.cpp @@ -0,0 +1,128 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/index/haystack_access_method.h" + +#include "mongo/base/status.h" +#include "mongo/db/geo/hash.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/index/haystack_access_method_internal.h" +#include "mongo/db/index/haystack_key_generator.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/internal_plans.h" + +namespace mongo { + + // ------------------- + + HaystackKeyGenerator::HaystackKeyGenerator( const std::string& geoField, + const std::vector& otherFields, + double bucketSize ) + : _geoField( geoField ), + _otherFields( otherFields ), + _bucketSize( bucketSize ) { + } + + namespace { + /** + * Build a new BSONObj with root in it. If e is non-empty, append that to the key. + * Insert the BSONObj into keys. + * Used by getHaystackKeys. + */ + void addKey(const string& root, const BSONElement& e, BSONObjSet* keys) { + BSONObjBuilder buf; + buf.append("", root); + + if (e.eoo()) + buf.appendNull(""); + else + buf.appendAs(e, ""); + + keys->insert(buf.obj()); + } + + } + + void HaystackKeyGenerator::getKeys( const BSONObj& obj, BSONObjSet* keys) const { + + BSONElement loc = obj.getFieldDotted(_geoField); + + if (loc.eoo()) { return; } + + uassert(16775, "latlng not an array", loc.isABSONObj()); + string root; + { + BSONObjIterator i(loc.Obj()); + BSONElement x = i.next(); + BSONElement y = i.next(); + root = makeHaystackString(hashHaystackElement(x, _bucketSize), + hashHaystackElement(y, _bucketSize)); + } + + verify(_otherFields.size() == 1); + + BSONElementSet all; + + // This is getFieldsDotted (plural not singular) since the object we're indexing + // may be an array. + obj.getFieldsDotted(_otherFields[0], all); + + if (all.size() == 0) { + // We're indexing a document that doesn't have the secondary non-geo field present. + // XXX: do we want to add this even if all.size() > 0? result:empty search terms + // match everything instead of only things w/empty search terms) + addKey(root, BSONElement(), keys); + } else { + // Ex:If our secondary field is type: "foo" or type: {a:"foo", b:"bar"}, + // all.size()==1. We can query on the complete field. + // Ex: If our secondary field is type: ["A", "B"] all.size()==2 and all has values + // "A" and "B". The query looks for any of the fields in the array. + for (BSONElementSet::iterator i = all.begin(); i != all.end(); ++i) { + addKey(root, *i, keys); + } + } + } + + // static + int HaystackKeyGenerator::hashHaystackElement(const BSONElement& e, double bucketSize) { + uassert(16776, "geo field is not a number", e.isNumber()); + double d = e.numberDouble(); + d += 180; + d /= bucketSize; + return static_cast(d); + } + + // static + std::string HaystackKeyGenerator::makeHaystackString(int hashedX, int hashedY) { + mongoutils::str::stream ss; + ss << hashedX << "_" << hashedY; + return ss; + } + +} // namespace mongo diff --git a/src/mongo/db/index/haystack_key_generator.h b/src/mongo/db/index/haystack_key_generator.h new file mode 100644 index 00000000000..c540e161346 --- /dev/null +++ b/src/mongo/db/index/haystack_key_generator.h @@ -0,0 +1,65 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class HaystackKeyGenerator : public KeyGenerator { + public: + HaystackKeyGenerator( const std::string& geoField, + const std::vector& otherFields, + double bucketSize ); + virtual ~HaystackKeyGenerator() {} + + virtual void getKeys( const BSONObj& obj, BSONObjSet* keys ) const; + + /** + * Returns a hash of a BSON element. + * Used by getHaystackKeys and HaystackAccessMethod::searchCommand. + */ + static int hashHaystackElement(const BSONElement& e, double bucketSize); + + /** + * Joins two strings using underscore as separator. + * Used by getHaystackKeys and HaystackAccessMethod::searchCommand. + */ + static std::string makeHaystackString(int hashedX, int hashedY); + + private: + std::string _geoField; + std::vector _otherFields; + double _bucketSize; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h new file mode 100644 index 00000000000..4a0bdccc038 --- /dev/null +++ b/src/mongo/db/index/index_access_method.h @@ -0,0 +1,238 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/diskloc.h" +#include "mongo/db/index/index_cursor.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index/key_generator.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class UpdateTicket; + class InsertTicket; + + struct InsertDeleteOptions; + struct PregeneratedKeysOnIndex; + + /** + * An IndexAccessMethod is the interface through which all the mutation, lookup, and + * traversal of index entries is done. The class is designed so that the underlying index + * data structure is opaque to the caller. + * + * IndexAccessMethods for existing indices are obtained through the system catalog. + * + * We assume the caller has whatever locks required. This interface is not thread safe. + * + */ + class IndexAccessMethod { + public: + virtual ~IndexAccessMethod() { } + + // + // Lookup, traversal, and mutation support + // + + /** + * Internally generate the keys {k1, ..., kn} for 'obj'. For each key k, insert (k -> + * 'loc') into the index. 'obj' is the object at the location 'loc'. If not NULL, + * 'numInserted' will be set to the number of keys added to the index for the document. If + * there is more than one key for 'obj', either all keys will be inserted or none will. + * + * The behavior of the insertion can be specified through 'options'. + * + * prepared: if you generated keys before, you can pass the generator you used + * and the keys you got. If the generator matches, the keys are used. Otherwise we + * generate our own keys and you do not have to do anything. + */ + virtual Status insert(const BSONObj& obj, + const DiskLoc& loc, + const InsertDeleteOptions& options, + int64_t* numInserted, + const PregeneratedKeysOnIndex* prepared = NULL ) = 0; + + /** + * Analogous to above, but remove the records instead of inserting them. If not NULL, + * numDeleted will be set to the number of keys removed from the index for the document. + */ + virtual Status remove(const BSONObj& obj, + const DiskLoc& loc, + const InsertDeleteOptions& options, + int64_t* numDeleted) = 0; + + /** + * Checks whether the index entries for the document 'from', which is placed at location + * 'loc' on disk, can be changed to the index entries for the doc 'to'. Provides a ticket + * for actually performing the update. + * + * Returns an error if the update is invalid. The ticket will also be marked as invalid. + * Returns OK if the update should proceed without error. The ticket is marked as valid. + * + * There is no obligation to perform the update after performing validation. + */ + virtual Status validateUpdate(const BSONObj& from, + const BSONObj& to, + const DiskLoc& loc, + const InsertDeleteOptions& options, + UpdateTicket* ticket) = 0; + + /** + * Perform a validated update. The keys for the 'from' object will be removed, and the keys + * for the object 'to' will be added. Returns OK if the update succeeded, failure if it did + * not. If an update does not succeed, the index will be unmodified, and the keys for + * 'from' will remain. Assumes that the index has not changed since validateUpdate was + * called. If the index was changed, we may return an error, as our ticket may have been + * invalidated. + */ + virtual Status update(const UpdateTicket& ticket, int64_t* numUpdated) = 0; + + /** + * Fills in '*out' with an IndexCursor. Return a status indicating success or reason of + * failure. If the latter, '*out' contains NULL. See index_cursor.h for IndexCursor usage. + */ + virtual Status newCursor(IndexCursor **out) const = 0; + + // ------ index level operations ------ + + + /** + * initializes this index + * only called once for the lifetime of the index + * if called multiple times, is an error + */ + virtual Status initializeAsEmpty() = 0; + + /** + * Try to page-in the pages that contain the keys generated from 'obj'. + * This can be used to speed up future accesses to an index by trying to ensure the + * appropriate pages are not swapped out. + * See prefetch.cpp. + */ + virtual Status touch(const BSONObj& obj) = 0; + + /** + * Try to page-in the pages that contain the keys. + * This can be used to speed up future accesses to an index by trying to ensure the + * appropriate pages are not swapped out. + * See prefetch.cpp. + */ + virtual Status touch(const BSONObjSet& keys) = 0; + + /** + * Walk the entire index, checking the internal structure for consistency. + * Set numKeys to the number of keys in the index. + * + * Return OK if the index is valid. + * + * Currently wasserts that the index is invalid. This could/should be changed in + * the future to return a Status. + */ + virtual Status validate(int64_t* numKeys) = 0; + + // + // Bulk operations support + // + + /** + * Starts a bulk operation. + * You work on the returned IndexAccessMethod and then call commitBulk. + * This can return NULL, meaning bulk mode is not available. + * + * Long term, you'll eventually be able to mix/match bulk, not bulk, + * have as many as you want, etc.. + * + * For now (1/8/14) you can only do bulk when the index is empty + * it will fail if you try other times. + */ + virtual IndexAccessMethod* initiateBulk() = 0; + + /** + * Call this when you are ready to finish your bulk work. + * Pass in the IndexAccessMethod gotten from initiateBulk. + * After this method is called, the bulk index access method is invalid + * and should not be used. + * @param bulk - something created from initiateBulk + * @param mayInterrupt - is this commit interruptable (will cancel) + * @param dups - if NULL, error out on dups if not allowed + * if not NULL, put the bad DiskLocs there + */ + virtual Status commitBulk( IndexAccessMethod* bulk, + bool mayInterrupt, + std::set* dups ) = 0; + + /** + * this returns a shared_ptr so that someone can get all the generators in a lock, + * then unlock, generate keys, and then re-lock and use those keys + */ + virtual shared_ptr getKeyGenerator() const = 0; + }; + + /** + * Updates are two steps: verify that it's a valid update, and perform it. + * validateUpdate fills out the UpdateStatus and update actually applies it. + */ + class UpdateTicket { + public: + UpdateTicket() : _isValid(false) { } + + protected: + // These friends are the classes that actually fill out an UpdateStatus. + friend class BtreeBasedAccessMethod; + + class PrivateUpdateData; + + bool _isValid; + + // This is meant to be filled out only by the friends above. + scoped_ptr _indexSpecificUpdateData; + }; + + class UpdateTicket::PrivateUpdateData { + public: + virtual ~PrivateUpdateData() { } + }; + + /** + * Flags we can set for inserts and deletes (and updates, which are kind of both). + */ + struct InsertDeleteOptions { + InsertDeleteOptions() : logIfError(false), dupsAllowed(false), ignoreKeyTooLong(false) { } + + // If there's an error, log() it. + bool logIfError; + + // Are duplicate keys allowed in the index? + bool dupsAllowed; + + // Ignore key too long failures. + bool ignoreKeyTooLong; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/index_cursor.h b/src/mongo/db/index/index_cursor.h new file mode 100644 index 00000000000..6d50f243f7b --- /dev/null +++ b/src/mongo/db/index/index_cursor.h @@ -0,0 +1,148 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + struct CursorOptions; + + /** + * An IndexCursor is the interface through which one traverses the entries of a given + * index. The internal structure of an index is kept isolated. + * + * The cursor must be initiailized by seek()ing to a given entry in the index. The index is + * traversed by calling next() or skip()-ping ahead. + * + * The set of predicates a given index can understand is known a priori. These predicates may + * be simple (a key location for a Btree index) or rich ($within for a geo index). + * + * Locking is the responsibility of the caller. The IndexCursor keeps state. If the caller + * wishes to yield or unlock, it must call savePosition() first. When it decides to unyield it + * must call restorePosition(). The cursor may be EOF after a restorePosition(). + */ + class IndexCursor { + public: + virtual ~IndexCursor() { } + + /** + * Set options on the cursor (direction). See CursorOptions below. + */ + virtual Status setOptions(const CursorOptions& options) = 0; + + /** + * A cursor doesn't point anywhere by default. You must seek to the start position. + * The provided position must be a predicate that the index understands. The + * predicate must describe one value, though there may be several instances + * + * Possible return values: + * 1. Success: seeked to the position. + * 2. Success: seeked to 'closest' key oriented according to the cursor's direction. + * 3. Error: can't seek to the position. + */ + virtual Status seek(const BSONObj& position) = 0; + + // + // Iteration support + // + + // Are we out of documents? + virtual bool isEOF() const = 0; + + // Move to the next key/value pair. Assumes !isEOF(). + virtual void next() = 0; + + // + // Accessors + // + + // Current key we point at. Assumes !isEOF(). + virtual BSONObj getKey() const = 0; + + // Current value we point at. Assumes !isEOF(). + virtual DiskLoc getValue() const = 0; + + // + // Yielding support + // + + /** + * Yielding semantics: + * If the entry that a cursor points at is not deleted during a yield, the cursor will + * point at that entry after a restore. + * An entry inserted during a yield may or may not be returned by an in-progress scan. + * An entry deleted during a yield may or may not be returned by an in-progress scan. + * An entry modified during a yield may or may not be returned by an in-progress scan. + * An entry that is not inserted or deleted during a yield will be returned, and only once. + * If the index returns entries in a given order (Btree), this order will be mantained even + * if the entry corresponding to a saved position is deleted during a yield. + */ + + /** + * Save our current position in the index. Assumes that we are currently pointing to a + * valid position in the index. + * If not, we error. Otherwise, succeed. + */ + virtual Status savePosition() = 0; + + /** + * Restore the saved position. Errors if there is no saved position. + * The cursor may be EOF after a restore. + */ + virtual Status restorePosition() = 0; + + // Return a string describing the cursor. + virtual string toString() = 0; + + /** + * Add debugging info to the provided builder. + * TODO(hk): We can do this better, perhaps with a more structured format. + */ + virtual void explainDetails(BSONObjBuilder* b) { } + }; + + // All the options we might want to set on a cursor. + struct CursorOptions { + // Set the direction of the scan. Ignored if the cursor doesn't have directions (geo). + enum Direction { + DECREASING = -1, + INCREASING = 1, + }; + + Direction direction; + + // 2d indices need to know exactly how many results you want beforehand. + // Ignored by every other index. + int numWanted; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/index_descriptor.cpp b/src/mongo/db/index/index_descriptor.cpp new file mode 100644 index 00000000000..b2b1b9b41de --- /dev/null +++ b/src/mongo/db/index/index_descriptor.cpp @@ -0,0 +1,85 @@ +// index_descriptor.cpp + +/** +* Copyright (C) 2014 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/index_descriptor.h" + +namespace mongo { + + namespace { + void populateOptionsMap( std::map& theMap, + const BSONObj& spec ) { + + BSONObjIterator it( spec ); + while ( it.more() ) { + const BSONElement e = it.next(); + + StringData fieldName = e.fieldNameStringData(); + if ( fieldName == "key" || + fieldName == "ns" || + fieldName == "name" || + fieldName == "v" || // not considered for equivalence + fieldName == "textIndexVersion" || // same as "v" + fieldName == "2dsphereIndexVersion" || // same as "v" + fieldName == "background" || // this is a creation time option only + fieldName == "dropDups" || // this is a creation time option only + fieldName == "sparse" || // checked specially + fieldName == "unique" // check specially + ) { + continue; + } + theMap[ fieldName ] = e; + } + } + } + + bool IndexDescriptor::areIndexOptionsEquivalent( const IndexDescriptor* other ) const { + + if ( isSparse() != other->isSparse() ) { + return false; + } + + if ( !isIdIndex() && + unique() != other->unique() ) { + // Note: { _id: 1 } or { _id: -1 } implies unique: true. + return false; + } + + // Then compare the rest of the options. + + std::map existingOptionsMap; + populateOptionsMap( existingOptionsMap, infoObj() ); + + std::map newOptionsMap; + populateOptionsMap( newOptionsMap, other->infoObj() ); + + return existingOptionsMap == newOptionsMap; + } + +} diff --git a/src/mongo/db/index/index_descriptor.h b/src/mongo/db/index/index_descriptor.h new file mode 100644 index 00000000000..18cf465a060 --- /dev/null +++ b/src/mongo/db/index/index_descriptor.h @@ -0,0 +1,205 @@ +// index_descriptor.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/db/structure/catalog/index_details.h" // For IndexDetails. +#include "mongo/db/jsobj.h" +#include "mongo/db/structure/catalog/namespace_details.h" // For NamespaceDetails. +#include "mongo/db/catalog/collection.h" + +#include "mongo/util/stacktrace.h" + +namespace mongo { + + class IndexCatalog; + class IndexCatalogEntry; + class IndexCatalogEntryContainer; + + /** + * A cache of information computed from the memory-mapped per-index data (OnDiskIndexData). + * Contains accessors for the various immutable index parameters, and an accessor for the + * mutable "head" pointer which is index-specific. + * + * All synchronization is the responsibility of the caller. + */ + class IndexDescriptor { + public: + /** + * OnDiskIndexData is a pointer to the memory mapped per-index data. + * infoObj is a copy of the index-describing BSONObj contained in the OnDiskIndexData. + */ + IndexDescriptor(Collection* collection, const std::string& accessMethodName, BSONObj infoObj) + : _magic(123987), + _collection(collection), + _accessMethodName(accessMethodName), + _infoObj(infoObj.getOwned()), + _numFields(infoObj.getObjectField("key").nFields()), + _keyPattern(infoObj.getObjectField("key").getOwned()), + _indexName(infoObj.getStringField("name")), + _parentNS(infoObj.getStringField("ns")), + _isIdIndex(IndexDetails::isIdIndexPattern( _keyPattern )), + _sparse(infoObj["sparse"].trueValue()), + _dropDups(infoObj["dropDups"].trueValue()), + _unique( _isIdIndex || infoObj["unique"].trueValue() ), + _cachedEntry( NULL ) + { + _indexNamespace = _parentNS + ".$" + _indexName; + + _version = 0; + BSONElement e = _infoObj["v"]; + if ( e.isNumber() ) { + _version = e.numberInt(); + } + } + + ~IndexDescriptor() { + _magic = 555; + } + + // + // Information about the key pattern. + // + + /** + * Return the user-provided index key pattern. + * Example: {geo: "2dsphere", nonGeo: 1} + * Example: {foo: 1, bar: -1} + */ + const BSONObj& keyPattern() const { _checkOk(); return _keyPattern; } + + // How many fields do we index / are in the key pattern? + int getNumFields() const { _checkOk(); return _numFields; } + + // + // Information about the index's namespace / collection. + // + + // Return the name of the index. + const std::string& indexName() const { _checkOk(); return _indexName; } + + // Return the name of the indexed collection. + const std::string& parentNS() const { return _parentNS; } + + // Return the name of this index's storage area (database.table.$index) + const std::string& indexNamespace() const { return _indexNamespace; } + + // Return the name of the access method we must use to access this index's data. + const std::string& getAccessMethodName() const { return _accessMethodName; } + + // + // Properties every index has + // + + // Return what version of index this is. + int version() const { return _version; } + + // May each key only occur once? + bool unique() const { return _unique; } + + // Is dropDups set on this index? + bool dropDups() const { return _dropDups; } + + // Is this index sparse? + bool isSparse() const { return _sparse; } + + // Is this index multikey? + bool isMultikey() const { _checkOk(); return _collection->getIndexCatalog()->isMultikey( this ); } + + bool isIdIndex() const { _checkOk(); return _isIdIndex; } + + // + // Properties that are Index-specific. + // + + // Allow access to arbitrary fields in the per-index info object. Some indices stash + // index-specific data there. + BSONElement getInfoElement(const string& name) const { return _infoObj[name]; } + + // + // "Internals" of accessing the index, used by IndexAccessMethod(s). + // + + // Return a (rather compact) string representation. + std::string toString() const { _checkOk(); return _infoObj.toString(); } + + // Return the info object. + const BSONObj& infoObj() const { _checkOk(); return _infoObj; } + + // this is the owner of this IndexDescriptor + IndexCatalog* getIndexCatalog() const { return _collection->getIndexCatalog(); } + + bool areIndexOptionsEquivalent( const IndexDescriptor* other ) const; + + private: + + void _checkOk() const { + if ( _magic == 123987 ) + return; + log() << "uh oh: " << (void*)(this) << " " << _magic; + verify(0); + } + + int _magic; + + // Related catalog information of the parent collection + Collection* _collection; + + // What access method should we use for this index? + std::string _accessMethodName; + + // The BSONObj describing the index. Accessed through the various members above. + const BSONObj _infoObj; + + // --- cached data from _infoObj + + int64_t _numFields; // How many fields are indexed? + BSONObj _keyPattern; + std::string _indexName; + std::string _parentNS; + std::string _indexNamespace; + bool _isIdIndex; + bool _sparse; + bool _dropDups; + bool _unique; + int _version; + + // only used by IndexCatalogEntryContainer to do caching for perf + // users not allowed to touch, and not part of API + IndexCatalogEntry* _cachedEntry; + + friend class IndexCatalog; + friend class IndexCatalogEntry; + friend class IndexCatalogEntryContainer; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/key_generator.cpp b/src/mongo/db/index/key_generator.cpp new file mode 100644 index 00000000000..009aef8a931 --- /dev/null +++ b/src/mongo/db/index/key_generator.cpp @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/index/key_generator.h" + +namespace mongo { + namespace { + AtomicUInt64 NEXT; + } + + KeyGenerator::KeyGenerator() { + _id = NEXT.fetchAndAdd(1); + } + +} diff --git a/src/mongo/db/index/key_generator.h b/src/mongo/db/index/key_generator.h new file mode 100644 index 00000000000..0cfbd3df5ef --- /dev/null +++ b/src/mongo/db/index/key_generator.h @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonmisc.h" + +namespace mongo { + + class KeyGenerator { + public: + KeyGenerator(); + virtual ~KeyGenerator() {} + + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys) const = 0; + + int64_t getId() const { return _id; } + private: + int64_t _id; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/s2_access_method.cpp b/src/mongo/db/index/s2_access_method.cpp new file mode 100644 index 00000000000..50df4f34711 --- /dev/null +++ b/src/mongo/db/index/s2_access_method.cpp @@ -0,0 +1,114 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/s2_access_method.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/db/geo/geoconstants.h" +#include "mongo/db/geo/geoparser.h" +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/geo/s2.h" +#include "mongo/db/geo/s2common.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/index/s2_key_generator.h" +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" + +#include "third_party/s2/s2cell.h" +#include "third_party/s2/s2regioncoverer.h" + +namespace mongo { + + static const string kIndexVersionFieldName("2dsphereIndexVersion"); + + S2AccessMethod::S2AccessMethod(IndexCatalogEntry* btreeState) + : BtreeBasedAccessMethod(btreeState) { + + const IndexDescriptor* descriptor = btreeState->descriptor(); + + ExpressionParams::parse2dsphereParams(descriptor->infoObj(), + &_params); + + _keyGenerator.reset( new S2KeyGenerator( descriptor->keyPattern(), _params ) ); + + int geoFields = 0; + + // Categorize the fields we're indexing and make sure we have a geo field. + BSONObjIterator i(descriptor->keyPattern()); + while (i.more()) { + BSONElement e = i.next(); + if (e.type() == String && IndexNames::GEO_2DSPHERE == e.String() ) { + ++geoFields; + } + else { + // We check for numeric in 2d, so that's the check here + uassert( 16823, (string)"Cannot use " + IndexNames::GEO_2DSPHERE + + " index with other special index types: " + e.toString(), + e.isNumber() ); + } + } + + uassert(16750, "Expect at least one geo field, spec=" + descriptor->keyPattern().toString(), + geoFields >= 1); + + if (descriptor->isSparse()) { + warning() << "Sparse option ignored for index spec " + << descriptor->keyPattern().toString() << "\n"; + } + } + + // static + BSONObj S2AccessMethod::fixSpec(const BSONObj& specObj) { + // If the spec object has the field "2dsphereIndexVersion", validate it. If it doesn't, add + // {2dsphereIndexVersion: 2}, which is the default for newly-built indexes. + + BSONElement indexVersionElt = specObj[kIndexVersionFieldName]; + if (indexVersionElt.eoo()) { + BSONObjBuilder bob; + bob.appendElements(specObj); + bob.append(kIndexVersionFieldName, S2_INDEX_VERSION_2); + return bob.obj(); + } + + const int indexVersion = indexVersionElt.numberInt(); + uassert(17394, + str::stream() << "unsupported geo index version { " << kIndexVersionFieldName + << " : " << indexVersionElt << " }, only support versions: [" + << S2_INDEX_VERSION_1 << "," << S2_INDEX_VERSION_2 << "]", + indexVersionElt.isNumber() && (indexVersion == S2_INDEX_VERSION_2 + || indexVersion == S2_INDEX_VERSION_1)); + return specObj; + } + + void S2AccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) { + return _keyGenerator->getKeys( obj, keys ); + } + +} // namespace mongo diff --git a/src/mongo/db/index/s2_access_method.h b/src/mongo/db/index/s2_access_method.h new file mode 100644 index 00000000000..efc79ef9df4 --- /dev/null +++ b/src/mongo/db/index/s2_access_method.h @@ -0,0 +1,65 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/geo/s2common.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class IndexCursor; + struct S2IndexingParams; + + class S2AccessMethod : public BtreeBasedAccessMethod { + public: + using BtreeBasedAccessMethod::_descriptor; + + S2AccessMethod(IndexCatalogEntry* btreeState); + virtual ~S2AccessMethod() { } + + /** + * Takes an index spec object for this index and returns a copy tweaked to conform to the + * expected format. When an index build is initiated, this function is called on the spec + * object the user provides, and the return value of this function is the final spec object + * that gets saved in the index catalog. Throws a UserException if 'specObj' is invalid. + */ + static BSONObj fixSpec(const BSONObj& specObj); + + virtual shared_ptr getKeyGenerator() const { return _keyGenerator; } + private: + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys); + + S2IndexingParams _params; + shared_ptr _keyGenerator; + }; + +} // namespace mongo diff --git a/src/mongo/db/index/s2_key_generator.cpp b/src/mongo/db/index/s2_key_generator.cpp new file mode 100644 index 00000000000..2b925964bd8 --- /dev/null +++ b/src/mongo/db/index/s2_key_generator.cpp @@ -0,0 +1,272 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index/s2_access_method.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/db/geo/geoconstants.h" +#include "mongo/db/geo/geoparser.h" +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/geo/s2.h" +#include "mongo/db/geo/s2common.h" +#include "mongo/db/index/expression_params.h" +#include "mongo/db/index/s2_key_generator.h" +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" + +#include "third_party/s2/s2cell.h" +#include "third_party/s2/s2regioncoverer.h" + +namespace mongo { + + S2KeyGenerator::S2KeyGenerator( const BSONObj& keyPattern, const S2IndexingParams& params ) + : _keyPattern( keyPattern.getOwned() ), _params( params ) { + } + + // + // Helper functions for getS2Keys + // + + static void S2KeysFromRegion(S2RegionCoverer *coverer, const S2Region ®ion, + vector *out) { + vector covering; + coverer->GetCovering(region, &covering); + for (size_t i = 0; i < covering.size(); ++i) { + out->push_back(covering[i].toString()); + } + } + + bool S2GetKeysForObject(const BSONObj& obj, + const S2IndexingParams& params, + vector* out) { + S2RegionCoverer coverer; + params.configureCoverer(&coverer); + + GeometryContainer geoContainer; + if (!geoContainer.parseFrom(obj)) { return false; } + + // Only certain geometries can be indexed in the old index format S2_INDEX_VERSION_1. See + // definition of S2IndexVersion for details. + if (params.indexVersion == S2_INDEX_VERSION_1 && !geoContainer.isSimpleContainer()) { + return false; + } + + if (!geoContainer.hasS2Region()) { return false; } + + S2KeysFromRegion(&coverer, geoContainer.getRegion(), out); + + return true; + } + + /** + * Get the index keys for elements that are GeoJSON. + * Used by getS2Keys. + */ + void getS2GeoKeys(const BSONObj& document, const BSONElementSet& elements, + const S2IndexingParams& params, + BSONObjSet* out) { + for (BSONElementSet::iterator i = elements.begin(); i != elements.end(); ++i) { + uassert(16754, "Can't parse geometry from element: " + i->toString(), + i->isABSONObj()); + const BSONObj &geoObj = i->Obj(); + + vector cells; + bool succeeded = S2GetKeysForObject(geoObj, params, &cells); + uassert(16755, "Can't extract geo keys from object, malformed geometry?: " + + document.toString(), succeeded); + + uassert(16756, "Unable to generate keys for (likely malformed) geometry: " + + document.toString(), + cells.size() > 0); + + for (vector::const_iterator it = cells.begin(); it != cells.end(); ++it) { + BSONObjBuilder b; + b.append("", *it); + out->insert(b.obj()); + } + } + + if (0 == out->size()) { + BSONObjBuilder b; + b.appendNull(""); + out->insert(b.obj()); + } + } + + /** + * Expands array and appends items to 'out'. + * Used by getOneLiteralKey. + */ + void getS2LiteralKeysArray(const BSONObj& obj, BSONObjSet* out) { + BSONObjIterator objIt(obj); + if (!objIt.more()) { + // Empty arrays are indexed as undefined. + BSONObjBuilder b; + b.appendUndefined(""); + out->insert(b.obj()); + } else { + // Non-empty arrays are exploded. + while (objIt.more()) { + BSONObjBuilder b; + b.appendAs(objIt.next(), ""); + out->insert(b.obj()); + } + } + } + + /** + * If 'elt' is an array, expands elt and adds items to 'out'. + * Otherwise, adds 'elt' as a single element. + * Used by getLiteralKeys. + */ + void getS2OneLiteralKey(const BSONElement& elt, BSONObjSet* out) { + if (Array == elt.type()) { + getS2LiteralKeysArray(elt.Obj(), out); + } else { + // One thing, not an array, index as-is. + BSONObjBuilder b; + b.appendAs(elt, ""); + out->insert(b.obj()); + } + } + + /** + * elements is a non-geo field. Add the values literally, expanding arrays. + * Used by getS2Keys. + */ + void getS2LiteralKeys(const BSONElementSet& elements, BSONObjSet* out) { + if (0 == elements.size()) { + // Missing fields are indexed as null. + BSONObjBuilder b; + b.appendNull(""); + out->insert(b.obj()); + } else { + for (BSONElementSet::iterator i = elements.begin(); i != elements.end(); ++i) { + getS2OneLiteralKey(*i, out); + } + } + } + + void S2KeyGenerator::getKeys(const BSONObj& obj, BSONObjSet* keys) const { + BSONObjSet keysToAdd; + + // Does one of our documents have a geo field? + bool haveGeoField = false; + + // We output keys in the same order as the fields we index. + BSONObjIterator i(_keyPattern); + while (i.more()) { + BSONElement e = i.next(); + + // First, we get the keys that this field adds. Either they're added literally from + // the value of the field, or they're transformed if the field is geo. + BSONElementSet fieldElements; + // false means Don't expand the last array, duh. + obj.getFieldsDotted(e.fieldName(), fieldElements, false); + + BSONObjSet keysForThisField; + if (IndexNames::GEO_2DSPHERE == e.valuestr()) { + if (S2_INDEX_VERSION_2 == _params.indexVersion) { + // For V2, + // geo: null, + // geo: undefined + // geo: [] + // should all behave like there is no geo field. So we look for these cases + // and throw out the field elements if we find them. + if (1 == fieldElements.size()) { + BSONElement elt = *fieldElements.begin(); + // Get the :null and :undefined cases. + if (elt.isNull() || Undefined == elt.type()) { + fieldElements.clear(); + } + else if (elt.isABSONObj()) { + // And this is the :[] case. + BSONObj obj = elt.Obj(); + if (0 == obj.nFields()) { + fieldElements.clear(); + } + } + } + + // V2 2dsphere indices require that at least one geo field to be present in a + // document in order to index it. + if (fieldElements.size() > 0) { + haveGeoField = true; + } + } + + getS2GeoKeys(obj, fieldElements, _params, &keysForThisField); + } else { + getS2LiteralKeys(fieldElements, &keysForThisField); + } + + // We expect there to be the missing field element present in the keys if data is + // missing. So, this should be non-empty. + verify(!keysForThisField.empty()); + + // We take the Cartesian product of all of the keys. This requires that we have + // some keys to take the Cartesian product with. If keysToAdd.empty(), we + // initialize it. + if (keysToAdd.empty()) { + keysToAdd = keysForThisField; + continue; + } + + BSONObjSet updatedKeysToAdd; + for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); + ++it) { + for (BSONObjSet::const_iterator newIt = keysForThisField.begin(); + newIt!= keysForThisField.end(); ++newIt) { + BSONObjBuilder b; + b.appendElements(*it); + b.append(newIt->firstElement()); + updatedKeysToAdd.insert(b.obj()); + } + } + keysToAdd = updatedKeysToAdd; + } + + // Make sure that if we're V2 there's at least one geo field present in the doc. + if (S2_INDEX_VERSION_2 == _params.indexVersion) { + if (!haveGeoField) { + return; + } + } + + if (keysToAdd.size() > _params.maxKeysPerInsert) { + warning() << "insert of geo object generated lots of keys (" << keysToAdd.size() + << ") consider creating larger buckets. obj=" + << obj; + } + + *keys = keysToAdd; + } + +} // namespace mongo diff --git a/src/mongo/db/index/s2_key_generator.h b/src/mongo/db/index/s2_key_generator.h new file mode 100644 index 00000000000..2b96ea8187b --- /dev/null +++ b/src/mongo/db/index/s2_key_generator.h @@ -0,0 +1,56 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/geo/s2common.h" +#include "mongo/db/index/btree_based_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class IndexCursor; + struct S2IndexingParams; + + class S2KeyGenerator : public KeyGenerator { + public: + S2KeyGenerator( const BSONObj& keyPattern, const S2IndexingParams& params ); + + virtual ~S2KeyGenerator(){} + + virtual void getKeys(const BSONObj& obj, BSONObjSet* keys) const; + + private: + BSONObj _keyPattern; + S2IndexingParams _params; + + }; + +} // namespace mongo diff --git a/src/mongo/db/index_builder.cpp b/src/mongo/db/index_builder.cpp new file mode 100644 index 00000000000..5a064e57877 --- /dev/null +++ b/src/mongo/db/index_builder.cpp @@ -0,0 +1,141 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/index_builder.h" + +#include "mongo/db/client.h" +#include "mongo/db/curop.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/db.h" +#include "mongo/db/d_concurrency.h" +#include "mongo/db/repl/rs.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + AtomicUInt IndexBuilder::_indexBuildCount = 0; + +namespace { + // Synchronization tools when replication spawns a background index in a new thread. + // The bool is 'true' when a new background index has started in a new thread but the + // parent thread has not yet synchronized with it. + bool _bgIndexStarting(false); + boost::mutex _bgIndexStartingMutex; + boost::condition_variable _bgIndexStartingCondVar; + + void _setBgIndexStarting() { + boost::mutex::scoped_lock lk(_bgIndexStartingMutex); + invariant(_bgIndexStarting == false); + _bgIndexStarting = true; + _bgIndexStartingCondVar.notify_one(); + } +} // namespace + + IndexBuilder::IndexBuilder(const BSONObj& index) : + BackgroundJob(true /* self-delete */), _index(index.getOwned()), + _name(str::stream() << "repl index builder " << (_indexBuildCount++).get()) { + } + + IndexBuilder::~IndexBuilder() {} + + std::string IndexBuilder::name() const { + return _name; + } + + void IndexBuilder::run() { + LOG(2) << "IndexBuilder building index " << _index; + + Client::initThread(name().c_str()); + Lock::ParallelBatchWriterMode::iAmABatchParticipant(); + + replLocalAuth(); + + cc().curop()->reset(HostAndPort(), dbInsert); + NamespaceString ns(_index["ns"].String()); + Client::WriteContext ctx(ns.getSystemIndexesCollection()); + + // Show which index we're background building in the curop display, and unblock waiters. + cc().curop()->setQuery(_index); + _setBgIndexStarting(); + + Status status = build( ctx.ctx() ); + if ( !status.isOK() ) { + log() << "IndexBuilder could not build index: " << status.toString(); + } + + cc().shutdown(); + } + + Status IndexBuilder::build( Client::Context& context ) const { + string ns = _index["ns"].String(); + Database* db = context.db(); + Collection* c = db->getCollection( ns ); + if ( !c ) { + c = db->getOrCreateCollection( ns ); + verify(c); + } + + // Show which index we're building in the curop display. + context.getClient()->curop()->setQuery(_index); + + Status status = c->getIndexCatalog()->createIndex( _index, + true, + IndexCatalog::SHUTDOWN_LEAVE_DIRTY ); + if ( status.code() == ErrorCodes::IndexAlreadyExists ) + return Status::OK(); + return status; + } + + void IndexBuilder::waitForBgIndexStarting() { + boost::unique_lock lk(_bgIndexStartingMutex); + while (_bgIndexStarting == false) { + _bgIndexStartingCondVar.wait(lk); + } + // Reset for next time. + _bgIndexStarting = false; + } + + std::vector + IndexBuilder::killMatchingIndexBuilds(Collection* collection, + const IndexCatalog::IndexKillCriteria& criteria) { + invariant(collection); + return collection->getIndexCatalog()->killMatchingIndexBuilds(criteria); + } + + void IndexBuilder::restoreIndexes(const std::vector& indexes) { + log() << "restarting " << indexes.size() << " index build(s)" << endl; + for (int i = 0; i < static_cast(indexes.size()); i++) { + IndexBuilder* indexBuilder = new IndexBuilder(indexes[i]); + // This looks like a memory leak, but indexBuilder deletes itself when it finishes + indexBuilder->go(); + dbtemprelease release; + IndexBuilder::waitForBgIndexStarting(); + } + } +} + diff --git a/src/mongo/db/index_builder.h b/src/mongo/db/index_builder.h new file mode 100644 index 00000000000..6730d98ecf3 --- /dev/null +++ b/src/mongo/db/index_builder.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/client.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/background.h" + +/** + * Forks off a thread to build an index. + */ +namespace mongo { + + class Collection; + + class IndexBuilder : public BackgroundJob { + public: + IndexBuilder(const BSONObj& index); + virtual ~IndexBuilder(); + + virtual void run(); + + /** + * name of the builder, not the index + */ + virtual std::string name() const; + + Status build( Client::Context& context ) const; + + /** + * Kill all in-progress indexes matching criteria, if non-empty: + * index ns, index name, and/or index key spec. + * Returns a vector of the indexes that were killed. + */ + static std::vector + killMatchingIndexBuilds(Collection* collection, + const IndexCatalog::IndexKillCriteria& criteria); + + /** + * Retry all index builds in the list. Builds each index in a separate thread. If ns does + * not match the ns field in the indexes list, the BSONObj's ns field is changed before the + * index is built (to handle rename). + */ + static void restoreIndexes(const std::vector& indexes); + + /** + * Waits for a background index build to register itself. This function must be called + * after starting a background index build via a BackgroundJob and before starting a + * subsequent one. + */ + static void waitForBgIndexStarting(); + + private: + const BSONObj _index; + std::string _name; // name of this builder, not related to the index + static AtomicUInt _indexBuildCount; + }; + +} diff --git a/src/mongo/db/index_insertion_continuation.h b/src/mongo/db/index_insertion_continuation.h deleted file mode 100644 index 67493138296..00000000000 --- a/src/mongo/db/index_insertion_continuation.h +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright (C) 2008 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#pragma once - -#include "mongo/db/diskloc.h" -#include "mongo/db/jsobj.h" - -namespace mongo { - - class IndexDetails; - template class BtreeBucket; - - /** - * This class represents the write phase of the two-phase index insertion. - */ - class IndexInsertionContinuation : private boost::noncopyable { - public: - enum Op { Nothing, SetUsed, InsertHere }; - - virtual ~IndexInsertionContinuation(); - virtual void doIndexInsertionWrites() const = 0; - }; - - template< class V > - struct IndexInsertionContinuationImpl : public IndexInsertionContinuation { - - IndexInsertionContinuationImpl(DiskLoc thisLoc, DiskLoc _recordLoc, const BSONObj &_key, - Ordering _order, IndexDetails& _idx) : - bLoc(thisLoc), recordLoc(_recordLoc), key(_key), order(_order), idx(_idx) { - op = Nothing; - } - - DiskLoc bLoc; - DiskLoc recordLoc; - typename V::KeyOwned key; - const Ordering order; - IndexDetails& idx; - Op op; - - int pos; - const BtreeBucket *b; - - void doIndexInsertionWrites() const { - if( op == Nothing ) - return; - else if( op == SetUsed ) { - const typename V::_KeyNode& kn = b->k(pos); - kn.writing().setUsed(); - } - else { - b->insertHere(bLoc, pos, recordLoc, key, order, DiskLoc(), DiskLoc(), idx); - } - } - }; - - -} // namespace mongo diff --git a/src/mongo/db/index_legacy.cpp b/src/mongo/db/index_legacy.cpp new file mode 100644 index 00000000000..3199c5958da --- /dev/null +++ b/src/mongo/db/index_legacy.cpp @@ -0,0 +1,96 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index_legacy.h" + +#include "mongo/db/client.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/db/index/hash_key_generator.h" +#include "mongo/db/index/s2_access_method.h" +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/structure/catalog/namespace_details.h" + +namespace mongo { + + // static + BSONObj IndexLegacy::adjustIndexSpecObject(const BSONObj& obj) { + string pluginName = IndexNames::findPluginName(obj.getObjectField("key")); + + if (IndexNames::TEXT == pluginName) { + return fts::FTSSpec::fixSpec(obj); + } + + if (IndexNames::GEO_2DSPHERE == pluginName) { + return S2AccessMethod::fixSpec(obj); + } + + return obj; + } + + // static + BSONObj IndexLegacy::getMissingField(Collection* collection, const BSONObj& infoObj) { + BSONObj keyPattern = infoObj.getObjectField( "key" ); + string accessMethodName; + if ( collection ) + accessMethodName = collection->getIndexCatalog()->getAccessMethodName(keyPattern); + else + accessMethodName = IndexNames::findPluginName(keyPattern); + + if (IndexNames::HASHED == accessMethodName ) { + int hashVersion = infoObj["hashVersion"].numberInt(); + HashSeed seed = infoObj["seed"].numberInt(); + + // Explicit null valued fields and missing fields are both represented in hashed indexes + // using the hash value of the null BSONElement. This is partly for historical reasons + // (hash of null was used in the initial release of hashed indexes and changing would + // alter the data format). Additionally, in certain places the hashed index code and + // the index bound calculation code assume null and missing are indexed identically. + BSONObj nullObj = BSON("" << BSONNULL); + return BSON("" << HashKeyGenerator::makeSingleHashKey(nullObj.firstElement(), seed, hashVersion)); + } + else { + BSONObjBuilder b; + b.appendNull(""); + return b.obj(); + } + } + + // static + void IndexLegacy::postBuildHook(Collection* collection, const BSONObj& keyPattern) { + // If it's an FTS index, we want to set the power of 2 flag. + string pluginName = collection->getIndexCatalog()->getAccessMethodName(keyPattern); + if (IndexNames::TEXT == pluginName) { + NamespaceDetails* nsd = collection->details(); + if (nsd->setUserFlag(NamespaceDetails::Flag_UsePowerOf2Sizes)) { + nsd->syncUserFlags(collection->ns().ns()); + } + } + } + +} // namespace mongo diff --git a/src/mongo/db/index_legacy.h b/src/mongo/db/index_legacy.h new file mode 100644 index 00000000000..942c53f6487 --- /dev/null +++ b/src/mongo/db/index_legacy.h @@ -0,0 +1,80 @@ + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/jsobj.h" + +namespace mongo { + + class Collection; + class IndexDescriptor; + + /** + * There has been some behavior concerning indexed access patterns -- both pre and post-index + * construction -- that does not quite fit in the access pattern model implemented in + * index/index_access_pattern.h. Such behavior can't be changed in the current implementation of + * the code. + * + * We grouped such exception/legacy behavior here. + */ + class IndexLegacy { + public: + /** + * Adjust the provided index spec BSONObj depending on the type of index obj describes. + * + * This is a no-op unless the object describes a TEXT or a GEO_2DSPHERE index. TEXT and + * GEO_2DSPHERE provide additional validation on the index spec, and tweak the index spec + * object to conform to their expected format. + */ + static BSONObj adjustIndexSpecObject(const BSONObj& obj); + + /** + * Returns the BSONObj that is inserted into an index when the object is missing the keys + * the index is over. + * + * For every index *except hash*, this is the BSON equivalent of jstNULL. + * For the hash index, it's the hash of BSON("" << BSONNULL). + * + * s/d_split.cpp needs to know this. + * + * This is a significant leak of index functionality out of the index layer. + */ + static BSONObj getMissingField(Collection* collection, const BSONObj& infoObj); + + /** + * Perform any post-build steps for this index. + * + * This is a no-op unless the index is a FTS index. In that case, we set the flag for using + * power of 2 sizes for space allocation. + */ + static void postBuildHook(Collection* collection, const BSONObj& keyPattern ); + }; + +} // namespace mongo diff --git a/src/mongo/db/index_names.cpp b/src/mongo/db/index_names.cpp new file mode 100644 index 00000000000..d3f79886e36 --- /dev/null +++ b/src/mongo/db/index_names.cpp @@ -0,0 +1,95 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/index_names.h" + +#include "mongo/db/jsobj.h" + +namespace mongo { + + const string IndexNames::GEO_2D = "2d"; + const string IndexNames::GEO_HAYSTACK = "geoHaystack"; + const string IndexNames::GEO_2DSPHERE = "2dsphere"; + const string IndexNames::TEXT = "text"; + const string IndexNames::HASHED = "hashed"; + const string IndexNames::BTREE = ""; + + // static + string IndexNames::findPluginName(const BSONObj& keyPattern) { + BSONObjIterator i(keyPattern); + + while (i.more()) { + BSONElement e = i.next(); + if (String != e.type()) { continue; } + return e.String(); + } + + return IndexNames::BTREE; + } + + // static + bool IndexNames::existedBefore24(const string& name) { + return name == IndexNames::BTREE + || name == IndexNames::GEO_2D + || name == IndexNames::GEO_HAYSTACK + || name == IndexNames::HASHED; + } + + // static + bool IndexNames::isKnownName(const string& name) { + return name == IndexNames::GEO_2D + || name == IndexNames::GEO_2DSPHERE + || name == IndexNames::GEO_HAYSTACK + || name == IndexNames::TEXT + || name == IndexNames::HASHED + || name == IndexNames::BTREE; + } + + // static + IndexType IndexNames::nameToType(const string& accessMethod) { + if (IndexNames::GEO_2D == accessMethod) { + return INDEX_2D; + } + else if (IndexNames::GEO_HAYSTACK == accessMethod) { + return INDEX_HAYSTACK; + } + else if (IndexNames::GEO_2DSPHERE == accessMethod) { + return INDEX_2DSPHERE; + } + else if (IndexNames::TEXT == accessMethod) { + return INDEX_TEXT; + } + else if (IndexNames::HASHED == accessMethod) { + return INDEX_HASHED; + } + else { + return INDEX_BTREE; + } + } + +} // namespace mongo diff --git a/src/mongo/db/index_names.h b/src/mongo/db/index_names.h new file mode 100644 index 00000000000..fd7af42675b --- /dev/null +++ b/src/mongo/db/index_names.h @@ -0,0 +1,96 @@ +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +namespace mongo { + + using std::string; + + class BSONObj; + + /** + * We need to know what 'type' an index is in order to plan correctly. We can't entirely rely + * on the key pattern to tell us what kind of index we have. + * + * An example of the Bad Thing That We Must Avoid: + * 1. Create a 2dsphere index in 2.4, insert some docs. + * 2. Downgrade to 2.2. Insert some more docs into the collection w/the 2dsphere + * index. 2.2 treats the index as a normal btree index and creates keys accordingly. + * 3. Using the 2dsphere index in 2.4 gives wrong results or assert-fails or crashes as + * the data isn't what we expect. + */ + enum IndexType { + INDEX_BTREE, + INDEX_2D, + INDEX_HAYSTACK, + INDEX_2DSPHERE, + INDEX_TEXT, + INDEX_HASHED, + }; + + /** + * We use the string representation of index names all over the place, so we declare them all + * once here. + */ + class IndexNames { + public: + static const string GEO_2D; + static const string GEO_HAYSTACK; + static const string GEO_2DSPHERE; + static const string TEXT; + static const string HASHED; + static const string BTREE; + + /** + * True if is a regular (non-plugin) index or uses a plugin that existed before 2.4. + * These plugins are grandfathered in and allowed to exist in DBs with + * PDFILE_MINOR_VERSION_22_AND_OLDER + */ + static bool existedBefore24(const string& name); + + /** + * Return the first string value in the provided object. For an index key pattern, + * a field with a non-string value indicates a "special" (not straight Btree) index. + */ + static string findPluginName(const BSONObj& keyPattern); + + /** + * Is the provided access method name one we recognize? + */ + static bool isKnownName(const string& name); + + /** + * Convert an index name to an IndexType. + */ + static IndexType nameToType(const string& accessMethod); + }; + +} // namespace mongo diff --git a/src/mongo/db/index_rebuilder.cpp b/src/mongo/db/index_rebuilder.cpp index ac2132ed56e..d713226b318 100644 --- a/src/mongo/db/index_rebuilder.cpp +++ b/src/mongo/db/index_rebuilder.cpp @@ -12,135 +12,132 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/db/index_rebuilder.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/client.h" #include "mongo/db/instance.h" #include "mongo/db/pdfile.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/catalog/collection.h" #include "mongo/util/scopeguard.h" namespace mongo { - // Disabled until locking at startup can be ironed out. - // See SERVER-8344 and SERVER-8536 - //IndexRebuilder indexRebuilder; + IndexRebuilder indexRebuilder; IndexRebuilder::IndexRebuilder() {} - /** - * This resets memory tracking to its original value after all indexes are rebuilt. - * - * Before the server starts listening, all memory accesses are counted as taking 0 time (because - * the timer hasn't started yet). The Record class warns about these 0-time accesses (actually, - * the warning is in Rolling, which is used by Record) so run() turns off the tracking to - * silence these warnings. We want to make sure that they're turned back on, though, no matter - * how run() exits. - */ - static void resetMemoryTracking(bool originalTracking) { - Record::MemoryTrackingEnabled = originalTracking; + std::string IndexRebuilder::name() const { + return "IndexRebuilder"; } void IndexRebuilder::run() { - // Disable record access timer warnings - ON_BLOCK_EXIT(resetMemoryTracking, Record::MemoryTrackingEnabled); - Record::MemoryTrackingEnabled = false; - - Client::GodScope gs; - Lock::GlobalWrite lk; + Client::initThread(name().c_str()); + ON_BLOCK_EXIT_OBJ(cc(), &Client::shutdown); + cc().getAuthorizationSession()->grantInternalAuthorization(); - bool firstTime = true; std::vector dbNames; getDatabaseNames(dbNames); - for (std::vector::const_iterator it = dbNames.begin(); - it < dbNames.end(); - it++) { - checkDB(*it, &firstTime); + try { + std::list collNames; + for (std::vector::const_iterator dbName = dbNames.begin(); + dbName < dbNames.end(); + dbName++) { + Client::ReadContext ctx(*dbName); + Database* db = cc().database(); + db->namespaceIndex().getNamespaces(collNames, /* onlyCollections */ true); + } + checkNS(collNames); } - - cc().shutdown(); + catch (const DBException& e) { + warning() << "Index rebuilding did not complete: " << e.what() << endl; + } + boost::unique_lock lk(ReplSet::rss.mtx); + ReplSet::rss.indexRebuildDone = true; + ReplSet::rss.cond.notify_all(); + LOG(1) << "checking complete" << endl; } - void IndexRebuilder::checkDB(const std::string& dbName, bool* firstTime) { - const std::string systemNS = dbName + ".system.namespaces"; - DBDirectClient cli; - scoped_ptr cursor(cli.query(systemNS, Query())); + void IndexRebuilder::checkNS(const std::list& nsToCheck) { + bool firstTime = true; + for (std::list::const_iterator it = nsToCheck.begin(); + it != nsToCheck.end(); + ++it) { + + string ns = *it; - // This depends on system.namespaces not changing while we iterate - while (cursor->more()) { - BSONObj nsDoc = cursor->next(); - const char* ns = nsDoc["name"].valuestrsafe(); + LOG(3) << "IndexRebuilder::checkNS: " << ns; + // This write lock is held throughout the index building process + // for this namespace. Client::WriteContext ctx(ns); - NamespaceDetails* nsd = nsdetails(ns); + Collection* collection = ctx.ctx().db()->getCollection( ns ); + if ( collection == NULL ) + continue; - if (!nsd || !nsd->indexBuildsInProgress) { + IndexCatalog* indexCatalog = collection->getIndexCatalog(); + + if ( collection->ns().isOplog() && indexCatalog->numIndexesTotal() > 0 ) { + warning() << ns << " had illegal indexes, removing"; + indexCatalog->dropAllIndexes( true ); continue; } - log() << "Found interrupted index build on " << ns << endl; - if (*firstTime) { - log() << "Restart the server with --noIndexBuildRetry to skip index rebuilds" - << endl; - *firstTime = false; + vector indexesToBuild = indexCatalog->getAndClearUnfinishedIndexes(); + + // The indexes have now been removed from system.indexes, so the only record is + // in-memory. If there is a journal commit between now and when insert() rewrites + // the entry and the db crashes before the new system.indexes entry is journalled, + // the index will be lost forever. Thus, we're assuming no journaling will happen + // between now and the entry being re-written. + + if ( indexesToBuild.size() == 0 ) { + continue; } - // If the indexBuildRetry flag isn't set, just clear the inProg flag - if (!cmdLine.indexBuildRetry) { - // If we crash between unsetting the inProg flag and cleaning up the index, the - // index space will be lost. - int inProg = nsd->indexBuildsInProgress; - getDur().writingInt(nsd->indexBuildsInProgress) = 0; + log() << "found " << indexesToBuild.size() + << " interrupted index build(s) on " << ns; - for (int i = 0; i < inProg; i++) { - nsd->idx(nsd->nIndexes+i).kill_idx(); - } + if (firstTime) { + log() << "note: restart the server with --noIndexBuildRetry to skip index rebuilds"; + firstTime = false; + } + if (!serverGlobalParams.indexBuildRetry) { + log() << " not rebuilding interrupted indexes"; continue; } - // We go from right to left building these indexes, so that indexBuildInProgress-- has - // the correct effect of "popping" an index off the list. - while (nsd->indexBuildsInProgress > 0) { - retryIndexBuild(dbName, nsd, nsd->nIndexes+nsd->indexBuildsInProgress-1); - } - } - } + // TODO: these can/should/must be done in parallel + for ( size_t i = 0; i < indexesToBuild.size(); i++ ) { + BSONObj indexObj = indexesToBuild[i]; - void IndexRebuilder::retryIndexBuild(const std::string& dbName, - NamespaceDetails* nsd, - const int index) { - // details.info is always a valid system.indexes entry because DataFileMgr::insert journals - // creating the index doc and then insert_makeIndex durably assigns its DiskLoc to info. - // indexBuildsInProgress is set after that, so if it is set, info must be set. - IndexDetails& details = nsd->idx(index); - - // First, clean up the in progress index build. Save the system.indexes entry so that we - // can add it again afterwards. - BSONObj indexObj = details.info.obj().getOwned(); - - // Clean up the in-progress index build - getDur().writingInt(nsd->indexBuildsInProgress) -= 1; - details.kill_idx(); - // The index has now been removed from system.indexes, so the only record of it is in- - // memory. If there is a journal commit between now and when insert() rewrites the entry and - // the db crashes before the new system.indexes entry is journalled, the index will be lost - // forever. Thus, we're assuming no journaling will happen between now and the entry being - // re-written. - - // We need to force a foreground index build to prevent replication from replaying an - // incompatible op (like a drop) during a yield. - // TODO: once commands can interrupt/wait for index builds, this can be removed. - indexObj = indexObj.removeField("background"); + log() << "going to rebuild: " << indexObj; - try { - const std::string ns = dbName + ".system.indexes"; - theDataFileMgr.insert(ns.c_str(), indexObj.objdata(), indexObj.objsize(), false, true); - } - catch (const DBException& e) { - log() << "Rebuilding index failed: " << e.what() << " (" << e.getCode() << ")" - << endl; + Status status = indexCatalog->createIndex( indexObj, false ); + if ( !status.isOK() ) { + log() << "building index failed: " << status.toString() << " index: " << indexObj; + } + + } } } + } diff --git a/src/mongo/db/index_rebuilder.h b/src/mongo/db/index_rebuilder.h index 2b327fbc048..bf008f59b39 100644 --- a/src/mongo/db/index_rebuilder.h +++ b/src/mongo/db/index_rebuilder.h @@ -12,15 +12,32 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once -#include "mongo/db/namespace_details.h" +#include +#include + +#include "mongo/util/background.h" namespace mongo { - class IndexRebuilder { + // This is a job that's only run at startup. It finds all incomplete indices and + // finishes rebuilding them. After they complete rebuilding, the thread terminates. + class IndexRebuilder : public BackgroundJob { public: IndexRebuilder(); @@ -29,18 +46,10 @@ namespace mongo { private: /** - * Check each collection in a database to see if it has any in-progress index builds that - * need to be retried. If so, calls retryIndexBuild. - */ - void checkDB(const std::string& dbname, bool* firstTime); - - /** - * Actually retry an index build on a given namespace. - * @param dbName the name of the database for accessing db.system.indexes - * @param nsd the namespace details of the namespace building the index - * @param index the offset into nsd's index array of the partially-built index + * Check each collection in the passed in list to see if it has any in-progress index + * builds that need to be retried. If so, calls retryIndexBuild. */ - void retryIndexBuild(const std::string& dbName, NamespaceDetails* nsd, const int index); + void checkNS(const std::list& nsToCheck); }; extern IndexRebuilder indexRebuilder; diff --git a/src/mongo/db/index_set.cpp b/src/mongo/db/index_set.cpp deleted file mode 100644 index 65f74ed54ea..00000000000 --- a/src/mongo/db/index_set.cpp +++ /dev/null @@ -1,134 +0,0 @@ -// index_set.cpp - -/** -* Copyright (C) 2013 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/bson/util/builder.h" -#include "mongo/db/index_set.h" - -namespace mongo { - - void IndexPathSet::addPath( const StringData& path ) { - string s; - if ( getCanonicalIndexField( path, &s ) ) { - _canonical.insert( s ); - } - else { - _canonical.insert( path.toString() ); - } - } - - void IndexPathSet::clear() { - _canonical.clear(); - } - - bool IndexPathSet::mightBeIndexed( const StringData& path ) const { - StringData use = path; - string x; - if ( getCanonicalIndexField( path, &x ) ) - use = StringData( x ); - - for ( std::set::const_iterator i = _canonical.begin(); - i != _canonical.end(); - ++i ) { - - StringData idx( *i ); - - if ( _startsWith( use, idx ) ) - return true; - - if ( _startsWith( idx, use ) ) - return true; - } - - return false; - } - - bool IndexPathSet::_startsWith( const StringData& a, const StringData& b ) const { - if ( !a.startsWith( b ) ) - return false; - - // make sure there is a dot or EOL right after - - if ( a.size() == b.size() ) - return true; - - return a[b.size()] == '.'; - } - - bool getCanonicalIndexField( const StringData& fullName, string* out ) { - // check if fieldName contains ".$" or ".###" substrings (#=digit) and skip them - // however do not skip the first field even if it meets these criteria - - if ( fullName.find( '.' ) == string::npos ) - return false; - - bool modified = false; - - StringBuilder buf; - for ( size_t i=0; i= fullName.size() || fullName[i+2] == '.' ) { - i++; - modified = true; - continue; - } - } - - // check for ".###" for any number of digits (no letters) - if ( isdigit( fullName[i+1] ) ) { - size_t j = i; - // skip digits - while ( j+1 < fullName.size() && isdigit( fullName[j+1] ) ) - j++; - - if ( j+1 == fullName.size() || fullName[j+1] == '.' ) { - // only digits found, skip forward - i = j; - modified = true; - continue; - } - } - - buf << c; - } - - if ( !modified ) - return false; - - *out = buf.str(); - return true; - } - - -} - diff --git a/src/mongo/db/index_set.h b/src/mongo/db/index_set.h deleted file mode 100644 index 1ff99780f07..00000000000 --- a/src/mongo/db/index_set.h +++ /dev/null @@ -1,48 +0,0 @@ -// index_set.h - -/** -* Copyright (C) 2013 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include - -#include "mongo/base/string_data.h" - -namespace mongo { - - /** - * a.$ -> a - * @return true if out is set and we made a change - */ - bool getCanonicalIndexField( const StringData& fullName, std::string* out ); - - class IndexPathSet { - public: - void addPath( const StringData& path ); - - void clear(); - - bool mightBeIndexed( const StringData& path ) const; - - private: - - bool _startsWith( const StringData& a, const StringData& b ) const; - - std::set _canonical; - }; - -} diff --git a/src/mongo/db/index_set_test.cpp b/src/mongo/db/index_set_test.cpp deleted file mode 100644 index 28a1d9fe68e..00000000000 --- a/src/mongo/db/index_set_test.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// index_set_tests.cpp - -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/unittest/unittest.h" - -#include "mongo/db/index_set.h" - -namespace mongo { - - TEST( IndexPathSetTest, Simple1 ) { - IndexPathSet a; - a.addPath( "a.b" ); - ASSERT_TRUE( a.mightBeIndexed( "a.b" ) ); - ASSERT_TRUE( a.mightBeIndexed( "a" ) ); - ASSERT_TRUE( a.mightBeIndexed( "a.b.c" ) ); - ASSERT_TRUE( a.mightBeIndexed( "a.$.b" ) ); - - ASSERT_FALSE( a.mightBeIndexed( "b" ) ); - ASSERT_FALSE( a.mightBeIndexed( "a.c" ) ); - } - - TEST( IndexPathSetTest, Simple2 ) { - IndexPathSet a; - a.addPath( "ab" ); - ASSERT_FALSE( a.mightBeIndexed( "a" ) ); - } - - - TEST( IndexPathSetTest, getCanonicalIndexField1 ) { - string x; - - ASSERT_FALSE( getCanonicalIndexField( "a", &x ) ); - ASSERT_FALSE( getCanonicalIndexField( "aaa", &x ) ); - ASSERT_FALSE( getCanonicalIndexField( "a.b", &x ) ); - - ASSERT_TRUE( getCanonicalIndexField( "a.$", &x ) ); - ASSERT_EQUALS( x, "a" ); - ASSERT_TRUE( getCanonicalIndexField( "a.0", &x ) ); - ASSERT_EQUALS( x, "a" ); - ASSERT_TRUE( getCanonicalIndexField( "a.123", &x ) ); - ASSERT_EQUALS( x, "a" ); - - ASSERT_TRUE( getCanonicalIndexField( "a.$.b", &x ) ); - ASSERT_EQUALS( x, "a.b" ); - ASSERT_TRUE( getCanonicalIndexField( "a.0.b", &x ) ); - ASSERT_EQUALS( x, "a.b" ); - ASSERT_TRUE( getCanonicalIndexField( "a.123.b", &x ) ); - ASSERT_EQUALS( x, "a.b" ); - - ASSERT_FALSE( getCanonicalIndexField( "a.$ref", &x ) ); - ASSERT_FALSE( getCanonicalIndexField( "a.$ref.b", &x ) ); - - - ASSERT_FALSE( getCanonicalIndexField( "a.c$d.b", &x ) ); - - ASSERT_FALSE( getCanonicalIndexField( "a.123a", &x ) ); - ASSERT_FALSE( getCanonicalIndexField( "a.a123", &x ) ); - ASSERT_FALSE( getCanonicalIndexField( "a.123a.b", &x ) ); - ASSERT_FALSE( getCanonicalIndexField( "a.a123.b", &x ) ); - - ASSERT_FALSE( getCanonicalIndexField( "a.", &x ) ); - } - -} diff --git a/src/mongo/db/index_update.cpp b/src/mongo/db/index_update.cpp deleted file mode 100644 index 741b1285c51..00000000000 --- a/src/mongo/db/index_update.cpp +++ /dev/null @@ -1,704 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/index_update.h" - -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/background.h" -#include "mongo/db/btreebuilder.h" -#include "mongo/db/clientcursor.h" -#include "mongo/db/extsort.h" -#include "mongo/db/index.h" -#include "mongo/db/kill_current_op.h" -#include "mongo/db/namespace_details.h" -#include "mongo/db/pdfile_private.h" -#include "mongo/db/replutil.h" -#include "mongo/db/repl/rs.h" -#include "mongo/db/sort_phase_one.h" -#include "mongo/util/processinfo.h" -#include "mongo/util/startup_test.h" - -namespace mongo { - - /* unindex all keys in index for this record. */ - static void _unindexRecord(IndexDetails& id, BSONObj& obj, const DiskLoc& dl, bool logMissing = true) { - BSONObjSet keys; - id.getKeysFromObject(obj, keys); - IndexInterface& ii = id.idxInterface(); - for ( BSONObjSet::iterator i=keys.begin(); i != keys.end(); i++ ) { - BSONObj j = *i; - - bool ok = false; - try { - ok = ii.unindex(id.head, id, j, dl); - } - catch (AssertionException& e) { - problem() << "Assertion failure: _unindex failed " << id.indexNamespace() << endl; - out() << "Assertion failure: _unindex failed: " << e.what() << '\n'; - out() << " obj:" << obj.toString() << '\n'; - out() << " key:" << j.toString() << '\n'; - out() << " dl:" << dl.toString() << endl; - logContext(); - } - - if ( !ok && logMissing ) { - log() << "unindex failed (key too big?) " << id.indexNamespace() << " key: " << j << " " << obj["_id"] << endl; - } - } - } - -//zzz - /* unindex all keys in all indexes for this record. */ - void unindexRecord(NamespaceDetails *d, - Record *todelete, - const DiskLoc& dl, - bool noWarn /* = false */) { - BSONObj obj = BSONObj::make(todelete); - int n = d->nIndexes; - for ( int i = 0; i < n; i++ ) - _unindexRecord(d->idx(i), obj, dl, !noWarn); - - for (int i = 0; i < d->indexBuildsInProgress; i++) { // background index - // Always pass nowarn here, as this one may be missing for valid reasons as we are - // concurrently building it - _unindexRecord(d->idx(n+i), obj, dl, false); - } - } - - /* step one of adding keys to index idxNo for a new record - */ - void fetchIndexInserters(BSONObjSet & /*out*/keys, - IndexInterface::IndexInserter &inserter, - NamespaceDetails *d, - int idxNo, - const BSONObj& obj, - DiskLoc recordLoc, - const bool allowDups) { - IndexDetails &idx = d->idx(idxNo); - idx.getKeysFromObject(obj, keys); - if( keys.empty() ) - return; - bool dupsAllowed = !idx.unique() || allowDups; - Ordering ordering = Ordering::make(idx.keyPattern()); - - try { - // we can't do the two step method with multi keys as insertion of one key changes the indexes - // structure. however we can do the first key of the set so we go ahead and do that FWIW - inserter.addInsertionContinuation( - idx.idxInterface().beginInsertIntoIndex( - idxNo, idx, recordLoc, *keys.begin(), ordering, dupsAllowed)); - } - catch (AssertionException& e) { - if( e.getCode() == 10287 && idxNo >= d->nIndexes ) { - DEV log() << "info: caught key already in index on bg indexing (ok)" << endl; - } - else { - throw; - } - } - } - - /** add index keys for a newly inserted record - done in two steps/phases to allow potential deferal of write lock portion in the future - */ - void indexRecordUsingTwoSteps(const char *ns, NamespaceDetails *d, BSONObj obj, - DiskLoc loc, bool shouldBeUnlocked) { - vector multi; - vector multiKeys; - - IndexInterface::IndexInserter inserter; - - // Step 1, read phase. - int n = d->getTotalIndexCount(); - { - BSONObjSet keys; - for ( int i = 0; i < n; i++ ) { - // this call throws on unique constraint violation. we haven't done any writes yet so that is fine. - fetchIndexInserters(/*out*/keys, - inserter, - d, - i, - obj, - loc, - ignoreUniqueIndex(d->idx(i))); - if( keys.size() > 1 ) { - multi.push_back(i); - multiKeys.push_back(BSONObjSet()); - multiKeys[multiKeys.size()-1].swap(keys); - } - keys.clear(); - } - } - - inserter.finishAllInsertions(); // Step 2, write phase. - - // now finish adding multikeys - for( unsigned j = 0; j < multi.size(); j++ ) { - unsigned i = multi[j]; - BSONObjSet& keys = multiKeys[j]; - IndexDetails& idx = d->idx(i); - bool dupsAllowed = !idx.unique() || ignoreUniqueIndex(idx); - IndexInterface& ii = idx.idxInterface(); - Ordering ordering = Ordering::make(idx.keyPattern()); - d->setIndexIsMultikey(ns, i); - for( BSONObjSet::iterator k = ++keys.begin()/*skip 1*/; k != keys.end(); k++ ) { - try { - ii.bt_insert(idx.head, loc, *k, ordering, dupsAllowed, idx); - } catch (AssertionException& e) { - if( e.getCode() == 10287 && (int) i >= d->nIndexes ) { - DEV log() << "info: caught key already in index on bg indexing (ok)" << endl; - } - else { - /* roll back previously added index entries - note must do self index as it is multikey and could require some cleanup itself - */ - for( int j = 0; j < n; j++ ) { - try { - _unindexRecord(d->idx(j), obj, loc, false); - } - catch(...) { - LOG(3) << "unindex fails on rollback after unique key constraint prevented insert\n"; - } - } - throw; - } - } - } - } - } - - /* add keys to index idxNo for a new record */ - static void addKeysToIndex(const char *ns, NamespaceDetails *d, int idxNo, BSONObj& obj, - DiskLoc recordLoc, bool dupsAllowed) { - IndexDetails& idx = d->idx(idxNo); - BSONObjSet keys; - idx.getKeysFromObject(obj, keys); - if( keys.empty() ) - return; - BSONObj order = idx.keyPattern(); - IndexInterface& ii = idx.idxInterface(); - Ordering ordering = Ordering::make(order); - int n = 0; - for ( BSONObjSet::iterator i=keys.begin(); i != keys.end(); i++ ) { - if( ++n == 2 ) { - d->setIndexIsMultikey(ns, idxNo); - } - verify( !recordLoc.isNull() ); - try { - ii.bt_insert(idx.head, recordLoc, *i, ordering, dupsAllowed, idx); - } - catch (AssertionException& e) { - if( e.getCode() == 10287 && idxNo >= d->nIndexes ) { - DEV log() << "info: caught key already in index on bg indexing (ok)" << endl; - continue; - } - if( !dupsAllowed ) { - // dup key exception, presumably. - throw; - } - problem() << " caught assertion addKeysToIndex " << idx.indexNamespace() << " " << obj["_id"] << endl; - } - } - } - - void addKeysToPhaseOne( const char* ns, - const IndexDetails& idx, - const BSONObj& order, - SortPhaseOne* phaseOne, - int64_t nrecords, - ProgressMeter* progressMeter, - bool mayInterrupt ) { - shared_ptr cursor = theDataFileMgr.findAll( ns ); - phaseOne->sorter.reset( new BSONObjExternalSorter( idx.idxInterface(), order ) ); - phaseOne->sorter->hintNumObjects( nrecords ); - const IndexSpec& spec = idx.getSpec(); - while ( cursor->ok() ) { - RARELY killCurrentOp.checkForInterrupt( !mayInterrupt ); - BSONObj o = cursor->current(); - DiskLoc loc = cursor->currLoc(); - phaseOne->addKeys( spec, o, loc, mayInterrupt ); - cursor->advance(); - progressMeter->hit(); - if ( logLevel > 1 && phaseOne->n % 10000 == 0 ) { - printMemInfo( "\t iterating objects" ); - } - } - } - - template< class V > - void buildBottomUpPhases2And3( bool dupsAllowed, - IndexDetails& idx, - BSONObjExternalSorter& sorter, - bool dropDups, - set& dupsToDrop, - CurOp* op, - SortPhaseOne* phase1, - ProgressMeterHolder& pm, - Timer& t, - bool mayInterrupt ) { - BtreeBuilder btBuilder(dupsAllowed, idx); - BSONObj keyLast; - auto_ptr i = sorter.iterator(); - // verifies that pm and op refer to the same ProgressMeter - verify(pm == op->setMessage("index: (2/3) btree bottom up", - "Index: (2/3) BTree Bottom Up Progress", - phase1->nkeys, - 10)); - while( i->more() ) { - RARELY killCurrentOp.checkForInterrupt( !mayInterrupt ); - BSONObjExternalSorter::Data d = i->next(); - - try { - if ( !dupsAllowed && dropDups ) { - LastError::Disabled led( lastError.get() ); - btBuilder.addKey(d.first, d.second); - } - else { - btBuilder.addKey(d.first, d.second); - } - } - catch( AssertionException& e ) { - if ( dupsAllowed ) { - // unknown exception?? - throw; - } - - if( e.interrupted() ) { - killCurrentOp.checkForInterrupt(); - } - - if ( ! dropDups ) - throw; - - /* we could queue these on disk, but normally there are very few dups, so instead we - keep in ram and have a limit. - */ - dupsToDrop.insert(d.second); - uassert( 10092 , "too may dups on index build with dropDups=true", dupsToDrop.size() < 1000000 ); - } - pm.hit(); - } - pm.finished(); - op->setMessage("index: (3/3) btree-middle", "Index: (3/3) BTree Middle Progress"); - LOG(t.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit" << endl; - btBuilder.commit( mayInterrupt ); - if ( btBuilder.getn() != phase1->nkeys && ! dropDups ) { - warning() << "not all entries were added to the index, probably some keys were too large" << endl; - } - } - - void doDropDups( const char* ns, - NamespaceDetails* d, - const set& dupsToDrop, - bool mayInterrupt ) { - for( set::const_iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); ++i ) { - RARELY killCurrentOp.checkForInterrupt( !mayInterrupt ); - theDataFileMgr.deleteRecord( d, - ns, - i->rec(), - *i, - false /* cappedOk */, - true /* noWarn */, - isMaster( ns ) /* logOp */ ); - getDur().commitIfNeeded(); - } - } - - // throws DBException - uint64_t fastBuildIndex(const char* ns, - NamespaceDetails* d, - IndexDetails& idx, - bool mayInterrupt) { - CurOp * op = cc().curop(); - - Timer t; - - tlog(1) << "fastBuildIndex " << ns << ' ' << idx.info.obj().toString() << endl; - - bool dupsAllowed = !idx.unique() || ignoreUniqueIndex(idx); - bool dropDups = idx.dropDups() || inDBRepair; - BSONObj order = idx.keyPattern(); - - getDur().writingDiskLoc(idx.head).Null(); - - if ( logLevel > 1 ) printMemInfo( "before index start" ); - - /* get and sort all the keys ----- */ - ProgressMeterHolder pm(op->setMessage("index: (1/3) external sort", - "Index: (1/3) External Sort Progress", - d->stats.nrecords, - 10)); - SortPhaseOne _ours; - SortPhaseOne *phase1 = theDataFileMgr.getPrecalced(); - if( phase1 == 0 ) { - phase1 = &_ours; - addKeysToPhaseOne( ns, idx, order, phase1, d->stats.nrecords, pm.get(), mayInterrupt ); - } - pm.finished(); - - BSONObjExternalSorter& sorter = *(phase1->sorter); - // Ensure the index and external sorter have a consistent index interface (and sort order). - fassert( 16408, &idx.idxInterface() == &sorter.getIndexInterface() ); - - if( phase1->multi ) { - int idxNo = IndexBuildsInProgress::get(ns, idx.info.obj()["name"].valuestr()); - d->setIndexIsMultikey(ns, idxNo); - } - - if ( logLevel > 1 ) printMemInfo( "before final sort" ); - phase1->sorter->sort( mayInterrupt ); - if ( logLevel > 1 ) printMemInfo( "after final sort" ); - - LOG(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles() << " files " << " in " << t.seconds() << " secs" << endl; - - set dupsToDrop; - - /* build index --- */ - if( idx.version() == 0 ) - buildBottomUpPhases2And3(dupsAllowed, - idx, - sorter, - dropDups, - dupsToDrop, - op, - phase1, - pm, - t, - mayInterrupt); - else if( idx.version() == 1 ) - buildBottomUpPhases2And3(dupsAllowed, - idx, - sorter, - dropDups, - dupsToDrop, - op, - phase1, - pm, - t, - mayInterrupt); - else - verify(false); - - if( dropDups ) - log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl; - - doDropDups(ns, d, dupsToDrop, mayInterrupt); - - return phase1->n; - } - - class BackgroundIndexBuildJob : public BackgroundOperation { - - unsigned long long addExistingToIndex(const char *ns, NamespaceDetails *d, - IndexDetails& idx) { - bool dupsAllowed = !idx.unique(); - bool dropDups = idx.dropDups(); - - ProgressMeter& progress = cc().curop()->setMessage("bg index build", - "Background Index Build Progress", - d->stats.nrecords); - - unsigned long long n = 0; - unsigned long long numDropped = 0; - auto_ptr cc; - { - shared_ptr c = theDataFileMgr.findAll(ns); - cc.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, ns) ); - } - - std::string idxName = idx.indexName(); - int idxNo = IndexBuildsInProgress::get(ns, idxName); - massert(16574, "Couldn't find index being built", idxNo != -1); - - // After this yields in the loop, idx may point at a different index (if indexes get - // flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should - // depend on idx. idxNo should be recalculated after each yield. - - while ( cc->ok() ) { - BSONObj js = cc->current(); - try { - { - if ( !dupsAllowed && dropDups ) { - LastError::Disabled led( lastError.get() ); - addKeysToIndex(ns, d, idxNo, js, cc->currLoc(), dupsAllowed); - } - else { - addKeysToIndex(ns, d, idxNo, js, cc->currLoc(), dupsAllowed); - } - } - cc->advance(); - } - catch( AssertionException& e ) { - if( e.interrupted() ) { - killCurrentOp.checkForInterrupt(); - } - - if ( dropDups ) { - DiskLoc toDelete = cc->currLoc(); - bool ok = cc->advance(); - ClientCursor::YieldData yieldData; - massert( 16093, "after yield cursor deleted" , cc->prepareToYield( yieldData ) ); - theDataFileMgr.deleteRecord( d, ns, toDelete.rec(), toDelete, false, true , true ); - if( !cc->recoverFromYield( yieldData ) ) { - cc.release(); - if( !ok ) { - /* we were already at the end. normal. */ - } - else { - uasserted(12585, "cursor gone during bg index; dropDups"); - } - break; - } - - // Recalculate idxNo if we yielded - idxNo = IndexBuildsInProgress::get(ns, idxName); - // This index must still be around, because this is thread that would clean - // it up - massert(16575, "cannot find index build anymore", idxNo != -1); - - numDropped++; - } - else { - log() << "background addExistingToIndex exception " << e.what() << endl; - throw; - } - } - n++; - progress.hit(); - - getDur().commitIfNeeded(); - - if ( cc->yieldSometimes( ClientCursor::WillNeed ) ) { - progress.setTotalWhileRunning( d->stats.nrecords ); - - // Recalculate idxNo if we yielded - idxNo = IndexBuildsInProgress::get(ns, idxName); - // Someone may have interrupted the index build - massert(16576, "cannot find index build anymore", idxNo != -1); - } - else { - idxNo = -1; - cc.release(); - uasserted(12584, "cursor gone during bg index"); - break; - } - } - progress.finished(); - if ( dropDups ) - log() << "\t backgroundIndexBuild dupsToDrop: " << numDropped << endl; - return n; - } - - /* we do set a flag in the namespace for quick checking, but this is our authoritative info - - that way on a crash/restart, we don't think we are still building one. */ - set bgJobsInProgress; - - void prep(const char *ns, NamespaceDetails *d) { - Lock::assertWriteLocked(ns); - uassert( 13130 , "can't start bg index b/c in recursive lock (db.eval?)" , !Lock::nested() ); - bgJobsInProgress.insert(d); - } - void done(const char *ns) { - NamespaceDetailsTransient::get(ns).addedIndex(); // clear query optimizer cache - Lock::assertWriteLocked(ns); - } - - public: - BackgroundIndexBuildJob(const char *ns) : BackgroundOperation(ns) { } - - unsigned long long go(string ns, NamespaceDetails *d, IndexDetails& idx) { - - // clear cached things since we are changing state - // namely what fields are indexed - NamespaceDetailsTransient::get(ns.c_str()).addedIndex(); - - unsigned long long n = 0; - - prep(ns.c_str(), d); - try { - idx.head.writing() = idx.idxInterface().addBucket(idx); - n = addExistingToIndex(ns.c_str(), d, idx); - // idx may point at an invalid index entry at this point - } - catch(...) { - if( cc().database() && nsdetails(ns) == d ) { - done(ns.c_str()); - } - else { - log() << "ERROR: db gone during bg index?" << endl; - } - throw; - } - done(ns.c_str()); - return n; - } - }; - - // throws DBException - void buildAnIndex(const std::string& ns, - NamespaceDetails* d, - IndexDetails& idx, - bool background, - bool mayInterrupt) { - tlog() << "build index " << ns << ' ' << idx.keyPattern() << ( background ? " background" : "" ) << endl; - Timer t; - unsigned long long n; - - verify( Lock::isWriteLocked(ns) ); - - // Build index spec here in case the collection is empty and the index details are invalid - idx.getSpec(); - - if( inDBRepair || !background ) { - n = fastBuildIndex(ns.c_str(), d, idx, mayInterrupt); - verify( !idx.head.isNull() ); - } - else { - BackgroundIndexBuildJob j(ns.c_str()); - n = j.go(ns, d, idx); - } - tlog() << "build index done. scanned " << n << " total records. " << t.millis() / 1000.0 << " secs" << endl; - } - - extern BSONObj id_obj; // { _id : 1 } - - void ensureHaveIdIndex(const char* ns, bool mayInterrupt) { - NamespaceDetails *d = nsdetails(ns); - if ( d == 0 || d->isSystemFlagSet(NamespaceDetails::Flag_HaveIdIndex) ) - return; - - d->setSystemFlag( NamespaceDetails::Flag_HaveIdIndex ); - - { - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - if( i.next().isIdIndex() ) - return; - } - } - - string system_indexes = cc().database()->name + ".system.indexes"; - - BSONObjBuilder b; - b.append("name", "_id_"); - b.append("ns", ns); - b.append("key", id_obj); - BSONObj o = b.done(); - - /* edge case: note the insert could fail if we have hit maxindexes already */ - theDataFileMgr.insert(system_indexes.c_str(), o.objdata(), o.objsize(), mayInterrupt, true); - } - - /* remove bit from a bit array - actually remove its slot, not a clear - note: this function does not work with x == 63 -- that is ok - but keep in mind in the future if max indexes were extended to - exactly 64 it would be a problem - */ - unsigned long long removeBit(unsigned long long b, int x) { - unsigned long long tmp = b; - return - (tmp & ((((unsigned long long) 1) << x)-1)) | - ((tmp >> (x+1)) << x); - } - - class IndexUpdateTest : public StartupTest { - public: - void run() { - verify( removeBit(1, 0) == 0 ); - verify( removeBit(2, 0) == 1 ); - verify( removeBit(2, 1) == 0 ); - verify( removeBit(255, 1) == 127 ); - verify( removeBit(21, 2) == 9 ); - verify( removeBit(0x4000000000000001ULL, 62) == 1 ); - } - } iu_unittest; - - bool dropIndexes( NamespaceDetails *d, const char *ns, const char *name, string &errmsg, BSONObjBuilder &anObjBuilder, bool mayDeleteIdIndex ) { - - BackgroundOperation::assertNoBgOpInProgForNs(ns); - - d = d->writingWithExtra(); - d->aboutToDeleteAnIndex(); - - /* there may be pointers pointing at keys in the btree(s). kill them. */ - ClientCursor::invalidate(ns); - - // delete a specific index or all? - if ( *name == '*' && name[1] == 0 ) { - LOG(4) << " d->nIndexes was " << d->nIndexes << '\n'; - anObjBuilder.append("nIndexesWas", (double)d->nIndexes); - IndexDetails *idIndex = 0; - if( d->nIndexes ) { - for ( int i = 0; i < d->nIndexes; i++ ) { - if ( !mayDeleteIdIndex && d->idx(i).isIdIndex() ) { - idIndex = &d->idx(i); - } - else { - d->idx(i).kill_idx(); - } - } - d->nIndexes = 0; - } - if ( idIndex ) { - d->getNextIndexDetails(ns) = *idIndex; - d->addIndex(ns); - wassert( d->nIndexes == 1 ); - } - /* assuming here that id index is not multikey: */ - d->multiKeyIndexBits = 0; - assureSysIndexesEmptied(ns, idIndex); - anObjBuilder.append("msg", mayDeleteIdIndex ? - "indexes dropped for collection" : - "non-_id indexes dropped for collection"); - } - else { - // delete just one index - int x = d->findIndexByName(name); - if ( x >= 0 ) { - LOG(4) << " d->nIndexes was " << d->nIndexes << endl; - anObjBuilder.append("nIndexesWas", (double)d->nIndexes); - - /* note it is important we remove the IndexDetails with this - call, otherwise, on recreate, the old one would be reused, and its - IndexDetails::info ptr would be bad info. - */ - IndexDetails *id = &d->idx(x); - if ( !mayDeleteIdIndex && id->isIdIndex() ) { - errmsg = "may not delete _id index"; - return false; - } - id->kill_idx(); - d->multiKeyIndexBits = removeBit(d->multiKeyIndexBits, x); - d->nIndexes--; - for ( int i = x; i < d->nIndexes; i++ ) - d->idx(i) = d->idx(i+1); - } - else { - int n = removeFromSysIndexes(ns, name); // just in case an orphaned listing there - i.e. should have been repaired but wasn't - if( n ) { - log() << "info: removeFromSysIndexes cleaned up " << n << " entries" << endl; - } - log() << "dropIndexes: " << name << " not found" << endl; - errmsg = "index not found"; - return false; - } - } - return true; - } - -} diff --git a/src/mongo/db/index_update.h b/src/mongo/db/index_update.h deleted file mode 100644 index ad71d293ce0..00000000000 --- a/src/mongo/db/index_update.h +++ /dev/null @@ -1,99 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/db/diskloc.h" -#include "mongo/db/index.h" -#include "mongo/db/jsobj.h" -#include "mongo/platform/cstdint.h" - -namespace mongo { - class NamespaceDetails; - class Record; - - // unindex all keys in index for this record. - void unindexRecord(NamespaceDetails *d, Record *todelete, const DiskLoc& dl, bool noWarn = false); - - // Build an index in the foreground - // If background is false, uses fast index builder - // If background is true, uses background index builder; blocks until done. - void buildAnIndex(const std::string& ns, - NamespaceDetails *d, - IndexDetails& idx, - bool background, - bool mayInterrupt); - - // add index keys for a newly inserted record - // done in two steps/phases to allow potential deferal of write lock portion in the future - void indexRecordUsingTwoSteps(const char *ns, NamespaceDetails *d, BSONObj obj, - DiskLoc loc, bool shouldBeUnlocked); - - // Given an object, populate "inserter" with information necessary to update indexes. - void fetchIndexInserters(BSONObjSet & /*out*/keys, - IndexInterface::IndexInserter &inserter, - NamespaceDetails *d, - int idxNo, - const BSONObj& obj, - DiskLoc recordLoc, - const bool allowDups = false); - - bool dropIndexes( NamespaceDetails *d, const char *ns, const char *name, string &errmsg, BSONObjBuilder &anObjBuilder, bool maydeleteIdIndex ); - - /** - * Add an _id index to namespace @param 'ns' if not already present. - * @param mayInterrupt When true, killop may interrupt the function call. - */ - void ensureHaveIdIndex(const char* ns, bool mayInterrupt); - - ////// The remaining functions are only included in this header file for unit testing. - - class BSONObjExternalSorter; - class CurOp; - class ProgressMeter; - class ProgressMeterHolder; - struct SortPhaseOne; - class Timer; - - /** Extract index keys from the @param 'ns' to the external sorter in @param 'phaseOne'. */ - void addKeysToPhaseOne( const char* ns, - const IndexDetails& idx, - const BSONObj& order, - SortPhaseOne* phaseOne, - int64_t nrecords, - ProgressMeter* progressMeter, - bool mayInterrupt ); - - /** Popuate the index @param 'idx' using the keys contained in @param 'sorter'. */ - template< class V > - void buildBottomUpPhases2And3( bool dupsAllowed, - IndexDetails& idx, - BSONObjExternalSorter& sorter, - bool dropDups, - set& dupsToDrop, - CurOp* op, - SortPhaseOne* phase1, - ProgressMeterHolder& pm, - Timer& t, - bool mayInterrupt ); - - /** Drop duplicate documents from the set @param 'dupsToDrop'. */ - void doDropDups( const char* ns, - NamespaceDetails* d, - const set& dupsToDrop, - bool mayInterrupt ); - -} // namespace mongo diff --git a/src/mongo/db/indexkey.cpp b/src/mongo/db/indexkey.cpp deleted file mode 100644 index 616dbd9a499..00000000000 --- a/src/mongo/db/indexkey.cpp +++ /dev/null @@ -1,518 +0,0 @@ -// index_key.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" -#include "namespace-inl.h" -#include "index.h" -#include "background.h" -#include "../util/stringutils.h" -#include "mongo/util/mongoutils/str.h" -#include "../util/text.h" -#include "mongo/db/client.h" -#include "mongo/db/database.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/queryutil.h" - -namespace mongo { - - /** old (<= v1.8) : 0 - 1 is new version - */ - const int DefaultIndexVersionNumber = 1; - - map * IndexPlugin::_plugins; - - IndexType::IndexType( const IndexPlugin * plugin , const IndexSpec * spec ) - : _plugin( plugin ) , _spec( spec ) { - - } - - IndexType::~IndexType() { - } - - BSONElement IndexType::missingField() const { - return _spec->_nullElt; - } - - const BSONObj& IndexType::keyPattern() const { - return _spec->keyPattern; - } - - IndexPlugin::IndexPlugin( const string& name ) - : _name( name ) { - if ( ! _plugins ) - _plugins = new map(); - (*_plugins)[name] = this; - } - - string IndexPlugin::findPluginName( const BSONObj& keyPattern ) { - string pluginName = ""; - - BSONObjIterator i( keyPattern ); - - while( i.more() ) { - BSONElement e = i.next(); - if ( e.type() != String ) - continue; - - uassert( 13007 , "can only have 1 index plugin / bad index key pattern" , pluginName.size() == 0 || pluginName == e.String() ); - pluginName = e.String(); - } - - return pluginName; - } - - int IndexType::compare( const BSONObj& l , const BSONObj& r ) const { - return l.woCompare( r , _spec->keyPattern ); - } - - void IndexSpec::_init(PluginRules rules) { - verify( keyPattern.objsize() ); - - // some basics - _nFields = keyPattern.nFields(); - _sparse = info["sparse"].trueValue(); - uassert( 13529 , "sparse only works for single field keys" , ! _sparse || _nFields ); - - - { - // build _nullKey - - BSONObjBuilder b; - BSONObjIterator i( keyPattern ); - - while( i.more() ) { - BSONElement e = i.next(); - _fieldNames.push_back( e.fieldName() ); - _fixed.push_back( BSONElement() ); - b.appendNull( "" ); - } - _nullKey = b.obj(); - } - - { - // _nullElt - BSONObjBuilder b; - b.appendNull( "" ); - _nullObj = b.obj(); - _nullElt = _nullObj.firstElement(); - } - - { - // _undefinedElt - BSONObjBuilder b; - b.appendUndefined( "" ); - _undefinedObj = b.obj(); - _undefinedElt = _undefinedObj.firstElement(); - } - - { - // handle plugins - string pluginName = IndexPlugin::findPluginName( keyPattern ); - if ( pluginName.size() ) { - IndexPlugin * plugin = IndexPlugin::get( pluginName ); - - switch (rules) { - case NoPlugins: - uasserted(16735, - str::stream() - << "Attempting to use index type '" << pluginName << "' " - << "where index types are not allowed (1 or -1 only)."); - break; - - case RulesFor22: { - if ( ! plugin ) { - log() << "warning: can't find plugin [" << pluginName << "]" << endl; - } - - if (!IndexPlugin::existedBefore24(pluginName)) { - warning() << "Treating index " << info << " as ascending since " - << "it was created before 2.4 and '" << pluginName << "' " - << "was not a valid type at that time." - << endl; - - plugin = NULL; - } - break; - } - case RulesFor24: - // This assert will be triggered when downgrading from a future version that - // supports an index plugin unsupported by this version. - uassert(16736, str::stream() << "Invalid index type '" << pluginName << "' " - << "in index " << info - , plugin); - break; - } - - if (plugin) - _indexType.reset( plugin->generate( this ) ); - } - } - - _finishedInit = true; - } - - string IndexSpec::toString() const { - stringstream s; - s << "IndexSpec @ " << hex << this << dec << ", " - << "Details @ " << hex << _details << dec << ", " - << "Type: " << getTypeName() << ", " - << "nFields: " << _nFields << ", " - << "KeyPattern: " << keyPattern << ", " - << "Info: " << info; - return s.str(); - } - - void assertParallelArrays( const char *first, const char *second ) { - stringstream ss; - ss << "cannot index parallel arrays [" << first << "] [" << second << "]"; - uasserted( ParallelArraysCode , ss.str() ); - } - - class KeyGeneratorV0 { - public: - KeyGeneratorV0( const IndexSpec &spec ) : _spec( spec ) {} - - void getKeys( const BSONObj &obj, BSONObjSet &keys ) const { - if ( _spec._indexType.get() ) { //plugin (eg geo) - _spec._indexType->getKeys( obj , keys ); - return; - } - vector fieldNames( _spec._fieldNames ); - vector fixed( _spec._fixed ); - _getKeys( fieldNames , fixed , obj, keys ); - if ( keys.empty() && ! _spec._sparse ) - keys.insert( _spec._nullKey ); - } - - private: - void _getKeys( vector fieldNames , vector fixed , const BSONObj &obj, BSONObjSet &keys ) const { - BSONElement arrElt; - unsigned arrIdx = ~0; - int numNotFound = 0; - - for( unsigned i = 0; i < fieldNames.size(); ++i ) { - if ( *fieldNames[ i ] == '\0' ) - continue; - - BSONElement e = obj.getFieldDottedOrArray( fieldNames[ i ] ); - - if ( e.eoo() ) { - e = _spec._nullElt; // no matching field - numNotFound++; - } - - if ( e.type() != Array ) - fieldNames[ i ] = ""; // no matching field or non-array match - - if ( *fieldNames[ i ] == '\0' ) - fixed[ i ] = e; // no need for further object expansion (though array expansion still possible) - - if ( e.type() == Array && arrElt.eoo() ) { // we only expand arrays on a single path -- track the path here - arrIdx = i; - arrElt = e; - } - - // enforce single array path here - if ( e.type() == Array && e.rawdata() != arrElt.rawdata() ) { - assertParallelArrays( e.fieldName(), arrElt.fieldName() ); - } - } - - bool allFound = true; // have we found elements for all field names in the key spec? - for( vector::const_iterator i = fieldNames.begin(); i != fieldNames.end(); ++i ) { - if ( **i != '\0' ) { - allFound = false; - break; - } - } - - if ( _spec._sparse && numNotFound == _spec._nFields ) { - // we didn't find any fields - // so we're not going to index this document - return; - } - - bool insertArrayNull = false; - - if ( allFound ) { - if ( arrElt.eoo() ) { - // no terminal array element to expand - BSONObjBuilder b(_spec._sizeTracker); - for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i ) - b.appendAs( *i, "" ); - keys.insert( b.obj() ); - } - else { - // terminal array element to expand, so generate all keys - BSONObjIterator i( arrElt.embeddedObject() ); - if ( i.more() ) { - while( i.more() ) { - BSONObjBuilder b(_spec._sizeTracker); - for( unsigned j = 0; j < fixed.size(); ++j ) { - if ( j == arrIdx ) - b.appendAs( i.next(), "" ); - else - b.appendAs( fixed[ j ], "" ); - } - keys.insert( b.obj() ); - } - } - else if ( fixed.size() > 1 ) { - insertArrayNull = true; - } - } - } - else { - // nonterminal array element to expand, so recurse - verify( !arrElt.eoo() ); - BSONObjIterator i( arrElt.embeddedObject() ); - if ( i.more() ) { - while( i.more() ) { - BSONElement e = i.next(); - if ( e.type() == Object ) { - _getKeys( fieldNames, fixed, e.embeddedObject(), keys ); - } - } - } - else { - insertArrayNull = true; - } - } - - if ( insertArrayNull ) { - // x : [] - need to insert undefined - BSONObjBuilder b(_spec._sizeTracker); - for( unsigned j = 0; j < fixed.size(); ++j ) { - if ( j == arrIdx ) { - b.appendUndefined( "" ); - } - else { - BSONElement e = fixed[j]; - if ( e.eoo() ) - b.appendNull( "" ); - else - b.appendAs( e , "" ); - } - } - keys.insert( b.obj() ); - } - } - - const IndexSpec &_spec; - }; - - class KeyGeneratorV1 { - public: - KeyGeneratorV1( const IndexSpec &spec ) : _spec( spec ) {} - - void getKeys( const BSONObj &obj, BSONObjSet &keys ) const { - if ( _spec._indexType.get() ) { //plugin (eg geo) - _spec._indexType->getKeys( obj , keys ); - return; - } - vector fieldNames( _spec._fieldNames ); - vector fixed( _spec._fixed ); - _getKeys( fieldNames , fixed , obj, keys ); - if ( keys.empty() && ! _spec._sparse ) - keys.insert( _spec._nullKey ); - } - - private: - /** - * @param arrayNestedArray - set if the returned element is an array nested directly within arr. - */ - BSONElement extractNextElement( const BSONObj &obj, const BSONObj &arr, const char *&field, bool &arrayNestedArray ) const { - string firstField = mongoutils::str::before( field, '.' ); - bool haveObjField = !obj.getField( firstField ).eoo(); - BSONElement arrField = arr.getField( firstField ); - bool haveArrField = !arrField.eoo(); - - // An index component field name cannot exist in both a document array and one of that array's children. - uassert( 15855, - mongoutils::str::stream() << - "Ambiguous field name found in array (do not use numeric field names in " - "embedded elements in an array), field: '" << arrField.fieldName() << - "' for array: " << arr, - !haveObjField || !haveArrField ); - - arrayNestedArray = false; - if ( haveObjField ) { - return obj.getFieldDottedOrArray( field ); - } - else if ( haveArrField ) { - if ( arrField.type() == Array ) { - arrayNestedArray = true; - } - return arr.getFieldDottedOrArray( field ); - } - return BSONElement(); - } - - void _getKeysArrEltFixed( vector &fieldNames , vector &fixed , const BSONElement &arrEntry, BSONObjSet &keys, int numNotFound, const BSONElement &arrObjElt, const set< unsigned > &arrIdxs, bool mayExpandArrayUnembedded ) const { - // set up any terminal array values - for( set::const_iterator j = arrIdxs.begin(); j != arrIdxs.end(); ++j ) { - if ( *fieldNames[ *j ] == '\0' ) { - fixed[ *j ] = mayExpandArrayUnembedded ? arrEntry : arrObjElt; - } - } - // recurse - _getKeys( fieldNames, fixed, ( arrEntry.type() == Object ) ? arrEntry.embeddedObject() : BSONObj(), keys, numNotFound, arrObjElt.embeddedObject() ); - } - - /** - * @param fieldNames - fields to index, may be postfixes in recursive calls - * @param fixed - values that have already been identified for their index fields - * @param obj - object from which keys should be extracted, based on names in fieldNames - * @param keys - set where index keys are written - * @param numNotFound - number of index fields that have already been identified as missing - * @param array - array from which keys should be extracted, based on names in fieldNames - * If obj and array are both nonempty, obj will be one of the elements of array. - */ - void _getKeys( vector fieldNames , vector fixed , const BSONObj &obj, BSONObjSet &keys, int numNotFound = 0, const BSONObj &array = BSONObj() ) const { - BSONElement arrElt; - set arrIdxs; - bool mayExpandArrayUnembedded = true; - for( unsigned i = 0; i < fieldNames.size(); ++i ) { - if ( *fieldNames[ i ] == '\0' ) { - continue; - } - - bool arrayNestedArray; - // Extract element matching fieldName[ i ] from object xor array. - BSONElement e = extractNextElement( obj, array, fieldNames[ i ], arrayNestedArray ); - - if ( e.eoo() ) { - // if field not present, set to null - fixed[ i ] = _spec._nullElt; - // done expanding this field name - fieldNames[ i ] = ""; - numNotFound++; - } - else if ( e.type() == Array ) { - arrIdxs.insert( i ); - if ( arrElt.eoo() ) { - // we only expand arrays on a single path -- track the path here - arrElt = e; - } - else if ( e.rawdata() != arrElt.rawdata() ) { - // enforce single array path here - assertParallelArrays( e.fieldName(), arrElt.fieldName() ); - } - if ( arrayNestedArray ) { - mayExpandArrayUnembedded = false; - } - } - else { - // not an array - no need for further expansion - fixed[ i ] = e; - } - } - - if ( arrElt.eoo() ) { - // No array, so generate a single key. - if ( _spec._sparse && numNotFound == _spec._nFields ) { - return; - } - BSONObjBuilder b(_spec._sizeTracker); - for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i ) { - b.appendAs( *i, "" ); - } - keys.insert( b.obj() ); - } - else if ( arrElt.embeddedObject().firstElement().eoo() ) { - // Empty array, so set matching fields to undefined. - _getKeysArrEltFixed( fieldNames, fixed, _spec._undefinedElt, keys, numNotFound, arrElt, arrIdxs, true ); - } - else { - // Non empty array that can be expanded, so generate a key for each member. - BSONObj arrObj = arrElt.embeddedObject(); - BSONObjIterator i( arrObj ); - while( i.more() ) { - _getKeysArrEltFixed( fieldNames, fixed, i.next(), keys, numNotFound, arrElt, arrIdxs, mayExpandArrayUnembedded ); - } - } - } - - const IndexSpec &_spec; - }; - - void IndexSpec::getKeys( const BSONObj &obj, BSONObjSet &keys ) const { - switch( indexVersion() ) { - case 0: { - KeyGeneratorV0 g( *this ); - g.getKeys( obj, keys ); - break; - } - case 1: { - KeyGeneratorV1 g( *this ); - g.getKeys( obj, keys ); - break; - } - default: - massert( 15869, "Invalid index version for key generation.", false ); - } - } - - - IndexSuitability IndexSpec::suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const { - if ( _indexType.get() ) - return _indexType->suitability( queryConstraints , order ); - return _suitability( queryConstraints , order ); - } - - IndexSuitability IndexSpec::_suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const { - // This is a quick first pass to determine the suitability of the index. It produces some - // false positives (returns HELPFUL for some indexes which are not particularly). When we - // return HELPFUL a more precise determination of utility is done by the query optimizer. - - // check whether any field in the index is constrained at all by the query - BSONForEach( elt, keyPattern ){ - const FieldRange& frange = queryConstraints.range( elt.fieldName() ); - if( ! frange.universal() ) - return HELPFUL; - } - // or whether any field in the desired sort order is in the index - set orderFields; - order.getFieldNames( orderFields ); - BSONForEach( k, keyPattern ) { - if ( orderFields.find( k.fieldName() ) != orderFields.end() ) - return HELPFUL; - } - return USELESS; - } - - IndexSuitability IndexType::suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const { - return _spec->_suitability( queryConstraints , order ); - } - - int IndexSpec::indexVersion() const { - if ( !info.hasField( "v" ) ) { - return DefaultIndexVersionNumber; - } - return IndexDetails::versionForIndexObj( info ); - } - - bool IndexType::scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const { - return ! order.isEmpty(); - } - -} diff --git a/src/mongo/db/indexkey.h b/src/mongo/db/indexkey.h deleted file mode 100644 index 187f2fe48bc..00000000000 --- a/src/mongo/db/indexkey.h +++ /dev/null @@ -1,267 +0,0 @@ -// index_key.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/pch.h" -#include "diskloc.h" -#include "jsobj.h" -#include - -namespace mongo { - - extern const int DefaultIndexVersionNumber; - - const int ParallelArraysCode = 10088; - - class Cursor; - class IndexSpec; - class IndexType; // TODO: this name sucks - class IndexPlugin; - class IndexDetails; - class FieldRangeSet; - - enum IndexSuitability { USELESS = 0 , HELPFUL = 1 , OPTIMAL = 2 }; - - /** - * this represents an instance of a index plugin - * done this way so parsing, etc... can be cached - * so if there is a FTS IndexPlugin, for each index using FTS - * there will be 1 of these, and it can have things pre-parsed, etc... - */ - class IndexType : boost::noncopyable { - public: - IndexType( const IndexPlugin * plugin , const IndexSpec * spec ); - virtual ~IndexType(); - - virtual void getKeys( const BSONObj &obj, BSONObjSet &keys ) const = 0; - - /** - * Returns the element placed in an index key when indexing a field absent from a document. - * By default this is a null BSONElement. - */ - virtual BSONElement missingField() const; - - /* Full semantics of numWanted: - * numWanted == 0 : Return any number of results, but try to return in batches of 101. - * numWanted == 1 : Return exactly one result. - * numWanted > 1 : Return any number of results, but try to return in batches of numWanted. - * - * In practice, your cursor can ignore numWanted, as enforcement of limits is done - * by the caller. - */ - virtual shared_ptr newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const = 0; - - /** optional op : changes query to match what's in the index */ - virtual BSONObj fixKey( const BSONObj& in ) { return in; } - - /** optional op : compare 2 objects with regards to this index */ - virtual int compare( const BSONObj& l , const BSONObj& r ) const; - - /** @return plugin */ - const IndexPlugin * getPlugin() const { return _plugin; } - - const BSONObj& keyPattern() const; - - /* Determines the suitability level of this index for answering a given query. The query is - * represented as a set of constraints given by a FieldRangeSet, and a desired ordering of - * the output. - * - * Note: it is the responsibility of the caller to pass in the correct FieldRangeSet, which - * may depend upon whether this is a single or multi-key index at the time of calling. - */ - virtual IndexSuitability suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const; - - virtual bool scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const ; - - protected: - const IndexPlugin * _plugin; - const IndexSpec * _spec; - }; - - /** - * this represents a plugin - * a plugin could be something like full text search, sparse index, etc... - * 1 of these exists per type of index per server - * 1 IndexType is created per index using this plugin - */ - class IndexPlugin : boost::noncopyable { - public: - IndexPlugin( const string& name ); - virtual ~IndexPlugin() {} - - virtual IndexType* generate( const IndexSpec * spec ) const = 0; - - string getName() const { return _name; } - - /** - * @return new keyPattern - * if nothing changes, should return keyPattern - */ - virtual BSONObj adjustIndexSpec( const BSONObj& spec ) const { return spec; } - - /** - * Hook function to run after an index that uses this plugin is built. - * - * This will be called with an active write context (and lock) on the database. - * - * @param spec The IndexSpec of the newly built index. - */ - virtual void postBuildHook( const IndexSpec& spec ) const { } - - // ------- static below ------- - - static IndexPlugin* get( const string& name ) { - if ( ! _plugins ) - return 0; - map::iterator i = _plugins->find( name ); - if ( i == _plugins->end() ) - return 0; - return i->second; - } - - /** - * @param keyPattern { x : "fts" } - * @return "" or the name - */ - static string findPluginName( const BSONObj& keyPattern ); - - /** - * True if is a regular (non-plugin) index or uses a plugin that existed before 2.4. - * These plugins are grandfathered in and allowed to exist in DBs with - * PDFILE_MINOR_VERSION_22_AND_OLDER - */ - static bool existedBefore24(const string& name) { - return name.empty() - || name == "2d" - || name == "geoHaystack" - || name == "hashed" - ; - } - - private: - string _name; - static map * _plugins; - }; - - /* precomputed details about an index, used for inserting keys on updates - stored/cached in NamespaceDetailsTransient, or can be used standalone - */ - class IndexSpec { - public: - enum PluginRules { - NoPlugins, - RulesFor22, // if !IndexPlugin::existedBefore24() treat as ascending - RulesFor24, // allow new plugins but error if unknown - }; - - BSONObj keyPattern; // e.g., { name : 1 } - BSONObj info; // this is the same as IndexDetails::info.obj() - - IndexSpec() - : _details(0) , _finishedInit(false) { - } - - explicit IndexSpec(const BSONObj& k, const BSONObj& m=BSONObj(), - PluginRules rules=RulesFor24) - : keyPattern(k) , info(m) , _details(0) , _finishedInit(false) { - _init(rules); - } - - /** - this is a DiscLoc of an IndexDetails info - should have a key field - */ - explicit IndexSpec(const DiskLoc& loc, PluginRules rules=RulesFor24) { - reset(loc, rules); - } - - void reset(const BSONObj& info, PluginRules rules=RulesFor24); - void reset(const IndexDetails * details); // determines rules based on pdfile version - void reset(const DiskLoc& infoLoc, PluginRules rules=RulesFor24) { - reset(infoLoc.obj(), rules); - } - - void getKeys( const BSONObj &obj, BSONObjSet &keys ) const; - - /** - * Returns the element placed in an index key when indexing a field absent from a document. - * By default this is a null BSONElement. - */ - BSONElement missingField() const { - if ( _indexType.get() ) - return _indexType->missingField(); - return _nullElt; - } - - string getTypeName() const { - if ( _indexType.get() ) - return _indexType->getPlugin()->getName(); - return ""; - } - - IndexType* getType() const { - return _indexType.get(); - } - - const IndexDetails * getDetails() const { - return _details; - } - - IndexSuitability suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const ; - - bool isSparse() const { return _sparse; } - - string toString() const; - - protected: - - int indexVersion() const; - - IndexSuitability _suitability( const FieldRangeSet& queryConstraints , - const BSONObj& order ) const ; - - BSONSizeTracker _sizeTracker; - vector _fieldNames; - vector _fixed; - - BSONObj _nullKey; // a full key with all fields null - BSONObj _nullObj; // only used for _nullElt - BSONElement _nullElt; // jstNull - - BSONObj _undefinedObj; // only used for _undefinedElt - BSONElement _undefinedElt; // undefined - - int _nFields; // number of fields in the index - bool _sparse; // if the index is sparse - shared_ptr _indexType; - const IndexDetails * _details; - - void _init(PluginRules rules); - - friend class IndexType; - friend class KeyGeneratorV0; - friend class KeyGeneratorV1; - public: - bool _finishedInit; - }; - - -} // namespace mongo diff --git a/src/mongo/db/initialize_server_global_state.cpp b/src/mongo/db/initialize_server_global_state.cpp index f439c9e8a79..df9f77fdb2e 100644 --- a/src/mongo/db/initialize_server_global_state.cpp +++ b/src/mongo/db/initialize_server_global_state.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -19,17 +31,32 @@ #include "mongo/db/initialize_server_global_state.h" #include +#include #ifndef _WIN32 +#include #include #include #endif +#include "mongo/base/init.h" +#include "mongo/client/sasl_client_authenticate.h" #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/auth/security_key.h" -#include "mongo/db/cmdline.h" +#include "mongo/logger/logger.h" +#include "mongo/logger/console_appender.h" +#include "mongo/logger/message_event.h" +#include "mongo/logger/message_event_utf8_encoder.h" +#include "mongo/logger/ramlog.h" +#include "mongo/logger/rotatable_file_appender.h" +#include "mongo/logger/rotatable_file_manager.h" +#include "mongo/logger/rotatable_file_writer.h" +#include "mongo/logger/syslog_appender.h" +#include "mongo/platform/process_id.h" #include "mongo/util/log.h" #include "mongo/util/net/listen.h" +#include "mongo/util/net/ssl_manager.h" #include "mongo/util/processinfo.h" namespace fs = boost::filesystem; @@ -40,9 +67,9 @@ namespace mongo { // support for exit value propagation with fork void launchSignal( int sig ) { if ( sig == SIGUSR2 ) { - pid_t cur = getpid(); + ProcessId cur = ProcessId::getCurrent(); - if ( cur == cmdLine.parentProc || cur == cmdLine.leaderProc ) { + if (cur == serverGlobalParams.parentProc || cur == serverGlobalParams.leaderProc) { // signal indicates successful start allowing us to exit _exit(0); } @@ -53,31 +80,25 @@ namespace mongo { verify( signal(SIGUSR2 , launchSignal ) != SIG_ERR ); } - void CmdLine::launchOk() { - if ( cmdLine.doFork ) { + void signalForkSuccess() { + if (serverGlobalParams.doFork) { // killing leader will propagate to parent - verify( kill( cmdLine.leaderProc, SIGUSR2 ) == 0 ); + verify(kill(serverGlobalParams.leaderProc.toNative(), SIGUSR2) == 0); } } #endif - bool initializeServerGlobalState(bool isMongodShutdownSpecialCase) { - - Listener::globalTicketHolder.resize( cmdLine.maxConns ); + static bool forkServer() { #ifndef _WIN32 - if (!fs::is_directory(cmdLine.socket)) { - cout << cmdLine.socket << " must be a directory" << endl; - return false; - } - - if (cmdLine.doFork) { - fassert(16447, !cmdLine.logpath.empty() || cmdLine.logWithSyslog); + if (serverGlobalParams.doFork) { + fassert(16447, !serverGlobalParams.logpath.empty() || + serverGlobalParams.logWithSyslog); cout.flush(); cerr.flush(); - cmdLine.parentProc = getpid(); + serverGlobalParams.parentProc = ProcessId::getCurrent(); // facilitate clean exit when child starts successfully setupLaunchSignals(); @@ -116,7 +137,7 @@ namespace mongo { } setsid(); - cmdLine.leaderProc = getpid(); + serverGlobalParams.leaderProc = ProcessId::getCurrent(); pid_t child2 = fork(); if (child2 == -1) { @@ -138,14 +159,13 @@ namespace mongo { // this is run in the final child process (the server) - // stdout handled in initLogging - //fclose(stdout); - //freopen("/dev/null", "w", stdout); - - fclose(stderr); - fclose(stdin); + FILE* f = freopen("/dev/null", "w", stdout); + if ( f == NULL ) { + cout << "Cant reassign stdout while forking server process: " << strerror(errno) << endl; + return false; + } - FILE* f = freopen("/dev/null", "w", stderr); + f = freopen("/dev/null", "w", stderr); if ( f == NULL ) { cout << "Cant reassign stderr while forking server process: " << strerror(errno) << endl; return false; @@ -157,47 +177,193 @@ namespace mongo { return false; } } +#endif // !defined(_WIN32) + return true; + } + + void forkServerOrDie() { + if (!forkServer()) + _exit(EXIT_FAILURE); + } + + MONGO_INITIALIZER_GENERAL(ServerLogRedirection, + ("GlobalLogManager", "EndStartupOptionHandling", "ForkServer"), + ("default"))( + InitializerContext*) { + + using logger::LogManager; + using logger::MessageEventEphemeral; + using logger::MessageEventDetailsEncoder; + using logger::MessageEventWithContextEncoder; + using logger::MessageLogDomain; + using logger::RotatableFileAppender; + using logger::StatusWithRotatableFileWriter; + + if (serverGlobalParams.logWithSyslog) { +#ifdef _WIN32 + return Status(ErrorCodes::InternalError, + "Syslog requested in Windows build; command line processor logic error"); +#else + using logger::SyslogAppender; - if (cmdLine.logWithSyslog) { StringBuilder sb; - sb << cmdLine.binaryName << "." << cmdLine.port; - Logstream::useSyslog( sb.str().c_str() ); + sb << serverGlobalParams.binaryName << "." << serverGlobalParams.port; + openlog(strdup(sb.str().c_str()), + LOG_PID | LOG_CONS, + serverGlobalParams.syslogFacility); + LogManager* manager = logger::globalLogManager(); + manager->getGlobalDomain()->clearAppenders(); + manager->getGlobalDomain()->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new SyslogAppender( + new logger::MessageEventWithContextEncoder))); + manager->getNamedDomain("javascriptOutput")->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new SyslogAppender( + new logger::MessageEventWithContextEncoder))); +#endif // defined(_WIN32) } -#endif - if (!cmdLine.logpath.empty() && !isMongodShutdownSpecialCase) { - fassert(16448, !cmdLine.logWithSyslog); - string absoluteLogpath = boost::filesystem::absolute( - cmdLine.logpath, cmdLine.cwd).string(); - if (!initLogging(absoluteLogpath, cmdLine.logAppend)) { - cout << "Bad logpath value: \"" << absoluteLogpath << "\"; terminating." << endl; - return false; + else if (!serverGlobalParams.logpath.empty()) { + fassert(16448, !serverGlobalParams.logWithSyslog); + std::string absoluteLogpath = boost::filesystem::absolute( + serverGlobalParams.logpath, serverGlobalParams.cwd).string(); + + bool exists; + + try{ + exists = boost::filesystem::exists(absoluteLogpath); + } catch(boost::filesystem::filesystem_error& e) { + return Status(ErrorCodes::FileNotOpen, mongoutils::str::stream() << + "Failed probe for \"" << absoluteLogpath << "\": " << + e.code().message()); + } + + if (exists) { + if (boost::filesystem::is_directory(absoluteLogpath)) { + return Status(ErrorCodes::FileNotOpen, mongoutils::str::stream() << + "logpath \"" << absoluteLogpath << + "\" should name a file, not a directory."); + } + + if (!serverGlobalParams.logAppend && + boost::filesystem::is_regular(absoluteLogpath)) { + std::string renameTarget = absoluteLogpath + "." + terseCurrentTime(false); + if (0 == rename(absoluteLogpath.c_str(), renameTarget.c_str())) { + log() << "log file \"" << absoluteLogpath + << "\" exists; moved to \"" << renameTarget << "\"."; + } + else { + return Status(ErrorCodes::FileRenameFailed, mongoutils::str::stream() << + "Could not rename preexisting log file \"" << + absoluteLogpath << "\" to \"" << renameTarget << + "\"; run with --logappend or manually remove file: " << + errnoWithDescription()); + } + } } + + StatusWithRotatableFileWriter writer = + logger::globalRotatableFileManager()->openFile(absoluteLogpath, + serverGlobalParams.logAppend); + if (!writer.isOK()) { + return writer.getStatus(); + } + + LogManager* manager = logger::globalLogManager(); + manager->getGlobalDomain()->clearAppenders(); + manager->getGlobalDomain()->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new RotatableFileAppender( + new MessageEventDetailsEncoder, writer.getValue()))); + manager->getNamedDomain("javascriptOutput")->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new RotatableFileAppender( + new MessageEventDetailsEncoder, writer.getValue()))); + + if (serverGlobalParams.logAppend && exists) { + log() << "***** SERVER RESTARTED *****" << endl; + Status status = + logger::RotatableFileWriter::Use(writer.getValue()).status(); + if (!status.isOK()) + return status; + } + } + else { + logger::globalLogManager()->getNamedDomain("javascriptOutput")->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new logger::ConsoleAppender( + new MessageEventDetailsEncoder))); } - if (!cmdLine.pidFile.empty()) { - writePidFile(cmdLine.pidFile); + logger::globalLogDomain()->attachAppender( + logger::MessageLogDomain::AppenderAutoPtr( + new RamLogAppender(RamLog::get("global")))); + + return Status::OK(); + } + + /** + * atexit handler to terminate the process before static destructors run. + * + * Mongo server processes cannot safely call ::exit() or std::exit(), but + * some third-party libraries may call one of those functions. In that + * case, to avoid static-destructor problems in the server, this exits the + * process immediately with code EXIT_FAILURE. + * + * TODO: Remove once exit() executes safely in mongo server processes. + */ + static void shortCircuitExit() { _exit(EXIT_FAILURE); } + + MONGO_INITIALIZER(RegisterShortCircuitExitHandler)(InitializerContext*) { + if (std::atexit(&shortCircuitExit) != 0) + return Status(ErrorCodes::InternalError, "Failed setting short-circuit exit handler."); + return Status::OK(); + } + + bool initializeServerGlobalState() { + + Listener::globalTicketHolder.resize(serverGlobalParams.maxConns); + +#ifndef _WIN32 + if (!fs::is_directory(serverGlobalParams.socket)) { + cout << serverGlobalParams.socket << " must be a directory" << endl; + return false; } +#endif - if (!cmdLine.keyFile.empty()) { + if (!serverGlobalParams.pidFile.empty()) { + if (!writePidFile(serverGlobalParams.pidFile)) { + // error message logged in writePidFile + return false; + } + } - if (!setUpSecurityKey(cmdLine.keyFile)) { + int clusterAuthMode = serverGlobalParams.clusterAuthMode.load(); + if (!serverGlobalParams.keyFile.empty() && + clusterAuthMode != ServerGlobalParams::ClusterAuthMode_x509) { + if (!setUpSecurityKey(serverGlobalParams.keyFile)) { // error message printed in setUpPrivateKey return false; } - - noauth = false; } - return true; - } + // Auto-enable auth except if clusterAuthMode is not set. + // clusterAuthMode is automatically set if a --keyFile parameter is provided. + if (clusterAuthMode != ServerGlobalParams::ClusterAuthMode_undefined) { + getGlobalAuthorizationManager()->setAuthEnabled(true); + } - static void ignoreSignal( int sig ) {} +#ifdef MONGO_SSL - void setupCoreSignals() { -#if !defined(_WIN32) - verify( signal(SIGHUP , ignoreSignal ) != SIG_ERR ); - verify( signal(SIGUSR2, ignoreSignal ) != SIG_ERR ); + if (clusterAuthMode == ServerGlobalParams::ClusterAuthMode_x509 || + clusterAuthMode == ServerGlobalParams::ClusterAuthMode_sendX509) { + setInternalUserAuthParams(BSON(saslCommandMechanismFieldName << "MONGODB-X509" << + saslCommandUserDBFieldName << "$external" << + saslCommandUserFieldName << + getSSLManager()->getClientSubjectName())); + } #endif + return true; } } // namespace mongo diff --git a/src/mongo/db/initialize_server_global_state.h b/src/mongo/db/initialize_server_global_state.h index 4ef164ba99b..7085274d7af 100644 --- a/src/mongo/db/initialize_server_global_state.h +++ b/src/mongo/db/initialize_server_global_state.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -22,14 +34,21 @@ namespace mongo { * Perform initialization activity common across all mongo server types. * * Set up logging, daemonize the process, configure SSL, etc. + */ + bool initializeServerGlobalState(); + + /** + * Forks and detaches the server, on platforms that support it, if serverGlobalParams.doFork is + * true. * - * If isMongodShutdownSpecialCase, perform this processing knowing that - * we're only bringing this process up to kill another mongod. - * - * TODO: Untie the knot that requires the isMongodShutdownSpecialCase parameter. + * Call after processing the command line but before running mongo initializers. */ - bool initializeServerGlobalState(bool isMongodShutdownSpecialCase = false); + void forkServerOrDie(); - void setupCoreSignals(); + /** + * Notify the parent that we forked from that we have successfully completed basic + * initialization so it can stop waiting and exit. + */ + void signalForkSuccess(); } // namespace mongo diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp index 926cdeecdb6..40fe6d07c33 100644 --- a/src/mongo/db/instance.cpp +++ b/src/mongo/db/instance.cpp @@ -14,53 +14,80 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" +#include #include #include -#include #if defined(_WIN32) #include #else #include #endif -#include "mongo/util/time_support.h" #include "mongo/base/status.h" #include "mongo/bson/util/atomic_int.h" +#include "mongo/db/audit.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/background.h" -#include "mongo/db/cmdline.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/commands/fsync.h" #include "mongo/db/d_concurrency.h" #include "mongo/db/db.h" +#include "mongo/db/dbhelpers.h" #include "mongo/db/dbmessage.h" #include "mongo/db/dur_commitjob.h" #include "mongo/db/dur_journal.h" #include "mongo/db/dur_recover.h" #include "mongo/db/instance.h" #include "mongo/db/introspect.h" +#include "mongo/db/jsobjmanipulator.h" #include "mongo/db/json.h" #include "mongo/db/kill_current_op.h" #include "mongo/db/lasterror.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/matcher.h" +#include "mongo/db/mongod_options.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/ops/count.h" -#include "mongo/db/ops/delete.h" -#include "mongo/db/ops/query.h" -#include "mongo/db/ops/update.h" +#include "mongo/db/ops/delete_executor.h" +#include "mongo/db/ops/delete_request.h" +#include "mongo/db/ops/insert.h" +#include "mongo/db/ops/update_lifecycle_impl.h" +#include "mongo/db/ops/update_driver.h" +#include "mongo/db/ops/update_executor.h" +#include "mongo/db/ops/update_request.h" #include "mongo/db/pagefault.h" -#include "mongo/db/repl.h" -#include "mongo/db/replutil.h" +#include "mongo/db/query/new_find.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" #include "mongo/db/stats/counters.h" +#include "mongo/db/storage_options.h" +#include "mongo/platform/process_id.h" #include "mongo/s/d_logic.h" #include "mongo/s/stale_exception.h" // for SendStaleConfigException +#include "mongo/scripting/engine.h" #include "mongo/util/fail_point_service.h" #include "mongo/util/file_allocator.h" +#include "mongo/util/gcov.h" #include "mongo/util/goodies.h" #include "mongo/util/mongoutils/str.h" +#include "mongo/util/time_support.h" namespace mongo { @@ -79,8 +106,6 @@ namespace mongo { string dbExecCommand; - bool useHints = true; - KillCurrentOp killCurrentOp; int lockFile = 0; @@ -90,41 +115,6 @@ namespace mongo { MONGO_FP_DECLARE(rsStopGetMore); - /*static*/ OpTime OpTime::_now() { - OpTime result; - unsigned t = (unsigned) time(0); - if ( last.secs == t ) { - last.i++; - result = last; - } - else if ( t < last.secs ) { - result = skewed(); // separate function to keep out of the hot code path - } - else { - last = OpTime(t, 1); - result = last; - } - notifier.notify_all(); - return last; - } - OpTime OpTime::now(const mongo::mutex::scoped_lock&) { - return _now(); - } - OpTime OpTime::getLast(const mongo::mutex::scoped_lock&) { - return last; - } - boost::condition OpTime::notifier; - mongo::mutex OpTime::m("optime"); - - // OpTime::now() uses mutex, thus it is in this file not in the cpp files used by drivers and such - void BSONElementManipulator::initTimestamp() { - massert( 10332 , "Expected CurrentTime type", _element.type() == Timestamp ); - unsigned long long ×tamp = *( reinterpret_cast< unsigned long long* >( value() ) ); - if ( timestamp == 0 ) { - mutex::scoped_lock lk(OpTime::m); - timestamp = OpTime::now(lk).asDate(); - } - } void BSONElementManipulator::SetNumber(double d) { if ( _element.type() == NumberDouble ) *getDur().writing( reinterpret_cast< double * >( value() ) ) = d; @@ -140,34 +130,41 @@ namespace mongo { verify( _element.type() == NumberInt ); getDur().writingInt( *reinterpret_cast< int * >( value() ) ) = n; } - /* dur:: version */ - void BSONElementManipulator::ReplaceTypeAndValue( const BSONElement &e ) { - char *d = data(); - char *v = value(); - int valsize = e.valuesize(); - int ofs = (int) (v-d); - dassert( ofs > 0 ); - char *p = (char *) getDur().writingPtr(d, valsize + ofs); - *p = e.type(); - memcpy( p + ofs, e.value(), valsize ); - } void inProgCmd( Message &m, DbResponse &dbresponse ) { + DbMessage d(m); + QueryMessage q(d); BSONObjBuilder b; - if (!cc().getAuthorizationManager()->checkAuthorization( - AuthorizationManager::SERVER_RESOURCE_NAME, ActionType::inprog)) { + const bool isAuthorized = cc().getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::inprog); + + audit::logInProgAuthzCheck( + &cc(), q.query, isAuthorized ? ErrorCodes::OK : ErrorCodes::Unauthorized); + + if (!isAuthorized) { b.append("err", "unauthorized"); } else { - DbMessage d(m); - QueryMessage q(d); bool all = q.query["$all"].trueValue(); vector vals; { + BSONObj filter; + { + BSONObjBuilder b; + BSONObjIterator i( q.query ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( str::equals( "$all", e.fieldName() ) ) + continue; + b.append( e ); + } + filter = b.obj(); + } + Client& me = cc(); scoped_lock bl(Client::clientsMutex); - scoped_ptr m(new Matcher(q.query)); + scoped_ptr m(new Matcher(filter)); for( set::iterator i = Client::clients.begin(); i != Client::clients.end(); i++ ) { Client *c = *i; verify( c ); @@ -195,17 +192,21 @@ namespace mongo { } void killOp( Message &m, DbResponse &dbresponse ) { + DbMessage d(m); + QueryMessage q(d); BSONObj obj; - if (!cc().getAuthorizationManager()->checkAuthorization( - AuthorizationManager::SERVER_RESOURCE_NAME, ActionType::killop)) { + const bool isAuthorized = cc().getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::killop); + audit::logKillOpAuthzCheck(&cc(), + q.query, + isAuthorized ? ErrorCodes::OK : ErrorCodes::Unauthorized); + if (!isAuthorized) { obj = fromjson("{\"err\":\"unauthorized\"}"); } /*else if( !dbMutexInfo.isLocked() ) obj = fromjson("{\"info\":\"no op in progress/not locked\"}"); */ else { - DbMessage d(m); - QueryMessage q(d); BSONElement e = q.query.getField("op"); if( !e.isNumber() ) { obj = fromjson("{\"err\":\"no op number field specified?\"}"); @@ -222,8 +223,11 @@ namespace mongo { bool _unlockFsync(); void unlockFsync(const char *ns, Message& m, DbResponse &dbresponse) { BSONObj obj; - if (!cc().getAuthorizationManager()->checkAuthorization( - AuthorizationManager::SERVER_RESOURCE_NAME, ActionType::unlock)) { + const bool isAuthorized = cc().getAuthorizationSession()->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::unlock); + audit::logFsyncUnlockAuthzCheck( + &cc(), isAuthorized ? ErrorCodes::OK : ErrorCodes::Unauthorized); + if (!isAuthorized) { obj = fromjson("{\"err\":\"unauthorized\"}"); } else if (strncmp(ns, "admin.", 6) != 0 ) { @@ -254,12 +258,15 @@ namespace mongo { shared_ptr ex; try { - if (!NamespaceString(d.getns()).isCommand()) { + NamespaceString ns(d.getns()); + if (!ns.isCommand()) { // Auth checking for Commands happens later. - Status status = cc().getAuthorizationManager()->checkAuthForQuery(d.getns()); - uassert(16550, status.reason(), status.isOK()); + Client* client = &cc(); + Status status = client->getAuthorizationSession()->checkAuthForQuery(ns, q.query); + audit::logQueryAuthzCheck(client, ns, q.query, status.code()); + uassertStatusOK(status); } - dbresponse.exhaustNS = runQuery(m, q, op, *resp); + dbresponse.exhaustNS = newRunQuery(m, q, op, *resp); verify( !resp->empty() ); } catch ( SendStaleConfigException& e ){ @@ -324,12 +331,13 @@ namespace mongo { return ok; } + // Mongod on win32 defines a value for this function. In all other executables it is NULL. void (*reportEventToSystem)(const char *msg) = 0; - void mongoAbort(const char *msg) { - if( reportEventToSystem ) + void mongoAbort(const char *msg) { + if( reportEventToSystem ) reportEventToSystem(msg); - rawOut(msg); + severe() << msg; ::abort(); } @@ -339,10 +347,19 @@ namespace mongo { // before we lock... int op = m.operation(); bool isCommand = false; - const char *ns = m.singleData()->_data + 4; + + DbMessage dbmsg(m); + + Client& c = cc(); + if (!c.isGod()) + c.getAuthorizationSession()->startRequest(); + + c.setIsWriteCmd(false); if ( op == dbQuery ) { - if( strstr(ns, ".$cmd") ) { + const char *ns = dbmsg.getns(); + + if (strstr(ns, ".$cmd")) { isCommand = true; opwrite(m); if( strstr(ns, ".$cmd.sys.") ) { @@ -371,10 +388,31 @@ namespace mongo { opwrite(m); } - globalOpCounters.gotOp( op , isCommand ); - - Client& c = cc(); - c.getAuthorizationManager()->startRequest(); + // Increment op counters. + switch (op) { + case dbQuery: + if (!isCommand) { + globalOpCounters.gotQuery(); + } + else { + // Command counting is deferred, since it is not known yet whether the command + // needs counting. + } + break; + case dbGetMore: + globalOpCounters.gotGetMore(); + break; + case dbInsert: + // Insert counting is deferred, since it is not known yet whether the insert contains + // multiple documents (each of which needs to be counted). + break; + case dbUpdate: + globalOpCounters.gotUpdate(); + break; + case dbDelete: + globalOpCounters.gotDelete(); + break; + } auto_ptr nestedOp; CurOp* currentOpP = c.curop(); @@ -392,8 +430,8 @@ namespace mongo { OpDebug& debug = currentOp.debug(); debug.op = op; - long long logThreshold = cmdLine.slowMS; - bool shouldLog = logLevel >= 1; + long long logThreshold = serverGlobalParams.slowMS; + bool shouldLog = logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1)); if ( op == dbQuery ) { if ( handlePossibleShardedMessage( m , &dbresponse ) ) @@ -406,7 +444,8 @@ namespace mongo { } else if ( op == dbMsg ) { // deprecated - replaced by commands - char *p = m.singleData()->_data; + const char *p = dbmsg.getns(); + int len = strlen(p); if ( len > 400 ) out() << curTimeMillis64() % 10000 << @@ -423,8 +462,6 @@ namespace mongo { } else { try { - const NamespaceString nsString( ns ); - // The following operations all require authorization. // dbInsert, dbUpdate and dbDelete can be easily pre-authorized, // here, but dbKillCursors cannot. @@ -433,32 +470,40 @@ namespace mongo { logThreshold = 10; receivedKillCursors(m); } - else if ( !nsString.isValid() ) { - // Only killCursors doesn't care about namespaces - uassert( 16257, str::stream() << "Invalid ns [" << ns << "]", false ); - } - else if ( op == dbInsert ) { - receivedInsert(m, currentOp); - } - else if ( op == dbUpdate ) { - receivedUpdate(m, currentOp); - } - else if ( op == dbDelete ) { - receivedDelete(m, currentOp); - } - else { + else if (op != dbInsert && op != dbUpdate && op != dbDelete) { mongo::log() << " operation isn't supported: " << op << endl; currentOp.done(); shouldLog = true; } - } - catch ( UserException& ue ) { - tlog(3) << " Caught Assertion in " << opToString(op) << ", continuing " - << ue.toString() << endl; + else { + const char* ns = dbmsg.getns(); + const NamespaceString nsString(ns); + + if (!nsString.isValid()) { + uassert(16257, str::stream() << "Invalid ns [" << ns << "]", false); + } + else if (op == dbInsert) { + receivedInsert(m, currentOp); + } + else if (op == dbUpdate) { + receivedUpdate(m, currentOp); + } + else if (op == dbDelete) { + receivedDelete(m, currentOp); + } + else { + invariant(false); + } + } + } + catch (const UserException& ue) { + setLastError(ue.getCode(), ue.getInfo().msg.c_str()); + LOG(3) << " Caught Assertion in " << opToString(op) << ", continuing " + << ue.toString() << endl; debug.exceptionInfo = ue.getInfo(); } catch ( AssertionException& e ) { - tlog(3) << " Caught Assertion in " << opToString(op) << ", continuing " + MONGO_TLOG(3) << " Caught Assertion in " << opToString(op) << ", continuing " << e.toString() << endl; debug.exceptionInfo = e.getInfo(); shouldLog = true; @@ -471,7 +516,7 @@ namespace mongo { logThreshold += currentOp.getExpectedLatencyMs(); if ( shouldLog || debug.executionTime > logThreshold ) { - mongo::tlog() << debug.report( currentOp ) << endl; + MONGO_TLOG(0) << debug.report( currentOp ) << endl; } if ( currentOp.shouldDBProfile( debug.executionTime ) ) { @@ -492,22 +537,23 @@ namespace mongo { } /* assembleResponse() */ void receivedKillCursors(Message& m) { - int *x = (int *) m.singleData()->_data; - x++; // reserved - int n = *x++; + DbMessage dbmessage(m); + int n = dbmessage.pullInt(); uassert( 13659 , "sent 0 cursors to kill" , n != 0 ); massert( 13658 , str::stream() << "bad kill cursors size: " << m.dataSize() , m.dataSize() == 8 + ( 8 * n ) ); uassert( 13004 , str::stream() << "sent negative cursors to kill: " << n , n >= 1 ); if ( n > 2000 ) { - LOG( n < 30000 ? LL_WARNING : LL_ERROR ) << "receivedKillCursors, n=" << n << endl; + ( n < 30000 ? warning() : error() ) << "receivedKillCursors, n=" << n << endl; verify( n < 30000 ); } - int found = ClientCursor::eraseIfAuthorized(n, (long long *) x); + const long long* cursorArray = dbmessage.getArray(n); - if ( logLevel > 0 || found != n ) { + int found = CollectionCursorCache::eraseCursorGlobalIfAuthorized(n, cursorArray); + + if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1)) || found != n ) { LOG( found == n ? 1 : 0 ) << "killcursors: found " << found << " of " << n << endl; } @@ -516,14 +562,14 @@ namespace mongo { /* db - database name path - db directory */ - /*static*/ void Database::closeDatabase( const char *db, const string& path ) { + /*static*/ void Database::closeDatabase( const string& db, const string& path ) { verify( Lock::isW() ); Client::Context * ctx = cc().getContext(); verify( ctx ); verify( ctx->inDB( db , path ) ); Database *database = ctx->db(); - verify( database->name == db ); + verify( database->name() == db ); oplogCheckCloseDatabase( database ); // oplog caches some things, dirty its caches @@ -534,9 +580,6 @@ namespace mongo { /* important: kill all open cursors on the database */ string prefix(db); prefix += '.'; - ClientCursor::invalidate(prefix.c_str()); - - NamespaceDetailsTransient::eraseDB( prefix ); dbHolderW().erase( db, path ); ctx->_clear(); @@ -545,8 +588,9 @@ namespace mongo { void receivedUpdate(Message& m, CurOp& op) { DbMessage d(m); - const char *ns = d.getns(); - op.debug().ns = ns; + NamespaceString ns(d.getns()); + uassertStatusOK( userAllowedWriteNS( ns ) ); + op.debug().ns = ns.ns(); int flags = d.pullInt(); BSONObj query = d.nextJsObj(); @@ -560,90 +604,96 @@ namespace mongo { bool multi = flags & UpdateOption_Multi; bool broadcast = flags & UpdateOption_Broadcast; - Status status = cc().getAuthorizationManager()->checkAuthForUpdate(ns, upsert); - uassert(16538, status.reason(), status.isOK()); + Status status = cc().getAuthorizationSession()->checkAuthForUpdate(ns, + query, + toupdate, + upsert); + audit::logUpdateAuthzCheck(&cc(), ns, query, toupdate, upsert, multi, status.code()); + uassertStatusOK(status); op.debug().query = query; op.setQuery(query); - PageFaultRetryableSection s; - while ( 1 ) { - try { - Lock::DBWrite lk(ns); - - // void ReplSetImpl::relinquish() uses big write lock so - // this is thus synchronized given our lock above. - uassert( 10054 , "not master", isMasterNs( ns ) ); - - // if this ever moves to outside of lock, need to adjust check Client::Context::_finishInit - if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) ) - return; - - Client::Context ctx( ns ); - - UpdateResult res = updateObjects(ns, toupdate, query, upsert, multi, true, op.debug() ); - lastError.getSafe()->recordUpdate( res.existing , res.num , res.upserted ); // for getlasterror - break; - } - catch ( PageFaultException& e ) { - e.touch(); - } - } + UpdateRequest request(ns); + + request.setUpsert(upsert); + request.setMulti(multi); + request.setQuery(query); + request.setUpdates(toupdate); + request.setUpdateOpLog(); // TODO: This is wasteful if repl is not active. + UpdateLifecycleImpl updateLifecycle(broadcast, ns); + request.setLifecycle(&updateLifecycle); + UpdateExecutor executor(&request, &op.debug()); + uassertStatusOK(executor.prepare()); + + Lock::DBWrite lk(ns.ns()); + + // if this ever moves to outside of lock, need to adjust check + // Client::Context::_finishInit + if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) ) + return; + + Client::Context ctx( ns ); + + UpdateResult res = executor.execute(); + + // for getlasterror + lastError.getSafe()->recordUpdate( res.existing , res.numMatched , res.upserted ); } void receivedDelete(Message& m, CurOp& op) { DbMessage d(m); - const char *ns = d.getns(); - - Status status = cc().getAuthorizationManager()->checkAuthForDelete(ns); - uassert(16542, status.reason(), status.isOK()); + NamespaceString ns(d.getns()); + uassertStatusOK( userAllowedWriteNS( ns ) ); - op.debug().ns = ns; + op.debug().ns = ns.ns(); int flags = d.pullInt(); bool justOne = flags & RemoveOption_JustOne; bool broadcast = flags & RemoveOption_Broadcast; verify( d.moreJSObjs() ); BSONObj pattern = d.nextJsObj(); - + + Status status = cc().getAuthorizationSession()->checkAuthForDelete(ns, pattern); + audit::logDeleteAuthzCheck(&cc(), ns, pattern, status.code()); + uassertStatusOK(status); + op.debug().query = pattern; op.setQuery(pattern); - PageFaultRetryableSection s; - while ( 1 ) { - try { - Lock::DBWrite lk(ns); - - // writelock is used to synchronize stepdowns w/ writes - uassert( 10056 , "not master", isMasterNs( ns ) ); - - // if this ever moves to outside of lock, need to adjust check Client::Context::_finishInit - if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) ) - return; - - Client::Context ctx(ns); - - long long n = deleteObjects(ns, pattern, justOne, true); - lastError.getSafe()->recordDelete( n ); - op.debug().ndeleted = n; - break; - } - catch ( PageFaultException& e ) { - LOG(2) << "recordDelete got a PageFaultException" << endl; - e.touch(); + { + PageFaultRetryableSection s; + while ( 1 ) { + try { + DeleteRequest request(ns); + request.setQuery(pattern); + request.setMulti(!justOne); + request.setUpdateOpLog(true); + DeleteExecutor executor(&request); + uassertStatusOK(executor.prepare()); + Lock::DBWrite lk(ns.ns()); + + // if this ever moves to outside of lock, need to adjust check + // Client::Context::_finishInit + if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) ) + return; + + Client::Context ctx(ns); + + long long n = executor.execute(); + lastError.getSafe()->recordDelete( n ); + op.debug().ndeleted = n; + break; + } + catch ( PageFaultException& e ) { + LOG(2) << "recordDelete got a PageFaultException" << endl; + e.touch(); + } } - } + } // end PageFaultRetryableSection } QueryResult* emptyMoreResult(long long); - void OpTime::waitForDifferent(unsigned millis){ - mutex::scoped_lock lk(m); - while (*this == last) { - if (!notifier.timed_wait(lk.boost(), boost::posix_time::milliseconds(millis))) - return; // timed out - } - } - bool receivedGetMore(DbResponse& dbresponse, Message& m, CurOp& curop ) { bool ok = true; @@ -669,8 +719,10 @@ namespace mongo { const NamespaceString nsString( ns ); uassert( 16258, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid() ); - Status status = cc().getAuthorizationManager()->checkAuthForGetMore(ns); - uassert(16543, status.reason(), status.isOK()); + Status status = cc().getAuthorizationSession()->checkAuthForGetMore( + nsString, cursorid); + audit::logGetMoreAuthzCheck(&cc(), nsString, cursorid, status.code()); + uassertStatusOK(status); if (str::startsWith(ns, "local.oplog.")){ while (MONGO_FAIL_POINT(rsStopGetMore)) { @@ -686,13 +738,13 @@ namespace mongo { } } - msgdata = processGetMore(ns, - ntoreturn, - cursorid, - curop, - pass, - exhaust, - &isCursorAuthorized); + msgdata = newGetMore(ns, + ntoreturn, + cursorid, + curop, + pass, + exhaust, + &isCursorAuthorized); } catch ( AssertionException& e ) { if ( isCursorAuthorized ) { @@ -701,7 +753,7 @@ namespace mongo { // because it may now be out of sync with the client's iteration state. // SERVER-7952 // TODO Temporary code, see SERVER-4563 for a cleanup overview. - ClientCursor::erase( cursorid ); + CollectionCursorCache::eraseCursorGlobal( cursorid ); } ex.reset( new AssertionException( e.getInfo().msg, e.getCode() ) ); ok = false; @@ -738,24 +790,16 @@ namespace mongo { }; if (ex) { - exhaust = false; - BSONObjBuilder err; ex->getInfo().append( err ); BSONObj errObj = err.done(); - log() << errObj << endl; - curop.debug().exceptionInfo = ex->getInfo(); - if (ex->getCode() == 13436) { - replyToQuery(ResultFlag_ErrSet, m, dbresponse, errObj); - curop.debug().responseLength = dbresponse.response->header()->dataLen(); - curop.debug().nreturned = 1; - return ok; - } - - msgdata = emptyMoreResult(cursorid); + replyToQuery(ResultFlag_ErrSet, m, dbresponse, errObj); + curop.debug().responseLength = dbresponse.response->header()->dataLen(); + curop.debug().nreturned = 1; + return ok; } Message *resp = new Message(); @@ -774,41 +818,57 @@ namespace mongo { return ok; } - void checkAndInsert(const char *ns, /*modifies*/BSONObj& js) { - uassert( 10059 , "object to insert too large", js.objsize() <= BSONObjMaxUserSize); - { - BSONObjIterator i( js ); - while ( i.more() ) { - BSONElement e = i.next(); - - // check no $ modifiers. note we only check top level. - // (scanning deep would be quite expensive) - uassert( 13511, "document to insert can't have $ fields", e.fieldName()[0] != '$' ); - - // check no regexp for _id (SERVER-9502) - if (str::equals(e.fieldName(), "_id")) { - uassert(16824, "can't use a regex for _id", e.type() != RegEx); - } + void checkAndInsert(Client::Context& ctx, const char *ns, /*modifies*/BSONObj& js, + PregeneratedKeys* preGen ) { + if ( nsToCollectionSubstring( ns ) == "system.indexes" ) { + string targetNS = js["ns"].String(); + uassertStatusOK( userAllowedWriteNS( targetNS ) ); + + Collection* collection = ctx.db()->getCollection( targetNS ); + if ( !collection ) { + // implicitly create + collection = ctx.db()->createCollection( targetNS ); + verify( collection ); } + + // Only permit interrupting an (index build) insert if the + // insert comes from a socket client request rather than a + // parent operation using the client interface. The parent + // operation might not support interrupts. + bool mayInterrupt = cc().curop()->parent() == NULL; + + cc().curop()->setQuery(js); + Status status = collection->getIndexCatalog()->createIndex( js, mayInterrupt ); + + if ( status.code() == ErrorCodes::IndexAlreadyExists ) + return; + + uassertStatusOK( status ); + logOp( "i", ns, js ); + return; + } + + StatusWith fixed = fixDocumentForInsert( js ); + uassertStatusOK( fixed.getStatus() ); + if ( !fixed.getValue().isEmpty() ) + js = fixed.getValue(); + + Collection* collection = ctx.db()->getCollection( ns ); + if ( !collection ) { + collection = ctx.db()->createCollection( ns ); + verify( collection ); } - theDataFileMgr.insertWithObjMod(ns, - // May be modified in the call to add an _id field. - js, - // Only permit interrupting an (index build) insert if the - // insert comes from a socket client request rather than a - // parent operation using the client interface. The parent - // operation might not support interrupts. - cc().curop()->parent() == NULL, - false); + StatusWith status = collection->insertDocument( js, true, preGen ); + uassertStatusOK( status.getStatus() ); logOp("i", ns, js); } - NOINLINE_DECL void insertMulti(bool keepGoing, const char *ns, vector& objs, CurOp& op) { + NOINLINE_DECL void insertMulti(Client::Context& ctx, bool keepGoing, const char *ns, vector& objs, CurOp& op) { size_t i; for (i=0; icheckAuthForInsert(ns); - uassert(16544, status.reason(), status.isOK()); - } + uassertStatusOK( userAllowedWriteNS( ns ) ); if( !d.moreJSObjs() ) { // strange. should we complain? @@ -845,51 +899,63 @@ namespace mongo { while (d.moreJSObjs()){ BSONObj obj = d.nextJsObj(); multi.push_back(obj); - if (isIndexWrite) { - string indexNS = obj.getStringField("ns"); - uassert(16548, - mongoutils::str::stream() << "not authorized to create index on " - << indexNS, - cc().getAuthorizationManager()->checkAuthorization( - indexNS, ActionType::ensureIndex)); - } + + // Check auth for insert (also handles checking if this is an index build and checks + // for the proper privileges in that case). + const NamespaceString nsString(ns); + Status status = cc().getAuthorizationSession()->checkAuthForInsert(nsString, obj); + audit::logInsertAuthzCheck(&cc(), nsString, obj, status.code()); + uassertStatusOK(status); } - PageFaultRetryableSection s; - while ( true ) { - try { - Lock::DBWrite lk(ns); - - // CONCURRENCY TODO: is being read locked in big log sufficient here? - // writelock is used to synchronize stepdowns w/ writes - uassert( 10058 , "not master", isMasterNs(ns) ); - - if ( handlePossibleShardedMessage( m , 0 ) ) + PregeneratedKeys tempHack; + if ( multi.size() == 1 ) { + StatusWith fixed = fixDocumentForInsert( multi[0] ); + uassertStatusOK( fixed.getStatus() ); + if ( !fixed.getValue().isEmpty() ) + multi[0] = fixed.getValue(); + + GeneratorHolder::getInstance()->prepare( ns, multi[0], &tempHack ); + } + + { + PageFaultRetryableSection s; + while ( true ) { + try { + Lock::DBWrite lk(ns); + + // CONCURRENCY TODO: is being read locked in big log sufficient here? + // writelock is used to synchronize stepdowns w/ writes + uassert( 10058 , "not master", isMasterNs(ns) ); + + if ( handlePossibleShardedMessage( m , 0 ) ) + return; + + Client::Context ctx(ns); + + if (multi.size() > 1) { + const bool keepGoing = d.reservedField() & InsertOption_ContinueOnError; + insertMulti(ctx, keepGoing, ns, multi, op); + } + else { + checkAndInsert(ctx, ns, multi[0], &tempHack); + globalOpCounters.incInsertInWriteLock(1); + op.debug().ninserted = 1; + } return; - - Client::Context ctx(ns); - - if (multi.size() > 1) { - const bool keepGoing = d.reservedField() & InsertOption_ContinueOnError; - insertMulti(keepGoing, ns, multi, op); - } else { - checkAndInsert(ns, multi[0]); - globalOpCounters.incInsertInWriteLock(1); - op.debug().ninserted = 1; } - return; - } - catch ( PageFaultException& e ) { - e.touch(); + catch ( PageFaultException& e ) { + e.touch(); + } } - } + } // end PageFaultRetryableSection } void getDatabaseNames( vector< string > &names , const string& usePath ) { boost::filesystem::path path( usePath ); for ( boost::filesystem::directory_iterator i( path ); i != boost::filesystem::directory_iterator(); ++i ) { - if ( directoryperdb ) { + if (storageGlobalParams.directoryperdb) { boost::filesystem::path p = *i; string dbName = p.leaf().string(); p /= ( dbName + ".ns" ); @@ -931,7 +997,21 @@ namespace mongo { return QueryOptions(DBClientBase::_lookupAvailableOptions() & ~QueryOption_Exhaust); } +namespace { + class GodScope { + MONGO_DISALLOW_COPYING(GodScope); + public: + GodScope() { + _prev = cc().setGod(true); + } + ~GodScope() { cc().setGod(_prev); } + private: + bool _prev; + }; +} // namespace + bool DBDirectClient::call( Message &toSend, Message &response, bool assertOk , string * actualServer ) { + GodScope gs; if ( lastError._get() ) lastError.startRequest( toSend, lastError._get() ); DbResponse dbResponse; @@ -944,6 +1024,7 @@ namespace mongo { } void DBDirectClient::say( Message &toSend, bool isRetry, string * actualServer ) { + GodScope gs; if ( lastError._get() ) lastError.startRequest( toSend, lastError._get() ); DbResponse dbResponse; @@ -962,7 +1043,7 @@ namespace mongo { } void DBDirectClient::killCursor( long long id ) { - ClientCursor::erase( id ); + CollectionCursorCache::eraseCursorGlobal( id ); } HostAndPort DBDirectClient::_clientHost = HostAndPort( "0.0.0.0" , 0 ); @@ -976,7 +1057,7 @@ namespace mongo { Lock::DBRead lk( ns ); string errmsg; int errCode; - long long res = runCount( ns.c_str() , _countCmd( ns , query , options , limit , skip ) , errmsg, errCode ); + long long res = runCount( ns, _countCmd( ns , query , options , limit , skip ) , errmsg, errCode ); if ( res == -1 ) { // namespace doesn't exist return 0; @@ -989,6 +1070,14 @@ namespace mongo { return new DBDirectClient(); } + MONGO_INITIALIZER(CreateJSDirectClient) + (InitializerContext* context) { + + directDBClient = createDirectClient(); + + return Status::OK(); + } + mongo::mutex exitMutex("exit"); AtomicUInt numExitCalls = 0; @@ -996,22 +1085,6 @@ namespace mongo { return numExitCalls > 0; } - void tryToOutputFatal( const string& s ) { - try { - rawOut( s ); - return; - } - catch ( ... ) {} - - try { - cerr << s << endl; - return; - } - catch ( ... ) {} - - // uh - oh, not sure there is anything else we can do... - } - static void shutdownServer() { log() << "shutdown: going to close listening sockets..." << endl; @@ -1030,7 +1103,7 @@ namespace mongo { log() << "shutdown: waiting for fs preallocator..." << endl; FileAllocator::get()->waitUntilFinished(); - if( cmdLine.dur ) { + if (storageGlobalParams.dur) { log() << "shutdown: lock for final commit..." << endl; { int n = 10; @@ -1058,7 +1131,7 @@ namespace mongo { MemoryMappedFile::closeAllFiles( ss3 ); log() << ss3.str() << endl; - if( cmdLine.dur ) { + if (storageGlobalParams.dur) { dur::journalCleanup(true); } @@ -1097,7 +1170,10 @@ namespace mongo { /* not using log() herein in case we are already locked */ NOINLINE_DECL void dbexit( ExitCode rc, const char *why ) { + flushForGcov(); + Client * c = currentClient.get(); + audit::logShutdown(c); { scoped_lock lk( exitMutex ); if ( numExitCalls++ > 0 ) { @@ -1105,25 +1181,19 @@ namespace mongo { // this means something horrible has happened ::_exit( rc ); } - stringstream ss; - ss << "dbexit: " << why << "; exiting immediately"; - tryToOutputFatal( ss.str() ); + log() << "dbexit: " << why << "; exiting immediately"; if ( c ) c->shutdown(); ::_exit( rc ); } } - { - stringstream ss; - ss << "dbexit: " << why; - tryToOutputFatal( ss.str() ); - } + log() << "dbexit: " << why; try { shutdownServer(); // gracefully shutdown instance } catch ( ... ) { - tryToOutputFatal( "shutdown failed with exception" ); + severe() << "shutdown failed with exception"; } #if defined(_DEBUG) @@ -1146,7 +1216,7 @@ namespace mongo { return; } #endif - tryToOutputFatal( "dbexit: really exiting now" ); + log() << "dbexit: really exiting now"; if ( c ) c->shutdown(); ::_exit(rc); } @@ -1154,7 +1224,7 @@ namespace mongo { #if !defined(__sunos__) void writePid(int fd) { stringstream ss; - ss << getpid() << endl; + ss << ProcessId::getCurrent() << endl; string s = ss.str(); const char * data = s.c_str(); #ifdef _WIN32 @@ -1165,7 +1235,7 @@ namespace mongo { } void acquirePathLock(bool doingRepair) { - string name = ( boost::filesystem::path( dbpath ) / "mongod.lock" ).string(); + string name = (boost::filesystem::path(storageGlobalParams.dbpath) / "mongod.lock").string(); bool oldFile = false; @@ -1214,7 +1284,7 @@ namespace mongo { "run with --repair again.\n" "**************"; } - else if (cmdLine.dur) { + else if (storageGlobalParams.dur) { if (!dur::haveJournalFiles(/*anyFiles=*/true)) { // Passing anyFiles=true as we are trying to protect against starting in an // unclean state with the journal directory unmounted. If there are any files, @@ -1242,7 +1312,6 @@ namespace mongo { << "*************"; } - } } else { @@ -1268,7 +1337,7 @@ namespace mongo { } // Not related to lock file, but this is where we handle unclean shutdown - if( !cmdLine.dur && dur::haveJournalFiles() ) { + if (!storageGlobalParams.dur && dur::haveJournalFiles()) { cout << "**************" << endl; cout << "Error: journal files are present in journal directory, yet starting without journaling enabled." << endl; cout << "It is recommended that you start with journaling enabled so that recovery may occur." << endl; @@ -1292,7 +1361,7 @@ namespace mongo { // TODO - this is very bad that the code above not running here. // Not related to lock file, but this is where we handle unclean shutdown - if( !cmdLine.dur && dur::haveJournalFiles() ) { + if (!storageGlobalParams.dur && dur::haveJournalFiles()) { cout << "**************" << endl; cout << "Error: journal files are present in journal directory, yet starting without --journal enabled." << endl; cout << "It is recommended that you start with journaling enabled so that recovery may occur." << endl; @@ -1310,7 +1379,7 @@ namespace mongo { void DiagLog::openFile() { verify( f == 0 ); stringstream ss; - ss << dbpath << "/diaglog." << hex << time(0); + ss << storageGlobalParams.dbpath << "/diaglog." << hex << time(0); string name = ss.str(); f = new ofstream(name.c_str(), ios::out | ios::binary); if ( ! f->good() ) { diff --git a/src/mongo/db/instance.h b/src/mongo/db/instance.h index 1a5d3a9e8f7..d848f6f1c09 100644 --- a/src/mongo/db/instance.h +++ b/src/mongo/db/instance.h @@ -15,15 +15,27 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once - -#include "curop-inl.h" -#include "cmdline.h" -#include "client.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/db/client.h" +#include "mongo/db/curop-inl.h" +#include "mongo/db/dbmessage.h" +#include "mongo/db/storage_options.h" namespace mongo { @@ -55,23 +67,10 @@ namespace mongo { extern DiagLog _diaglog; - /* we defer response until we unlock. don't want a blocked socket to - keep things locked. - */ - struct DbResponse { - Message *response; - MSGID responseTo; - string exhaustNS; /* points to ns if exhaust mode. 0=normal mode*/ - DbResponse(Message *r, MSGID rt) : response(r), responseTo(rt){ } - DbResponse() { - response = 0; - } - ~DbResponse() { delete response; } - }; - void assembleResponse( Message &m, DbResponse &dbresponse, const HostAndPort &client ); - void getDatabaseNames( vector< string > &names , const string& usePath = dbpath ); + void getDatabaseNames(vector &names, + const std::string& usePath = storageGlobalParams.dbpath); /* returns true if there is no data on this server. useful when starting replication. local database does NOT count. @@ -91,7 +90,12 @@ namespace mongo { virtual bool isFailed() const { return false; } - virtual string toString() { + + virtual bool isStillConnected() { + return true; + } + + virtual string toString() const { return "DBDirectClient"; } virtual string getServerAddress() const { diff --git a/src/mongo/db/interrupt_status.h b/src/mongo/db/interrupt_status.h index 90f09878e13..8aa418ced16 100644 --- a/src/mongo/db/interrupt_status.h +++ b/src/mongo/db/interrupt_status.h @@ -12,12 +12,22 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once -#include "mongo/pch.h" - namespace mongo { /** @@ -38,7 +48,7 @@ namespace mongo { @throws a uassert if the process has received an interrupt (SIGINT) */ - virtual void checkForInterrupt() = 0; + virtual void checkForInterrupt() const =0; /** Check for interrupt. @@ -47,7 +57,7 @@ namespace mongo { "" if there hasn't been an interrupt. These strings are static and don't need to be freed. */ - virtual const char *checkForInterruptNoAssert() = 0; + virtual const char *checkForInterruptNoAssert() const =0; protected: /** diff --git a/src/mongo/db/interrupt_status_mongod.cpp b/src/mongo/db/interrupt_status_mongod.cpp index 915a183651c..53f04562203 100644 --- a/src/mongo/db/interrupt_status_mongod.cpp +++ b/src/mongo/db/interrupt_status_mongod.cpp @@ -12,9 +12,21 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/db/interrupt_status_mongod.h" @@ -23,16 +35,16 @@ namespace mongo { - InterruptStatusMongod InterruptStatusMongod::status; + const InterruptStatusMongod InterruptStatusMongod::status; InterruptStatusMongod::InterruptStatusMongod() { } - void InterruptStatusMongod::checkForInterrupt() { + void InterruptStatusMongod::checkForInterrupt() const { killCurrentOp.checkForInterrupt(); } - const char *InterruptStatusMongod::checkForInterruptNoAssert() { + const char *InterruptStatusMongod::checkForInterruptNoAssert() const { return killCurrentOp.checkForInterruptNoAssert(); } diff --git a/src/mongo/db/interrupt_status_mongod.h b/src/mongo/db/interrupt_status_mongod.h index 0111571ec71..7e85c08c866 100644 --- a/src/mongo/db/interrupt_status_mongod.h +++ b/src/mongo/db/interrupt_status_mongod.h @@ -12,12 +12,23 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once -#include "mongo/pch.h" -#include "db/interrupt_status.h" +#include "mongo/db/interrupt_status.h" namespace mongo { @@ -26,13 +37,13 @@ namespace mongo { boost::noncopyable { public: // virtuals from InterruptStatus - virtual void checkForInterrupt(); - virtual const char *checkForInterruptNoAssert(); + virtual void checkForInterrupt() const; + virtual const char *checkForInterruptNoAssert() const; /* Static singleton instance. */ - static InterruptStatusMongod status; + static const InterruptStatusMongod status; private: InterruptStatusMongod(); diff --git a/src/mongo/db/intervalbtreecursor.cpp b/src/mongo/db/intervalbtreecursor.cpp deleted file mode 100644 index 58190a08280..00000000000 --- a/src/mongo/db/intervalbtreecursor.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "mongo/db/intervalbtreecursor.h" - -#include "mongo/db/btree.h" -#include "mongo/db/kill_current_op.h" -#include "mongo/db/namespace_details-inl.h" -#include "mongo/db/pdfile.h" - -namespace mongo { - - /** - * Advance 'loc' until it does not reference an unused key, or the end of the btree is reached. - */ - static void skipUnused( BtreeKeyLocation* loc ) { - - // While loc points to an unused key ... - while( !loc->bucket.isNull() && - loc->bucket.btree()->k( loc->pos ).isUnused() ) { - - // ... advance loc to the next key in the btree. - loc->bucket = loc->bucket.btree()->advance( loc->bucket, - loc->pos, - 1, - __FUNCTION__ ); - } - } - - IntervalBtreeCursor* IntervalBtreeCursor::make( NamespaceDetails* namespaceDetails, - const IndexDetails& indexDetails, - const BSONObj& lowerBound, - bool lowerBoundInclusive, - const BSONObj& upperBound, - bool upperBoundInclusive ) { - if ( indexDetails.version() != 1 ) { - // Only v1 indexes are supported. - return NULL; - } - auto_ptr ret( new IntervalBtreeCursor( namespaceDetails, - indexDetails, - lowerBound, - lowerBoundInclusive, - upperBound, - upperBoundInclusive ) ); - ret->init(); - return ret.release(); - } - - IntervalBtreeCursor::IntervalBtreeCursor( NamespaceDetails* namespaceDetails, - const IndexDetails& indexDetails, - const BSONObj& lowerBound, - bool lowerBoundInclusive, - const BSONObj& upperBound, - bool upperBoundInclusive ) : - _namespaceDetails( *namespaceDetails ), - _indexNo( namespaceDetails->idxNo( indexDetails ) ), - _indexDetails( indexDetails ), - _ordering( Ordering::make( _indexDetails.keyPattern() ) ), - _lowerBound( lowerBound ), - _lowerBoundInclusive( lowerBoundInclusive ), - _upperBound( upperBound ), - _upperBoundInclusive( upperBoundInclusive ), - _currRecoverable( _indexDetails, _ordering, _curr ), - _nscanned(), - _multikeyFlag() { - } - - void IntervalBtreeCursor::init() { - _multikeyFlag = _namespaceDetails.isMultikey( _indexNo ); - _curr = locateKey( _lowerBound, !_lowerBoundInclusive ); - skipUnused( &_curr ); - relocateEnd(); - if ( ok() ) { - _nscanned = 1; - } - } - - bool IntervalBtreeCursor::ok() { - return !_curr.bucket.isNull(); - } - - DiskLoc IntervalBtreeCursor::currLoc() { - if ( eof() ) { - return DiskLoc(); - } - return _curr.bucket.btree()->keyNode( _curr.pos ).recordLoc; - } - - bool IntervalBtreeCursor::advance() { - RARELY killCurrentOp.checkForInterrupt(); - if ( eof() ) { - return false; - } - // Advance _curr to the next key in the btree. - _curr.bucket = _curr.bucket.btree()->advance( _curr.bucket, - _curr.pos, - 1, - __FUNCTION__ ); - skipUnused( &_curr ); - if ( _curr == _end ) { - // _curr has reached _end, so iteration is complete. - _curr.bucket.Null(); - } - else { - ++_nscanned; - } - return ok(); - } - - BSONObj IntervalBtreeCursor::currKey() const { - if ( _curr.bucket.isNull() ) { - return BSONObj(); - } - return _curr.bucket.btree()->keyNode( _curr.pos ).key.toBson(); - } - - void IntervalBtreeCursor::aboutToDeleteBucket( const DiskLoc& b ) { - if ( b == _curr.bucket ) { - _currRecoverable.invalidateInitialLocation(); - } - } - - void IntervalBtreeCursor::noteLocation() { - _currRecoverable = LogicalBtreePosition( _indexDetails, _ordering, _curr ); - _currRecoverable.init(); - } - - void IntervalBtreeCursor::checkLocation() { - _multikeyFlag = _namespaceDetails.isMultikey( _indexNo ); - _curr = _currRecoverable.currentLocation(); - skipUnused( &_curr ); - relocateEnd(); - } - - bool IntervalBtreeCursor::getsetdup( DiskLoc loc ) { - // TODO _multikeyFlag may be set part way through an iteration by checkLocation(). In this - // case results returned earlier, when _multikeyFlag was false, will not be deduped. This - // is an old issue with all mongo btree cursor implementations. - return _multikeyFlag && !_dups.insert( loc ).second; - } - - BSONObj IntervalBtreeCursor::prettyIndexBounds() const { - return BSON( "lower" << _lowerBound.replaceFieldNames( _indexDetails.keyPattern() ) << - "upper" << _upperBound.replaceFieldNames( _indexDetails.keyPattern() ) ); - } - - BtreeKeyLocation IntervalBtreeCursor::locateKey( const BSONObj& key, bool afterKey ) { - bool found; - BtreeKeyLocation ret; - - // To find the first btree location equal to the specified key, specify a record location of - // minDiskLoc, which is below any actual Record location. To find the first btree location - // greater than the specified key, specify a record location of maxDiskLoc, which is above - // any actual Record location. - DiskLoc targetRecord = afterKey ? maxDiskLoc : minDiskLoc; - - // Find the requested location in the btree. - ret.bucket = _indexDetails.head.btree()->locate( _indexDetails, - _indexDetails.head, - key, - _ordering, - ret.pos, - found, - targetRecord, - 1 ); - return ret; - } - - void IntervalBtreeCursor::relocateEnd() { - if ( eof() ) { - return; - } - - // If the current key is above the upper bound ... - int32_t cmp = currKey().woCompare( _upperBound, _ordering, false ); - if ( cmp > 0 || ( cmp == 0 && !_upperBoundInclusive ) ) { - - // ... then iteration is complete. - _curr.bucket.Null(); - return; - } - - // Otherwise, relocate _end. - _end = locateKey( _upperBound, _upperBoundInclusive ); - skipUnused( &_end ); - } - -} // namespace mongo diff --git a/src/mongo/db/intervalbtreecursor.h b/src/mongo/db/intervalbtreecursor.h deleted file mode 100644 index 01c1d26e919..00000000000 --- a/src/mongo/db/intervalbtreecursor.h +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "mongo/db/btreeposition.h" -#include "mongo/db/cursor.h" -#include "mongo/db/namespace_details.h" -#include "mongo/platform/cstdint.h" -#include "mongo/platform/unordered_set.h" - -namespace mongo { - - /** - * An optimized btree cursor that iterates through all btree keys between a lower bound and an - * upper bound. The contents of the individual keys are not examined by the implementation, - * which simply advances through the tree until reaching a predetermined end location. The goal - * is to optimize count operations where the keys must only be counted, not tested for matching. - * - * Limitations compared to a standard BtreeCursor (partial list): - * - Only supports index constraints consisting of a single interval within an index. - * - Only supports forward direction iteration. - * - Does not support covered index projections. - * - Does not support get more. - * - Only supports V1 indexes (not V0). - */ - class IntervalBtreeCursor : public Cursor { - public: - - /** - * @return a cursor, or NULL if no cursor can be created. - * @param namespaceDetails - Collection metadata that will not be modified. - * @param indexDetails - Index metadata, if not a v1 index then make() will return NULL. - * @param lowerBound - Lower bound of the key range to iterate, according to the index's - * native ordering. - * @param lowerBoundInclusive - If true, the lower bound includes the endpoint. - * @param upperBound - Upper bound of the key range to iterate. - * @param upperBoundInclusive - If true, the upper bound includes the endpoint. - */ - static IntervalBtreeCursor* make( /* const */ NamespaceDetails* namespaceDetails, - const IndexDetails& indexDetails, - const BSONObj& lowerBound, - bool lowerBoundInclusive, - const BSONObj& upperBound, - bool upperBoundInclusive ); - - /** Virtuals from Cursor. */ - - virtual bool ok(); - - virtual Record* _current() { return currLoc().rec(); } - - virtual BSONObj current() { return currLoc().obj(); } - - virtual DiskLoc currLoc(); - - virtual bool advance(); - - virtual BSONObj currKey() const; - - virtual DiskLoc refLoc() { return currLoc(); } - - virtual void aboutToDeleteBucket( const DiskLoc& b ); - - virtual BSONObj indexKeyPattern() { return _indexDetails.keyPattern(); } - - virtual bool supportGetMore() { return false; } - - virtual void noteLocation(); - - virtual void checkLocation(); - - virtual bool supportYields() { return true; } - - virtual string toString() { return "IntervalBtreeCursor"; } - - virtual bool getsetdup( DiskLoc loc ); - - virtual bool isMultiKey() const { return _multikeyFlag; } - - virtual bool modifiedKeys() const { return _multikeyFlag; } - - virtual BSONObj prettyIndexBounds() const; - - virtual long long nscanned() { return _nscanned; } - - virtual CoveredIndexMatcher* matcher() const { return _matcher.get(); } - - virtual void setMatcher( shared_ptr matcher ) { _matcher = matcher; } - - private: - IntervalBtreeCursor( NamespaceDetails* namespaceDetails, - const IndexDetails& indexDetails, - const BSONObj& lowerBound, - bool lowerBoundInclusive, - const BSONObj& upperBound, - bool upperBoundInclusive ); - - void init(); - - /** - * @return a location in the btree, determined by the parameters specified. - * @param key - The key to search for. - * @param afterKey - If true, return the first btree key greater than the supplied 'key'. - * If false, return the first key equal to the supplied 'key'. - */ - BtreeKeyLocation locateKey( const BSONObj& key, bool afterKey ); - - /** Find the iteration end location and set _end to it. */ - void relocateEnd(); - - const NamespaceDetails& _namespaceDetails; - const int32_t _indexNo; - const IndexDetails& _indexDetails; - const Ordering _ordering; - const BSONObj _lowerBound; - const bool _lowerBoundInclusive; - const BSONObj _upperBound; - const bool _upperBoundInclusive; - - BtreeKeyLocation _curr; // Current position in the btree. - LogicalBtreePosition _currRecoverable; // Helper to track the position of _curr if the - // btree is modified during a mutex yield. - BtreeKeyLocation _end; // Exclusive end location in the btree. - int64_t _nscanned; - - shared_ptr _matcher; - bool _multikeyFlag; - unordered_set _dups; - }; - -} // namespace mongo diff --git a/src/mongo/db/introspect.cpp b/src/mongo/db/introspect.cpp index 2c6fa6c5600..674b43ee8f3 100644 --- a/src/mongo/db/introspect.cpp +++ b/src/mongo/db/introspect.cpp @@ -14,18 +14,33 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" #include "mongo/bson/util/builder.h" #include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/principal_set.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/user_set.h" #include "mongo/db/curop.h" -#include "mongo/db/databaseholder.h" +#include "mongo/db/catalog/database_holder.h" #include "mongo/db/introspect.h" #include "mongo/db/jsobj.h" #include "mongo/db/pdfile.h" +#include "mongo/db/storage_options.h" +#include "mongo/db/catalog/collection.h" #include "mongo/util/goodies.h" namespace { @@ -35,10 +50,12 @@ namespace { namespace mongo { namespace { - void _appendUserInfo(const Client& c, BSONObjBuilder& builder, AuthorizationManager* authManager) { - PrincipalSet::NameIterator nameIter = authManager->getAuthenticatedPrincipalNames(); + void _appendUserInfo(const Client& c, + BSONObjBuilder& builder, + AuthorizationSession* authSession) { + UserNameIterator nameIter = authSession->getAuthenticatedUserNames(); - PrincipalName bestUser; + UserName bestUser; if (nameIter.more()) bestUser = *nameIter; @@ -48,7 +65,7 @@ namespace { for ( ; nameIter.more(); nameIter.next()) { BSONObjBuilder nextUser(allUsers.subobjStart()); nextUser.append(AuthorizationManager::USER_NAME_FIELD_NAME, nameIter->getUser()); - nextUser.append(AuthorizationManager::USER_SOURCE_FIELD_NAME, nameIter->getDB()); + nextUser.append(AuthorizationManager::USER_DB_FIELD_NAME, nameIter->getDB()); nextUser.doneFast(); if (nameIter->getDB() == opdb) { @@ -65,7 +82,6 @@ namespace { static void _profile(const Client& c, CurOp& currentOp, BufBuilder& profileBufBuilder) { Database *db = c.database(); DEV verify( db ); - const char *ns = db->profileName.c_str(); // build object BSONObjBuilder b(profileBufBuilder); @@ -76,8 +92,8 @@ namespace { b.appendDate("ts", jsTime()); b.append("client", c.clientAddress()); - AuthorizationManager* authManager = c.getAuthorizationManager(); - _appendUserInfo(c, b, authManager); + AuthorizationSession * authSession = c.getAuthorizationSession(); + _appendUserInfo(c, b, authSession); BSONObj p = b.done(); @@ -90,7 +106,7 @@ namespace { BSONObjBuilder b(profileBufBuilder); b.appendDate("ts", jsTime()); b.append("client", c.clientAddress() ); - _appendUserInfo(c, b, authManager); + _appendUserInfo(c, b, authSession); b.append("err", "profile line too large (max is 100KB)"); @@ -104,11 +120,9 @@ namespace { // write: not replicated // get or create the profiling collection - NamespaceDetails *details = getOrCreateProfileCollection(db); - if (details) { - int len = p.objsize(); - Record *r = theDataFileMgr.fast_oplog_insert(details, ns, len); - memcpy(getDur().writingPtr(r->data(), len), p.objdata(), len); + Collection* profileCollection = getOrCreateProfileCollection(db); + if ( profileCollection ) { + profileCollection->insertDocument( p, false ); } } @@ -118,9 +132,11 @@ namespace { BufBuilder profileBufBuilder(1024); try { + // NOTE: It's kind of weird that we lock the op's namespace, but have to for now since + // we're sometimes inside the lock already Lock::DBWrite lk( currentOp.getNS() ); - if ( dbHolder()._isLoaded( nsToDatabase( currentOp.getNS() ) , dbpath ) ) { - Client::Context cx(currentOp.getNS(), dbpath); + if (dbHolder()._isLoaded(nsToDatabase(currentOp.getNS()), storageGlobalParams.dbpath)) { + Client::Context cx(currentOp.getNS(), storageGlobalParams.dbpath, false); _profile(c, currentOp, profileBufBuilder); } else { @@ -135,41 +151,49 @@ namespace { } } - NamespaceDetails* getOrCreateProfileCollection(Database *db, bool force, string* errmsg ) { + Collection* getOrCreateProfileCollection(Database *db, bool force, string* errmsg ) { fassert(16372, db); - const char* profileName = db->profileName.c_str(); - NamespaceDetails* details = db->namespaceIndex.details(profileName); - if (!details && (cmdLine.defaultProfile || force)) { - // system.profile namespace doesn't exist; create it - log() << "creating profile collection: " << profileName << endl; - string myerrmsg; - if (!userCreateNS(db->profileName.c_str(), - BSON("capped" << true << "size" << 1024 * 1024), myerrmsg , false)) { - myerrmsg = str::stream() << "could not create ns " << db->profileName << ": " << myerrmsg; + const char* profileName = db->getProfilingNS(); + Collection* collection = db->getCollection( profileName ); + + if ( collection ) { + if ( !collection->isCapped() ) { + string myerrmsg = str::stream() << profileName << " exists but isn't capped"; log() << myerrmsg << endl; if ( errmsg ) *errmsg = myerrmsg; return NULL; } - details = db->namespaceIndex.details(profileName); - } - else if ( details && !details->isCapped() ) { - string myerrmsg = str::stream() << profileName << " exists but isn't capped"; - log() << myerrmsg << endl; - if ( errmsg ) - *errmsg = myerrmsg; - return NULL; + return collection; } - if (!details) { - // failed to get or create profile collection + // does not exist! + + if ( force == false && serverGlobalParams.defaultProfile == false ) { + // we don't want it, so why are we here? static time_t last = time(0) - 10; // warn the first time if( time(0) > last+10 ) { log() << "profile: warning ns " << profileName << " does not exist" << endl; last = time(0); } + return NULL; + } + + // system.profile namespace doesn't exist; create it + log() << "creating profile collection: " << profileName << endl; + string myerrmsg; + if (!userCreateNS(profileName, + BSON("capped" << true << "size" << 1024 * 1024), myerrmsg , false)) { + myerrmsg = str::stream() << "could not create ns " << profileName << ": " << myerrmsg; + log() << myerrmsg << endl; + if ( errmsg ) + *errmsg = myerrmsg; + return NULL; } - return details; + + collection = db->getCollection( profileName ); + verify( collection ); + return collection; } } // namespace mongo diff --git a/src/mongo/db/introspect.h b/src/mongo/db/introspect.h index 980d7c72b1c..50b98656bc2 100644 --- a/src/mongo/db/introspect.h +++ b/src/mongo/db/introspect.h @@ -15,16 +15,32 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "mongo/pch.h" -#include "jsobj.h" -#include "pdfile.h" +#include + +#include "mongo/db/curop.h" +#include "mongo/db/jsobj.h" namespace mongo { + class Collection; + class Database; + /* --- profiling -------------------------------------------- do when database->profile is set */ @@ -36,8 +52,10 @@ namespace mongo { * * @param db Database in which to create the profile collection * @param force Always create the collection if it does not exist - * @return NamespaceDetails for the newly created collection, or NULL on error + * @return Collection for the newly created collection, or NULL on error **/ - NamespaceDetails* getOrCreateProfileCollection(Database *db, bool force = false, string* errmsg = NULL); + Collection* getOrCreateProfileCollection(Database *db, + bool force = false, + std::string* errmsg = NULL); } // namespace mongo diff --git a/src/mongo/db/invalidation_type.h b/src/mongo/db/invalidation_type.h new file mode 100644 index 00000000000..b9aa7dde213 --- /dev/null +++ b/src/mongo/db/invalidation_type.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { + + enum InvalidationType { + // The DiskLoc is about to be deleted. The receiver of this invalidate call cannot use + // the DiskLoc after it returns from the invalidate. + INVALIDATION_DELETION, + + // The DiskLoc's contents are about to change. + INVALIDATION_MUTATION, + }; + +} // namespace mongo diff --git a/src/mongo/db/javatest.cpp b/src/mongo/db/javatest.cpp index 22f2bdf8d3c..9171407e3b8 100644 --- a/src/mongo/db/javatest.cpp +++ b/src/mongo/db/javatest.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "javajs.h" diff --git a/src/mongo/db/jsobj.cpp b/src/mongo/db/jsobj.cpp index ea3248e42da..ea5e1e48fb0 100644 --- a/src/mongo/db/jsobj.cpp +++ b/src/mongo/db/jsobj.cpp @@ -17,7 +17,8 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" + #include "mongo/db/jsobj.h" #include @@ -29,16 +30,16 @@ #include "mongo/bson/bson_validate.h" #include "mongo/bson/oid.h" #include "mongo/bson/util/atomic_int.h" -#include "mongo/db/jsobjmanipulator.h" #include "mongo/db/json.h" +#include "mongo/bson/optime.h" #include "mongo/platform/float_utils.h" #include "mongo/util/base64.h" #include "mongo/util/embedded_builder.h" #include "mongo/util/md5.hpp" #include "mongo/util/mongoutils/str.h" -#include "mongo/util/optime.h" #include "mongo/util/startup_test.h" #include "mongo/util/stringutils.h" +#include "mongo/util/time_support.h" // make sure our assumptions are valid @@ -51,6 +52,8 @@ BOOST_STATIC_ASSERT( sizeof(mongo::OID) == 12 ); namespace mongo { + namespace str = mongoutils::str; + BSONElement eooElement; GENOIDLabeler GENOID; @@ -75,15 +78,27 @@ namespace mongo { s << '"' << escape( string(valuestr(), valuestrsize()-1) ) << '"'; break; case NumberLong: - s << _numberLong(); + if (format == TenGen) { + s << "NumberLong(" << _numberLong() << ")"; + } + else { + s << "{ \"$numberLong\" : \"" << _numberLong() << "\" }"; + } break; case NumberInt: + if(format == JS) { + s << "NumberInt(" << _numberInt() << ")"; + break; + } case NumberDouble: if ( number() >= -numeric_limits< double >::max() && number() <= numeric_limits< double >::max() ) { s.precision( 16 ); s << number(); } + // This is not valid JSON, but according to RFC-4627, "Numeric values that cannot be + // represented as sequences of digits (such as Infinity and NaN) are not permitted." so + // we are accepting the fact that if we have such values we cannot output valid JSON. else if ( mongo::isNaN(number()) ) { s << "NaN"; } @@ -193,22 +208,47 @@ namespace mongo { break; } case mongo::Date: - if ( format == Strict ) - s << "{ \"$date\" : "; - else - s << "Date( "; - if( pretty ) { + if (format == Strict) { Date_t d = date(); - if( d == 0 ) s << '0'; - else - s << '"' << date().toString() << '"'; - } - else - s << date().asInt64(); - if ( format == Strict ) + s << "{ \"$date\" : "; + // The two cases in which we cannot convert Date_t::millis to an ISO Date string are + // when the date is too large to format (SERVER-13760), and when the date is before + // the epoch (SERVER-11273). Since Date_t internally stores millis as an unsigned + // long long, despite the fact that it is logically signed (SERVER-8573), this check + // handles both the case where Date_t::millis is too large, and the case where + // Date_t::millis is negative (before the epoch). + if (d.isFormatable()) { + s << "\"" << dateToISOStringLocal(date()) << "\""; + } + else { + s << "{ \"$numberLong\" : \"" << static_cast(d.millis) << "\" }"; + } s << " }"; - else + } + else { + s << "Date( "; + if (pretty) { + Date_t d = date(); + // The two cases in which we cannot convert Date_t::millis to an ISO Date string + // are when the date is too large to format (SERVER-13760), and when the date is + // before the epoch (SERVER-11273). Since Date_t internally stores millis as an + // unsigned long long, despite the fact that it is logically signed + // (SERVER-8573), this check handles both the case where Date_t::millis is too + // large, and the case where Date_t::millis is negative (before the epoch). + if (d.isFormatable()) { + s << "\"" << dateToISOStringLocal(date()) << "\""; + } + else { + // FIXME: This is not parseable by the shell, since it may not fit in a + // float + s << d.millis; + } + } + else { + s << date().asInt64(); + } s << " )"; + } break; case RegEx: if ( format == Strict ) { @@ -234,14 +274,14 @@ namespace mongo { case CodeWScope: { BSONObj scope = codeWScopeObject(); if ( ! scope.isEmpty() ) { - s << "{ \"$code\" : " << _asCode() << " , " - << " \"$scope\" : " << scope.jsonString() << " }"; + s << "{ \"$code\" : \"" << escape(_asCode()) << "\" , " + << "\"$scope\" : " << scope.jsonString() << " }"; break; } } case Code: - s << _asCode(); + s << "\"" << escape(_asCode()) << "\""; break; case Timestamp: @@ -318,11 +358,11 @@ namespace mongo { return BSONObj::opOPTIONS; else if ( fn[1] == 'w' && fn[2] == 'i' && fn[3] == 't' && fn[4] == 'h' && fn[5] == 'i' && fn[6] == 'n' && fn[7] == 0 ) return BSONObj::opWITHIN; - else if (mongoutils::str::equals(fn + 1, "geoIntersects")) + else if (str::equals(fn + 1, "geoIntersects")) return BSONObj::opGEO_INTERSECTS; - else if (mongoutils::str::equals(fn + 1, "geoNear")) + else if (str::equals(fn + 1, "geoNear")) return BSONObj::opNEAR; - else if (mongoutils::str::equals(fn + 1, "geoWithin")) + else if (str::equals(fn + 1, "geoWithin")) return BSONObj::opWITHIN; } return def; @@ -606,7 +646,7 @@ namespace mongo { while ( a.more() && b.more() ) { BSONElement x = a.next(); BSONElement y = b.next(); - if ( ! mongoutils::str::equals( x.fieldName() , y.fieldName() ) ) { + if ( ! str::equals( x.fieldName() , y.fieldName() ) ) { return false; } } @@ -742,14 +782,15 @@ namespace mongo { return b.obj(); } - BSONElement BSONObj::getFieldUsingIndexNames(const char *fieldName, const BSONObj &indexKey) const { + BSONElement BSONObj::getFieldUsingIndexNames(const StringData& fieldName, + const BSONObj &indexKey) const { BSONObjIterator i( indexKey ); int j = 0; while( i.moreWithEOO() ) { BSONElement f = i.next(); if ( f.eoo() ) return BSONElement(); - if ( strcmp( f.fieldName(), fieldName ) == 0 ) + if ( f.fieldName() == fieldName ) break; ++j; } @@ -825,7 +866,7 @@ namespace mongo { if( e.eoo() ) break; // TODO: If actually important, may be able to do int->char* much faster - if( strcmp( e.fieldName(), ((string)( mongoutils::str::stream() << index )).c_str() ) != 0 ) + if( strcmp( e.fieldName(), ((string)( str::stream() << index )).c_str() ) != 0 ) return false; index++; } @@ -879,55 +920,100 @@ namespace mongo { return b.obj(); } - bool BSONObj::okForStorage() const { + Status BSONObj::_okForStorage(bool root, bool deep) const { BSONObjIterator i( *this ); + + // The first field is special in the case of a DBRef where the first field must be $ref + bool first = true; while ( i.more() ) { BSONElement e = i.next(); - const char * name = e.fieldName(); - - if ( strchr( name , '.' ) || - strchr( name , '$' ) ) { - return - strcmp( name , "$ref" ) == 0 || - strcmp( name , "$id" ) == 0 - ; - } + const char* name = e.fieldName(); + + // Cannot start with "$", unless dbref which must start with ($ref, $id) + if (str::startsWith(name, '$')) { + if ( first && + // $ref is a collection name and must be a String + str::equals(name, "$ref") && e.type() == String && + str::equals(i.next().fieldName(), "$id") ) { + + first = false; + // keep inspecting fields for optional "$db" + e = i.next(); + name = e.fieldName(); // "" if eoo() + + // optional $db field must be a String + if (str::equals(name, "$db") && e.type() == String) { + continue; //this element is fine, so continue on to siblings (if any more) + } - // check no regexp for _id (SERVER-9502) - if (mongoutils::str::equals(e.fieldName(), "_id")) { - if (e.type() == RegEx) { - return false; + // Can't start with a "$", all other checks are done below (outside if blocks) + if (str::startsWith(name, '$')) { + return Status(ErrorCodes::DollarPrefixedFieldName, + str::stream() << name << " is not valid for storage."); + } } + else { + // not an okay, $ prefixed field name. + return Status(ErrorCodes::DollarPrefixedFieldName, + str::stream() << name << " is not valid for storage."); + } + } + + // Do not allow "." in the field name + if (strchr(name, '.')) { + return Status(ErrorCodes::DottedFieldName, + str::stream() << name << " is not valid for storage."); + } + + // (SERVER-9502) Do not allow storing an _id field with a RegEx type or + // Array type in a root document + if (root && (e.type() == RegEx || e.type() == Array || e.type() == Undefined) + && str::equals(name,"_id")) { + return Status(ErrorCodes::InvalidIdField, + str::stream() << name + << " is not valid for storage because it is of type " + << typeName(e.type())); } - if ( e.mayEncapsulate() ) { + if ( deep && e.mayEncapsulate() ) { switch ( e.type() ) { case Object: case Array: - if ( ! e.embeddedObject().okForStorage() ) - return false; + { + Status s = e.embeddedObject()._okForStorage(false, true); + // TODO: combine field names for better error messages + if ( ! s.isOK() ) + return s; + } break; case CodeWScope: - if ( ! e.codeWScopeObject().okForStorage() ) - return false; + { + Status s = e.codeWScopeObject()._okForStorage(false, true); + // TODO: combine field names for better error messages + if ( ! s.isOK() ) + return s; + } break; default: uassert( 12579, "unhandled cases in BSONObj okForStorage" , 0 ); } - } + + // After we have processed one field, we are no longer on the first field + first = false; } - return true; + return Status::OK(); } void BSONObj::dump() const { - out() << hex; + LogstreamBuilder builder = out(); + builder << hex; const char *p = objdata(); for ( int i = 0; i < objsize(); i++ ) { - out() << i << '\t' << ( 0xff & ( (unsigned) *p ) ); + builder << i << '\t' << ( 0xff & ( (unsigned) *p ) ); if ( *p >= 'A' && *p <= 'z' ) - out() << '\t' << *p; - out() << endl; + builder << '\t' << *p; + builder << endl; p++; } } @@ -1208,7 +1294,7 @@ namespace mongo { case Date: appendDate( fieldName , numeric_limits::max() ); return; case Timestamp: // TODO integrate with Date SERVER-3304 - appendTimestamp( fieldName , numeric_limits::max() ); return; + append( fieldName , OpTime::max() ); return; case Undefined: // shared with EOO appendUndefined( fieldName ); return; diff --git a/src/mongo/db/jsobj.h b/src/mongo/db/jsobj.h index b108caaf314..ab5e03daed2 100644 --- a/src/mongo/db/jsobj.h +++ b/src/mongo/db/jsobj.h @@ -29,17 +29,18 @@ #pragma once #include "mongo/pch.h" -#include "../bson/util/builder.h" -#include "../util/optime.h" -#include "../bson/bsontypes.h" -#include "../bson/oid.h" -#include "../bson/bsonelement.h" -#include "../bson/bsonobj.h" -#include "../bson/bsonmisc.h" -#include "../bson/bsonobjbuilder.h" -#include "../bson/bsonobjiterator.h" -#include "../bson/bson-inl.h" -#include "../bson/ordering.h" + +#include "mongo/bson/util/builder.h" +#include "mongo/bson/optime.h" +#include "mongo/bson/bsontypes.h" +#include "mongo/bson/oid.h" +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/bson/bson-inl.h" +#include "mongo/bson/ordering.h" #include "mongo/base/string_data.h" -#include "../bson/bson_db.h" +#include "mongo/bson/bson_db.h" diff --git a/src/mongo/db/jsobjmanipulator.h b/src/mongo/db/jsobjmanipulator.h index 05666409e62..85f2b5cc65a 100644 --- a/src/mongo/db/jsobjmanipulator.h +++ b/src/mongo/db/jsobjmanipulator.h @@ -14,12 +14,23 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once -#include "jsobj.h" -//#include "dur.h" +#include "mongo/db/jsobj.h" namespace mongo { @@ -32,10 +43,6 @@ namespace mongo { _element( element ) { verify( !_element.eoo() ); } - /** Replace a Timestamp type with a Date type initialized to - OpTime::now().asDate() - */ - void initTimestamp(); // Note the ones with a capital letter call getDur().writing and journal @@ -64,26 +71,6 @@ namespace mongo { memcpy( value(), e.value(), e.valuesize() ); } - /* dur:: version */ - void ReplaceTypeAndValue( const BSONElement &e ); - - static void lookForTimestamps( const BSONObj& obj ) { - // If have a Timestamp field as the first or second element, - // update it to a Date field set to OpTime::now().asDate(). The - // replacement policy is a work in progress. - - BSONObjIterator i( obj ); - for( int j = 0; i.moreWithEOO() && j < 2; ++j ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - if ( e.type() == Timestamp ) { - // performance note, this locks a mutex: - BSONElementManipulator( e ).initTimestamp(); - break; - } - } - } private: char *data() { return nonConst( _element.rawdata() ); } char *value() { return nonConst( _element.value() ); } diff --git a/src/mongo/db/json.cpp b/src/mongo/db/json.cpp index afe08f3b0f5..0203bc82932 100644 --- a/src/mongo/db/json.cpp +++ b/src/mongo/db/json.cpp @@ -15,12 +15,14 @@ #include "mongo/db/json.h" +#include "mongo/base/parse_number.h" #include "mongo/db/jsobj.h" #include "mongo/platform/cstdint.h" #include "mongo/platform/strtoll.h" #include "mongo/util/base64.h" #include "mongo/util/hex.h" #include "mongo/util/mongoutils/str.h" +#include "mongo/util/time_support.h" namespace mongo { @@ -45,7 +47,10 @@ namespace mongo { STRINGVAL_RESERVE_SIZE = 4096, BINDATA_RESERVE_SIZE = 4096, BINDATATYPE_RESERVE_SIZE = 4096, - NS_RESERVE_SIZE = 64 + NS_RESERVE_SIZE = 64, + DB_RESERVE_SIZE = 64, + NUMBERLONG_RESERVE_SIZE = 64, + DATE_RESERVE_SIZE = 64 }; static const char* LBRACE = "{", @@ -68,60 +73,74 @@ namespace mongo { ossmsg << msg; ossmsg << ": offset:"; ossmsg << offset(); + ossmsg << " of:"; + ossmsg << _buf; return Status(ErrorCodes::FailedToParse, ossmsg.str()); } Status JParse::value(const StringData& fieldName, BSONObjBuilder& builder) { MONGO_JSON_DEBUG("fieldName: " << fieldName); - if (accept(LBRACE, false)) { + if (peekToken(LBRACE)) { Status ret = object(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept(LBRACKET, false)) { + else if (peekToken(LBRACKET)) { Status ret = array(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept("new")) { + else if (readToken("new")) { Status ret = constructor(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept("Date")) { + else if (readToken("Date")) { Status ret = date(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept("Timestamp")) { + else if (readToken("Timestamp")) { Status ret = timestamp(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept("ObjectId")) { + else if (readToken("ObjectId")) { Status ret = objectId(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept("Dbref") || accept("DBRef")) { + else if (readToken("NumberLong")) { + Status ret = numberLong(fieldName, builder); + if (ret != Status::OK()) { + return ret; + } + } + else if (readToken("NumberInt")) { + Status ret = numberInt(fieldName, builder); + if (ret != Status::OK()) { + return ret; + } + } + else if (readToken("Dbref") || readToken("DBRef")) { Status ret = dbRef(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept(FORWARDSLASH, false)) { + else if (peekToken(FORWARDSLASH)) { Status ret = regex(fieldName, builder); if (ret != Status::OK()) { return ret; } } - else if (accept(DOUBLEQUOTE, false) || accept(SINGLEQUOTE, false)) { + else if (peekToken(DOUBLEQUOTE) || peekToken(SINGLEQUOTE)) { std::string valueString; valueString.reserve(STRINGVAL_RESERVE_SIZE); Status ret = quotedString(&valueString); @@ -130,25 +149,25 @@ namespace mongo { } builder.append(fieldName, valueString); } - else if (accept("true")) { + else if (readToken("true")) { builder.append(fieldName, true); } - else if (accept("false")) { + else if (readToken("false")) { builder.append(fieldName, false); } - else if (accept("null")) { + else if (readToken("null")) { builder.appendNull(fieldName); } - else if (accept("undefined")) { + else if (readToken("undefined")) { builder.appendUndefined(fieldName); } - else if (accept("NaN")) { + else if (readToken("NaN")) { builder.append(fieldName, std::numeric_limits::quiet_NaN()); } - else if (accept("Infinity")) { + else if (readToken("Infinity")) { builder.append(fieldName, std::numeric_limits::infinity()); } - else if (accept("-Infinity")) { + else if (readToken("-Infinity")) { builder.append(fieldName, -std::numeric_limits::infinity()); } else { @@ -162,12 +181,12 @@ namespace mongo { Status JParse::object(const StringData& fieldName, BSONObjBuilder& builder, bool subObject) { MONGO_JSON_DEBUG("fieldName: " << fieldName); - if (!accept(LBRACE)) { + if (!readToken(LBRACE)) { return parseError("Expecting '{'"); } // Empty object - if (accept(RBRACE)) { + if (readToken(RBRACE)) { if (subObject) { BSONObjBuilder empty(builder.subobjStart(fieldName)); empty.done(); @@ -246,6 +265,15 @@ namespace mongo { return ret; } } + else if (firstField == "$numberLong") { + if (!subObject) { + return parseError("Reserved field name in base object: $numberLong"); + } + Status ret = numberLongObject(fieldName, builder); + if (ret != Status::OK()) { + return ret; + } + } else { // firstField != // Normal object @@ -257,21 +285,21 @@ namespace mongo { objBuilder = subObjBuilder.get(); } - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } Status valueRet = value(firstField, *objBuilder); if (valueRet != Status::OK()) { return valueRet; } - while (accept(COMMA)) { + while (readToken(COMMA)) { std::string fieldName; fieldName.reserve(FIELD_RESERVE_SIZE); Status fieldRet = field(&fieldName); if (fieldRet != Status::OK()) { return fieldRet; } - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } Status valueRet = value(fieldName, *objBuilder); @@ -280,14 +308,14 @@ namespace mongo { } } } - if (!accept(RBRACE)) { + if (!readToken(RBRACE)) { return parseError("Expecting '}' or ','"); } return Status::OK(); } Status JParse::objectIdObject(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expected ':'"); } std::string id; @@ -307,7 +335,7 @@ namespace mongo { } Status JParse::binaryObject(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expected ':'"); } std::string binDataString; @@ -323,14 +351,14 @@ namespace mongo { return parseError("Invalid character in base64 encoded string"); } const std::string& binData = base64::decode(binDataString); - if (!accept(COMMA)) { + if (!readToken(COMMA)) { return parseError("Expected ','"); } - if (!acceptField("$type")) { + if (!readField("$type")) { return parseError("Expected second field name: \"$type\", in \"$binary\" object"); } - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expected ':'"); } std::string binDataType; @@ -349,48 +377,102 @@ namespace mongo { } Status JParse::dateObject(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expected ':'"); } errno = 0; char* endptr; - Date_t date = static_cast(strtoll(_input, &endptr, 10)); - if (_input == endptr) { - return parseError("Date expecting integer milliseconds"); + Date_t date; + + if (peekToken(DOUBLEQUOTE)) { + std::string dateString; + dateString.reserve(DATE_RESERVE_SIZE); + Status ret = quotedString(&dateString); + if (!ret.isOK()) { + return ret; + } + StatusWith dateRet = dateFromISOString(dateString); + if (!dateRet.isOK()) { + return dateRet.getStatus(); + } + date = dateRet.getValue(); } - if (errno == ERANGE) { - /* Need to handle this because jsonString outputs the value of Date_t as unsigned. - * See SERVER-8330 and SERVER-8573 */ - errno = 0; - date = strtoull(_input, &endptr, 10); + else if (readToken(LBRACE)) { + std::string fieldName; + fieldName.reserve(FIELD_RESERVE_SIZE); + Status ret = field(&fieldName); + if (ret != Status::OK()) { + return ret; + } + if (fieldName != "$numberLong") { + return parseError("Expected field name: $numberLong for $date value object"); + } + if (!readToken(COLON)) { + return parseError("Expecting ':'"); + } + + // The number must be a quoted string, since large long numbers could overflow a double + // and thus may not be valid JSON + std::string numberLongString; + numberLongString.reserve(NUMBERLONG_RESERVE_SIZE); + ret = quotedString(&numberLongString); + if (!ret.isOK()) { + return ret; + } + + long long numberLong; + ret = parseNumberFromString(numberLongString, &numberLong); + if (!ret.isOK()) { + return ret; + } + date = numberLong; + } + else { + // SERVER-11920: We should use parseNumberFromString here, but that function requires + // that we know ahead of time where the number ends, which is not currently the case. + date = static_cast(strtoll(_input, &endptr, 10)); + if (_input == endptr) { + return parseError("Date expecting integer milliseconds"); + } if (errno == ERANGE) { - return parseError("Date milliseconds overflow"); + /* Need to handle this because jsonString outputs the value of Date_t as unsigned. + * See SERVER-8330 and SERVER-8573 */ + errno = 0; + // SERVER-11920: We should use parseNumberFromString here, but that function + // requires that we know ahead of time where the number ends, which is not currently + // the case. + date = strtoull(_input, &endptr, 10); + if (errno == ERANGE) { + return parseError("Date milliseconds overflow"); + } } + _input = endptr; } - _input = endptr; builder.appendDate(fieldName, date); return Status::OK(); } Status JParse::timestampObject(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } - if (!accept(LBRACE)) { + if (!readToken(LBRACE)) { return parseError("Expecting '{' to start \"$timestamp\" object"); } - if (!acceptField("t")) { + if (!readField("t")) { return parseError("Expected field name \"t\" in \"$timestamp\" sub object"); } - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } - if (accept("-")) { + if (readToken("-")) { return parseError("Negative seconds in \"$timestamp\""); } errno = 0; char* endptr; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. uint32_t seconds = strtoul(_input, &endptr, 10); if (errno == ERANGE) { return parseError("Timestamp seconds overflow"); @@ -399,20 +481,22 @@ namespace mongo { return parseError("Expecting unsigned integer seconds in \"$timestamp\""); } _input = endptr; - if (!accept(COMMA)) { + if (!readToken(COMMA)) { return parseError("Expecting ','"); } - if (!acceptField("i")) { + if (!readField("i")) { return parseError("Expected field name \"i\" in \"$timestamp\" sub object"); } - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } - if (accept("-")) { + if (readToken("-")) { return parseError("Negative increment in \"$timestamp\""); } errno = 0; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. uint32_t count = strtoul(_input, &endptr, 10); if (errno == ERANGE) { return parseError("Timestamp increment overflow"); @@ -422,7 +506,7 @@ namespace mongo { } _input = endptr; - if (!accept(RBRACE)) { + if (!readToken(RBRACE)) { return parseError("Expecting '}'"); } builder.appendTimestamp(fieldName, (static_cast(seconds))*1000, count); @@ -430,7 +514,7 @@ namespace mongo { } Status JParse::regexObject(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } std::string pat; @@ -439,11 +523,11 @@ namespace mongo { if (patRet != Status::OK()) { return patRet; } - if (accept(COMMA)) { - if (!acceptField("$options")) { + if (readToken(COMMA)) { + if (!readField("$options")) { return parseError("Expected field name: \"$options\" in \"$regex\" object"); } - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } std::string opt; @@ -468,8 +552,8 @@ namespace mongo { BSONObjBuilder subBuilder(builder.subobjStart(fieldName)); - if (!accept(COLON)) { - return parseError("Expecting ':'"); + if (!readToken(COLON)) { + return parseError("DBRef: Expecting ':'"); } std::string ns; ns.reserve(NS_RESERVE_SIZE); @@ -479,54 +563,94 @@ namespace mongo { } subBuilder.append("$ref", ns); - if (!accept(COMMA)) { - return parseError("Expecting ','"); + if (!readToken(COMMA)) { + return parseError("DBRef: Expecting ','"); } - if (!acceptField("$id")) { - return parseError("Expected field name: \"$id\" in \"$ref\" object"); + if (!readField("$id")) { + return parseError("DBRef: Expected field name: \"$id\" in \"$ref\" object"); } - if (!accept(COLON)) { - return parseError("Expecting ':'"); + if (!readToken(COLON)) { + return parseError("DBRef: Expecting ':'"); } Status valueRet = value("$id", subBuilder); if (valueRet != Status::OK()) { return valueRet; } + if (readToken(COMMA)) { + if (!readField("$db")) { + return parseError("DBRef: Expected field name: \"$db\" in \"$ref\" object"); + } + if (!readToken(COLON)) { + return parseError("DBRef: Expecting ':'"); + } + std::string db; + db.reserve(DB_RESERVE_SIZE); + ret = quotedString(&db); + if (ret != Status::OK()) { + return ret; + } + subBuilder.append("$db", db); + } + subBuilder.done(); return Status::OK(); } Status JParse::undefinedObject(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(COLON)) { + if (!readToken(COLON)) { return parseError("Expecting ':'"); } - if (!accept("true")) { + if (!readToken("true")) { return parseError("Reserved field \"$undefined\" requires value of true"); } builder.appendUndefined(fieldName); return Status::OK(); } + Status JParse::numberLongObject(const StringData& fieldName, BSONObjBuilder& builder) { + if (!readToken(COLON)) { + return parseError("Expecting ':'"); + } + + // The number must be a quoted string, since large long numbers could overflow a double and + // thus may not be valid JSON + std::string numberLongString; + numberLongString.reserve(NUMBERLONG_RESERVE_SIZE); + Status ret = quotedString(&numberLongString); + if (!ret.isOK()) { + return ret; + } + + long long numberLong; + ret = parseNumberFromString(numberLongString, &numberLong); + if (!ret.isOK()) { + return ret; + } + + builder.appendNumber(fieldName, numberLong); + return Status::OK(); + } + Status JParse::array(const StringData& fieldName, BSONObjBuilder& builder) { MONGO_JSON_DEBUG("fieldName: " << fieldName); uint32_t index(0); - if (!accept(LBRACKET)) { + if (!readToken(LBRACKET)) { return parseError("Expecting '['"); } BSONObjBuilder subBuilder(builder.subarrayStart(fieldName)); - if (!accept(RBRACKET, false)) { + if (!peekToken(RBRACKET)) { do { Status ret = value(builder.numStr(index), subBuilder); if (ret != Status::OK()) { return ret; } index++; - } while (accept(COMMA)); + } while (readToken(COMMA)); } subBuilder.done(); - if (!accept(RBRACKET)) { + if (!readToken(RBRACKET)) { return parseError("Expecting ']' or ','"); } return Status::OK(); @@ -537,7 +661,7 @@ namespace mongo { * Also note that unlike the interactive shell "Date(x)" and "new Date(x)" * have the same behavior. XXX: this may not be desired. */ Status JParse::constructor(const StringData& fieldName, BSONObjBuilder& builder) { - if (accept("Date")) { + if (readToken("Date")) { date(fieldName, builder); } else { @@ -547,11 +671,13 @@ namespace mongo { } Status JParse::date(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(LPAREN)) { + if (!readToken(LPAREN)) { return parseError("Expecting '('"); } errno = 0; char* endptr; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. Date_t date = static_cast(strtoll(_input, &endptr, 10)); if (_input == endptr) { return parseError("Date expecting integer milliseconds"); @@ -560,13 +686,15 @@ namespace mongo { /* Need to handle this because jsonString outputs the value of Date_t as unsigned. * See SERVER-8330 and SERVER-8573 */ errno = 0; + // SERVER-11920: We should use parseNumberFromString here, but that function requires + // that we know ahead of time where the number ends, which is not currently the case. date = strtoull(_input, &endptr, 10); if (errno == ERANGE) { return parseError("Date milliseconds overflow"); } } _input = endptr; - if (!accept(RPAREN)) { + if (!readToken(RPAREN)) { return parseError("Expecting ')'"); } builder.appendDate(fieldName, date); @@ -574,14 +702,16 @@ namespace mongo { } Status JParse::timestamp(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(LPAREN)) { + if (!readToken(LPAREN)) { return parseError("Expecting '('"); } - if (accept("-")) { + if (readToken("-")) { return parseError("Negative seconds in \"$timestamp\""); } errno = 0; char* endptr; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. uint32_t seconds = strtoul(_input, &endptr, 10); if (errno == ERANGE) { return parseError("Timestamp seconds overflow"); @@ -590,13 +720,15 @@ namespace mongo { return parseError("Expecting unsigned integer seconds in \"$timestamp\""); } _input = endptr; - if (!accept(COMMA)) { + if (!readToken(COMMA)) { return parseError("Expecting ','"); } - if (accept("-")) { + if (readToken("-")) { return parseError("Negative seconds in \"$timestamp\""); } errno = 0; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. uint32_t count = strtoul(_input, &endptr, 10); if (errno == ERANGE) { return parseError("Timestamp increment overflow"); @@ -605,7 +737,7 @@ namespace mongo { return parseError("Expecting unsigned integer increment in \"$timestamp\""); } _input = endptr; - if (!accept(RPAREN)) { + if (!readToken(RPAREN)) { return parseError("Expecting ')'"); } builder.appendTimestamp(fieldName, (static_cast(seconds))*1000, count); @@ -613,7 +745,7 @@ namespace mongo { } Status JParse::objectId(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(LPAREN)) { + if (!readToken(LPAREN)) { return parseError("Expecting '('"); } std::string id; @@ -622,7 +754,7 @@ namespace mongo { if (ret != Status::OK()) { return ret; } - if (!accept(RPAREN)) { + if (!readToken(RPAREN)) { return parseError("Expecting ')'"); } if (id.size() != 24) { @@ -635,10 +767,57 @@ namespace mongo { return Status::OK(); } + Status JParse::numberLong(const StringData& fieldName, BSONObjBuilder& builder) { + if (!readToken(LPAREN)) { + return parseError("Expecting '('"); + } + errno = 0; + char* endptr; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. + int64_t val = strtoll(_input, &endptr, 10); + if (errno == ERANGE) { + return parseError("NumberLong out of range"); + } + if (_input == endptr) { + return parseError("Expecting number in NumberLong"); + } + _input = endptr; + if (!readToken(RPAREN)) { + return parseError("Expecting ')'"); + } + builder.appendNumber(fieldName, static_cast(val)); + return Status::OK(); + } + + Status JParse::numberInt(const StringData& fieldName, BSONObjBuilder& builder) { + if (!readToken(LPAREN)) { + return parseError("Expecting '('"); + } + errno = 0; + char* endptr; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. + int32_t val = strtol(_input, &endptr, 10); + if (errno == ERANGE) { + return parseError("NumberInt out of range"); + } + if (_input == endptr) { + return parseError("Expecting unsigned number in NumberInt"); + } + _input = endptr; + if (!readToken(RPAREN)) { + return parseError("Expecting ')'"); + } + builder.appendNumber(fieldName, static_cast(val)); + return Status::OK(); + } + + Status JParse::dbRef(const StringData& fieldName, BSONObjBuilder& builder) { BSONObjBuilder subBuilder(builder.subobjStart(fieldName)); - if (!accept(LPAREN)) { + if (!readToken(LPAREN)) { return parseError("Expecting '('"); } std::string ns; @@ -649,7 +828,7 @@ namespace mongo { } subBuilder.append("$ref", ns); - if (!accept(COMMA)) { + if (!readToken(COMMA)) { return parseError("Expecting ','"); } @@ -658,7 +837,17 @@ namespace mongo { return valueRet; } - if (!accept(RPAREN)) { + if (readToken(COMMA)) { + std::string db; + db.reserve(DB_RESERVE_SIZE); + Status dbRet = quotedString(&db); + if (dbRet != Status::OK()) { + return dbRet; + } + subBuilder.append("$db", db); + } + + if (!readToken(RPAREN)) { return parseError("Expecting ')'"); } @@ -667,7 +856,7 @@ namespace mongo { } Status JParse::regex(const StringData& fieldName, BSONObjBuilder& builder) { - if (!accept(FORWARDSLASH)) { + if (!readToken(FORWARDSLASH)) { return parseError("Expecting '/'"); } std::string pat; @@ -676,7 +865,7 @@ namespace mongo { if (patRet != Status::OK()) { return patRet; } - if (!accept(FORWARDSLASH)) { + if (!readToken(FORWARDSLASH)) { return parseError("Expecting '/'"); } std::string opt; @@ -722,6 +911,8 @@ namespace mongo { // reset errno to make sure that we are getting it from strtod errno = 0; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. retd = strtod(_input, &endptrd); // if pointer does not move, we found no digits if (_input == endptrd) { @@ -732,6 +923,8 @@ namespace mongo { } // reset errno to make sure that we are getting it from strtoll errno = 0; + // SERVER-11920: We should use parseNumberFromString here, but that function requires that + // we know ahead of time where the number ends, which is not currently the case. retll = strtoll(_input, &endptrll, 10); if (endptrll < endptrd || errno == ERANGE) { // The number either had characters only meaningful for a double or @@ -758,14 +951,20 @@ namespace mongo { Status JParse::field(std::string* result) { MONGO_JSON_DEBUG(""); - if (accept(DOUBLEQUOTE, false) || accept(SINGLEQUOTE, false)) { + if (peekToken(DOUBLEQUOTE) || peekToken(SINGLEQUOTE)) { // Quoted key // TODO: make sure quoted field names cannot contain null characters return quotedString(result); } else { // Unquoted key - while (_input < _input_end && isspace(*_input)) ++_input; + // 'isspace()' takes an 'int' (signed), so (default signed) 'char's get sign-extended + // and therefore 'corrupted' unless we force them to be unsigned ... 0x80 becomes + // 0xffffff80 as seen by isspace when sign-extended ... we want it to be 0x00000080 + while (_input < _input_end && + isspace(*reinterpret_cast(_input))) { + ++_input; + } if (_input >= _input_end) { return parseError("Field name expected"); } @@ -778,21 +977,21 @@ namespace mongo { Status JParse::quotedString(std::string* result) { MONGO_JSON_DEBUG(""); - if (accept(DOUBLEQUOTE, true)) { + if (readToken(DOUBLEQUOTE)) { Status ret = chars(result, "\""); if (ret != Status::OK()) { return ret; } - if (!accept(DOUBLEQUOTE)) { + if (!readToken(DOUBLEQUOTE)) { return parseError("Expecting '\"'"); } } - else if (accept(SINGLEQUOTE, true)) { + else if (readToken(SINGLEQUOTE)) { Status ret = chars(result, "'"); if (ret != Status::OK()) { return ret; } - if (!accept(SINGLEQUOTE)) { + if (!readToken(SINGLEQUOTE)) { return parseError("Expecting '''"); } } @@ -901,13 +1100,24 @@ namespace mongo { return oss.str(); } - bool JParse::accept(const char* token, bool advance) { + inline bool JParse::peekToken(const char* token) { + return readTokenImpl(token, false); + } + + inline bool JParse::readToken(const char* token) { + return readTokenImpl(token, true); + } + + bool JParse::readTokenImpl(const char* token, bool advance) { MONGO_JSON_DEBUG("token: " << token); const char* check = _input; if (token == NULL) { return false; } - while (check < _input_end && isspace(*check)) { + // 'isspace()' takes an 'int' (signed), so (default signed) 'char's get sign-extended + // and therefore 'corrupted' unless we force them to be unsigned ... 0x80 becomes + // 0xffffff80 as seen by isspace when sign-extended ... we want it to be 0x00000080 + while (check < _input_end && isspace(*reinterpret_cast(check))) { ++check; } while (*token != '\0') { @@ -922,7 +1132,7 @@ namespace mongo { return true; } - bool JParse::acceptField(const StringData& expectedField) { + bool JParse::readField(const StringData& expectedField) { MONGO_JSON_DEBUG("expectedField: " << expectedField); std::string nextField; nextField.reserve(FIELD_RESERVE_SIZE); @@ -976,7 +1186,16 @@ namespace mongo { } JParse jparse(jsonString); BSONObjBuilder builder; - Status ret = jparse.object("UNUSED", builder, false); + Status ret = Status::OK(); + try { + ret = jparse.object("UNUSED", builder, false); + } + catch(std::exception& e) { + std::ostringstream message; + message << "caught exception from within JSON parser: " << e.what(); + throw MsgAssertionException(17031, message.str()); + } + if (ret != Status::OK()) { ostringstream message; message << "code " << ret.code() << ": " << ret.codeString() << ": " << ret.reason(); diff --git a/src/mongo/db/json.h b/src/mongo/db/json.h index f344ebe1f90..04c881fa77d 100644 --- a/src/mongo/db/json.h +++ b/src/mongo/db/json.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -20,6 +32,7 @@ #include "mongo/bson/bsonobj.h" #include "mongo/base/status.h" +#include "mongo/client/export_macros.h" namespace mongo { @@ -36,10 +49,10 @@ namespace mongo { * @throws MsgAssertionException if parsing fails. The message included with * this assertion includes the character offset where parsing failed. */ - BSONObj fromjson(const std::string& str); + MONGO_CLIENT_API BSONObj fromjson(const std::string& str); /** @param len will be size of JSON object in text chars. */ - BSONObj fromjson(const char* str, int* len=NULL); + MONGO_CLIENT_API BSONObj fromjson(const char* str, int* len=NULL); /** * Parser class. A BSONObj is constructed incrementally by passing a @@ -59,6 +72,8 @@ namespace mongo { * VALUE : * STRING * | NUMBER + * | NUMBERINT + * | NUMBERLONG * | OBJECT * | ARRAY * @@ -103,6 +118,7 @@ namespace mongo { * | REGEXOBJECT * | REFOBJECT * | UNDEFINEDOBJECT + * | NUMBERLONGOBJECT * */ public: @@ -165,6 +181,12 @@ namespace mongo { */ Status undefinedObject(const StringData& fieldName, BSONObjBuilder&); + /* + * NUMBERLONGOBJECT : + * { FIELD("$numberLong") : "" } + */ + Status numberLongObject(const StringData& fieldName, BSONObjBuilder&); + /* * ARRAY : * [] @@ -204,6 +226,18 @@ namespace mongo { */ Status objectId(const StringData& fieldName, BSONObjBuilder&); + /* + * NUMBERLONG : + * NumberLong( ) + */ + Status numberLong(const StringData& fieldName, BSONObjBuilder&); + + /* + * NUMBERINT : + * NumberInt( ) + */ + Status numberInt(const StringData& fieldName, BSONObjBuilder&); + /* * DBREF : * Dbref( , <24 character hex string> ) @@ -322,19 +356,35 @@ namespace mongo { */ std::string encodeUTF8(unsigned char first, unsigned char second) const; + /** + * @return true if the given token matches the next non whitespace + * sequence in our buffer, and false if the token doesn't match or + * we reach the end of our buffer. Do not update the pointer to our + * buffer (same as calling readTokenImpl with advance=false). + */ + inline bool peekToken(const char* token); + + /** + * @return true if the given token matches the next non whitespace + * sequence in our buffer, and false if the token doesn't match or + * we reach the end of our buffer. Updates the pointer to our + * buffer (same as calling readTokenImpl with advance=true). + */ + inline bool readToken(const char* token); + /** * @return true if the given token matches the next non whitespace * sequence in our buffer, and false if the token doesn't match or * we reach the end of our buffer. Do not update the pointer to our * buffer if advance is false. */ - bool accept(const char* token, bool advance=true); + bool readTokenImpl(const char* token, bool advance=true); /** * @return true if the next field in our stream matches field. * Handles single quoted, double quoted, and unquoted field names */ - bool acceptField(const StringData& field); + bool readField(const StringData& field); /** * @return true if matchChar is in matchSet diff --git a/src/mongo/db/key.cpp b/src/mongo/db/key.cpp deleted file mode 100644 index 3d9eaa7e82c..00000000000 --- a/src/mongo/db/key.cpp +++ /dev/null @@ -1,683 +0,0 @@ -// @file key.cpp - -/** -* Copyright (C) 2011 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/key.h" - -#include "mongo/bson/util/builder.h" -#include "mongo/platform/float_utils.h" -#include "mongo/util/startup_test.h" - - -namespace mongo { - - extern const Ordering nullOrdering = Ordering::make(BSONObj()); - - // KeyBson is for V0 (version #0) indexes - - int oldCompare(const BSONObj& l,const BSONObj& r, const Ordering &o); - - // "old" = pre signed dates & such; i.e. btree V0 - /* must be same canon type when called */ - int oldCompareElementValues(const BSONElement& l, const BSONElement& r) { - dassert( l.canonicalType() == r.canonicalType() ); - int f; - double x; - - switch ( l.type() ) { - case EOO: - case Undefined: // EOO and Undefined are same canonicalType - case jstNULL: - case MaxKey: - case MinKey: - return 0; - case Bool: - return *l.value() - *r.value(); - case Timestamp: - case Date: - // unsigned dates for old version - if ( l.date() < r.date() ) - return -1; - return l.date() == r.date() ? 0 : 1; - case NumberLong: - if( r.type() == NumberLong ) { - long long L = l._numberLong(); - long long R = r._numberLong(); - if( L < R ) return -1; - if( L == R ) return 0; - return 1; - } - // else fall through - case NumberInt: - case NumberDouble: { - double left = l.number(); - double right = r.number(); - bool lNan = !( left <= numeric_limits< double >::max() && - left >= -numeric_limits< double >::max() ); - bool rNan = !( right <= numeric_limits< double >::max() && - right >= -numeric_limits< double >::max() ); - if ( lNan ) { - if ( rNan ) { - return 0; - } - else { - return -1; - } - } - else if ( rNan ) { - return 1; - } - x = left - right; - if ( x < 0 ) return -1; - return x == 0 ? 0 : 1; - } - case jstOID: - return memcmp(l.value(), r.value(), 12); - case Code: - case Symbol: - case String: - // nulls not allowed in the middle of strings in the old version - return strcmp(l.valuestr(), r.valuestr()); - case Object: - case Array: - return oldCompare(l.embeddedObject(), r.embeddedObject(), nullOrdering); - case DBRef: { - int lsz = l.valuesize(); - int rsz = r.valuesize(); - if ( lsz - rsz != 0 ) return lsz - rsz; - return memcmp(l.value(), r.value(), lsz); - } - case BinData: { - int lsz = l.objsize(); // our bin data size in bytes, not including the subtype byte - int rsz = r.objsize(); - if ( lsz - rsz != 0 ) return lsz - rsz; - return memcmp(l.value()+4, r.value()+4, lsz+1); - } - case RegEx: { - int c = strcmp(l.regex(), r.regex()); - if ( c ) - return c; - return strcmp(l.regexFlags(), r.regexFlags()); - } - case CodeWScope : { - f = l.canonicalType() - r.canonicalType(); - if ( f ) - return f; - f = strcmp( l.codeWScopeCode() , r.codeWScopeCode() ); - if ( f ) - return f; - f = strcmp( l.codeWScopeScopeDataUnsafe() , r.codeWScopeScopeDataUnsafe() ); - if ( f ) - return f; - return 0; - } - default: - out() << "oldCompareElementValues: bad type " << (int) l.type() << endl; - verify(false); - } - return -1; - } - - int oldElemCompare(const BSONElement&l , const BSONElement& r) { - int lt = (int) l.canonicalType(); - int rt = (int) r.canonicalType(); - int x = lt - rt; - if( x ) - return x; - return oldCompareElementValues(l, r); - } - - // pre signed dates & such - int oldCompare(const BSONObj& l,const BSONObj& r, const Ordering &o) { - BSONObjIterator i(l); - BSONObjIterator j(r); - unsigned mask = 1; - while ( 1 ) { - // so far, equal... - - BSONElement l = i.next(); - BSONElement r = j.next(); - if ( l.eoo() ) - return r.eoo() ? 0 : -1; - if ( r.eoo() ) - return 1; - - int x; - { - x = oldElemCompare(l, r); - if( o.descending(mask) ) - x = -x; - } - if ( x != 0 ) - return x; - mask <<= 1; - } - return -1; - } - - /* old style compares: - - dates are unsigned - - strings no nulls - */ - int KeyBson::woCompare(const KeyBson& r, const Ordering &o) const { - return oldCompare(_o, r._o, o); - } - - // woEqual could be made faster than woCompare but this is for backward compatibility so not worth a big effort - bool KeyBson::woEqual(const KeyBson& r) const { - return oldCompare(_o, r._o, nullOrdering) == 0; - } - - // [ ][HASMORE][x][y][canontype_4bits] - enum CanonicalsEtc { - cminkey=1, - cnull=2, - cdouble=4, - cstring=6, - cbindata=7, - coid=8, - cfalse=10, - ctrue=11, - cdate=12, - cmaxkey=14, - cCANONTYPEMASK = 0xf, - cY = 0x10, - cint = cY | cdouble, - cX = 0x20, - clong = cX | cdouble, - cHASMORE = 0x40, - cNOTUSED = 0x80 // but see IsBSON sentinel - this bit not usable without great care - }; - - // bindata bson type - const unsigned BinDataLenMask = 0xf0; // lengths are powers of 2 of this value - const unsigned BinDataTypeMask = 0x0f; // 0-7 as you would expect, 8-15 are 128+value. see BinDataType. - const int BinDataLenMax = 32; - const int BinDataLengthToCode[] = { - 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, - 0x80, -1/*9*/, 0x90/*10*/, -1/*11*/, 0xa0/*12*/, -1/*13*/, 0xb0/*14*/, -1/*15*/, - 0xc0/*16*/, -1, -1, -1, 0xd0/*20*/, -1, -1, -1, - 0xe0/*24*/, -1, -1, -1, -1, -1, -1, -1, - 0xf0/*32*/ - }; - const int BinDataCodeToLength[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 32 - }; - - int binDataCodeToLength(int codeByte) { - return BinDataCodeToLength[codeByte >> 4]; - } - - /** object cannot be represented in compact format. so store in traditional bson format - with a leading sentinel byte IsBSON to indicate it's in that format. - - Given that the KeyV1Owned constructor already grabbed a bufbuilder, we reuse it here - so that we don't have to do an extra malloc. - */ - void KeyV1Owned::traditional(const BSONObj& obj) { - b.reset(); - b.appendUChar(IsBSON); - b.appendBuf(obj.objdata(), obj.objsize()); - _keyData = (const unsigned char *) b.buf(); - } - - KeyV1Owned::KeyV1Owned(const KeyV1& rhs) { - b.appendBuf( rhs.data(), rhs.dataSize() ); - _keyData = (const unsigned char *) b.buf(); - dassert( b.len() == dataSize() ); // check datasize method is correct - dassert( (*_keyData & cNOTUSED) == 0 ); - } - - // fromBSON to Key format - KeyV1Owned::KeyV1Owned(const BSONObj& obj) { - BSONObj::iterator i(obj); - unsigned char bits = 0; - while( 1 ) { - BSONElement e = i.next(); - if( i.more() ) - bits |= cHASMORE; - switch( e.type() ) { - case MinKey: - b.appendUChar(cminkey|bits); - break; - case jstNULL: - b.appendUChar(cnull|bits); - break; - case MaxKey: - b.appendUChar(cmaxkey|bits); - break; - case Bool: - b.appendUChar( (e.boolean()?ctrue:cfalse) | bits ); - break; - case jstOID: - b.appendUChar(coid|bits); - b.appendBuf(&e.__oid(), sizeof(OID)); - break; - case BinData: - { - int t = e.binDataType(); - // 0-7 and 0x80 to 0x87 are supported by Key - if( (t & 0x78) == 0 && t != ByteArrayDeprecated ) { - int len; - const char * d = e.binData(len); - if( len <= BinDataLenMax ) { - int code = BinDataLengthToCode[len]; - if( code >= 0 ) { - if( t >= 128 ) - t = (t-128) | 0x08; - dassert( (code&t) == 0 ); - b.appendUChar( cbindata|bits ); - b.appendUChar( code | t ); - b.appendBuf(d, len); - break; - } - } - } - traditional(obj); - return; - } - case Date: - b.appendUChar(cdate|bits); - b.appendStruct(e.date()); - break; - case String: - { - b.appendUChar(cstring|bits); - // note we do not store the terminating null, to save space. - unsigned x = (unsigned) e.valuestrsize() - 1; - if( x > 255 ) { - traditional(obj); - return; - } - b.appendUChar(x); - b.appendBuf(e.valuestr(), x); - break; - } - case NumberInt: - b.appendUChar(cint|bits); - b.appendNum((double) e._numberInt()); - break; - case NumberLong: - { - long long n = e._numberLong(); - long long m = 2LL << 52; - DEV { - long long d = m-1; - verify( ((long long) ((double) -d)) == -d ); - } - if( n >= m || n <= -m ) { - // can't represent exactly as a double - traditional(obj); - return; - } - b.appendUChar(clong|bits); - b.appendNum((double) n); - break; - } - case NumberDouble: - { - double d = e._numberDouble(); - if( isNaN(d) ) { - traditional(obj); - return; - } - b.appendUChar(cdouble|bits); - b.appendNum(d); - break; - } - default: - // if other types involved, store as traditional BSON - traditional(obj); - return; - } - if( !i.more() ) - break; - bits = 0; - } - _keyData = (const unsigned char *) b.buf(); - dassert( b.len() == dataSize() ); // check datasize method is correct - dassert( (*_keyData & cNOTUSED) == 0 ); - } - - BSONObj KeyV1::toBson() const { - verify( _keyData != 0 ); - if( !isCompactFormat() ) - return bson(); - - BSONObjBuilder b(512); - const unsigned char *p = _keyData; - while( 1 ) { - unsigned bits = *p++; - - switch( bits & 0x3f ) { - case cminkey: b.appendMinKey(""); break; - case cnull: b.appendNull(""); break; - case cfalse: b.appendBool("", false); break; - case ctrue: b.appendBool("", true); break; - case cmaxkey: - b.appendMaxKey(""); - break; - case cstring: - { - unsigned sz = *p++; - // we build the element ourself as we have to null terminate it - BufBuilder &bb = b.bb(); - bb.appendNum((char) String); - bb.appendUChar(0); // fieldname "" - bb.appendNum(sz+1); - bb.appendBuf(p, sz); - bb.appendUChar(0); // null char at end of string - p += sz; - break; - } - case coid: - b.appendOID("", (OID *) p); - p += sizeof(OID); - break; - case cbindata: - { - int len = binDataCodeToLength(*p); - int subtype = (*p) & BinDataTypeMask; - if( subtype & 0x8 ) { - subtype = (subtype & 0x7) | 0x80; - } - b.appendBinData("", len, (BinDataType) subtype, ++p); - p += len; - break; - } - case cdate: - b.appendDate("", (Date_t&) *p); - p += 8; - break; - case cdouble: - b.append("", (double&) *p); - p += sizeof(double); - break; - case cint: - b.append("", static_cast< int >((reinterpret_cast< const PackedDouble& >(*p)).d)); - p += sizeof(double); - break; - case clong: - b.append("", static_cast< long long>((reinterpret_cast< const PackedDouble& >(*p)).d)); - p += sizeof(double); - break; - default: - verify(false); - } - - if( (bits & cHASMORE) == 0 ) - break; - } - return b.obj(); - } - - static int compare(const unsigned char *&l, const unsigned char *&r) { - int lt = (*l & cCANONTYPEMASK); - int rt = (*r & cCANONTYPEMASK); - int x = lt - rt; - if( x ) - return x; - - l++; r++; - - // same type - switch( lt ) { - case cdouble: - { - double L = (reinterpret_cast< const PackedDouble* >(l))->d; - double R = (reinterpret_cast< const PackedDouble* >(r))->d; - if( L < R ) - return -1; - if( L != R ) - return 1; - l += 8; r += 8; - break; - } - case cstring: - { - int lsz = *l; - int rsz = *r; - int common = min(lsz, rsz); - l++; r++; // skip the size byte - // use memcmp as we (will) allow zeros in UTF8 strings - int res = memcmp(l, r, common); - if( res ) - return res; - // longer string is the greater one - int diff = lsz-rsz; - if( diff ) - return diff; - l += lsz; r += lsz; - break; - } - case cbindata: - { - int L = *l; - int R = *r; - int llen = binDataCodeToLength(L); - int diff = L-R; // checks length and subtype simultaneously - if( diff ) { - // unfortunately nibbles are backwards to do subtype and len in one check (could bit swap...) - int rlen = binDataCodeToLength(R); - if( llen != rlen ) - return llen - rlen; - return diff; - } - // same length, same type - l++; r++; - int res = memcmp(l, r, llen); - if( res ) - return res; - l += llen; r += llen; - break; - } - case cdate: - { - long long L = *((long long *) l); - long long R = *((long long *) r); - if( L < R ) - return -1; - if( L > R ) - return 1; - l += 8; r += 8; - break; - } - case coid: - { - int res = memcmp(l, r, sizeof(OID)); - if( res ) - return res; - l += 12; r += 12; - break; - } - default: - // all the others are a match -- e.g. null == null - ; - } - - return 0; - } - - // at least one of this and right are traditional BSON format - int NOINLINE_DECL KeyV1::compareHybrid(const KeyV1& right, const Ordering& order) const { - BSONObj L = toBson(); - BSONObj R = right.toBson(); - return L.woCompare(R, order, /*considerfieldname*/false); - } - - int KeyV1::woCompare(const KeyV1& right, const Ordering &order) const { - const unsigned char *l = _keyData; - const unsigned char *r = right._keyData; - - if( (*l|*r) == IsBSON ) // only can do this if cNOTUSED maintained - return compareHybrid(right, order); - - unsigned mask = 1; - while( 1 ) { - char lval = *l; - char rval = *r; - { - int x = compare(l, r); // updates l and r pointers - if( x ) { - if( order.descending(mask) ) - x = -x; - return x; - } - } - - { - int x = ((int)(lval & cHASMORE)) - ((int)(rval & cHASMORE)); - if( x ) - return x; - if( (lval & cHASMORE) == 0 ) - break; - } - - mask <<= 1; - } - - return 0; - } - - static unsigned sizes[] = { - 0, - 1, //cminkey=1, - 1, //cnull=2, - 0, - 9, //cdouble=4, - 0, - 0, //cstring=6, - 0, - 13, //coid=8, - 0, - 1, //cfalse=10, - 1, //ctrue=11, - 9, //cdate=12, - 0, - 1, //cmaxkey=14, - 0 - }; - - inline unsigned sizeOfElement(const unsigned char *p) { - unsigned type = *p & cCANONTYPEMASK; - unsigned sz = sizes[type]; - if( sz == 0 ) { - if( type == cstring ) { - sz = ((unsigned) p[1]) + 2; - } - else { - verify( type == cbindata ); - sz = binDataCodeToLength(p[1]) + 2; - } - } - return sz; - } - - int KeyV1::dataSize() const { - const unsigned char *p = _keyData; - if( !isCompactFormat() ) { - return bson().objsize() + 1; - } - - bool more; - do { - unsigned z = sizeOfElement(p); - more = (*p & cHASMORE) != 0; - p += z; - } while( more ); - return p - _keyData; - } - - bool KeyV1::woEqual(const KeyV1& right) const { - const unsigned char *l = _keyData; - const unsigned char *r = right._keyData; - - if( (*l|*r) == IsBSON ) { - return toBson().equal(right.toBson()); - } - - while( 1 ) { - char lval = *l; - char rval = *r; - if( (lval&(cCANONTYPEMASK|cHASMORE)) != (rval&(cCANONTYPEMASK|cHASMORE)) ) - return false; - l++; r++; - switch( lval&cCANONTYPEMASK ) { - case coid: - if( *((unsigned*) l) != *((unsigned*) r) ) - return false; - l += 4; r += 4; - case cdate: - if( *((unsigned long long *) l) != *((unsigned long long *) r) ) - return false; - l += 8; r += 8; - break; - case cdouble: - if( (reinterpret_cast< const PackedDouble* > (l))->d != (reinterpret_cast< const PackedDouble* >(r))->d ) - return false; - l += 8; r += 8; - break; - case cstring: - { - if( *l != *r ) - return false; // not same length - unsigned sz = ((unsigned) *l) + 1; - if( memcmp(l, r, sz) ) - return false; - l += sz; r += sz; - break; - } - case cbindata: - { - if( *l != *r ) - return false; // len or subtype mismatch - int len = binDataCodeToLength(*l) + 1; - if( memcmp(l, r, len) ) - return false; - l += len; r += len; - break; - } - case cminkey: - case cnull: - case cfalse: - case ctrue: - case cmaxkey: - break; - default: - verify(false); - } - if( (lval&cHASMORE) == 0 ) - break; - } - return true; - } - - struct CmpUnitTest : public StartupTest { - void run() { - char a[2]; - char b[2]; - a[0] = -3; - a[1] = 0; - b[0] = 3; - b[1] = 0; - verify( strcmp(a,b)>0 && memcmp(a,b,2)>0 ); - } - } cunittest; - -} diff --git a/src/mongo/db/key.h b/src/mongo/db/key.h deleted file mode 100644 index e31d4108100..00000000000 --- a/src/mongo/db/key.h +++ /dev/null @@ -1,118 +0,0 @@ -// @file key.h class(es) representing individual keys in a btree - -/** -* Copyright (C) 2011 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "jsobj.h" - -namespace mongo { - - /** Key class for precomputing a small format index key that is denser than a traditional BSONObj. - - KeyBson is a legacy wrapper implementation for old BSONObj style keys for v:0 indexes. - - KeyV1 is the new implementation. - */ - class KeyBson /* "KeyV0" */ { - public: - KeyBson() { } - explicit KeyBson(const char *keyData) : _o(keyData) { } - explicit KeyBson(const BSONObj& obj) : _o(obj) { } - int woCompare(const KeyBson& r, const Ordering &o) const; - BSONObj toBson() const { return _o; } - string toString() const { return _o.toString(); } - int dataSize() const { return _o.objsize(); } - const char * data() const { return _o.objdata(); } - BSONElement _firstElement() const { return _o.firstElement(); } - bool isCompactFormat() const { return false; } - bool woEqual(const KeyBson& r) const; - void assign(const KeyBson& rhs) { *this = rhs; } - bool isValid() const { return true; } - private: - BSONObj _o; - }; - - class KeyV1Owned; - - // corresponding to BtreeData_V1 - class KeyV1 { - void operator=(const KeyV1&); // disallowed just to make people be careful as we don't own the buffer - KeyV1(const KeyV1Owned&); // disallowed as this is not a great idea as KeyV1Owned likely will go out of scope - public: - KeyV1() { _keyData = 0; } - ~KeyV1() { DEV _keyData = (const unsigned char *) 1; } - - KeyV1(const KeyV1& rhs) : _keyData(rhs._keyData) { - dassert( _keyData > (const unsigned char *) 1 ); - } - - // explicit version of operator= to be safe - void assign(const KeyV1& rhs) { - _keyData = rhs._keyData; - } - - /** @param keyData can be a buffer containing data in either BSON format, OR in KeyV1 format. - when BSON, we are just a wrapper - */ - explicit KeyV1(const char *keyData) : _keyData((unsigned char *) keyData) { } - - int woCompare(const KeyV1& r, const Ordering &o) const; - bool woEqual(const KeyV1& r) const; - BSONObj toBson() const; - string toString() const { return toBson().toString(); } - - /** get the key data we want to store in the btree bucket */ - const char * data() const { return (const char *) _keyData; } - - /** @return size of data() */ - int dataSize() const; - - /** only used by geo, which always has bson keys */ - BSONElement _firstElement() const { return bson().firstElement(); } - bool isCompactFormat() const { return *_keyData != IsBSON; } - - bool isValid() const { return _keyData > (const unsigned char*)1; } - protected: - enum { IsBSON = 0xff }; - const unsigned char *_keyData; - BSONObj bson() const { - dassert( !isCompactFormat() ); - return BSONObj((const char *) _keyData+1); - } - private: - int compareHybrid(const KeyV1& right, const Ordering& order) const; - }; - - class KeyV1Owned : public KeyV1 { - void operator=(const KeyV1Owned&); - public: - /** @obj a BSON object to be translated to KeyV1 format. If the object isn't - representable in KeyV1 format (which happens, intentionally, at times) - it will stay as bson herein. - */ - KeyV1Owned(const BSONObj& obj); - - /** makes a copy (memcpy's the whole thing) */ - KeyV1Owned(const KeyV1& rhs); - - private: - StackBufBuilder b; - void traditional(const BSONObj& obj); // store as traditional bson not as compact format - }; - -}; diff --git a/src/mongo/db/keypattern.cpp b/src/mongo/db/keypattern.cpp index 89eca1f16c6..c2a1f18cba2 100644 --- a/src/mongo/db/keypattern.cpp +++ b/src/mongo/db/keypattern.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/keypattern.h" @@ -40,6 +52,17 @@ namespace mongo { } } + bool KeyPattern::isIdKeyPattern(const BSONObj& pattern) { + BSONObjIterator i(pattern); + BSONElement e = i.next(); + // _id index must have form exactly {_id : 1} or {_id : -1}. + // Allows an index of form {_id : "hashed"} to exist but + // do not consider it to be the primary _id index + return (0 == strcmp(e.fieldName(), "_id")) + && (e.numberInt() == 1 || e.numberInt() == -1) + && i.next().eoo(); + } + BSONObj KeyPattern::extractSingleKey(const BSONObj& doc ) const { if ( _pattern.isEmpty() ) return BSONObj(); diff --git a/src/mongo/db/keypattern.h b/src/mongo/db/keypattern.h index a25ce2b87e5..b5ba777cecd 100644 --- a/src/mongo/db/keypattern.h +++ b/src/mongo/db/keypattern.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -72,13 +84,11 @@ namespace mongo { */ BSONElement getField( const char* fieldname ) const { return _pattern[ fieldname ]; } - /* - * Returns true if the key described by this KeyPattern is a prefix of - * the (potentially) compound key described by 'other' + /** + * Is the provided key pattern the index over the ID field? + * The always required ID index is always {_id: 1} or {_id: -1}. */ - bool isPrefixOf( const KeyPattern& other ) const { - return _pattern.isPrefixOf( other.toBSON() ); - } + static bool isIdKeyPattern(const BSONObj& pattern); /* Takes a BSONObj whose field names are a prefix of the fields in this keyPattern, and * outputs a new bound with MinKey values appended to match the fields in this keyPattern diff --git a/src/mongo/db/kill_current_op.cpp b/src/mongo/db/kill_current_op.cpp index d036d11c000..ed38bc175a8 100644 --- a/src/mongo/db/kill_current_op.cpp +++ b/src/mongo/db/kill_current_op.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/db/kill_current_op.h" @@ -21,10 +33,26 @@ #include "mongo/bson/util/atomic_int.h" #include "mongo/db/client.h" #include "mongo/db/curop.h" +#include "mongo/platform/random.h" #include "mongo/scripting/engine.h" +#include "mongo/util/fail_point_service.h" namespace mongo { + // Enabling the checkForInterruptFail fail point will start a game of random chance on the + // connection specified in the fail point data, generating an interrupt with a given fixed + // probability. Example invocation: + // + // {configureFailPoint: "checkForInterruptFail", + // mode: "alwaysOn", + // data: {conn: 17, chance: .01, allowNested: true}} + // + // All three data fields must be specified. In the above example, all interrupt points on + // connection 17 will generate a kill on the current operation with probability p(.01), + // including interrupt points of nested operations. If "allowNested" is false, nested + // operations are not targeted. "chance" must be a double between 0 and 1, inclusive. + MONGO_FP_DECLARE(checkForInterruptFail); + void KillCurrentOp::interruptJs( AtomicUInt *op ) { if ( !globalScriptEngine ) return; @@ -104,24 +132,84 @@ namespace mongo { _condvar.notify_all(); } - void KillCurrentOp::checkForInterrupt( bool heedMutex ) { +namespace { + + // Global state for checkForInterrupt fail point. + PseudoRandom checkForInterruptPRNG(static_cast(time(NULL))); + + // Helper function for checkForInterrupt fail point. Decides whether the operation currently + // being run by the given Client meet the (probabilistic) conditions for interruption as + // specified in the fail point info. + bool opShouldFail(const Client& c, const BSONObj& failPointInfo) { + // Only target the client with the specified connection number. + if (c.getConnectionId() != failPointInfo["conn"].safeNumberLong()) { + return false; + } + + // Only target nested operations if requested. + if (!failPointInfo["allowNested"].trueValue() && c.curop()->parent() != NULL) { + return false; + } + + // Return true with (approx) probability p = "chance". Recall: 0 <= chance <= 1. + double next = static_cast(std::abs(checkForInterruptPRNG.nextInt64())); + double upperBound = + std::numeric_limits::max() * failPointInfo["chance"].numberDouble(); + if (next > upperBound) { + return false; + } + return true; + } + +} // namespace + + void KillCurrentOp::checkForInterrupt(bool heedMutex) { Client& c = cc(); - if ( heedMutex && Lock::somethingWriteLocked() && c.hasWrittenThisPass() ) + + if (heedMutex && Lock::somethingWriteLocked() && c.hasWrittenSinceCheckpoint()) { return; - if( _globalKill ) - uasserted(11600,"interrupted at shutdown"); - if( c.curop()->killPending() ) { + } + + uassert(ErrorCodes::InterruptedAtShutdown, "interrupted at shutdown", !_globalKill); + + if (c.curop()->maxTimeHasExpired()) { + c.curop()->kill(); + notifyAllWaiters(); + uasserted(ErrorCodes::ExceededTimeLimit, "operation exceeded time limit"); + } + MONGO_FAIL_POINT_BLOCK(checkForInterruptFail, scopedFailPoint) { + if (opShouldFail(c, scopedFailPoint.getData())) { + log() << "set pending kill on " << (c.curop()->parent() ? "nested" : "top-level") + << " op " << c.curop()->opNum().get() << ", for checkForInterruptFail"; + c.curop()->kill(); + } + } + if (c.curop()->killPending()) { notifyAllWaiters(); - uasserted(11601,"operation was interrupted"); + uasserted(11601, "operation was interrupted"); } } const char * KillCurrentOp::checkForInterruptNoAssert() { Client& c = cc(); - if( _globalKill ) + + if (_globalKill) { return "interrupted at shutdown"; - if( c.curop()->killPending() ) + } + if (c.curop()->maxTimeHasExpired()) { + c.curop()->kill(); + return "exceeded time limit"; + } + MONGO_FAIL_POINT_BLOCK(checkForInterruptFail, scopedFailPoint) { + if (opShouldFail(c, scopedFailPoint.getData())) { + log() << "set pending kill on " << (c.curop()->parent() ? "nested" : "top-level") + << " op " << c.curop()->opNum().get() << ", for checkForInterruptFail"; + c.curop()->kill(); + } + } + if (c.curop()->killPending()) { return "interrupted"; + } return ""; } diff --git a/src/mongo/db/kill_current_op.h b/src/mongo/db/kill_current_op.h index d97c1a39594..4fd31ab3f7b 100644 --- a/src/mongo/db/kill_current_op.h +++ b/src/mongo/db/kill_current_op.h @@ -13,6 +13,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ diff --git a/src/mongo/db/lasterror.cpp b/src/mongo/db/lasterror.cpp index d56e0479f08..fd9129e7ed1 100644 --- a/src/mongo/db/lasterror.cpp +++ b/src/mongo/db/lasterror.cpp @@ -15,12 +15,12 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" -#include "mongo/util/net/message.h" +#include "mongo/db/lasterror.h" -#include "lasterror.h" -#include "jsobj.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/net/message.h" namespace mongo { @@ -77,9 +77,9 @@ namespace mongo { b.append( "code" , code ); if ( updatedExisting != NotUpdate ) b.appendBool( "updatedExisting", updatedExisting == True ); - if ( upsertedId.isSet() ) - b.append( "upserted" , upsertedId ); - + if ( !upsertedId.isEmpty() ) { + b.append( upsertedId[kUpsertedFieldName] ); + } b.appendNumber( "n", nObjects ); return ! msg.empty(); diff --git a/src/mongo/db/lasterror.h b/src/mongo/db/lasterror.h index afcb2a1bb52..31745ad4185 100644 --- a/src/mongo/db/lasterror.h +++ b/src/mongo/db/lasterror.h @@ -17,24 +17,35 @@ #pragma once +#include +#include + +#include "mongo/db/jsobj.h" #include "mongo/bson/oid.h" +#include "mongo/util/log.h" namespace mongo { class BSONObjBuilder; class Message; + static const char kUpsertedFieldName[] = "upserted"; + static const char kGLEStatsFieldName[] = "$gleStats"; + static const char kGLEStatsLastOpTimeFieldName[] = "lastOpTime"; + static const char kGLEStatsElectionIdFieldName[] = "electionId"; + struct LastError { int code; - string msg; + std::string msg; enum UpdatedExistingType { NotUpdate, True, False } updatedExisting; - OID upsertedId; + // _id field value from inserted doc, returned as kUpsertedFieldName (above) + BSONObj upsertedId; OID writebackId; // this shouldn't get reset so that old GLE are handled int writebackSince; long long nObjects; int nPrev; bool valid; bool disabled; - void writeback( OID& oid ) { + void writeback(const OID& oid) { reset( true ); writebackId = oid; writebackSince = 0; @@ -44,11 +55,11 @@ namespace mongo { code = _code; msg = _msg; } - void recordUpdate( bool _updateObjects , long long _nObjects , OID _upsertedId ) { + void recordUpdate( bool _updateObjects , long long _nObjects , BSONObj _upsertedId ) { reset( true ); nObjects = _nObjects; updatedExisting = _updateObjects ? True : False; - if ( _upsertedId.isSet() ) + if ( _upsertedId.valid() && _upsertedId.hasField(kUpsertedFieldName) ) upsertedId = _upsertedId; } @@ -68,7 +79,7 @@ namespace mongo { nPrev = 1; valid = _valid; disabled = false; - upsertedId.clear(); + upsertedId = BSONObj(); } /** diff --git a/src/mongo/db/lockstat.cpp b/src/mongo/db/lockstat.cpp index bd742ef4c34..152c31f0b35 100644 --- a/src/mongo/db/lockstat.cpp +++ b/src/mongo/db/lockstat.cpp @@ -14,11 +14,25 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" -#include "lockstat.h" + +#include "mongo/db/lockstat.h" + #include "mongo/db/jsobj.h" namespace mongo { @@ -42,7 +56,7 @@ namespace mongo { for ( int i=0; i < N; i++ ) { if ( timeLocked[i].load() == 0 ) continue; - + if ( ! prefixPrinted ) { builder << "locks(micros)"; prefixPrinted = true; diff --git a/src/mongo/db/lockstat.h b/src/mongo/db/lockstat.h index 5b44d7b2c9f..f2a5ea032cd 100644 --- a/src/mongo/db/lockstat.h +++ b/src/mongo/db/lockstat.h @@ -14,13 +14,25 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "util/timer.h" #include "mongo/platform/atomic_word.h" +#include "mongo/util/timer.h" namespace mongo { diff --git a/src/mongo/db/lockstate.cpp b/src/mongo/db/lockstate.cpp index ee56c1b9709..5f3540eec37 100644 --- a/src/mongo/db/lockstate.cpp +++ b/src/mongo/db/lockstate.cpp @@ -14,15 +14,29 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" + +#include "mongo/db/lockstate.h" + #include "mongo/db/d_concurrency.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/client.h" #include "mongo/util/mongoutils/str.h" -#include "lockstate.h" namespace mongo { @@ -211,16 +225,16 @@ namespace mongo { } void LockState::unlockedOther() { - _otherName = ""; + // we leave _otherName and _otherLock set as + // _otherLock exists to cache a pointer _otherCount = 0; - _otherLock = 0; } LockStat* LockState::getRelevantLockStat() { if ( _whichNestable ) return Lock::nestableLockStat( _whichNestable ); - if ( _otherLock ) + if ( _otherCount && _otherLock ) return &_otherLock->stats; if ( isRW() ) @@ -238,8 +252,10 @@ namespace mongo { Acquiring::~Acquiring() { _ls._lockPending = false; LockStat* stat = _ls.getRelevantLockStat(); - if ( stat && _lock ) + if ( stat && _lock ) { + // increment the global stats for this counter stat->recordAcquireTimeMicros( _ls.threadState(), _lock->acquireFinished( stat ) ); + } } AcquiringParallelWriter::AcquiringParallelWriter( LockState& ls ) diff --git a/src/mongo/db/lockstate.h b/src/mongo/db/lockstate.h index 5788807a002..6cb506f8d16 100644 --- a/src/mongo/db/lockstate.h +++ b/src/mongo/db/lockstate.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ @@ -111,16 +123,24 @@ namespace mongo { friend class AcquiringParallelWriter; }; - class WrapperForRWLock : boost::noncopyable { - SimpleRWLock r; + class WrapperForRWLock : boost::noncopyable { + SimpleRWLock rw; + SimpleMutex m; + bool sharedLatching; public: - string name() const { return r.name; } + string name() const { return rw.name; } LockStat stats; - WrapperForRWLock(const StringData& name) : r(name) { } - void lock() { r.lock(); } - void lock_shared() { r.lock_shared(); } - void unlock() { r.unlock(); } - void unlock_shared() { r.unlock_shared(); } + WrapperForRWLock(const StringData& name) + : rw(name), m(name) { + // For the local datbase, all operations are short, + // either writing one entry, or doing a tail. + // In tests, use a SimpleMutex is much faster for the local db. + sharedLatching = name != "local"; + } + void lock() { if ( sharedLatching ) { rw.lock(); } else { m.lock(); } } + void lock_shared() { if ( sharedLatching ) { rw.lock_shared(); } else { m.lock(); } } + void unlock() { if ( sharedLatching ) { rw.unlock(); } else { m.unlock(); } } + void unlock_shared() { if ( sharedLatching ) { rw.unlock_shared(); } else { m.unlock(); } } }; class ScopedLock; diff --git a/src/mongo/db/log_process_details.cpp b/src/mongo/db/log_process_details.cpp new file mode 100644 index 00000000000..a851444f5cd --- /dev/null +++ b/src/mongo/db/log_process_details.cpp @@ -0,0 +1,67 @@ +// @file log_process_details.cpp + +/** +* Copyright (C) 2013 MongoDB, Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. +*/ + +#include "mongo/platform/basic.h" + +#include "mongo/db/log_process_details.h" + +#include "mongo/db/server_options.h" +#include "mongo/db/server_options_helpers.h" +#include "mongo/util/net/sock.h" +#include "mongo/util/net/ssl_manager.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/version.h" +#include "mongo/util/version_reporting.h" + +namespace mongo { + + bool is32bit() { + return ( sizeof(int*) == 4 ); + } + + void logProcessDetails() { + log() << mongodVersion() << endl; + printGitVersion(); + printOpenSSLVersion(); + printSysInfo(); + printAllocator(); + printCommandLineOpts(); + } + + void logProcessDetailsForLogRotate() { + log() << "pid=" << ProcessId::getCurrent() + << " port=" << serverGlobalParams.port + << ( is32bit() ? " 32" : " 64" ) << "-bit " + << "host=" << getHostNameCached(); + + logProcessDetails(); + } + +} //mongo diff --git a/src/mongo/db/log_process_details.h b/src/mongo/db/log_process_details.h new file mode 100644 index 00000000000..a41837a6c49 --- /dev/null +++ b/src/mongo/db/log_process_details.h @@ -0,0 +1,44 @@ +/** +* Copyright (C) 2013 MongoDB, Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. +*/ + +#pragma once + +namespace mongo { + + /** + * Writes useful information about the running process to the diagnostic log on startup. + */ + void logProcessDetails(); + + /** + * Writes useful information about the running process to diagnostic log + * for after a log rotation. + */ + void logProcessDetailsForLogRotate(); + +} // namespace mongo diff --git a/src/mongo/db/matcher.cpp b/src/mongo/db/matcher.cpp deleted file mode 100644 index 2e679591f2c..00000000000 --- a/src/mongo/db/matcher.cpp +++ /dev/null @@ -1,1369 +0,0 @@ -// matcher.cpp - -/* Matcher is our boolean expression evaluator for "where" clauses */ - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" -#include "matcher.h" -#include "../util/goodies.h" -#include "../util/startup_test.h" -#include "diskloc.h" -#include "../scripting/engine.h" -#include "db.h" -#include "queryutil.h" -#include "client.h" - -#include "pdfile.h" - -namespace { - inline pcrecpp::RE_Options flags2options(const char* flags) { - pcrecpp::RE_Options options; - options.set_utf8(true); - while ( flags && *flags ) { - if ( *flags == 'i' ) - options.set_caseless(true); - else if ( *flags == 'm' ) - options.set_multiline(true); - else if ( *flags == 'x' ) - options.set_extended(true); - else if ( *flags == 's' ) - options.set_dotall(true); - flags++; - } - return options; - } -} - -//#define DEBUGMATCHER(x) cout << x << endl; -#define DEBUGMATCHER(x) - -namespace mongo { - - extern BSONObj staticNull; - - class Where : boost::noncopyable { - public: - - Where( const string& ns ) { - _ns = ns; - _func = 0; - _initCalled = false; - } - - ~Where() { - _func = 0; - } - - void init() { - if ( _initCalled ) - return; - _initCalled = true; - - NamespaceString ns( _ns ); - _scope = globalScriptEngine->getPooledScope( ns.db.c_str(), "where" ); - - massert( 10341 , "code has to be set first!" , ! _jsCode.empty() ); - - _func = _scope->createFunction( _jsCode.c_str() ); - } - - void setScope( const BSONObj& scope ) { - _jsScope = scope.copy(); - } - - void setCode( const string& code ) { - _jsCode = code; - } - - bool exec( const BSONObj& obj ) { - init(); - uassert( 10070 , "$where compile error", _func != 0 ); - - if ( ! _jsScope.isEmpty() ) { - _scope->init( &_jsScope ); - } - _scope->setObject( "obj", const_cast< BSONObj & >( obj ) ); - _scope->setBoolean( "fullObject" , true ); // this is a hack b/c fullObject used to be relevant - - - int err = _scope->invoke( _func , 0, &obj , 1000 * 60 , false ); - if ( err == -3 ) { // INVOKE_ERROR - stringstream ss; - ss << "error on invocation of $where function:\n" - << _scope->getError(); - uassert( 10071 , ss.str(), false); - } - else if ( err != 0 ) { // ! INVOKE_SUCCESS - uassert( 10072 , "unknown error in invocation of $where function", false); - } - - return _scope->getBoolean( "__returnValue" ) != 0; - } - - private: - bool _initCalled; - - string _ns; - - BSONObj _jsScope; - string _jsCode; - - auto_ptr _scope; - ScriptingFunction _func; - - - }; - - Matcher::~Matcher() { - delete _where; - _where = 0; - } - - ElementMatcher::ElementMatcher( BSONElement e , int op, bool isNot ) - : _toMatch( e ) , _compareOp( op ), _isNot( isNot ), _subMatcherOnPrimitives(false) { - if ( op == BSONObj::opMOD ) { - BSONObj o = e.embeddedObject(); - _mod = o["0"].numberInt(); - _modm = o["1"].numberInt(); - - uassert( 10073 , "mod can't be 0" , _mod ); - } - else if ( op == BSONObj::opTYPE ) { - _type = (BSONType)(e.numberInt()); - } - else if ( op == BSONObj::opELEM_MATCH ) { - BSONElement m = e; - uassert( 12517 , "$elemMatch needs an Object" , m.type() == Object ); - BSONObj x = m.embeddedObject(); - if ( x.firstElement().getGtLtOp() == BSONObj::Equality && - !str::equals( x.firstElement().fieldName(), "$not" ) ) { - _subMatcher.reset( new Matcher( x ) ); - _subMatcherOnPrimitives = false; - } - else { - // meant to act on primitives - _subMatcher.reset( new Matcher( BSON( "" << x ) ) ); - _subMatcherOnPrimitives = true; - } - } - } - - ElementMatcher::ElementMatcher( BSONElement e , int op , const BSONObj& array, bool isNot ) - : _toMatch( e ) , _compareOp( op ), _isNot( isNot ), _subMatcherOnPrimitives(false) { - - _myset.reset( new set() ); - - BSONObjIterator i( array ); - while ( i.more() ) { - BSONElement ie = i.next(); - if ( op == BSONObj::opALL && ie.type() == Object && ie.embeddedObject().firstElement().getGtLtOp() == BSONObj::opELEM_MATCH ) { - shared_ptr s; - s.reset( new Matcher( ie.embeddedObject().firstElement().embeddedObjectUserCheck() ) ); - _allMatchers.push_back( s ); - } - else if ( ie.type() == RegEx ) { - if ( !_myregex.get() ) { - _myregex.reset( new vector< RegexMatcher >() ); - } - _myregex->push_back( RegexMatcher() ); - RegexMatcher &rm = _myregex->back(); - rm._re.reset( new pcrecpp::RE( ie.regex(), flags2options( ie.regexFlags() ) ) ); - uassert(16431, "Regular expression is too long", - rm._re->pattern().size() <= RegexMatcher::MaxPatternSize); - - rm._fieldName = 0; // no need for field name - rm._regex = ie.regex(); - rm._flags = ie.regexFlags(); - rm._isNot = false; - bool purePrefix; - string prefix = simpleRegex(rm._regex, rm._flags, &purePrefix); - if (purePrefix) - rm._prefix = prefix; - } - else { - uassert( 15882, "$elemMatch not allowed within $in", - ie.type() != Object || - ie.embeddedObject().firstElement().getGtLtOp() != BSONObj::opELEM_MATCH ); - _myset->insert(ie); - } - } - - if ( _allMatchers.size() ) { - uassert( 13020 , "with $all, can't mix $elemMatch and others" , _myset->size() == 0 && !_myregex.get()); - } - - } - - int ElementMatcher::inverseOfNegativeCompareOp() const { - verify( negativeCompareOp() ); - return _compareOp == BSONObj::NE ? BSONObj::Equality : BSONObj::opIN; - } - - bool ElementMatcher::negativeCompareOpContainsNull() const { - verify( negativeCompareOp() ); - return (_compareOp == BSONObj::NE && _toMatch.type() != jstNULL) || - (_compareOp == BSONObj::NIN && _myset->count( staticNull.firstElement()) == 0 ); - } - - void ElementMatcher::visit( MatcherVisitor& visitor ) const { - if ( _subMatcher ) { - _subMatcher->visit( visitor ); - } - for( vector >::const_iterator j = _allMatchers.begin(); - j != _allMatchers.end(); ++j ) { - (*j)->visit( visitor ); - } - } - - MatchDetails::MatchDetails() : - _elemMatchKeyRequested() { - resetOutput(); - } - - void MatchDetails::resetOutput() { - _loadedRecord = false; - _elemMatchKeyFound = false; - _elemMatchKey = ""; - } - - string MatchDetails::toString() const { - stringstream ss; - ss << "loadedRecord: " << _loadedRecord << " "; - ss << "elemMatchKeyRequested: " << _elemMatchKeyRequested << " "; - ss << "elemMatchKey: " << ( _elemMatchKeyFound ? _elemMatchKey : "NONE" ) << " "; - return ss.str(); - } - - void Matcher::addRegex(const char *fieldName, const char *regex, const char *flags, bool isNot) { - - RegexMatcher rm; - rm._re.reset( new pcrecpp::RE(regex, flags2options(flags)) ); - uassert(16432, "Regular expression is too long", - rm._re->pattern().size() <= RegexMatcher::MaxPatternSize); - - rm._fieldName = fieldName; - rm._regex = regex; - rm._flags = flags; - rm._isNot = isNot; - _regexs.push_back(rm); - - if (!isNot) { //TODO something smarter - bool purePrefix; - string prefix = simpleRegex(regex, flags, &purePrefix); - if (purePrefix) - rm._prefix = prefix; - } - } - - bool Matcher::addOp( const BSONElement &e, const BSONElement &fe, bool isNot, const char *& regex, const char *&flags ) { - const char *fn = fe.fieldName(); - int op = fe.getGtLtOp( -1 ); - if ( op == -1 ) { - if ( !isNot && fn[1] == 'r' && fn[2] == 'e' && fn[3] == 'f' && fn[4] == 0 ) { - return false; // { $ref : xxx } - treat as normal object - } - uassert( 10068 , (string)"invalid operator: " + fn , op != -1 ); - } - - switch ( op ) { - case BSONObj::GT: - case BSONObj::GTE: - case BSONObj::LT: - case BSONObj::LTE: { - shared_ptr< BSONObjBuilder > b( new BSONObjBuilder() ); - _builders.push_back( b ); - b->appendAs(fe, e.fieldName()); - addBasic(b->done().firstElement(), op, isNot); - break; - } - case BSONObj::NE: { - _haveNeg = true; - shared_ptr< BSONObjBuilder > b( new BSONObjBuilder() ); - _builders.push_back( b ); - b->appendAs(fe, e.fieldName()); - addBasic(b->done().firstElement(), BSONObj::NE, isNot); - break; - } - case BSONObj::opALL: - _all = true; - case BSONObj::opIN: { - uassert( 13276 , "$in needs an array" , fe.isABSONObj() ); - _basics.push_back( ElementMatcher( e , op , fe.embeddedObject(), isNot ) ); - BSONObjIterator i( fe.embeddedObject() ); - while( i.more() ) { - if ( i.next().type() == Array ) { - _hasArray = true; - } - } - break; - } - case BSONObj::NIN: - uassert( 13277 , "$nin needs an array" , fe.isABSONObj() ); - _haveNeg = true; - _basics.push_back( ElementMatcher( e , op , fe.embeddedObject(), isNot ) ); - break; - case BSONObj::opMOD: - case BSONObj::opTYPE: - case BSONObj::opELEM_MATCH: { - shared_ptr< BSONObjBuilder > b( new BSONObjBuilder() ); - _builders.push_back( b ); - b->appendAs(fe, e.fieldName()); - // these are types where ElementMatcher has all the info - _basics.push_back( ElementMatcher( b->done().firstElement() , op, isNot ) ); - break; - } - case BSONObj::opSIZE: { - shared_ptr< BSONObjBuilder > b( new BSONObjBuilder() ); - _builders.push_back( b ); - b->appendAs(fe, e.fieldName()); - addBasic(b->done().firstElement(), BSONObj::opSIZE, isNot); - _haveSize = true; - break; - } - case BSONObj::opEXISTS: { - shared_ptr< BSONObjBuilder > b( new BSONObjBuilder() ); - _builders.push_back( b ); - b->appendAs(fe, e.fieldName()); - addBasic(b->done().firstElement(), BSONObj::opEXISTS, isNot); - break; - } - case BSONObj::opREGEX: { - uassert( 13032, "can't use $not with $regex, use BSON regex type instead", !isNot ); - if ( fe.type() == RegEx ) { - regex = fe.regex(); - flags = fe.regexFlags(); - } - else { - regex = fe.valuestrsafe(); - } - break; - } - case BSONObj::opOPTIONS: { - uassert( 13029, "can't use $not with $options, use BSON regex type instead", !isNot ); - flags = fe.valuestrsafe(); - break; - } - case BSONObj::opGEO_INTERSECTS: - case BSONObj::opWITHIN: { - uassert(16516, "Within must be provided a BSONObj", e.isABSONObj()); - BSONObj queryObj = e.Obj(); - if (isNot) { - // Get to the $within/$geoIntersects hiding inside the $not. - queryObj = queryObj.firstElement().embeddedObject(); - } - GeoQuery query(e.fieldName()); - uassert(16677, "Malformed geo query: " + queryObj.toString(), - query.parseFrom(queryObj)); - _geo.push_back(GeoMatcher(query, isNot)); - break; - } - case BSONObj::opNEAR: - case BSONObj::opMAX_DISTANCE: - break; - default: - uassert( 10069 , (string)"BUG - can't operator for: " + fn , 0 ); - } - return true; - } - - void Matcher::parseExtractedClause( const BSONElement &e, list< shared_ptr< Matcher > > &matchers ) { - uassert( 13086, "$and/$or/$nor must be a nonempty array", e.type() == Array && e.embeddedObject().nFields() > 0 ); - BSONObjIterator j( e.embeddedObject() ); - while( j.more() ) { - BSONElement f = j.next(); - uassert( 13087, "$and/$or/$nor match element must be an object", f.type() == Object ); - matchers.push_back( shared_ptr< Matcher >( new Matcher( f.embeddedObject(), true ) ) ); - } - } - - bool Matcher::parseClause( const BSONElement &e ) { - const char *ef = e.fieldName(); - - if ( ef[ 0 ] != '$' ) - return false; - - // $and - if ( ef[ 1 ] == 'a' && ef[ 2 ] == 'n' && ef[ 3 ] == 'd' ) { - parseExtractedClause( e, _andMatchers ); - return true; - } - - // $or - if ( ef[ 1 ] == 'o' && ef[ 2 ] == 'r' && ef[ 3 ] == 0 ) { - parseExtractedClause( e, _orMatchers ); - return true; - } - - // $nor - if ( ef[ 1 ] == 'n' && ef[ 2 ] == 'o' && ef[ 3 ] == 'r' && ef[ 4 ] == 0 ) { - parseExtractedClause( e, _norMatchers ); - return true; - } - - // $comment - if ( ef[ 1 ] == 'c' && ef[ 2 ] == 'o' && ef[ 3 ] == 'm' && str::equals( ef , "$comment" ) ) { - return true; - } - - return false; - } - bool inConstructorChain( bool printOffending=false ); - // $where: function()... - NOINLINE_DECL void Matcher::parseWhere( const BSONElement &e ) { - uassert(15902 , "$where expression has an unexpected type", e.type() == String || e.type() == CodeWScope || e.type() == Code ); - uassert( 10066 , "$where may only appear once in query", _where == 0 ); - uassert( 10067 , "$where query, but no script engine", globalScriptEngine ); - massert( 13089 , "no current client needed for $where" , haveClient() ); - _where = new Where( cc().ns() ); - - if ( e.type() == CodeWScope ) { - _where->setCode( e.codeWScopeCode() ); - _where->setScope( BSONObj( e.codeWScopeScopeDataUnsafe() ) ); - } - else { - _where->setCode( e.valuestr() ); - } - - } - - void Matcher::parseMatchExpressionElement( const BSONElement &e, bool nested ) { - - uassert( 13629 , "can't have undefined in a query expression" , e.type() != Undefined ); - - if ( parseClause( e ) ) { - return; - } - - const char *fn = e.fieldName(); - if ( str::equals(fn, "$where") ) { - parseWhere(e); - return; - } - - if ( e.type() == RegEx ) { - addRegex( fn, e.regex(), e.regexFlags() ); - return; - } - - // greater than / less than... - // e.g., e == { a : { $gt : 3 } } - // or - // { a : { $in : [1,2,3] } } - if ( e.type() == Object ) { - // support {$regex:"a|b", $options:"imx"} - const char* regex = NULL; - const char* flags = ""; - - // e.g., fe == { $gt : 3 } - BSONObjIterator j(e.embeddedObject()); - bool isOperator = false; - while ( j.more() ) { - BSONElement fe = j.next(); - const char *fn = fe.fieldName(); - - if ( fn[0] == '$' && fn[1] ) { - isOperator = true; - - if ( fn[1] == 'n' && fn[2] == 'o' && fn[3] == 't' && fn[4] == 0 ) { - _haveNeg = true; - switch( fe.type() ) { - case Object: { - BSONObjIterator k( fe.embeddedObject() ); - uassert( 13030, "$not cannot be empty", k.more() ); - while( k.more() ) { - addOp( e, k.next(), true, regex, flags ); - } - break; - } - case RegEx: - addRegex( e.fieldName(), fe.regex(), fe.regexFlags(), true ); - break; - default: - uassert( 13031, "invalid use of $not", false ); - } - } - else { - if ( !addOp( e, fe, false, regex, flags ) ) { - isOperator = false; - break; - } - } - } - else { - isOperator = false; - break; - } - } - if (regex) { - addRegex(e.fieldName(), regex, flags); - } - if ( isOperator ) - return; - } - - if ( e.type() == Array ) { - _hasArray = true; - } - else if( *fn == '$' ) { - if( str::equals(fn, "$atomic") || str::equals(fn, "$isolated") ) { - uassert( 14844, "$atomic specifier must be a top level field", !nested ); - _atomic = e.trueValue(); - return; - } - } - - // normal, simple case e.g. { a : "foo" } - addBasic(e, BSONObj::Equality, false); - } - - /* _jsobj - the query pattern - */ - Matcher::Matcher(const BSONObj &jsobj, bool nested) : - _where(0), _jsobj(jsobj), _haveSize(), _all(), _hasArray(0), _haveNeg(), _atomic(false) { - - BSONObjIterator i(_jsobj); - while ( i.more() ) { - parseMatchExpressionElement( i.next(), nested ); - } - } - - Matcher::Matcher( const Matcher &docMatcher, const BSONObj &key ) : - _where(0), _constrainIndexKey( key ), _haveSize(), _all(), _hasArray(0), _haveNeg(), _atomic(false) { - // Filter out match components that will provide an incorrect result - // given a key from a single key index. - for( vector< ElementMatcher >::const_iterator i = docMatcher._basics.begin(); i != docMatcher._basics.end(); ++i ) { - if ( key.hasField( i->_toMatch.fieldName() ) ) { - switch( i->_compareOp ) { - case BSONObj::opSIZE: - case BSONObj::opALL: - case BSONObj::NE: - case BSONObj::NIN: - case BSONObj::opEXISTS: // We can't match on index in this case. - case BSONObj::opTYPE: // For $type:10 (null), a null key could be a missing - // field or a null value field. - case BSONObj::opELEM_MATCH: // $elemMatch operates on arrays and no key match - // version has been implemented. - break; - case BSONObj::opIN: { - bool inContainsArray = false; - for( set::const_iterator j = i->_myset->begin(); j != i->_myset->end(); ++j ) { - if ( j->type() == Array ) { - inContainsArray = true; - break; - } - } - // Can't match an array to its first indexed element. - if ( !i->_isNot && !inContainsArray ) { - _basics.push_back( *i ); - } - break; - } - default: { - // Can't match an array to its first indexed element. - if ( !i->_isNot && i->_toMatch.type() != Array ) { - _basics.push_back( *i ); - } - } - } - } - } - for( vector::const_iterator it = docMatcher._regexs.begin(); - it != docMatcher._regexs.end(); - ++it) { - if ( !it->_isNot && key.hasField( it->_fieldName ) ) { - _regexs.push_back(*it); - } - } - // Recursively filter match components for and and or matchers. - for( list< shared_ptr< Matcher > >::const_iterator i = docMatcher._andMatchers.begin(); i != docMatcher._andMatchers.end(); ++i ) { - _andMatchers.push_back( shared_ptr< Matcher >( new Matcher( **i, key ) ) ); - } - for( list< shared_ptr< Matcher > >::const_iterator i = docMatcher._orMatchers.begin(); i != docMatcher._orMatchers.end(); ++i ) { - _orMatchers.push_back( shared_ptr< Matcher >( new Matcher( **i, key ) ) ); - } - } - - inline bool regexMatches(const RegexMatcher& rm, const BSONElement& e) { - switch (e.type()) { - case String: - case Symbol: - if (rm._prefix.empty()) - return rm._re->PartialMatch(e.valuestr()); - else - return !strncmp(e.valuestr(), rm._prefix.c_str(), rm._prefix.size()); - case RegEx: - return !strcmp(rm._regex, e.regex()) && !strcmp(rm._flags, e.regexFlags()); - default: - return false; - } - } - - inline int Matcher::valuesMatch(const BSONElement& l, const BSONElement& r, int op, const ElementMatcher& bm) const { - verify( op != BSONObj::NE && op != BSONObj::NIN && op != BSONObj::opELEM_MATCH ); - - if ( op == BSONObj::Equality ) { - return l.valuesEqual(r); - } - - if ( op == BSONObj::opIN ) { - // { $in : [1,2,3] } - int count = bm._myset->count(l); - if ( count ) - return count; - if ( bm._myregex.get() ) { - for( vector::const_iterator i = bm._myregex->begin(); i != bm._myregex->end(); ++i ) { - if ( regexMatches( *i, l ) ) { - return true; - } - } - } - } - - if ( op == BSONObj::opSIZE ) { - if ( l.type() != Array ) - return 0; - int count = 0; - BSONObjIterator i( l.embeddedObject() ); - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - ++count; - } - return count == r.number(); - } - - if ( op == BSONObj::opMOD ) { - if ( ! l.isNumber() ) - return false; - - return l.numberLong() % bm._mod == bm._modm; - } - - if ( op == BSONObj::opTYPE ) { - return bm._type == l.type(); - } - - /* check LT, GTE, ... */ - if ( l.canonicalType() != r.canonicalType() ) - return false; - int c = compareElementValues(l, r); - if ( c < -1 ) c = -1; - if ( c > 1 ) c = 1; - int z = 1 << (c+1); - return (op & z); - } - - int Matcher::inverseMatch(const char *fieldName, const BSONElement &toMatch, const BSONObj &obj, const ElementMatcher& bm , MatchDetails * details ) const { - int inverseRet = matchesDotted( fieldName, toMatch, obj, bm.inverseOfNegativeCompareOp(), bm , false , details ); - if ( bm.negativeCompareOpContainsNull() ) { - return ( inverseRet <= 0 ) ? 1 : 0; - } - return -inverseRet; - } - - int retExistsFound( const ElementMatcher &bm ) { - return bm._toMatch.trueValue() ? 1 : -1; - } - - /* Check if a particular field matches. - - fieldName - field to match "a.b" if we are reaching into an embedded object. - toMatch - element we want to match. - obj - database object to check against - compareOp - Equality, LT, GT, etc. This may be different than, and should supersede, the compare op in em. - isArr - - - Special forms: - - { "a.b" : 3 } means obj.a.b == 3 - { a : { $lt : 3 } } means obj.a < 3 - { a : { $in : [1,2] } } means [1,2].contains(obj.a) - - return value - -1 mismatch - 0 missing element - 1 match - */ - int Matcher::matchesDotted(const char *fieldName, const BSONElement& toMatch, const BSONObj& obj, int compareOp, const ElementMatcher& em , bool isArr, MatchDetails * details ) const { - DEBUGMATCHER( "\t matchesDotted : " << fieldName << " hasDetails: " << ( details ? "yes" : "no" ) ); - - if ( compareOp == BSONObj::opALL ) { - - if ( em._allMatchers.size() ) { - // $all query matching will not be performed against indexes, so the field - // to match is always extracted from the full document. - BSONElement e = obj.getFieldDotted( fieldName ); - // The $all/$elemMatch operator only matches arrays. - if ( e.type() != Array ) { - return -1; - } - - for ( unsigned i=0; imatches( f.embeddedObject() ) ) { - found = true; - break; - } - } - - if ( ! found ) - return -1; - } - - return 1; - } - - if ( em._myset->size() == 0 && !em._myregex.get() ) - return -1; // is this desired? - - BSONElementSet myValues; - obj.getFieldsDotted( fieldName , myValues ); - - for( set< BSONElement, element_lt >::const_iterator i = em._myset->begin(); i != em._myset->end(); ++i ) { - // ignore nulls - if ( i->type() == jstNULL ) - continue; - - if ( myValues.count( *i ) == 0 ) - return -1; - } - - if ( !em._myregex.get() ) - return 1; - - for( vector< RegexMatcher >::const_iterator i = em._myregex->begin(); i != em._myregex->end(); ++i ) { - bool match = false; - for( BSONElementSet::const_iterator j = myValues.begin(); j != myValues.end(); ++j ) { - if ( regexMatches( *i, *j ) ) { - match = true; - break; - } - } - if ( !match ) - return -1; - } - - return 1; - } // end opALL - - if ( compareOp == BSONObj::NE || compareOp == BSONObj::NIN ) { - return inverseMatch( fieldName, toMatch, obj, em , details ); - } - - BSONElement e; - bool indexed = !_constrainIndexKey.isEmpty(); - if ( indexed ) { - e = obj.getFieldUsingIndexNames(fieldName, _constrainIndexKey); - if( e.eoo() ) { - cout << "obj: " << obj << endl; - cout << "fieldName: " << fieldName << endl; - cout << "_constrainIndexKey: " << _constrainIndexKey << endl; - verify( !e.eoo() ); - } - } - else { - - const char *p = strchr(fieldName, '.'); - if ( p ) { - string left(fieldName, p-fieldName); - - BSONElement se = obj.getField(left.c_str()); - if ( se.eoo() ) - ; - else if ( se.type() != Object && se.type() != Array ) - ; - else { - BSONObj eo = se.embeddedObject(); - return matchesDotted(p+1, toMatch, eo, compareOp, em, se.type() == Array , details ); - } - } - - // An array was encountered while scanning for components of the field name. - if ( isArr ) { - DEBUGMATCHER( "\t\t isArr 1 : obj : " << obj ); - BSONObjIterator ai(obj); - bool found = false; - while ( ai.moreWithEOO() ) { - BSONElement z = ai.next(); - - if( strcmp(z.fieldName(),fieldName) == 0 ) { - if ( compareOp == BSONObj::opEXISTS ) { - return retExistsFound( em ); - } - if ( compareOp != BSONObj::opELEM_MATCH && - valuesMatch(z, toMatch, compareOp, em) ) { - // "field." array notation was used - if ( details ) - details->setElemMatchKey( z.fieldName() ); - return 1; - } - } - - if ( z.type() == Object ) { - BSONObj eo = z.embeddedObject(); - int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, em, false, details ); - if ( cmp > 0 ) { - if ( details ) - details->setElemMatchKey( z.fieldName() ); - return 1; - } - else if ( cmp < 0 ) { - found = true; - } - } - } - return found ? -1 : 0; - } - - if( p ) { - // Left portion of field name was not found or wrong type. - return 0; - } - else { - e = obj.getField(fieldName); - } - } - - if ( compareOp == BSONObj::opEXISTS ) { - if( e.eoo() ) { - return 0; - } else { - return retExistsFound( em ); - } - } - else if ( ( e.type() != Array || indexed || compareOp == BSONObj::opSIZE ) && - compareOp != BSONObj::opELEM_MATCH && - valuesMatch(e, toMatch, compareOp, em ) ) { - return 1; - } - else if ( e.type() == Array && compareOp != BSONObj::opSIZE ) { - BSONObjIterator ai(e.embeddedObject()); - - while ( ai.more() ) { - BSONElement z = ai.next(); - bool match = false; - - if ( compareOp == BSONObj::opELEM_MATCH ) { - if ( em._subMatcherOnPrimitives ) { - match = em._subMatcher->matches( z.wrap( "" ) ); - } - else { - match = ( z.isABSONObj() && em._subMatcher->matches( z.embeddedObject() ) ); - } - } - else { - match = valuesMatch( z, toMatch, compareOp, em ); - } - if ( match ) { - if ( details ) { - details->setElemMatchKey( z.fieldName() ); - } - return 1; - } - } - - // match an entire array to itself - if ( compareOp == BSONObj::Equality && e.woCompare( toMatch , false ) == 0 ) { - return 1; - } - if ( compareOp == BSONObj::opIN && valuesMatch( e, toMatch, compareOp, em ) ) { - return 1; - } - } - else if ( e.eoo() ) { - return 0; - } - return -1; - } - - extern int dump; - - /* See if an object matches the query. - */ - bool Matcher::matches(const BSONObj& jsobj , MatchDetails * details ) const { - /* - NB: if any modifications are made to how this operates, make sure - they are reflected in visitReferences(), whose implementation - parallels this. - */ - - LOG(5) << "Matcher::matches() " << jsobj.toString() << endl; - - /* assuming there is usually only one thing to match. if more this - could be slow sometimes. */ - - // check normal non-regex cases: - for ( unsigned i = 0; i < _basics.size(); i++ ) { - const ElementMatcher& bm = _basics[i]; - const BSONElement& m = bm._toMatch; - // -1=mismatch. 0=missing element. 1=match - int cmp = matchesDotted(m.fieldName(), m, jsobj, bm._compareOp, bm , false , details ); - if ( cmp == 0 && bm._compareOp == BSONObj::opEXISTS ) { - // If missing, match cmp is opposite of $exists spec. - cmp = -retExistsFound(bm); - } - if ( bm._isNot ) - cmp = -cmp; - if ( cmp < 0 ) - return false; - if ( cmp == 0 ) { - /* missing is ok iff we were looking for null */ - if ( m.type() == jstNULL || m.type() == Undefined || - ( ( bm._compareOp == BSONObj::opIN || bm._compareOp == BSONObj::NIN ) && bm._myset->count( staticNull.firstElement() ) > 0 ) ) { - if ( bm.negativeCompareOp() ^ bm._isNot ) { - return false; - } - } - else { - if ( !bm._isNot ) { - return false; - } - } - } - } - - for (vector::const_iterator it = _geo.begin(); it != _geo.end(); ++it) { - verify(_constrainIndexKey.isEmpty()); - BSONElementSet s; - jsobj.getFieldsDotted(it->getField().c_str(), s, false); - int matches = 0; - for (BSONElementSet::const_iterator i = s.begin(); i != s.end(); ++i) { - if (!i->isABSONObj()) { return false; } - GeometryContainer container; - if (container.parseFrom(i->Obj()) && it->matches(container)) { - ++matches; break; - } - // Maybe it's an array of geometries. - BSONObjIterator geoIt(i->Obj()); - while (geoIt.more()) { - BSONElement e = geoIt.next(); - if (!e.isABSONObj()) { return false; } - if (!container.parseFrom(e.embeddedObject())) { return false; } - if (it->matches(container)) { ++matches; break; } - } - } - if (0 == matches) { return false; } - } - - for (vector::const_iterator it = _regexs.begin(); - it != _regexs.end(); - ++it) { - BSONElementSet s; - if ( !_constrainIndexKey.isEmpty() ) { - BSONElement e = jsobj.getFieldUsingIndexNames(it->_fieldName, _constrainIndexKey); - - // Should only have keys nested one deep here, for geo-indices - // TODO: future indices may nest deeper? - if( e.type() == Array ){ - BSONObjIterator i( e.Obj() ); - while( i.more() ){ - s.insert( i.next() ); - } - } - else if ( !e.eoo() ) - s.insert( e ); - - } - else { - jsobj.getFieldsDotted( it->_fieldName, s ); - } - bool match = false; - for( BSONElementSet::const_iterator i = s.begin(); i != s.end(); ++i ) - if ( regexMatches(*it, *i) ) - match = true; - if ( !match ^ it->_isNot ) - return false; - } - - if ( _andMatchers.size() > 0 ) { - for( list< shared_ptr< Matcher > >::const_iterator i = _andMatchers.begin(); - i != _andMatchers.end(); ++i ) { - // SERVER-3192 Track field matched using details the same as for - // top level fields, at least for now. - if ( !(*i)->matches( jsobj, details ) ) { - return false; - } - } - } - - if ( _orMatchers.size() > 0 ) { - bool match = false; - for( list< shared_ptr< Matcher > >::const_iterator i = _orMatchers.begin(); - i != _orMatchers.end(); ++i ) { - // SERVER-205 don't submit details - we don't want to track field - // matched within $or - if ( (*i)->matches( jsobj ) ) { - match = true; - break; - } - } - if ( !match ) { - return false; - } - } - - if ( _norMatchers.size() > 0 ) { - for( list< shared_ptr< Matcher > >::const_iterator i = _norMatchers.begin(); - i != _norMatchers.end(); ++i ) { - // SERVER-205 don't submit details - we don't want to track field - // matched within $nor - if ( (*i)->matches( jsobj ) ) { - return false; - } - } - } - - if ( _where ) { - return _where->exec( jsobj ); - } - - return true; - } - -#ifdef MONGO_LATER_SERVER_4644 - void Matcher::visitReferences(FieldSink *pSink) const { - // check normal non-regex cases: - for ( unsigned i = 0; i < _basics.size(); i++ ) { - const ElementMatcher& bm = _basics[i]; - const BSONElement& m = bm._toMatch; - // -1=mismatch. 0=missing element. 1=match - int cmp = matchesDotted(m.fieldName(), m, jsobj, bm._compareOp, bm , false , details ); - if ( cmp == 0 && bm._compareOp == BSONObj::opEXISTS ) { - // If missing, match cmp is opposite of $exists spec. - cmp = -retExistsFound(bm); - } - if ( bm._isNot ) - cmp = -cmp; - if ( cmp < 0 ) - return false; - if ( cmp == 0 ) { - /* missing is ok iff we were looking for null */ - if ( m.type() == jstNULL || m.type() == Undefined || - ( ( bm._compareOp == BSONObj::opIN || bm._compareOp == BSONObj::NIN ) && bm._myset->count( staticNull.firstElement() ) > 0 ) ) { - if ( bm.negativeCompareOp() ^ bm._isNot ) { - return false; - } - } - else { - if ( !bm._isNot ) { - return false; - } - } - } - } - - for (vector::const_iterator it = _regexs.begin(); - it != _regexs.end(); - ++it) { - BSONElementSet s; - if ( !_constrainIndexKey.isEmpty() ) { - BSONElement e = jsobj.getFieldUsingIndexNames(it->_fieldName, _constrainIndexKey); - - // Should only have keys nested one deep here, for geo-indices - // TODO: future indices may nest deeper? - if( e.type() == Array ){ - BSONObjIterator i( e.Obj() ); - while( i.more() ){ - s.insert( i.next() ); - } - } - else if ( !e.eoo() ) - s.insert( e ); - - } - else { - jsobj.getFieldsDotted( it->_fieldName, s ); - } - bool match = false; - for( BSONElementSet::const_iterator i = s.begin(); i != s.end(); ++i ) - if ( regexMatches(*it, *i) ) - match = true; - if ( !match ^ it->_isNot ) - return false; - } - - if ( _andMatchers.size() > 0 ) { - for( list< shared_ptr< Matcher > >::const_iterator i = _andMatchers.begin(); - i != _andMatchers.end(); ++i ) { - // SERVER-3192 Track field matched using details the same as for - // top level fields, at least for now. - if ( !(*i)->matches( jsobj, details ) ) { - return false; - } - } - } - - if ( _orMatchers.size() > 0 ) { - bool match = false; - for( list< shared_ptr< Matcher > >::const_iterator i = _orMatchers.begin(); - i != _orMatchers.end(); ++i ) { - // SERVER-205 don't submit details - we don't want to track field - // matched within $or - if ( (*i)->matches( jsobj ) ) { - match = true; - break; - } - } - if ( !match ) { - return false; - } - } - - if ( _norMatchers.size() > 0 ) { - for( list< shared_ptr< Matcher > >::const_iterator i = _norMatchers.begin(); - i != _norMatchers.end(); ++i ) { - // SERVER-205 don't submit details - we don't want to track field - // matched within $nor - if ( (*i)->matches( jsobj ) ) { - return false; - } - } - } - - if ( _where ) { - if ( _where->func == 0 ) { - u_assert( 10070 , "$where compile error", false); - return false; // didn't compile - } - - if ( _where->jsScope ) { - _where->scope->init( _where->jsScope ); - } - _where->scope->setObject( "obj", const_cast< BSONObj & >( jsobj ) ); - _where->scope->setBoolean( "fullObject" , true ); // this is a hack b/c fullObject used to be relevant - - int err = _where->scope->invoke( _where->func , 0, &jsobj , 1000 * 60 , false ); - if ( err == -3 ) { // INVOKE_ERROR - stringstream ss; - ss << "error on invocation of $where function:\n" - << _where->scope->getError(); - u_assert( 10071 , ss.str(), false); - return false; - } - else if ( err != 0 ) { // ! INVOKE_SUCCESS - u_assert( 10072 , "unknown error in invocation of $where function", false); - return false; - } - return _where->scope->getBoolean( "__returnValue" ) != 0; - - } - } -#endif /* MONGO_LATER_SERVER_4644 */ - - static void visitList( MatcherVisitor& visitor, - const list >& matchers ) { - for( list >::const_iterator i = matchers.begin(); i != matchers.end(); - ++i ) { - (*i)->visit( visitor ); - } - } - - void Matcher::visit( MatcherVisitor& visitor ) const { - visitor.visitMatcher( *this ); - // Visit the _basics ElementMatchers. - for( vector::const_iterator i = _basics.begin(); i != _basics.end(); ++i ) { - visitor.visitElementMatcher( *i ); - } - // Visit Matchers contained within the _basics ElementMatchers. - for( vector::const_iterator i = _basics.begin(); i != _basics.end(); ++i ) { - i->visit( visitor ); - } - visitList( visitor, _andMatchers ); - visitList( visitor, _orMatchers ); - visitList( visitor, _norMatchers ); - } - - bool Matcher::keyMatch( const Matcher &docMatcher ) const { - // Quick check certain non key match cases. - if ( docMatcher._all - || docMatcher._haveSize - || docMatcher._hasArray // We can't match an array to its first indexed element using keymatch - || docMatcher._haveNeg ) { - return false; - } - - // Check that all match components are available in the index matcher. - if ( !( _basics.size() == docMatcher._basics.size() && _regexs.size() == docMatcher._regexs.size() && !docMatcher._where ) ) { - return false; - } - if (_geo.size() != docMatcher._geo.size()) { - return false; - } - if ( _andMatchers.size() != docMatcher._andMatchers.size() ) { - return false; - } - if ( _orMatchers.size() != docMatcher._orMatchers.size() ) { - return false; - } - if ( docMatcher._norMatchers.size() > 0 ) { - return false; - } - - // Recursively check that all submatchers support key match. - { - list< shared_ptr< Matcher > >::const_iterator i = _andMatchers.begin(); - list< shared_ptr< Matcher > >::const_iterator j = docMatcher._andMatchers.begin(); - while( i != _andMatchers.end() ) { - if ( !(*i)->keyMatch( **j ) ) { - return false; - } - ++i; ++j; - } - } - { - list< shared_ptr< Matcher > >::const_iterator i = _orMatchers.begin(); - list< shared_ptr< Matcher > >::const_iterator j = docMatcher._orMatchers.begin(); - while( i != _orMatchers.end() ) { - if ( !(*i)->keyMatch( **j ) ) { - return false; - } - ++i; ++j; - } - } - // Nor matchers and or dedup constraints aren't created for index matchers, - // so no need to check those here. - return true; - } - - - /*- just for testing -- */ -#pragma pack(1) - struct JSObj1 { - JSObj1() { - totsize=sizeof(JSObj1); - n = NumberDouble; - strcpy_s(nname, 5, "abcd"); - N = 3.1; - s = String; - strcpy_s(sname, 7, "abcdef"); - slen = 10; - strcpy_s(sval, 10, "123456789"); - eoo = EOO; - } - unsigned totsize; - - char n; - char nname[5]; - double N; - - char s; - char sname[7]; - unsigned slen; - char sval[10]; - - char eoo; - }; -#pragma pack() - - struct JSObj1 js1; - -#pragma pack(1) - struct JSObj2 { - JSObj2() { - totsize=sizeof(JSObj2); - s = String; - strcpy_s(sname, 7, "abcdef"); - slen = 10; - strcpy_s(sval, 10, "123456789"); - eoo = EOO; - } - unsigned totsize; - char s; - char sname[7]; - unsigned slen; - char sval[10]; - char eoo; - } js2; - - struct JSUnitTest : public StartupTest { - void run() { - - { - // a quick check that we are using our mongo assert macro - int x = 1; - verify( ++x ); - if( x != 2 ) { - log() << "bad build - wrong assert macro" << endl; - ::abort(); - } - } - - BSONObj j1((const char *) &js1); - BSONObj j2((const char *) &js2); - Matcher m(j2); - verify( m.matches(j1) ); - js2.sval[0] = 'z'; - verify( !m.matches(j1) ); - Matcher n(j1); - verify( n.matches(j1) ); - verify( !n.matches(j2) ); - - BSONObj j0 = BSONObj(); -// BSONObj j0((const char *) &js0); - Matcher p(j0); - verify( p.matches(j1) ); - verify( p.matches(j2) ); - } - } jsunittest; - -#pragma pack() - - struct RXTest : public StartupTest { - - RXTest() { - } - - void run() { - /* - static const boost::regex e("(\\d{4}[- ]){3}\\d{4}"); - static const boost::regex b("....."); - out() << "regex result: " << regex_match("hello", e) << endl; - out() << "regex result: " << regex_match("abcoo", b) << endl; - */ - - int ret = 0; - - pcre_config( PCRE_CONFIG_UTF8 , &ret ); - massert( 10342 , "pcre not compiled with utf8 support" , ret ); - - pcrecpp::RE re1(")({a}h.*o"); - pcrecpp::RE re("h.llo"); - verify( re.FullMatch("hello") ); - verify( !re1.FullMatch("hello") ); - - - pcrecpp::RE_Options options; - options.set_utf8(true); - pcrecpp::RE part("dwi", options); - verify( part.PartialMatch("dwight") ); - - pcre_config( PCRE_CONFIG_UNICODE_PROPERTIES , &ret ); - if ( ! ret ) - cout << "warning: some regex utf8 things will not work. pcre build doesn't have --enable-unicode-properties" << endl; - - } - } rxtest; - -} // namespace mongo diff --git a/src/mongo/db/matcher.h b/src/mongo/db/matcher.h index a11ff54efc2..f4ac8757a32 100644 --- a/src/mongo/db/matcher.h +++ b/src/mongo/db/matcher.h @@ -16,55 +16,29 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once -#include "jsobj.h" -#include "pcrecpp.h" -#include "mongo/db/geo/geoquery.h" +#include "mongo/db/jsobj.h" namespace mongo { class Cursor; - class CoveredIndexMatcher; - class ElementMatcher; - class Matcher; class FieldRangeVector; - class RegexMatcher { - public: - /** - * Maximum pattern size which pcre v8.3 can do matches correctly with - * LINK_SIZE define macro set to 2 @ pcre's config.h (based on - * experiments) - */ - static const size_t MaxPatternSize = 32764; - - const char *_fieldName; - const char *_regex; - const char *_flags; - string _prefix; - shared_ptr< pcrecpp::RE > _re; - bool _isNot; - RegexMatcher() : _isNot() {} - }; - - struct GeoMatcher { - public: - GeoMatcher(GeoQuery query, bool negated) : geoQuery(query), isNot(negated) {} - - string getField() const { return geoQuery.getField(); } - - bool matches(const GeometryContainer &container) const { - bool satisfied = geoQuery.satisfiesPredicate(container); - if (isNot) { return !satisfied; } - else { return satisfied; } - } - GeoQuery geoQuery; - bool isNot; - }; - struct element_lt { bool operator()(const BSONElement& l, const BSONElement& r) const { int x = (int) l.canonicalType() - (int) r.canonicalType(); @@ -73,287 +47,13 @@ namespace mongo { return compareElementValues(l,r) < 0; } }; +} - /** - * An interface for visiting a Matcher and all of its nested Matchers and ElementMatchers. - * RegexMatchers are not visited. - */ - class MatcherVisitor { - public: - virtual ~MatcherVisitor() {} - virtual void visitMatcher( const Matcher& matcher ) {} - virtual void visitElementMatcher( const ElementMatcher& elementMatcher ) {} - }; - - class ElementMatcher { - public: - - ElementMatcher() { - } - - ElementMatcher( BSONElement e , int op, bool isNot ); - - ElementMatcher( BSONElement e , int op , const BSONObj& array, bool isNot ); - - ~ElementMatcher() { } - - bool negativeCompareOp() const { return _compareOp == BSONObj::NE || _compareOp == BSONObj::NIN; } - int inverseOfNegativeCompareOp() const; - bool negativeCompareOpContainsNull() const; - - void visit( MatcherVisitor& visitor ) const; - - BSONElement _toMatch; - int _compareOp; - bool _isNot; - shared_ptr< set > _myset; - shared_ptr< vector > _myregex; - - // these are for specific operators - int _mod; - int _modm; - BSONType _type; - - shared_ptr _subMatcher; - bool _subMatcherOnPrimitives ; - - vector< shared_ptr > _allMatchers; - }; - - class Where; // used for $where javascript eval - class DiskLoc; - - /** Reports information about a match request. */ - class MatchDetails { - public: - MatchDetails(); - void resetOutput(); - string toString() const; - - /** Request that an elemMatchKey be recorded. */ - void requestElemMatchKey() { _elemMatchKeyRequested = true; } - - bool needRecord() const { return _elemMatchKeyRequested; } - - bool hasLoadedRecord() const { return _loadedRecord; } - bool hasElemMatchKey() const { return _elemMatchKeyFound; } - string elemMatchKey() const { - verify( hasElemMatchKey() ); - return _elemMatchKey; - } - - void setLoadedRecord( bool loadedRecord ) { _loadedRecord = loadedRecord; } - void setElemMatchKey( const string &elemMatchKey ) { - if ( _elemMatchKeyRequested ) { - _elemMatchKeyFound = true; - _elemMatchKey = elemMatchKey; - } - } - - private: - bool _loadedRecord; - bool _elemMatchKeyRequested; - bool _elemMatchKeyFound; - string _elemMatchKey; - }; - - /* Match BSON objects against a query pattern. - - e.g. - db.foo.find( { a : 3 } ); - - { a : 3 } is the pattern object. See manual for full info. - - GT/LT: - { a : { $gt : 3 } } - Not equal: - { a : { $ne : 3 } } - - TODO: we should rewrite the matcher to be more an AST style. - */ - class Matcher : boost::noncopyable { - int matchesDotted( - const char *fieldName, - const BSONElement& toMatch, const BSONObj& obj, - int compareOp, const ElementMatcher& bm, bool isArr , MatchDetails * details ) const; - - /** - * Perform a NE or NIN match by returning the inverse of the opposite matching operation. - * Missing values are considered matches unless the match must not equal null. - */ - int inverseMatch( - const char *fieldName, - const BSONElement &toMatch, const BSONObj &obj, - const ElementMatcher&bm, MatchDetails * details ) const; - - public: - static int opDirection(int op) { - return op <= BSONObj::LTE ? -1 : 1; - } - - Matcher(const BSONObj &pattern, bool nested=false); - - ~Matcher(); - - bool matches(const BSONObj& j, MatchDetails * details = 0 ) const; - -#ifdef MONGO_LATER_SERVER_4644 - class FieldSink { - public: - virtual ~FieldSink() {}; - virtual void referenceField(const string &fieldPath) = 0; - }; - - /** - Visit all of the fields that are referenced by this Matcher - (and any descendants). - - This can be used to gather a list of all the references made by - this matcher. The implementation of this parallels that of - matches() above. - - @param pSink a FieldSink that the caller will use to gather or - process the references - */ - void visitReferences(FieldSink *pSink) const; -#endif /* MONGO_LATER_SERVER_4644 */ - - /** - * Visit this Matcher and all of its nested Matchers and ElementMatchers. All top level - * ElementMatchers of a Matcher are visited immediately after the Matcher itself (before any - * other Matcher is visited). - */ - void visit( MatcherVisitor& visitor ) const; - - bool atomic() const { return _atomic; } - - string toString() const { - return _jsobj.toString(); - } - - /** - * @return true if this key matcher will return the same true/false - * value as the provided doc matcher. - */ - bool keyMatch( const Matcher &docMatcher ) const; - - bool singleSimpleCriterion() const { - if ( _where || - _basics.size() > 1 || - _haveNeg || - _haveSize || - _regexs.size() > 0 ) - return false; - - if ( _jsobj.nFields() > 1 ) - return false; - - if ( _basics.size() != 1 ) - return false; - - if ( strchr( _jsobj.firstElement().fieldName(), '.' ) ) - return false; - - return _basics[0]._compareOp == BSONObj::Equality; - } - - const BSONObj *getQuery() const { return &_jsobj; }; - - private: - /** - * Generate a matcher for the provided index key format using the - * provided full doc matcher. - */ - Matcher( const Matcher &docMatcher, const BSONObj &constrainIndexKey ); - - void addBasic(const BSONElement &e, int c, bool isNot) { - // TODO May want to selectively ignore these element types based on op type. - if ( e.type() == MinKey || e.type() == MaxKey ) - return; - _basics.push_back( ElementMatcher( e , c, isNot ) ); - } - - void addRegex(const char *fieldName, const char *regex, const char *flags, bool isNot = false); - bool addOp( const BSONElement &e, const BSONElement &fe, bool isNot, const char *& regex, const char *&flags ); - - int valuesMatch(const BSONElement& l, const BSONElement& r, int op, const ElementMatcher& bm) const; - - bool parseClause( const BSONElement &e ); - void parseExtractedClause( const BSONElement &e, list< shared_ptr< Matcher > > &matchers ); - - void parseWhere( const BSONElement &e ); - void parseMatchExpressionElement( const BSONElement &e, bool nested ); - - Where *_where; // set if query uses $where - BSONObj _jsobj; // the query pattern. e.g., { name: "joe" } - BSONObj _constrainIndexKey; - vector _basics; - bool _haveSize; - bool _all; - bool _hasArray; - bool _haveNeg; - - /* $atomic - if true, a multi document operation (some removes, updates) - should be done atomically. in that case, we do not yield - - i.e. we stay locked the whole time. - http://dochub.mongodb.org/core/remove - */ - bool _atomic; - - vector _regexs; - vector _geo; - - // so we delete the mem when we're done: - vector< shared_ptr< BSONObjBuilder > > _builders; - list< shared_ptr< Matcher > > _andMatchers; - list< shared_ptr< Matcher > > _orMatchers; - list< shared_ptr< Matcher > > _norMatchers; - - friend class CoveredIndexMatcher; - }; - - // If match succeeds on index key, then attempt to match full document. - class CoveredIndexMatcher : boost::noncopyable { - public: - CoveredIndexMatcher(const BSONObj &pattern, const BSONObj &indexKeyPattern); - bool matchesWithSingleKeyIndex( const BSONObj& key, const DiskLoc& recLoc, - MatchDetails* details = 0 ) const { - return matches( key, recLoc, details, true ); - } - /** - * This is the preferred method for matching against a cursor, as it - * can handle both multi and single key cursors. - */ - bool matchesCurrent( Cursor * cursor , MatchDetails * details = 0 ) const; - bool needRecord() const { return _needRecord; } +#include "mongo/db/matcher/matcher.h" - const Matcher &docMatcher() const { return *_docMatcher; } - - /** - * @return a matcher for a following $or clause. - * @param prevClauseFrs The index range scanned by the previous $or clause. May be empty. - * @param nextClauseIndexKeyPattern The index key of the following $or clause. - */ - CoveredIndexMatcher *nextClauseMatcher( const shared_ptr& prevClauseFrv, - const BSONObj& nextClauseIndexKeyPattern ) const { - return new CoveredIndexMatcher( *this, prevClauseFrv, nextClauseIndexKeyPattern ); - } - - string toString() const; - - private: - bool matches( const BSONObj& key, const DiskLoc& recLoc, MatchDetails* details = 0, - bool keyUsable = true ) const; - bool isOrClauseDup( const BSONObj &obj ) const; - CoveredIndexMatcher( const CoveredIndexMatcher &prevClauseMatcher, - const shared_ptr &prevClauseFrv, - const BSONObj &nextClauseIndexKeyPattern ); - void init(); - shared_ptr< Matcher > _docMatcher; - Matcher _keyMatcher; - vector > _orDedupConstraints; +namespace mongo { - bool _needRecord; // if the key itself isn't good enough to determine a positive match - }; + //typedef MatcherOld Matcher; + typedef Matcher2 Matcher; +} -} // namespace mongo diff --git a/src/mongo/db/matcher/expression.cpp b/src/mongo/db/matcher/expression.cpp new file mode 100644 index 00000000000..35e32c447a1 --- /dev/null +++ b/src/mongo/db/matcher/expression.cpp @@ -0,0 +1,72 @@ +// expression.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression.h" + +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/util/log.h" + +namespace mongo { + + MatchExpression::MatchExpression( MatchType type ) + : _matchType( type ) { } + + string MatchExpression::toString() const { + StringBuilder buf; + debugString( buf, 0 ); + return buf.str(); + } + + void MatchExpression::_debugAddSpace( StringBuilder& debug, int level ) const { + for ( int i = 0; i < level; i++ ) + debug << " "; + } + + bool MatchExpression::matchesBSON( const BSONObj& doc, MatchDetails* details ) const { + BSONMatchableDocument mydoc( doc ); + return matches( &mydoc, details ); + } + + + void AtomicMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "$atomic\n"; + } + + void FalseMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "$false\n"; + } + +} + + diff --git a/src/mongo/db/matcher/expression.h b/src/mongo/db/matcher/expression.h new file mode 100644 index 00000000000..73f3e48692d --- /dev/null +++ b/src/mongo/db/matcher/expression.h @@ -0,0 +1,257 @@ +// expression.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/matcher/matchable.h" +#include "mongo/db/matcher/match_details.h" + +namespace mongo { + + class TreeMatchExpression; + + class MatchExpression { + MONGO_DISALLOW_COPYING( MatchExpression ); + public: + enum MatchType { + // tree types + AND, OR, + + // array types + ELEM_MATCH_OBJECT, ELEM_MATCH_VALUE, SIZE, + + // leaf types + EQ, LTE, LT, GT, GTE, REGEX, MOD, EXISTS, MATCH_IN, + + // Negations. + NOT, NIN, NOR, + + // special types + TYPE_OPERATOR, GEO, WHERE, + + // things that maybe shouldn't even be nodes + ATOMIC, ALWAYS_FALSE, + + // Things that we parse but cannot be answered without an index. + GEO_NEAR, TEXT, + + // Expressions that are only created internally + INTERNAL_GEO_S2_KEYCHECK + }; + + MatchExpression( MatchType type ); + virtual ~MatchExpression(){} + + // + // Structural/AST information + // + + /** + * What type is the node? See MatchType above. + */ + MatchType matchType() const { return _matchType; } + + /** + * How many children does the node have? Most nodes are leaves so the default impl. is for + * a leaf. + */ + virtual size_t numChildren() const { return 0; } + + /** + * Get the i-th child. + */ + virtual MatchExpression* getChild( size_t i ) const { return NULL; } + + /** + * Get all the children of a node + */ + virtual std::vector* getChildVector() { return NULL; } + + /** + * Get the path of the leaf. Returns StringData() if there is no path (node is logical). + */ + virtual const StringData path() const { return StringData(); } + + /** + * Notes on structure: + * isLogical, isArray, and isLeaf define three partitions of all possible operators. + * + * isLogical can have children and its children can be arbitrary operators. + * + * isArray can have children and its children are predicates over one field. + * + * isLeaf is a predicate over one field. + */ + + /** + * Is this node a logical operator? All of these inherit from ListOfMatchExpression. + * AND, OR, NOT, NOR. + */ + bool isLogical() const { + return AND == _matchType || OR == _matchType || NOT == _matchType || NOR == _matchType; + } + + /** + * Is this node an array operator? Array operators have multiple clauses but operate on one + * field. + * + * ELEM_MATCH_VALUE, ELEM_MATCH_OBJECT, SIZE (ArrayMatchingMatchExpression) + */ + bool isArray() const { + return SIZE == _matchType + || ELEM_MATCH_VALUE == _matchType + || ELEM_MATCH_OBJECT == _matchType; + } + + /** + * Not-internal nodes, predicates over one field. Almost all of these inherit from + * LeafMatchExpression. + * + * Exceptions: WHERE, which doesn't have a field. + * TYPE_OPERATOR, which inherits from MatchExpression due to unique array + * semantics. + */ + bool isLeaf() const { + return !isArray() && !isLogical(); + } + + // XXX: document + virtual MatchExpression* shallowClone() const = 0; + + // XXX document + virtual bool equivalent( const MatchExpression* other ) const = 0; + + // + // Determine if a document satisfies the tree-predicate. + // + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const = 0; + + virtual bool matchesBSON( const BSONObj& doc, MatchDetails* details = 0 ) const; + + /** + * Determines if the element satisfies the tree-predicate. + * Not valid for all expressions (e.g. $where); in those cases, returns false. + */ + virtual bool matchesSingleElement( const BSONElement& e ) const = 0; + + // + // Tagging mechanism: Hang data off of the tree for retrieval later. + // + + class TagData { + public: + virtual ~TagData() { } + virtual void debugString(StringBuilder* builder) const = 0; + virtual TagData* clone() const = 0; + }; + + /** + * Takes ownership + */ + void setTag(TagData* data) { _tagData.reset(data); } + TagData* getTag() const { return _tagData.get(); } + virtual void resetTag() { + setTag(NULL); + for (size_t i = 0; i < numChildren(); ++i) { + getChild(i)->resetTag(); + } + } + + // + // Debug information + // + virtual string toString() const; + virtual void debugString( StringBuilder& debug, int level = 0 ) const = 0; + + protected: + void _debugAddSpace( StringBuilder& debug, int level ) const; + + private: + MatchType _matchType; + boost::scoped_ptr _tagData; + }; + + /** + * this isn't really an expression, but a hint to other things + * not sure where to put it in the end + */ + class AtomicMatchExpression : public MatchExpression { + public: + AtomicMatchExpression() : MatchExpression( ATOMIC ){} + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const { + return true; + } + + virtual bool matchesSingleElement( const BSONElement& e ) const { + return true; + } + + virtual MatchExpression* shallowClone() const { + return new AtomicMatchExpression(); + } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + virtual bool equivalent( const MatchExpression* other ) const { + return other->matchType() == ATOMIC; + } + }; + + class FalseMatchExpression : public MatchExpression { + public: + FalseMatchExpression() : MatchExpression( ALWAYS_FALSE ){} + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const { + return false; + } + + virtual bool matchesSingleElement( const BSONElement& e ) const { + return false; + } + + virtual MatchExpression* shallowClone() const { + return new FalseMatchExpression(); + } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + virtual bool equivalent( const MatchExpression* other ) const { + return other->matchType() == ALWAYS_FALSE; + } + }; + +} diff --git a/src/mongo/db/matcher/expression_array.cpp b/src/mongo/db/matcher/expression_array.cpp new file mode 100644 index 00000000000..73209d71d56 --- /dev/null +++ b/src/mongo/db/matcher/expression_array.cpp @@ -0,0 +1,231 @@ +// expression_array.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression_array.h" + +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/log.h" + +namespace mongo { + + Status ArrayMatchingMatchExpression::initPath( const StringData& path ) { + _path = path; + Status s = _elementPath.init( _path ); + _elementPath.setTraverseLeafArray( false ); + return s; + } + + bool ArrayMatchingMatchExpression::matches( const MatchableDocument* doc, MatchDetails* details ) const { + MatchableDocument::IteratorHolder cursor( doc, &_elementPath ); + + while ( cursor->more() ) { + ElementIterator::Context e = cursor->next(); + if ( e.element().type() != Array ) + continue; + + bool amIRoot = e.arrayOffset().eoo(); + + if ( !matchesArray( e.element().Obj(), amIRoot ? details : NULL ) ) + continue; + + if ( !amIRoot && details && details->needRecord() && !e.arrayOffset().eoo() ) { + details->setElemMatchKey( e.arrayOffset().fieldName() ); + } + return true; + } + return false; + } + + bool ArrayMatchingMatchExpression::matchesSingleElement( const BSONElement& e ) const { + if ( e.type() != Array ) + return false; + return matchesArray( e.Obj(), NULL ); + } + + + bool ArrayMatchingMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const ArrayMatchingMatchExpression* realOther = + static_cast( other ); + + if ( _path != realOther->_path ) + return false; + + if ( numChildren() != realOther->numChildren() ) + return false; + + for ( unsigned i = 0; i < numChildren(); i++ ) + if ( !getChild(i)->equivalent( realOther->getChild(i) ) ) + return false; + return true; + } + + + // ------- + + Status ElemMatchObjectMatchExpression::init( const StringData& path, MatchExpression* sub ) { + _sub.reset( sub ); + return initPath( path ); + } + + bool ElemMatchObjectMatchExpression::matchesArray( const BSONObj& anArray, MatchDetails* details ) const { + BSONObjIterator i( anArray ); + while ( i.more() ) { + BSONElement inner = i.next(); + if ( !inner.isABSONObj() ) + continue; + if ( _sub->matchesBSON( inner.Obj(), NULL ) ) { + if ( details && details->needRecord() ) { + details->setElemMatchKey( inner.fieldName() ); + } + return true; + } + } + return false; + } + + void ElemMatchObjectMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " $elemMatch (obj)"; + + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + _sub->debugString( debug, level + 1 ); + } + + + // ------- + + ElemMatchValueMatchExpression::~ElemMatchValueMatchExpression() { + for ( unsigned i = 0; i < _subs.size(); i++ ) + delete _subs[i]; + _subs.clear(); + } + + Status ElemMatchValueMatchExpression::init( const StringData& path, MatchExpression* sub ) { + init( path ); + add( sub ); + return Status::OK(); + } + + Status ElemMatchValueMatchExpression::init( const StringData& path ) { + return initPath( path ); + } + + + void ElemMatchValueMatchExpression::add( MatchExpression* sub ) { + verify( sub ); + _subs.push_back( sub ); + } + + bool ElemMatchValueMatchExpression::matchesArray( const BSONObj& anArray, MatchDetails* details ) const { + BSONObjIterator i( anArray ); + while ( i.more() ) { + BSONElement inner = i.next(); + + if ( _arrayElementMatchesAll( inner ) ) { + if ( details && details->needRecord() ) { + details->setElemMatchKey( inner.fieldName() ); + } + return true; + } + } + return false; + } + + bool ElemMatchValueMatchExpression::_arrayElementMatchesAll( const BSONElement& e ) const { + for ( unsigned i = 0; i < _subs.size(); i++ ) { + if ( !_subs[i]->matchesSingleElement( e ) ) + return false; + } + return true; + } + + void ElemMatchValueMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " $elemMatch (value)"; + + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + for ( unsigned i = 0; i < _subs.size(); i++ ) { + _subs[i]->debugString( debug, level + 1 ); + } + } + + + // --------- + + Status SizeMatchExpression::init( const StringData& path, int size ) { + _size = size; + return initPath( path ); + } + + bool SizeMatchExpression::matchesArray( const BSONObj& anArray, MatchDetails* details ) const { + if ( _size < 0 ) + return false; + return anArray.nFields() == _size; + } + + void SizeMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " $size : " << _size << "\n"; + + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + } + + bool SizeMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const SizeMatchExpression* realOther = static_cast( other ); + return path() == realOther->path() && _size == realOther->_size; + } + + + // ------------------ + + + +} diff --git a/src/mongo/db/matcher/expression_array.h b/src/mongo/db/matcher/expression_array.h new file mode 100644 index 00000000000..3f1dba5a88b --- /dev/null +++ b/src/mongo/db/matcher/expression_array.h @@ -0,0 +1,155 @@ +// expression_array.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + class ArrayMatchingMatchExpression : public MatchExpression { + public: + ArrayMatchingMatchExpression( MatchType matchType ) : MatchExpression( matchType ){} + virtual ~ArrayMatchingMatchExpression(){} + + Status initPath( const StringData& path ); + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details ) const; + + /** + * @param e - has to be an array. calls matchesArray with e as an array + */ + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual bool matchesArray( const BSONObj& anArray, MatchDetails* details ) const = 0; + + bool equivalent( const MatchExpression* other ) const; + + const StringData path() const { return _path; } + + private: + StringData _path; + ElementPath _elementPath; + }; + + class ElemMatchObjectMatchExpression : public ArrayMatchingMatchExpression { + public: + ElemMatchObjectMatchExpression() : ArrayMatchingMatchExpression( ELEM_MATCH_OBJECT ){} + Status init( const StringData& path, MatchExpression* sub ); + + bool matchesArray( const BSONObj& anArray, MatchDetails* details ) const; + + virtual ElemMatchObjectMatchExpression* shallowClone() const { + ElemMatchObjectMatchExpression* e = new ElemMatchObjectMatchExpression(); + e->init(path(), _sub->shallowClone()); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + virtual void debugString( StringBuilder& debug, int level ) const; + + virtual size_t numChildren() const { return 1; } + + virtual MatchExpression* getChild( size_t i ) const { return _sub.get(); } + + private: + boost::scoped_ptr _sub; + }; + + class ElemMatchValueMatchExpression : public ArrayMatchingMatchExpression { + public: + ElemMatchValueMatchExpression() : ArrayMatchingMatchExpression( ELEM_MATCH_VALUE ){} + virtual ~ElemMatchValueMatchExpression(); + + Status init( const StringData& path ); + Status init( const StringData& path, MatchExpression* sub ); + void add( MatchExpression* sub ); + + bool matchesArray( const BSONObj& anArray, MatchDetails* details ) const; + + virtual ElemMatchValueMatchExpression* shallowClone() const { + ElemMatchValueMatchExpression* e = new ElemMatchValueMatchExpression(); + e->init(path()); + for (size_t i = 0; i < _subs.size(); ++i) { + e->add(_subs[i]->shallowClone()); + } + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + virtual void debugString( StringBuilder& debug, int level ) const; + + virtual size_t numChildren() const { return _subs.size(); } + + virtual MatchExpression* getChild( size_t i ) const { return _subs[i]; } + + private: + bool _arrayElementMatchesAll( const BSONElement& e ) const; + + std::vector _subs; + }; + + class SizeMatchExpression : public ArrayMatchingMatchExpression { + public: + SizeMatchExpression() : ArrayMatchingMatchExpression( SIZE ){} + Status init( const StringData& path, int size ); + + virtual SizeMatchExpression* shallowClone() const { + SizeMatchExpression* e = new SizeMatchExpression(); + e->init(path(), _size); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + virtual bool matchesArray( const BSONObj& anArray, MatchDetails* details ) const; + + virtual void debugString( StringBuilder& debug, int level ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + int getData() const { return _size; } + + private: + int _size; // >= 0 real, < 0, nothing will match + }; + +} diff --git a/src/mongo/db/matcher/expression_array_test.cpp b/src/mongo/db/matcher/expression_array_test.cpp new file mode 100644 index 00000000000..626ac72409e --- /dev/null +++ b/src/mongo/db/matcher/expression_array_test.cpp @@ -0,0 +1,483 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** Unit tests for MatchMatchExpression operator implementations in match_operators.{h,cpp}. */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_array.h" +#include "mongo/db/matcher/expression_tree.h" + +namespace mongo { + + TEST( ElemMatchObjectMatchExpression, MatchesElementSingle ) { + BSONObj baseOperand = BSON( "b" << 5 ); + BSONObj match = BSON( "a" << BSON_ARRAY( BSON( "b" << 5.0 ) ) ); + BSONObj notMatch = BSON( "a" << BSON_ARRAY( BSON( "b" << 6 ) ) ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "b", baseOperand[ "b" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a", eq.release() ).isOK() ); + ASSERT( op.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !op.matchesSingleElement( notMatch[ "a" ] ) ); + } + + TEST( ElemMatchObjectMatchExpression, MatchesElementArray ) { + BSONObj baseOperand = BSON( "1" << 5 ); + BSONObj match = BSON( "a" << BSON_ARRAY( BSON_ARRAY( 's' << 5.0 ) ) ); + BSONObj notMatch = BSON( "a" << BSON_ARRAY( BSON_ARRAY( 5 << 6 ) ) ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "1", baseOperand[ "1" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a", eq.release() ).isOK() ); + ASSERT( op.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !op.matchesSingleElement( notMatch[ "a" ] ) ); + } + + TEST( ElemMatchObjectMatchExpression, MatchesElementMultiple ) { + BSONObj baseOperand1 = BSON( "b" << 5 ); + BSONObj baseOperand2 = BSON( "b" << 6 ); + BSONObj baseOperand3 = BSON( "c" << 7 ); + BSONObj notMatch1 = BSON( "a" << BSON_ARRAY( BSON( "b" << 5 << "c" << 7 ) ) ); + BSONObj notMatch2 = BSON( "a" << BSON_ARRAY( BSON( "b" << 6 << "c" << 7 ) ) ); + BSONObj notMatch3 = BSON( "a" << BSON_ARRAY( BSON( "b" << BSON_ARRAY( 5 << 6 ) ) ) ); + BSONObj match = + BSON( "a" << BSON_ARRAY( BSON( "b" << BSON_ARRAY( 5 << 6 ) << "c" << 7 ) ) ); + auto_ptr eq1( new EqualityMatchExpression() ); + ASSERT( eq1->init( "b", baseOperand1[ "b" ] ).isOK() ); + auto_ptr eq2( new EqualityMatchExpression() ); + ASSERT( eq2->init( "b", baseOperand2[ "b" ] ).isOK() ); + auto_ptr eq3( new EqualityMatchExpression() ); + ASSERT( eq3->init( "c", baseOperand3[ "c" ] ).isOK() ); + + auto_ptr andOp( new AndMatchExpression() ); + andOp->add( eq1.release() ); + andOp->add( eq2.release() ); + andOp->add( eq3.release() ); + + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a", andOp.release() ).isOK() ); + ASSERT( !op.matchesSingleElement( notMatch1[ "a" ] ) ); + ASSERT( !op.matchesSingleElement( notMatch2[ "a" ] ) ); + ASSERT( !op.matchesSingleElement( notMatch3[ "a" ] ) ); + ASSERT( op.matchesSingleElement( match[ "a" ] ) ); + } + + TEST( ElemMatchObjectMatchExpression, MatchesNonArray ) { + BSONObj baseOperand = BSON( "b" << 5 ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "b", baseOperand[ "b" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a", eq.release() ).isOK() ); + // Directly nested objects are not matched with $elemMatch. An intervening array is + // required. + ASSERT( !op.matchesBSON( BSON( "a" << BSON( "b" << 5 ) ), NULL ) ); + ASSERT( !op.matchesBSON( BSON( "a" << BSON( "0" << ( BSON( "b" << 5 ) ) ) ), NULL ) ); + ASSERT( !op.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( ElemMatchObjectMatchExpression, MatchesArrayObject ) { + BSONObj baseOperand = BSON( "b" << 5 ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "b", baseOperand[ "b" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a", eq.release() ).isOK() ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 5 ) ) ), NULL ) ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << BSON( "b" << 5 ) ) ), NULL ) ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( BSONObj() << BSON( "b" << 5 ) ) ), NULL ) ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 6 ) << BSON( "b" << 5 ) ) ), + NULL ) ); + } + + TEST( ElemMatchObjectMatchExpression, MatchesMultipleNamedValues ) { + BSONObj baseOperand = BSON( "c" << 5 ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "c", baseOperand[ "c" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a.b", eq.release() ).isOK() ); + ASSERT( op.matchesBSON( BSON( "a" << + BSON_ARRAY( BSON( "b" << + BSON_ARRAY( BSON( "c" << + 5 ) ) ) ) ), + NULL ) ); + ASSERT( op.matchesBSON( BSON( "a" << + BSON_ARRAY( BSON( "b" << + BSON_ARRAY( BSON( "c" << + 1 ) ) ) << + BSON( "b" << + BSON_ARRAY( BSON( "c" << + 5 ) ) ) ) ), + NULL ) ); + } + + TEST( ElemMatchObjectMatchExpression, ElemMatchKey ) { + BSONObj baseOperand = BSON( "c" << 6 ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "c", baseOperand[ "c" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a.b", eq.release() ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !op.matchesBSON( BSONObj(), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( !op.matchesBSON( BSON( "a" << BSON( "b" << BSON_ARRAY( BSON( "c" << 7 ) ) ) ), + &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( op.matchesBSON( BSON( "a" << BSON( "b" << BSON_ARRAY( 3 << BSON( "c" << 6 ) ) ) ), + &details ) ); + ASSERT( details.hasElemMatchKey() ); + // The entry within the $elemMatch array is reported. + ASSERT_EQUALS( "1", details.elemMatchKey() ); + ASSERT( op.matchesBSON( BSON( "a" << + BSON_ARRAY( 1 << 2 << + BSON( "b" << BSON_ARRAY( 3 << + 5 << + BSON( "c" << 6 ) ) ) ) ), + &details ) ); + ASSERT( details.hasElemMatchKey() ); + // The entry within a parent of the $elemMatch array is reported. + ASSERT_EQUALS( "2", details.elemMatchKey() ); + } + + /** + TEST( ElemMatchObjectMatchExpression, MatchesIndexKey ) { + BSONObj baseOperand = BSON( "b" << 5 ); + auto_ptr eq( new ComparisonMatchExpression() ); + ASSERT( eq->init( "b", baseOperand[ "b" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a", eq.release() ).isOK() ); + IndexSpec indexSpec( BSON( "a.b" << 1 ) ); + BSONObj indexKey = BSON( "" << "5" ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + op.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + + TEST( ElemMatchValueMatchExpression, MatchesElementSingle ) { + BSONObj baseOperand = BSON( "$gt" << 5 ); + BSONObj match = BSON( "a" << BSON_ARRAY( 6 ) ); + BSONObj notMatch = BSON( "a" << BSON_ARRAY( 4 ) ); + auto_ptr gt( new GTMatchExpression() ); + ASSERT( gt->init( "", baseOperand[ "$gt" ] ).isOK() ); + ElemMatchValueMatchExpression op; + ASSERT( op.init( "a", gt.release() ).isOK() ); + ASSERT( op.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !op.matchesSingleElement( notMatch[ "a" ] ) ); + } + + TEST( ElemMatchValueMatchExpression, MatchesElementMultiple ) { + BSONObj baseOperand1 = BSON( "$gt" << 1 ); + BSONObj baseOperand2 = BSON( "$lt" << 10 ); + BSONObj notMatch1 = BSON( "a" << BSON_ARRAY( 0 << 1 ) ); + BSONObj notMatch2 = BSON( "a" << BSON_ARRAY( 10 << 11 ) ); + BSONObj match = BSON( "a" << BSON_ARRAY( 0 << 5 << 11 ) ); + auto_ptr gt( new GTMatchExpression() ); + ASSERT( gt->init( "", baseOperand1[ "$gt" ] ).isOK() ); + auto_ptr lt( new LTMatchExpression() ); + ASSERT( lt->init( "", baseOperand2[ "$lt" ] ).isOK() ); + + ElemMatchValueMatchExpression op; + ASSERT( op.init( "a" ).isOK() ); + op.add( gt.release() ); + op.add( lt.release() ); + + ASSERT( !op.matchesSingleElement( notMatch1[ "a" ] ) ); + ASSERT( !op.matchesSingleElement( notMatch2[ "a" ] ) ); + ASSERT( op.matchesSingleElement( match[ "a" ] ) ); + } + + TEST( ElemMatchValueMatchExpression, MatchesNonArray ) { + BSONObj baseOperand = BSON( "$gt" << 5 ); + auto_ptr gt( new GTMatchExpression() ); + ASSERT( gt->init( "", baseOperand[ "$gt" ] ).isOK() ); + ElemMatchObjectMatchExpression op; + ASSERT( op.init( "a", gt.release() ).isOK() ); + // Directly nested objects are not matched with $elemMatch. An intervening array is + // required. + ASSERT( !op.matchesBSON( BSON( "a" << 6 ), NULL ) ); + ASSERT( !op.matchesBSON( BSON( "a" << BSON( "0" << 6 ) ), NULL ) ); + } + + TEST( ElemMatchValueMatchExpression, MatchesArrayScalar ) { + BSONObj baseOperand = BSON( "$gt" << 5 ); + auto_ptr gt( new GTMatchExpression() ); + ASSERT( gt->init( "", baseOperand[ "$gt" ] ).isOK() ); + ElemMatchValueMatchExpression op; + ASSERT( op.init( "a", gt.release() ).isOK() ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( 6 ) ), NULL ) ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 6 ) ), NULL ) ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( BSONObj() << 7 ) ), NULL ) ); + } + + TEST( ElemMatchValueMatchExpression, MatchesMultipleNamedValues ) { + BSONObj baseOperand = BSON( "$gt" << 5 ); + auto_ptr gt( new GTMatchExpression() ); + ASSERT( gt->init( "", baseOperand[ "$gt" ] ).isOK() ); + ElemMatchValueMatchExpression op; + ASSERT( op.init( "a.b", gt.release() ).isOK() ); + ASSERT( op.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << BSON_ARRAY( 6 ) ) ) ), NULL ) ); + ASSERT( op.matchesBSON( BSON( "a" << + BSON_ARRAY( BSON( "b" << BSON_ARRAY( 4 ) ) << + BSON( "b" << BSON_ARRAY( 4 << 6 ) ) ) ), + NULL ) ); + } + + TEST( ElemMatchValueMatchExpression, ElemMatchKey ) { + BSONObj baseOperand = BSON( "$gt" << 6 ); + auto_ptr gt( new GTMatchExpression() ); + ASSERT( gt->init( "", baseOperand[ "$gt" ] ).isOK() ); + ElemMatchValueMatchExpression op; + ASSERT( op.init( "a.b", gt.release() ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !op.matchesBSON( BSONObj(), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( !op.matchesBSON( BSON( "a" << BSON( "b" << BSON_ARRAY( 2 ) ) ), + &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( op.matchesBSON( BSON( "a" << BSON( "b" << BSON_ARRAY( 3 << 7 ) ) ), + &details ) ); + ASSERT( details.hasElemMatchKey() ); + // The entry within the $elemMatch array is reported. + ASSERT_EQUALS( "1", details.elemMatchKey() ); + ASSERT( op.matchesBSON( BSON( "a" << + BSON_ARRAY( 1 << 2 << + BSON( "b" << BSON_ARRAY( 3 << 7 ) ) ) ), + &details ) ); + ASSERT( details.hasElemMatchKey() ); + // The entry within a parent of the $elemMatch array is reported. + ASSERT_EQUALS( "2", details.elemMatchKey() ); + } + + /** + TEST( ElemMatchValueMatchExpression, MatchesIndexKey ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + auto_ptr lt( new ComparisonMatchExpression() ); + ASSERT( lt->init( "a", baseOperand[ "$lt" ] ).isOK() ); + ElemMatchValueMatchExpression op; + ASSERT( op.init( "a", lt.release() ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + BSONObj indexKey = BSON( "" << "3" ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + op.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + + TEST( AndOfElemMatch, MatchesElement ) { + + + BSONObj baseOperanda1 = BSON( "a" << 1 ); + auto_ptr eqa1( new EqualityMatchExpression() ); + ASSERT( eqa1->init( "a", baseOperanda1[ "a" ] ).isOK() ); + + BSONObj baseOperandb1 = BSON( "b" << 1 ); + auto_ptr eqb1( new EqualityMatchExpression() ); + ASSERT( eqb1->init( "b", baseOperandb1[ "b" ] ).isOK() ); + + auto_ptr and1( new AndMatchExpression() ); + and1->add( eqa1.release() ); + and1->add( eqb1.release() ); + // and1 = { a : 1, b : 1 } + + auto_ptr elemMatch1( new ElemMatchObjectMatchExpression() ); + elemMatch1->init( "x", and1.release() ); + // elemMatch1 = { x : { $elemMatch : { a : 1, b : 1 } } } + + BSONObj baseOperanda2 = BSON( "a" << 2 ); + auto_ptr eqa2( new EqualityMatchExpression() ); + ASSERT( eqa2->init( "a", baseOperanda2[ "a" ] ).isOK() ); + + BSONObj baseOperandb2 = BSON( "b" << 2 ); + auto_ptr eqb2( new EqualityMatchExpression() ); + ASSERT( eqb2->init( "b", baseOperandb2[ "b" ] ).isOK() ); + + auto_ptr and2( new AndMatchExpression() ); + and2->add( eqa2.release() ); + and2->add( eqb2.release() ); + // and2 = { a : 2, b : 2 } + + auto_ptr elemMatch2( new ElemMatchObjectMatchExpression() ); + elemMatch2->init( "x", and2.release() ); + // elemMatch2 = { x : { $elemMatch : { a : 2, b : 2 } } } + + auto_ptr andOfEM( new AndMatchExpression() ); + andOfEM->add( elemMatch1.release() ); + andOfEM->add( elemMatch2.release() ); + + BSONObj nonArray = BSON( "x" << 4 ); + ASSERT( !andOfEM->matchesSingleElement( nonArray[ "x" ] ) ); + BSONObj emptyArray = BSON( "x" << BSONArray() ); + ASSERT( !andOfEM->matchesSingleElement( emptyArray[ "x" ] ) ); + BSONObj nonObjArray = BSON( "x" << BSON_ARRAY( 4 ) ); + ASSERT( !andOfEM->matchesSingleElement( nonObjArray[ "x" ] ) ); + BSONObj singleObjMatch = BSON( "x" << BSON_ARRAY( BSON( "a" << 1 << "b" << 1 ) ) ); + ASSERT( !andOfEM->matchesSingleElement( singleObjMatch[ "x" ] ) ); + BSONObj otherObjMatch = BSON( "x" << BSON_ARRAY( BSON( "a" << 2 << "b" << 2 ) ) ); + ASSERT( !andOfEM->matchesSingleElement( otherObjMatch[ "x" ] ) ); + BSONObj bothObjMatch = BSON( "x" << BSON_ARRAY( BSON( "a" << 1 << "b" << 1 ) << + BSON( "a" << 2 << "b" << 2 ) ) ); + ASSERT( andOfEM->matchesSingleElement( bothObjMatch[ "x" ] ) ); + BSONObj noObjMatch = BSON( "x" << BSON_ARRAY( BSON( "a" << 1 << "b" << 2 ) << + BSON( "a" << 2 << "b" << 1 ) ) ); + ASSERT( !andOfEM->matchesSingleElement( noObjMatch[ "x" ] ) ); + } + + TEST( AndOfElemMatch, Matches ) { + BSONObj baseOperandgt1 = BSON( "$gt" << 1 ); + auto_ptr gt1( new GTMatchExpression() ); + ASSERT( gt1->init( "", baseOperandgt1[ "$gt" ] ).isOK() ); + + BSONObj baseOperandlt1 = BSON( "$lt" << 10 ); + auto_ptr lt1( new LTMatchExpression() ); + ASSERT( lt1->init( "", baseOperandlt1[ "$lt" ] ).isOK() ); + + auto_ptr elemMatch1( new ElemMatchValueMatchExpression() ); + elemMatch1->init( "x" ); + elemMatch1->add( gt1.release() ); + elemMatch1->add( lt1.release() ); + // elemMatch1 = { x : { $elemMatch : { $gt : 1 , $lt : 10 } } } + + BSONObj baseOperandgt2 = BSON( "$gt" << 101 ); + auto_ptr gt2( new GTMatchExpression() ); + ASSERT( gt2->init( "", baseOperandgt2[ "$gt" ] ).isOK() ); + + BSONObj baseOperandlt2 = BSON( "$lt" << 110 ); + auto_ptr lt2( new LTMatchExpression() ); + ASSERT( lt2->init( "", baseOperandlt2[ "$lt" ] ).isOK() ); + + auto_ptr elemMatch2( new ElemMatchValueMatchExpression() ); + elemMatch2->init( "x" ); + elemMatch2->add( gt2.release() ); + elemMatch2->add( lt2.release() ); + // elemMatch2 = { x : { $elemMatch : { $gt : 101 , $lt : 110 } } } + + auto_ptr andOfEM( new AndMatchExpression() ); + andOfEM->add( elemMatch1.release() ); + andOfEM->add( elemMatch2.release() ); + + BSONObj nonArray = BSON( "x" << 4 ); + ASSERT( !andOfEM->matchesBSON( nonArray, NULL ) ); + BSONObj emptyArray = BSON( "x" << BSONArray() ); + ASSERT( !andOfEM->matchesBSON( emptyArray, NULL ) ); + BSONObj nonNumberArray = BSON( "x" << BSON_ARRAY( "q" ) ); + ASSERT( !andOfEM->matchesBSON( nonNumberArray, NULL ) ); + BSONObj singleMatch = BSON( "x" << BSON_ARRAY( 5 ) ); + ASSERT( !andOfEM->matchesBSON( singleMatch, NULL ) ); + BSONObj otherMatch = BSON( "x" << BSON_ARRAY( 105 ) ); + ASSERT( !andOfEM->matchesBSON( otherMatch, NULL ) ); + BSONObj bothMatch = BSON( "x" << BSON_ARRAY( 5 << 105 ) ); + ASSERT( andOfEM->matchesBSON( bothMatch, NULL ) ); + BSONObj neitherMatch = BSON( "x" << BSON_ARRAY( 0 << 200 ) ); + ASSERT( !andOfEM->matchesBSON( neitherMatch, NULL ) ); + } + + TEST( SizeMatchExpression, MatchesElement ) { + BSONObj match = BSON( "a" << BSON_ARRAY( 5 << 6 ) ); + BSONObj notMatch = BSON( "a" << BSON_ARRAY( 5 ) ); + SizeMatchExpression size; + ASSERT( size.init( "", 2 ).isOK() ); + ASSERT( size.matchesSingleElement( match.firstElement() ) ); + ASSERT( !size.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( SizeMatchExpression, MatchesNonArray ) { + // Non arrays do not match. + BSONObj stringValue = BSON( "a" << "z" ); + BSONObj numberValue = BSON( "a" << 0 ); + BSONObj arrayValue = BSON( "a" << BSONArray() ); + SizeMatchExpression size; + ASSERT( size.init( "", 0 ).isOK() ); + ASSERT( !size.matchesSingleElement( stringValue.firstElement() ) ); + ASSERT( !size.matchesSingleElement( numberValue.firstElement() ) ); + ASSERT( size.matchesSingleElement( arrayValue.firstElement() ) ); + } + + TEST( SizeMatchExpression, MatchesArray ) { + SizeMatchExpression size; + ASSERT( size.init( "a", 2 ).isOK() ); + ASSERT( size.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5.5 ) ), NULL ) ); + // Arrays are not unwound to look for matching subarrays. + ASSERT( !size.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5.5 << BSON_ARRAY( 1 << 2 ) ) ), + NULL ) ); + } + + TEST( SizeMatchExpression, MatchesNestedArray ) { + SizeMatchExpression size; + ASSERT( size.init( "a.2", 2 ).isOK() ); + // A numerically referenced nested array is matched. + ASSERT( size.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5.5 << BSON_ARRAY( 1 << 2 ) ) ), + NULL ) ); + } + + TEST( SizeMatchExpression, ElemMatchKey ) { + SizeMatchExpression size; + ASSERT( size.init( "a.b", 3 ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !size.matchesBSON( BSON( "a" << 1 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( size.matchesBSON( BSON( "a" << BSON( "b" << BSON_ARRAY( 1 << 2 << 3 ) ) ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( size.matchesBSON( BSON( "a" << + BSON_ARRAY( 2 << + BSON( "b" << BSON_ARRAY( 1 << 2 << 3 ) ) ) ), + &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + TEST( SizeMatchExpression, Equivalent ) { + SizeMatchExpression e1; + SizeMatchExpression e2; + SizeMatchExpression e3; + + e1.init( "a", 5 ); + e2.init( "a", 6 ); + e3.init( "v", 5 ); + + ASSERT( e1.equivalent( &e1 ) ); + ASSERT( !e1.equivalent( &e2 ) ); + ASSERT( !e1.equivalent( &e3 ) ); + } + + /** + TEST( SizeMatchExpression, MatchesIndexKey ) { + BSONObj operand = BSON( "$size" << 4 ); + SizeMatchExpression size; + ASSERT( size.init( "a", operand[ "$size" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + BSONObj indexKey = BSON( "" << 1 ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + size.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + +} // namespace mongo diff --git a/src/mongo/db/matcher/expression_geo.cpp b/src/mongo/db/matcher/expression_geo.cpp new file mode 100644 index 00000000000..384a1ad5696 --- /dev/null +++ b/src/mongo/db/matcher/expression_geo.cpp @@ -0,0 +1,144 @@ +// expression_geo.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" +#include "mongo/db/matcher/expression_geo.h" + +namespace mongo { + + // + // Geo queries we don't need an index to answer: geoWithin and geoIntersects + // + + Status GeoMatchExpression::init( const StringData& path, const GeoQuery& query, + const BSONObj& rawObj ) { + _query = query; + _rawObj = rawObj; + return initPath( path ); + } + + bool GeoMatchExpression::matchesSingleElement( const BSONElement& e ) const { + if ( !e.isABSONObj()) + return false; + + GeometryContainer container; + if ( !container.parseFrom( e.Obj() ) ) + return false; + + return _query.satisfiesPredicate( container ); + } + + void GeoMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "GEO raw = " << _rawObj.toString(); + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + } + + bool GeoMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const GeoMatchExpression* realOther = static_cast( other ); + + if ( path() != realOther->path() ) + return false; + + // TODO: + // return _query == realOther->_query; + return false; + } + + LeafMatchExpression* GeoMatchExpression::shallowClone() const { + GeoMatchExpression* next = new GeoMatchExpression(); + next->init( path(), _query, _rawObj); + if (getTag()) { + next->setTag(getTag()->clone()); + } + return next; + } + + // + // Parse-only geo expressions: geoNear (formerly known as near). + // + + Status GeoNearMatchExpression::init( const StringData& path, const NearQuery& query, + const BSONObj& rawObj ) { + _query = query; + _rawObj = rawObj; + return initPath( path ); + } + + bool GeoNearMatchExpression::matchesSingleElement( const BSONElement& e ) const { + // See ops/update.cpp. + // This node is removed by the query planner. It's only ever called if we're getting an + // elemMatchKey. + return true; + } + + void GeoNearMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "GEONEAR " << _query.toString(); + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + } + + bool GeoNearMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const GeoNearMatchExpression* realOther = static_cast(other); + + if ( path() != realOther->path() ) + return false; + + // TODO: + // return _query == realOther->_query; + return false; + } + + LeafMatchExpression* GeoNearMatchExpression::shallowClone() const { + GeoNearMatchExpression* next = new GeoNearMatchExpression(); + next->init( path(), _query, _rawObj ); + if (getTag()) { + next->setTag(getTag()->clone()); + } + return next; + } + +} diff --git a/src/mongo/db/matcher/expression_geo.h b/src/mongo/db/matcher/expression_geo.h new file mode 100644 index 00000000000..d67edc91919 --- /dev/null +++ b/src/mongo/db/matcher/expression_geo.h @@ -0,0 +1,86 @@ +// expression_geo.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#pragma once + +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + class GeoMatchExpression : public LeafMatchExpression { + public: + GeoMatchExpression() : LeafMatchExpression( GEO ){} + virtual ~GeoMatchExpression(){} + + Status init( const StringData& path, const GeoQuery& query, const BSONObj& rawObj ); + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + virtual LeafMatchExpression* shallowClone() const; + + const GeoQuery& getGeoQuery() const { return _query; } + const BSONObj getRawObj() const { return _rawObj; } + + private: + BSONObj _rawObj; + GeoQuery _query; + }; + + class GeoNearMatchExpression : public LeafMatchExpression { + public: + GeoNearMatchExpression() : LeafMatchExpression( GEO_NEAR ){} + virtual ~GeoNearMatchExpression(){} + + Status init( const StringData& path, const NearQuery& query, const BSONObj& rawObj ); + + // This shouldn't be called and as such will crash. GeoNear always requires an index. + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + virtual LeafMatchExpression* shallowClone() const; + + const NearQuery& getData() const { return _query; } + const BSONObj getRawObj() const { return _rawObj; } + private: + NearQuery _query; + BSONObj _rawObj; + }; + +} // namespace mongo diff --git a/src/mongo/db/matcher/expression_geo_test.cpp b/src/mongo/db/matcher/expression_geo_test.cpp new file mode 100644 index 00000000000..f49b966c25b --- /dev/null +++ b/src/mongo/db/matcher/expression_geo_test.cpp @@ -0,0 +1,75 @@ +// expression_geo_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** Unit tests for MatchExpression operator implementations in match_operators.{h,cpp}. */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_geo.h" + +namespace mongo { + + TEST( ExpressionGeoTest, Geo1 ) { + BSONObj query = fromjson("{loc:{$within:{$box:[{x: 4, y:4},[6,6]]}}}"); + + GeoQuery gq; + ASSERT( gq.parseFrom( query["loc"].Obj() ) ); + + GeoMatchExpression ge; + ASSERT( ge.init("a", gq, query ).isOK() ); + + ASSERT(!ge.matchesBSON(fromjson("{a: [3,4]}"))); + ASSERT(ge.matchesBSON(fromjson("{a: [4,4]}"))); + ASSERT(ge.matchesBSON(fromjson("{a: [5,5]}"))); + ASSERT(ge.matchesBSON(fromjson("{a: [5,5.1]}"))); + ASSERT(ge.matchesBSON(fromjson("{a: {x: 5, y:5.1}}"))); + + } + + TEST(ExpressionGeoTest, GeoNear1) { + BSONObj query = fromjson("{loc:{$near:{$maxDistance:100, " + "$geometry:{type:\"Point\", coordinates:[0,0]}}}}"); + NearQuery nq; + ASSERT_OK(nq.parseFrom(query["loc"].Obj())); + + GeoNearMatchExpression gne; + ASSERT(gne.init("a", nq, query).isOK()); + + // We can't match the data but we can make sure it was parsed OK. + ASSERT_EQUALS(gne.getData().centroid.crs, SPHERE); + ASSERT_EQUALS(gne.getData().minDistance, 0); + ASSERT_EQUALS(gne.getData().maxDistance, 100); + } + +} diff --git a/src/mongo/db/matcher/expression_leaf.cpp b/src/mongo/db/matcher/expression_leaf.cpp new file mode 100644 index 00000000000..68002636fbc --- /dev/null +++ b/src/mongo/db/matcher/expression_leaf.cpp @@ -0,0 +1,529 @@ +// expression_leaf.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression_leaf.h" + +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/path.h" +#include "mongo/util/log.h" + +namespace mongo { + + Status LeafMatchExpression::initPath( const StringData& path ) { + _path = path; + return _elementPath.init( _path ); + } + + + bool LeafMatchExpression::matches( const MatchableDocument* doc, MatchDetails* details ) const { + MatchableDocument::IteratorHolder cursor( doc, &_elementPath ); + while ( cursor->more() ) { + ElementIterator::Context e = cursor->next(); + if ( !matchesSingleElement( e.element() ) ) + continue; + if ( details && details->needRecord() && !e.arrayOffset().eoo() ) { + details->setElemMatchKey( e.arrayOffset().fieldName() ); + } + return true; + } + return false; + } + + // ------------- + + bool ComparisonMatchExpression::equivalent( const MatchExpression* other ) const { + if ( other->matchType() != matchType() ) + return false; + const ComparisonMatchExpression* realOther = + static_cast( other ); + + return + path() == realOther->path() && + _rhs.valuesEqual( realOther->_rhs ); + } + + + Status ComparisonMatchExpression::init( const StringData& path, const BSONElement& rhs ) { + _rhs = rhs; + + if ( rhs.eoo() ) { + return Status( ErrorCodes::BadValue, "need a real operand" ); + } + + if ( rhs.type() == Undefined ) { + return Status( ErrorCodes::BadValue, "cannot compare to undefined" ); + } + + switch ( matchType() ) { + case LT: + case LTE: + case EQ: + case GT: + case GTE: + break; + default: + return Status( ErrorCodes::BadValue, "bad match type for ComparisonMatchExpression" ); + } + + return initPath( path ); + } + + + bool ComparisonMatchExpression::matchesSingleElement( const BSONElement& e ) const { + //log() << "\t ComparisonMatchExpression e: " << e << " _rhs: " << _rhs << "\n" + //<< toString() << std::endl; + + if ( e.canonicalType() != _rhs.canonicalType() ) { + // some special cases + // jstNULL and undefined are treated the same + if ( e.canonicalType() + _rhs.canonicalType() == 5 ) { + return matchType() == EQ || matchType() == LTE || matchType() == GTE; + } + + if ( _rhs.type() == MaxKey || _rhs.type() == MinKey ) { + return matchType() != EQ; + } + + return false; + } + + int x = compareElementValues( e, _rhs ); + + //log() << "\t\t" << x << endl; + + switch ( matchType() ) { + case LT: + return x < 0; + case LTE: + return x <= 0; + case EQ: + return x == 0; + case GT: + return x > 0; + case GTE: + return x >= 0; + default: + fassertFailed( 16828 ); + } + } + + void ComparisonMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " "; + switch ( matchType() ) { + case LT: debug << "$lt"; break; + case LTE: debug << "$lte"; break; + case EQ: debug << "=="; break; + case GT: debug << "$gt"; break; + case GTE: debug << "$gte"; break; + default: debug << " UNKNOWN - should be impossible"; break; + } + debug << " " << _rhs.toString( false ); + + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + + debug << "\n"; + } + + // --------------- + + // TODO: move + inline pcrecpp::RE_Options flags2options(const char* flags) { + pcrecpp::RE_Options options; + options.set_utf8(true); + while ( flags && *flags ) { + if ( *flags == 'i' ) + options.set_caseless(true); + else if ( *flags == 'm' ) + options.set_multiline(true); + else if ( *flags == 'x' ) + options.set_extended(true); + else if ( *flags == 's' ) + options.set_dotall(true); + flags++; + } + return options; + } + + bool RegexMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const RegexMatchExpression* realOther = static_cast( other ); + return + path() == realOther->path() && + _regex == realOther->_regex + && _flags == realOther->_flags; + } + + + Status RegexMatchExpression::init( const StringData& path, const BSONElement& e ) { + if ( e.type() != RegEx ) + return Status( ErrorCodes::BadValue, "regex not a regex" ); + return init( path, e.regex(), e.regexFlags() ); + } + + + Status RegexMatchExpression::init( const StringData& path, const StringData& regex, const StringData& options ) { + if ( regex.size() > MaxPatternSize ) { + return Status( ErrorCodes::BadValue, "Regular expression is too long" ); + } + + _regex = regex.toString(); + _flags = options.toString(); + _re.reset( new pcrecpp::RE( _regex.c_str(), flags2options( _flags.c_str() ) ) ); + + return initPath( path ); + } + + bool RegexMatchExpression::matchesSingleElement( const BSONElement& e ) const { + //log() << "RegexMatchExpression::matchesSingleElement _regex: " << _regex << " e: " << e << std::endl; + switch (e.type()) { + case String: + case Symbol: + // TODO + //if (rm._prefix.empty()) + return _re->PartialMatch(e.valuestr()); + //else + //return !strncmp(e.valuestr(), rm._prefix.c_str(), rm._prefix.size()); + case RegEx: + return _regex == e.regex() && _flags == e.regexFlags(); + default: + return false; + } + } + + void RegexMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " regex /" << _regex << "/" << _flags; + + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + } + + void RegexMatchExpression::shortDebugString( StringBuilder& debug ) const { + debug << "/" << _regex << "/" << _flags; + } + + // --------- + + Status ModMatchExpression::init( const StringData& path, int divisor, int remainder ) { + if ( divisor == 0 ) + return Status( ErrorCodes::BadValue, "divisor cannot be 0" ); + _divisor = divisor; + _remainder = remainder; + return initPath( path ); + } + + bool ModMatchExpression::matchesSingleElement( const BSONElement& e ) const { + if ( !e.isNumber() ) + return false; + return e.numberLong() % _divisor == _remainder; + } + + void ModMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " mod " << _divisor << " % x == " << _remainder; + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + } + + bool ModMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const ModMatchExpression* realOther = static_cast( other ); + return + path() == realOther->path() && + _divisor == realOther->_divisor && + _remainder == realOther->_remainder; + } + + + // ------------------ + + Status ExistsMatchExpression::init( const StringData& path ) { + return initPath( path ); + } + + bool ExistsMatchExpression::matchesSingleElement( const BSONElement& e ) const { + return !e.eoo(); + } + + void ExistsMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " exists"; + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + } + + bool ExistsMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const ExistsMatchExpression* realOther = static_cast( other ); + return path() == realOther->path(); + } + + + // ---- + + Status TypeMatchExpression::init( const StringData& path, int type ) { + _path = path; + _type = type; + return _elementPath.init( _path ); + } + + bool TypeMatchExpression::matchesSingleElement( const BSONElement& e ) const { + return e.type() == _type; + } + + bool TypeMatchExpression::matches( const MatchableDocument* doc, MatchDetails* details ) const { + MatchableDocument::IteratorHolder cursor( doc, &_elementPath ); + while ( cursor->more() ) { + ElementIterator::Context e = cursor->next(); + + // In the case where _elementPath is referring to an array, + // $type should match elements of that array only. + // outerArray() helps to identify elements of the array + // and the containing array itself. + // This matters when we are looking for {$type: Array}. + // Example (_elementPath refers to field 'a' and _type is Array): + // a : [ // outer array. should not match + // 123, // inner array + // [ 456 ], // inner array. should match + // ... + // ] + if ( _type == mongo::Array && e.outerArray() ) { + continue; + } + + if ( !matchesSingleElement( e.element() ) ) { + continue; + } + + if ( details && details->needRecord() && !e.arrayOffset().eoo() ) { + details->setElemMatchKey( e.arrayOffset().fieldName() ); + } + return true; + } + return false; + } + + void TypeMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << _path << " type: " << _type; + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + } + + + bool TypeMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const TypeMatchExpression* realOther = static_cast( other ); + return _path == realOther->_path && _type == realOther->_type; + } + + + // -------- + + ArrayFilterEntries::ArrayFilterEntries(){ + _hasNull = false; + _hasEmptyArray = false; + } + + ArrayFilterEntries::~ArrayFilterEntries() { + for ( unsigned i = 0; i < _regexes.size(); i++ ) + delete _regexes[i]; + _regexes.clear(); + } + + Status ArrayFilterEntries::addEquality( const BSONElement& e ) { + if ( e.type() == RegEx ) + return Status( ErrorCodes::BadValue, "ArrayFilterEntries equality cannot be a regex" ); + + if ( e.type() == Undefined ) { + return Status( ErrorCodes::BadValue, + "ArrayFilterEntries equality cannot be undefined" ); + } + + if ( e.type() == jstNULL ) { + _hasNull = true; + } + + if ( e.type() == Array && e.Obj().isEmpty() ) + _hasEmptyArray = true; + + _equalities.insert( e ); + return Status::OK(); + } + + Status ArrayFilterEntries::addRegex( RegexMatchExpression* expr ) { + _regexes.push_back( expr ); + return Status::OK(); + } + + bool ArrayFilterEntries::equivalent( const ArrayFilterEntries& other ) const { + if ( _hasNull != other._hasNull ) + return false; + + if ( _regexes.size() != other._regexes.size() ) + return false; + for ( unsigned i = 0; i < _regexes.size(); i++ ) + if ( !_regexes[i]->equivalent( other._regexes[i] ) ) + return false; + + return _equalities == other._equalities; + } + + void ArrayFilterEntries::copyTo( ArrayFilterEntries& toFillIn ) const { + toFillIn._hasNull = _hasNull; + toFillIn._hasEmptyArray = _hasEmptyArray; + toFillIn._equalities = _equalities; + for ( unsigned i = 0; i < _regexes.size(); i++ ) + toFillIn._regexes.push_back( static_cast(_regexes[i]->shallowClone()) ); + } + + void ArrayFilterEntries::debugString( StringBuilder& debug ) const { + debug << "[ "; + for (BSONElementSet::const_iterator it = _equalities.begin(); + it != _equalities.end(); ++it) { + debug << it->toString( false ) << " "; + } + for (size_t i = 0; i < _regexes.size(); ++i) { + _regexes[i]->shortDebugString( debug ); + debug << " "; + } + debug << "]"; + } + + // ----------- + + Status InMatchExpression::init( const StringData& path ) { + return initPath( path ); + } + + bool InMatchExpression::_matchesRealElement( const BSONElement& e ) const { + if ( _arrayEntries.contains( e ) ) + return true; + + for ( unsigned i = 0; i < _arrayEntries.numRegexes(); i++ ) { + if ( _arrayEntries.regex(i)->matchesSingleElement( e ) ) + return true; + } + + return false; + } + + bool InMatchExpression::matchesSingleElement( const BSONElement& e ) const { + if ( _arrayEntries.hasNull() && e.eoo() ) + return true; + + if ( _matchesRealElement( e ) ) + return true; + + /* + if ( e.type() == Array ) { + BSONObjIterator i( e.Obj() ); + while ( i.more() ) { + BSONElement sub = i.next(); + if ( _matchesRealElement( sub ) ) + return true; + } + } + */ + + return false; + } + + void InMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << path() << " $in "; + _arrayEntries.debugString(debug); + MatchExpression::TagData* td = getTag(); + if (NULL != td) { + debug << " "; + td->debugString(&debug); + } + debug << "\n"; + } + + bool InMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + const InMatchExpression* realOther = static_cast( other ); + return + path() == realOther->path() && + _arrayEntries.equivalent( realOther->_arrayEntries ); + } + + LeafMatchExpression* InMatchExpression::shallowClone() const { + InMatchExpression* next = new InMatchExpression(); + copyTo( next ); + if ( getTag() ) { + next->setTag(getTag()->clone()); + } + return next; + } + + void InMatchExpression::copyTo( InMatchExpression* toFillIn ) const { + toFillIn->init( path() ); + _arrayEntries.copyTo( toFillIn->_arrayEntries ); + } + +} + + diff --git a/src/mongo/db/matcher/expression_leaf.h b/src/mongo/db/matcher/expression_leaf.h new file mode 100644 index 00000000000..b7d88d06d14 --- /dev/null +++ b/src/mongo/db/matcher/expression_leaf.h @@ -0,0 +1,385 @@ +// expression_leaf.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/db/matcher/expression.h" + +namespace mongo { + + /** + * This file contains leaves in the parse tree that are not array-based. + * + * LeafMatchExpression: REGEX MOD EXISTS MATCH_IN + * ComparisonMatchExpression: EQ LTE LT GT GTE + * MatchExpression: TYPE_OPERATOR + */ + + /** + * Many operators subclass from this: + * REGEX, MOD, EXISTS, IN + * Everything that inherits from ComparisonMatchExpression. + */ + class LeafMatchExpression : public MatchExpression { + public: + LeafMatchExpression( MatchType matchType ) + : MatchExpression( matchType ) { + } + + virtual ~LeafMatchExpression(){} + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const; + + virtual bool matchesSingleElement( const BSONElement& e ) const = 0; + + virtual const StringData path() const { return _path; } + + protected: + Status initPath( const StringData& path ); + + private: + StringData _path; + ElementPath _elementPath; + }; + + /** + * EQ, LTE, LT, GT, GTE subclass from ComparisonMatchExpression. + */ + class ComparisonMatchExpression : public LeafMatchExpression { + public: + ComparisonMatchExpression( MatchType type ) : LeafMatchExpression( type ){} + + Status init( const StringData& path, const BSONElement& rhs ); + + virtual ~ComparisonMatchExpression(){} + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual const BSONElement& getRHS() const { return _rhs; } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + const BSONElement& getData() const { return _rhs; } + + protected: + BSONElement _rhs; + }; + + // + // ComparisonMatchExpression inheritors + // + + class EqualityMatchExpression : public ComparisonMatchExpression { + public: + EqualityMatchExpression() : ComparisonMatchExpression( EQ ){} + virtual LeafMatchExpression* shallowClone() const { + ComparisonMatchExpression* e = new EqualityMatchExpression(); + e->init( path(), _rhs ); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + }; + + class LTEMatchExpression : public ComparisonMatchExpression { + public: + LTEMatchExpression() : ComparisonMatchExpression( LTE ){} + virtual LeafMatchExpression* shallowClone() const { + ComparisonMatchExpression* e = new LTEMatchExpression(); + e->init( path(), _rhs ); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + }; + + class LTMatchExpression : public ComparisonMatchExpression { + public: + LTMatchExpression() : ComparisonMatchExpression( LT ){} + virtual LeafMatchExpression* shallowClone() const { + ComparisonMatchExpression* e = new LTMatchExpression(); + e->init( path(), _rhs ); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + }; + + class GTMatchExpression : public ComparisonMatchExpression { + public: + GTMatchExpression() : ComparisonMatchExpression( GT ){} + virtual LeafMatchExpression* shallowClone() const { + ComparisonMatchExpression* e = new GTMatchExpression(); + e->init( path(), _rhs ); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + }; + + class GTEMatchExpression : public ComparisonMatchExpression { + public: + GTEMatchExpression() : ComparisonMatchExpression( GTE ){} + virtual LeafMatchExpression* shallowClone() const { + ComparisonMatchExpression* e = new GTEMatchExpression(); + e->init( path(), _rhs ); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + }; + + // + // LeafMatchExpression inheritors + // + + class RegexMatchExpression : public LeafMatchExpression { + public: + /** + * Maximum pattern size which pcre v8.3 can do matches correctly with + * LINK_SIZE define macro set to 2 @ pcre's config.h (based on + * experiments) + */ + static const size_t MaxPatternSize = 32764; + + RegexMatchExpression() : LeafMatchExpression( REGEX ){} + + Status init( const StringData& path, const StringData& regex, const StringData& options ); + Status init( const StringData& path, const BSONElement& e ); + + virtual LeafMatchExpression* shallowClone() const { + RegexMatchExpression* e = new RegexMatchExpression(); + e->init( path(), _regex, _flags ); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual void debugString( StringBuilder& debug, int level ) const; + + void shortDebugString( StringBuilder& debug ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + const string& getString() const { return _regex; } + const string& getFlags() const { return _flags; } + + private: + std::string _regex; + std::string _flags; + boost::scoped_ptr _re; + }; + + class ModMatchExpression : public LeafMatchExpression { + public: + ModMatchExpression() : LeafMatchExpression( MOD ){} + + Status init( const StringData& path, int divisor, int remainder ); + + virtual LeafMatchExpression* shallowClone() const { + ModMatchExpression* m = new ModMatchExpression(); + m->init( path(), _divisor, _remainder ); + if ( getTag() ) { + m->setTag(getTag()->clone()); + } + return m; + } + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual void debugString( StringBuilder& debug, int level ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + int getDivisor() const { return _divisor; } + int getRemainder() const { return _remainder; } + + private: + int _divisor; + int _remainder; + }; + + class ExistsMatchExpression : public LeafMatchExpression { + public: + ExistsMatchExpression() : LeafMatchExpression( EXISTS ){} + + Status init( const StringData& path ); + + virtual LeafMatchExpression* shallowClone() const { + ExistsMatchExpression* e = new ExistsMatchExpression(); + e->init( path() ); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual void debugString( StringBuilder& debug, int level ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + }; + + /** + * INTERNAL + * terrible name + * holds the entries of an $in or $all + * either scalars or regex + */ + class ArrayFilterEntries { + MONGO_DISALLOW_COPYING( ArrayFilterEntries ); + public: + ArrayFilterEntries(); + ~ArrayFilterEntries(); + + Status addEquality( const BSONElement& e ); + Status addRegex( RegexMatchExpression* expr ); + + const BSONElementSet& equalities() const { return _equalities; } + bool contains( const BSONElement& elem ) const { return _equalities.count(elem) > 0; } + + size_t numRegexes() const { return _regexes.size(); } + RegexMatchExpression* regex( int idx ) const { return _regexes[idx]; } + + bool hasNull() const { return _hasNull; } + bool singleNull() const { return size() == 1 && _hasNull; } + bool hasEmptyArray() const { return _hasEmptyArray; } + int size() const { return _equalities.size() + _regexes.size(); } + + bool equivalent( const ArrayFilterEntries& other ) const; + + void copyTo( ArrayFilterEntries& toFillIn ) const; + + void debugString( StringBuilder& debug ) const; + + private: + bool _hasNull; // if _equalities has a jstNULL element in it + bool _hasEmptyArray; + BSONElementSet _equalities; + std::vector _regexes; + }; + + /** + * query operator: $in + */ + class InMatchExpression : public LeafMatchExpression { + public: + InMatchExpression() : LeafMatchExpression( MATCH_IN ){} + Status init( const StringData& path ); + + virtual LeafMatchExpression* shallowClone() const; + + ArrayFilterEntries* getArrayFilterEntries() { return &_arrayEntries; } + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual void debugString( StringBuilder& debug, int level ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + void copyTo( InMatchExpression* toFillIn ) const; + + const ArrayFilterEntries& getData() const { return _arrayEntries; } + + private: + bool _matchesRealElement( const BSONElement& e ) const; + ArrayFilterEntries _arrayEntries; + }; + + // + // The odd duck out, TYPE_OPERATOR. + // + + /** + * Type has some odd semantics with arrays and as such it can't inherit from + * LeafMatchExpression. + */ + class TypeMatchExpression : public MatchExpression { + public: + TypeMatchExpression() : MatchExpression( TYPE_OPERATOR ){} + + Status init( const StringData& path, int type ); + + virtual MatchExpression* shallowClone() const { + TypeMatchExpression* e = new TypeMatchExpression(); + e->init(_path, _type); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const; + + virtual void debugString( StringBuilder& debug, int level ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + /** + * What is the type we're matching against? + */ + int getData() const { return _type; } + + virtual const StringData path() const { return _path; } + + private: + bool _matches( const StringData& path, + const MatchableDocument* doc, + MatchDetails* details = 0 ) const; + + StringData _path; + ElementPath _elementPath; + int _type; + }; + +} // namespace mongo diff --git a/src/mongo/db/matcher/expression_leaf_test.cpp b/src/mongo/db/matcher/expression_leaf_test.cpp new file mode 100644 index 00000000000..b28f3f13713 --- /dev/null +++ b/src/mongo/db/matcher/expression_leaf_test.cpp @@ -0,0 +1,1653 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** Unit tests for MatchMatchExpression operator implementations in match_operators.{h,cpp}. */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + TEST( EqOp, MatchesElement ) { + BSONObj operand = BSON( "a" << 5 ); + BSONObj match = BSON( "a" << 5.0 ); + BSONObj notMatch = BSON( "a" << 6 ); + + EqualityMatchExpression eq; + eq.init( "", operand["a"] ); + ASSERT( eq.matchesSingleElement( match.firstElement() ) ); + ASSERT( !eq.matchesSingleElement( notMatch.firstElement() ) ); + + ASSERT( eq.equivalent( &eq ) ); + } + + TEST( EqOp, InvalidEooOperand ) { + BSONObj operand; + EqualityMatchExpression eq; + ASSERT( !eq.init( "", operand.firstElement() ).isOK() ); + } + + TEST( EqOp, MatchesScalar ) { + BSONObj operand = BSON( "a" << 5 ); + EqualityMatchExpression eq; + eq.init( "a", operand[ "a" ] ); + ASSERT( eq.matchesBSON( BSON( "a" << 5.0 ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( EqOp, MatchesArrayValue ) { + BSONObj operand = BSON( "a" << 5 ); + EqualityMatchExpression eq; + eq.init( "a", operand[ "a" ] ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON_ARRAY( 5.0 << 6 ) ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 7 ) ), NULL ) ); + } + + TEST( EqOp, MatchesReferencedObjectValue ) { + BSONObj operand = BSON( "a.b" << 5 ); + EqualityMatchExpression eq; + eq.init( "a.b", operand[ "a.b" ] ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON( "b" << 5 ) ), NULL ) ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON( "b" << BSON_ARRAY( 5 ) ) ), NULL ) ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 5 ) ) ), NULL ) ); + } + + TEST( EqOp, MatchesReferencedArrayValue ) { + BSONObj operand = BSON( "a.0" << 5 ); + EqualityMatchExpression eq; + eq.init( "a.0", operand[ "a.0" ] ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON_ARRAY( 5 ) ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 5 ) ) ), NULL ) ); + } + + TEST( EqOp, MatchesNull ) { + BSONObj operand = BSON( "a" << BSONNULL ); + EqualityMatchExpression eq; + eq.init( "a", operand[ "a" ] ); + ASSERT( eq.matchesBSON( BSONObj(), NULL ) ); + ASSERT( eq.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << 4 ), NULL ) ); + // A non-existent field is treated same way as an empty bson object + ASSERT( eq.matchesBSON( BSON( "b" << 4 ), NULL ) ); + } + + // This test documents how the matcher currently works, + // not necessarily how it should work ideally. + TEST( EqOp, MatchesNestedNull ) { + BSONObj operand = BSON( "a.b" << BSONNULL ); + EqualityMatchExpression eq; + eq.init( "a.b", operand[ "a.b" ] ); + // null matches any empty object that is on a subpath of a.b + ASSERT( eq.matchesBSON( BSONObj(), NULL ) ); + ASSERT( eq.matchesBSON( BSON( "a" << BSONObj() ), NULL ) ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON_ARRAY( BSONObj() ) ), NULL ) ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON( "b" << BSONNULL ) ), NULL ) ); + // b does not exist as an element in array under a. + ASSERT( !eq.matchesBSON( BSON( "a" << BSONArray() ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << BSON_ARRAY( BSONNULL ) ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 ) ), NULL ) ); + // a.b exists but is not null. + ASSERT( !eq.matchesBSON( BSON( "a" << BSON( "b" << 4 ) ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << BSON( "b" << BSONObj() ) ), NULL ) ); + // A non-existent field is treated same way as an empty bson object + ASSERT( eq.matchesBSON( BSON( "b" << 4 ), NULL ) ); + } + + TEST( EqOp, MatchesMinKey ) { + BSONObj operand = BSON( "a" << MinKey ); + EqualityMatchExpression eq; + eq.init( "a", operand[ "a" ] ); + ASSERT( eq.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + + + TEST( EqOp, MatchesMaxKey ) { + BSONObj operand = BSON( "a" << MaxKey ); + EqualityMatchExpression eq; + ASSERT( eq.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( eq.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( EqOp, MatchesFullArray ) { + BSONObj operand = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + EqualityMatchExpression eq; + ASSERT( eq.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 ) ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 << 3 ) ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) ), NULL ) ); + ASSERT( !eq.matchesBSON( BSON( "a" << 1 ), NULL ) ); + } + + TEST( EqOp, MatchesThroughNestedArray ) { + BSONObj operand = BSON( "a.b.c.d" << 3 ); + EqualityMatchExpression eq; + eq.init( "a.b.c.d", operand["a.b.c.d"] ); + BSONObj obj = fromjson("{a:{b:[{c:[{d:1},{d:2}]},{c:[{d:3}]}]}}"); + ASSERT( eq.matchesBSON( obj, NULL ) ); + } + + TEST( EqOp, ElemMatchKey ) { + BSONObj operand = BSON( "a" << 5 ); + EqualityMatchExpression eq; + ASSERT( eq.init( "a", operand[ "a" ] ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !eq.matchesBSON( BSON( "a" << 4 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( eq.matchesBSON( BSON( "a" << 5 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( eq.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 << 5 ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "2", details.elemMatchKey() ); + } + + // SERVER-14886: when an array is being traversed explictly at the same time that a nested array + // is being traversed implicitly, the elemMatch key should refer to the offset of the array + // being implicitly traversed. + TEST( EqOp, ElemMatchKeyWithImplicitAndExplicitTraversal ) { + BSONObj operand = BSON( "a.0.b" << 3 ); + BSONElement operandFirstElt = operand.firstElement(); + EqualityMatchExpression eq; + ASSERT( eq.init( operandFirstElt.fieldName(), operandFirstElt ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + BSONObj obj = fromjson("{a: [{b: [2, 3]}, {b: [4, 5]}]}"); + ASSERT( eq.matchesBSON( obj, &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + TEST( EqOp, Equality1 ) { + EqualityMatchExpression eq1; + EqualityMatchExpression eq2; + EqualityMatchExpression eq3; + + BSONObj operand = BSON( "a" << 5 << "b" << 5 << "c" << 4 ); + + eq1.init( "a", operand["a"] ); + eq2.init( "a", operand["b"] ); + eq3.init( "c", operand["c"] ); + + ASSERT( eq1.equivalent( &eq1 ) ); + ASSERT( eq1.equivalent( &eq2 ) ); + ASSERT( !eq1.equivalent( &eq3 ) ); + } + + /** + TEST( EqOp, MatchesIndexKeyScalar ) { + BSONObj operand = BSON( "a" << 6 ); + EqualityMatchExpression eq; + ASSERT( eq.init( "a", operand[ "a" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + eq.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + eq.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + eq.matchesIndexKey( BSON( "" << BSON_ARRAY( 6 ) ), indexSpec ) ); + } + + TEST( EqOp, MatchesIndexKeyMissing ) { + BSONObj operand = BSON( "a" << 6 ); + EqualityMatchExpression eq; + ASSERT( eq.init( "a", operand[ "a" ] ).isOK() ); + IndexSpec indexSpec( BSON( "b" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + eq.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + eq.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + eq.matchesIndexKey( BSON( "" << BSON_ARRAY( 8 << 6 ) ), indexSpec ) ); + } + + TEST( EqOp, MatchesIndexKeyArray ) { + BSONObj operand = BSON( "a" << BSON_ARRAY( 4 << 5 ) ); + ComparisonMatchExpression eq + ASSERT( eq.init( "a", operand[ "a" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + eq.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + } + + TEST( EqOp, MatchesIndexKeyArrayValue ) { + BSONObj operand = BSON( "a" << 6 ); + ComparisonMatchExpression eq + ASSERT( eq.init( "a", operand[ "a" ] ).isOK() ); + IndexSpec indexSpec( BSON( "loc" << "mockarrayvalue" << "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + eq.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + eq.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + eq.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << 6 ) ), indexSpec ) ); + } + */ + TEST( LtOp, MatchesElement ) { + BSONObj operand = BSON( "$lt" << 5 ); + BSONObj match = BSON( "a" << 4.5 ); + BSONObj notMatch = BSON( "a" << 6 ); + BSONObj notMatchEqual = BSON( "a" << 5 ); + BSONObj notMatchWrongType = BSON( "a" << "foo" ); + LTMatchExpression lt; + ASSERT( lt.init( "", operand[ "$lt" ] ).isOK() ); + ASSERT( lt.matchesSingleElement( match.firstElement() ) ); + ASSERT( !lt.matchesSingleElement( notMatch.firstElement() ) ); + ASSERT( !lt.matchesSingleElement( notMatchEqual.firstElement() ) ); + ASSERT( !lt.matchesSingleElement( notMatchWrongType.firstElement() ) ); + } + + TEST( LtOp, InvalidEooOperand ) { + BSONObj operand; + LTMatchExpression lt; + ASSERT( !lt.init( "", operand.firstElement() ).isOK() ); + } + + TEST( LtOp, MatchesScalar ) { + BSONObj operand = BSON( "$lt" << 5 ); + LTMatchExpression lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + ASSERT( lt.matchesBSON( BSON( "a" << 4.5 ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << 6 ), NULL ) ); + } + + TEST( LtOp, MatchesScalarEmptyKey ) { + BSONObj operand = BSON( "$lt" << 5 ); + LTMatchExpression lt; + ASSERT( lt.init( "", operand[ "$lt" ] ).isOK() ); + ASSERT( lt.matchesBSON( BSON( "" << 4.5 ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "" << 6 ), NULL ) ); + } + + TEST( LtOp, MatchesArrayValue ) { + BSONObj operand = BSON( "$lt" << 5 ); + LTMatchExpression lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + ASSERT( lt.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 4.5 ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 7 ) ), NULL ) ); + } + + TEST( LtOp, MatchesWholeArray ) { + BSONObj operand = BSON( "$lt" << BSON_ARRAY( 5 ) ); + LTMatchExpression lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + ASSERT( lt.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( 5 ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( 6 ) ), NULL ) ); + // Nested array. + ASSERT( lt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 4 ) ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 5 ) ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 6 ) ) ), NULL ) ); + } + + TEST( LtOp, MatchesNull ) { + BSONObj operand = BSON( "$lt" << BSONNULL ); + LTMatchExpression lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + ASSERT( !lt.matchesBSON( BSONObj(), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + // A non-existent field is treated same way as an empty bson object + ASSERT( !lt.matchesBSON( BSON( "b" << 4 ), NULL ) ); + } + + TEST( LtOp, MatchesDotNotationNull) { + BSONObj operand = BSON( "$lt" << BSONNULL ); + LTMatchExpression lt; + ASSERT( lt.init( "a.b", operand[ "$lt" ] ).isOK() ); + ASSERT( !lt.matchesBSON( BSONObj(), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSONObj() ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << BSONNULL ) ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "a" << 4 ) << BSON( "b" << 4 ) ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 4 ) ) ), NULL ) ); + } + + TEST( LtOp, MatchesMinKey ) { + BSONObj operand = BSON( "a" << MinKey ); + LTMatchExpression lt; + ASSERT( lt.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( !lt.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !lt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( LtOp, MatchesMaxKey ) { + BSONObj operand = BSON( "a" << MaxKey ); + LTMatchExpression lt; + ASSERT( lt.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( !lt.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( lt.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( lt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( LtOp, ElemMatchKey ) { + BSONObj operand = BSON( "$lt" << 5 ); + LTMatchExpression lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !lt.matchesBSON( BSON( "a" << 6 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( lt.matchesBSON( BSON( "a" << 4 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( lt.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 2 << 5 ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + /** + TEST( LtOp, MatchesIndexKeyScalar ) { + BSONObj operand = BSON( "$lt" << 6 ); + LtOp lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + lt.matchesIndexKey( BSON( "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + lt.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + lt.matchesIndexKey( BSON( "" << BSON_ARRAY( 5 ) ), indexSpec ) ); + } + + TEST( LtOp, MatchesIndexKeyMissing ) { + BSONObj operand = BSON( "$lt" << 6 ); + LtOp lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "b" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lt.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lt.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lt.matchesIndexKey( BSON( "" << BSON_ARRAY( 8 << 6 ) ), indexSpec ) ); + } + + TEST( LtOp, MatchesIndexKeyArray ) { + BSONObj operand = BSON( "$lt" << BSON_ARRAY( 4 << 5 ) ); + LtOp lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lt.matchesIndexKey( BSON( "" << 3 ), indexSpec ) ); + } + + TEST( LtOp, MatchesIndexKeyArrayValue ) { + BSONObj operand = BSON( "$lt" << 6 ); + LtOp lt; + ASSERT( lt.init( "a", operand[ "$lt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "loc" << "mockarrayvalue" << "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + lt.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + lt.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + lt.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << 6 << 4 ) ), indexSpec ) ); + } + */ + TEST( LteOp, MatchesElement ) { + BSONObj operand = BSON( "$lte" << 5 ); + BSONObj match = BSON( "a" << 4.5 ); + BSONObj equalMatch = BSON( "a" << 5 ); + BSONObj notMatch = BSON( "a" << 6 ); + BSONObj notMatchWrongType = BSON( "a" << "foo" ); + LTEMatchExpression lte; + ASSERT( lte.init( "", operand[ "$lte" ] ).isOK() ); + ASSERT( lte.matchesSingleElement( match.firstElement() ) ); + ASSERT( lte.matchesSingleElement( equalMatch.firstElement() ) ); + ASSERT( !lte.matchesSingleElement( notMatch.firstElement() ) ); + ASSERT( !lte.matchesSingleElement( notMatchWrongType.firstElement() ) ); + } + + TEST( LteOp, InvalidEooOperand ) { + BSONObj operand; + LTEMatchExpression lte; + ASSERT( !lte.init( "", operand.firstElement() ).isOK() ); + } + + TEST( LteOp, MatchesScalar ) { + BSONObj operand = BSON( "$lte" << 5 ); + LTEMatchExpression lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + ASSERT( lte.matchesBSON( BSON( "a" << 4.5 ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << 6 ), NULL ) ); + } + + TEST( LteOp, MatchesArrayValue ) { + BSONObj operand = BSON( "$lte" << 5 ); + LTEMatchExpression lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 4.5 ) ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 7 ) ), NULL ) ); + } + + TEST( LteOp, MatchesWholeArray ) { + BSONObj operand = BSON( "$lte" << BSON_ARRAY( 5 ) ); + LTEMatchExpression lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( 5 ) ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << BSON_ARRAY( 6 ) ), NULL ) ); + // Nested array. + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 4 ) ) ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 5 ) ) ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 6 ) ) ), NULL ) ); + } + + TEST( LteOp, MatchesNull ) { + BSONObj operand = BSON( "$lte" << BSONNULL ); + LTEMatchExpression lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + ASSERT( lte.matchesBSON( BSONObj(), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + // A non-existent field is treated same way as an empty bson object + ASSERT( lte.matchesBSON( BSON( "b" << 4 ), NULL ) ); + } + + TEST( LteOp, MatchesDotNotationNull) { + BSONObj operand = BSON( "$lte" << BSONNULL ); + LTEMatchExpression lte; + ASSERT( lte.init( "a.b", operand[ "$lte" ] ).isOK() ); + ASSERT( lte.matchesBSON( BSONObj(), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << BSONObj() ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << BSONNULL ) ) ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "a" << 4 ) << BSON( "b" << 4 ) ) ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 4 ) ) ), NULL ) ); + } + + TEST( LteOp, MatchesMinKey ) { + BSONObj operand = BSON( "a" << MinKey ); + LTEMatchExpression lte; + ASSERT( lte.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( lte.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !lte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( LteOp, MatchesMaxKey ) { + BSONObj operand = BSON( "a" << MaxKey ); + LTEMatchExpression lte; + ASSERT( lte.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( lte.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( lte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + + TEST( LteOp, ElemMatchKey ) { + BSONObj operand = BSON( "$lte" << 5 ); + LTEMatchExpression lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !lte.matchesBSON( BSON( "a" << 6 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( lte.matchesBSON( BSON( "a" << 4 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( lte.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 2 << 5 ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + /** + TEST( LteOp, MatchesIndexKeyScalar ) { + BSONObj operand = BSON( "$lte" << 6 ); + LteOp lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + lte.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + lte.matchesIndexKey( BSON( "" << 7 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + lte.matchesIndexKey( BSON( "" << BSON_ARRAY( 5 ) ), indexSpec ) ); + } + + TEST( LteOp, MatchesIndexKeyMissing ) { + BSONObj operand = BSON( "$lte" << 6 ); + LteOp lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "b" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lte.matchesIndexKey( BSON( "" << 7 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lte.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lte.matchesIndexKey( BSON( "" << BSON_ARRAY( 8 << 6 ) ), indexSpec ) ); + } + + TEST( LteOp, MatchesIndexKeyArray ) { + BSONObj operand = BSON( "$lte" << BSON_ARRAY( 4 << 5 ) ); + LteOp lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + lte.matchesIndexKey( BSON( "" << 3 ), indexSpec ) ); + } + + TEST( LteOp, MatchesIndexKeyArrayValue ) { + BSONObj operand = BSON( "$lte" << 6 ); + LteOp lte; + ASSERT( lte.init( "a", operand[ "$lte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "loc" << "mockarrayvalue" << "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + lte.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + lte.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 7 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + lte.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << 6 << 4 ) ), indexSpec ) ); + } + + TEST( GtOp, MatchesElement ) { + BSONObj operand = BSON( "$gt" << 5 ); + BSONObj match = BSON( "a" << 5.5 ); + BSONObj notMatch = BSON( "a" << 4 ); + BSONObj notMatchEqual = BSON( "a" << 5 ); + BSONObj notMatchWrongType = BSON( "a" << "foo" ); + GtOp gt; + ASSERT( gt.init( "", operand[ "$gt" ] ).isOK() ); + ASSERT( gt.matchesSingleElement( match.firstElement() ) ); + ASSERT( !gt.matchesSingleElement( notMatch.firstElement() ) ); + ASSERT( !gt.matchesSingleElement( notMatchEqual.firstElement() ) ); + ASSERT( !gt.matchesSingleElement( notMatchWrongType.firstElement() ) ); + } + */ + + TEST( GtOp, InvalidEooOperand ) { + BSONObj operand; + GTMatchExpression gt; + ASSERT( !gt.init( "", operand.firstElement() ).isOK() ); + } + + TEST( GtOp, MatchesScalar ) { + BSONObj operand = BSON( "$gt" << 5 ); + GTMatchExpression gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + ASSERT( gt.matchesBSON( BSON( "a" << 5.5 ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( GtOp, MatchesArrayValue ) { + BSONObj operand = BSON( "$gt" << 5 ); + GTMatchExpression gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + ASSERT( gt.matchesBSON( BSON( "a" << BSON_ARRAY( 3 << 5.5 ) ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSON_ARRAY( 2 << 4 ) ), NULL ) ); + } + + TEST( GtOp, MatchesWholeArray ) { + BSONObj operand = BSON( "$gt" << BSON_ARRAY( 5 ) ); + GTMatchExpression gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSON_ARRAY( 5 ) ), NULL ) ); + ASSERT( gt.matchesBSON( BSON( "a" << BSON_ARRAY( 6 ) ), NULL ) ); + // Nested array. + // XXX: The following assertion documents current behavior. + ASSERT( gt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 4 ) ) ), NULL ) ); + // XXX: The following assertion documents current behavior. + ASSERT( gt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 5 ) ) ), NULL ) ); + ASSERT( gt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 6 ) ) ), NULL ) ); + } + + TEST( GtOp, MatchesNull ) { + BSONObj operand = BSON( "$gt" << BSONNULL ); + GTMatchExpression gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + ASSERT( !gt.matchesBSON( BSONObj(), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + // A non-existent field is treated same way as an empty bson object + ASSERT( !gt.matchesBSON( BSON( "b" << 4 ), NULL ) ); + } + + TEST( GtOp, MatchesDotNotationNull) { + BSONObj operand = BSON( "$gt" << BSONNULL ); + GTMatchExpression gt; + ASSERT( gt.init( "a.b", operand[ "$gt" ] ).isOK() ); + ASSERT( !gt.matchesBSON( BSONObj(), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSONObj() ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << BSONNULL ) ) ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "a" << 4 ) << BSON( "b" << 4 ) ) ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 4 ) ) ), NULL ) ); + } + + TEST( GtOp, MatchesMinKey ) { + BSONObj operand = BSON( "a" << MinKey ); + GTMatchExpression gt; + ASSERT( gt.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( !gt.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( gt.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( gt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( GtOp, MatchesMaxKey ) { + BSONObj operand = BSON( "a" << MaxKey ); + GTMatchExpression gt; + ASSERT( gt.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( !gt.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !gt.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( GtOp, ElemMatchKey ) { + BSONObj operand = BSON( "$gt" << 5 ); + GTMatchExpression gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !gt.matchesBSON( BSON( "a" << 4 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( gt.matchesBSON( BSON( "a" << 6 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( gt.matchesBSON( BSON( "a" << BSON_ARRAY( 2 << 6 << 5 ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + /** + TEST( GtOp, MatchesIndexKeyScalar ) { + BSONObj operand = BSON( "$gt" << 6 ); + GtOp gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + gt.matchesIndexKey( BSON( "" << 7 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + gt.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + gt.matchesIndexKey( BSON( "" << BSON_ARRAY( 9 ) ), indexSpec ) ); + } + + TEST( GtOp, MatchesIndexKeyMissing ) { + BSONObj operand = BSON( "$gt" << 6 ); + GtOp gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "b" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gt.matchesIndexKey( BSON( "" << 7 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gt.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gt.matchesIndexKey( BSON( "" << BSON_ARRAY( 8 << 6 ) ), indexSpec ) ); + } + + TEST( GtOp, MatchesIndexKeyArray ) { + BSONObj operand = BSON( "$gt" << BSON_ARRAY( 4 << 5 ) ); + GtOp gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gt.matchesIndexKey( BSON( "" << 8 ), indexSpec ) ); + } + + TEST( GtOp, MatchesIndexKeyArrayValue ) { + BSONObj operand = BSON( "$gt" << 6 ); + GtOp gt; + ASSERT( gt.init( "a", operand[ "$gt" ] ).isOK() ); + IndexSpec indexSpec( BSON( "loc" << "mockarrayvalue" << "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + gt.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 7 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + gt.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + gt.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << 6 << 4 ) ), indexSpec ) ); + } + */ + + TEST( ComparisonMatchExpression, MatchesElement ) { + BSONObj operand = BSON( "$gte" << 5 ); + BSONObj match = BSON( "a" << 5.5 ); + BSONObj equalMatch = BSON( "a" << 5 ); + BSONObj notMatch = BSON( "a" << 4 ); + BSONObj notMatchWrongType = BSON( "a" << "foo" ); + GTEMatchExpression gte; + ASSERT( gte.init( "", operand[ "$gte" ] ).isOK() ); + ASSERT( gte.matchesSingleElement( match.firstElement() ) ); + ASSERT( gte.matchesSingleElement( equalMatch.firstElement() ) ); + ASSERT( !gte.matchesSingleElement( notMatch.firstElement() ) ); + ASSERT( !gte.matchesSingleElement( notMatchWrongType.firstElement() ) ); + } + + TEST( ComparisonMatchExpression, InvalidEooOperand ) { + BSONObj operand; + GTEMatchExpression gte; + ASSERT( !gte.init( "", operand.firstElement() ).isOK() ); + } + + TEST( ComparisonMatchExpression, MatchesScalar ) { + BSONObj operand = BSON( "$gte" << 5 ); + GTEMatchExpression gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + ASSERT( gte.matchesBSON( BSON( "a" << 5.5 ), NULL ) ); + ASSERT( !gte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( ComparisonMatchExpression, MatchesArrayValue ) { + BSONObj operand = BSON( "$gte" << 5 ); + GTEMatchExpression gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5.5 ) ), NULL ) ); + ASSERT( !gte.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 ) ), NULL ) ); + } + + TEST( ComparisonMatchExpression, MatchesWholeArray ) { + BSONObj operand = BSON( "$gte" << BSON_ARRAY( 5 ) ); + GTEMatchExpression gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + ASSERT( !gte.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( 5 ) ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( 6 ) ), NULL ) ); + // Nested array. + // XXX: The following assertion documents current behavior. + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 4 ) ) ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 5 ) ) ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 6 ) ) ), NULL ) ); + } + + TEST( ComparisonMatchExpression, MatchesNull ) { + BSONObj operand = BSON( "$gte" << BSONNULL ); + GTEMatchExpression gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + ASSERT( gte.matchesBSON( BSONObj(), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !gte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + // A non-existent field is treated same way as an empty bson object + ASSERT( gte.matchesBSON( BSON( "b" << 4 ), NULL ) ); + } + + TEST( ComparisonMatchExpression, MatchesDotNotationNull) { + BSONObj operand = BSON( "$gte" << BSONNULL ); + GTEMatchExpression gte; + ASSERT( gte.init( "a.b", operand[ "$gte" ] ).isOK() ); + ASSERT( gte.matchesBSON( BSONObj(), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSONObj() ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << BSONNULL ) ) ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "a" << 4 ) << BSON( "b" << 4 ) ) ), NULL ) ); + ASSERT( !gte.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( !gte.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 4 ) ) ), NULL ) ); + } + + TEST( ComparisonMatchExpression, MatchesMinKey ) { + BSONObj operand = BSON( "a" << MinKey ); + GTEMatchExpression gte; + ASSERT( gte.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( gte.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( gte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( ComparisonMatchExpression, MatchesMaxKey ) { + BSONObj operand = BSON( "a" << MaxKey ); + GTEMatchExpression gte; + ASSERT( gte.init( "a", operand[ "a" ] ).isOK() ); + ASSERT( gte.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !gte.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !gte.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( ComparisonMatchExpression, ElemMatchKey ) { + BSONObj operand = BSON( "$gte" << 5 ); + GTEMatchExpression gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !gte.matchesBSON( BSON( "a" << 4 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( gte.matchesBSON( BSON( "a" << 6 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( gte.matchesBSON( BSON( "a" << BSON_ARRAY( 2 << 6 << 5 ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + /** + TEST( GteOp, MatchesIndexKeyScalar ) { + BSONObj operand = BSON( "$gte" << 6 ); + GteOp gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + gte.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + gte.matchesIndexKey( BSON( "" << 5 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + gte.matchesIndexKey( BSON( "" << BSON_ARRAY( 7 ) ), indexSpec ) ); + } + + TEST( GteOp, MatchesIndexKeyMissing ) { + BSONObj operand = BSON( "$gte" << 6 ); + GteOp gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "b" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gte.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gte.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gte.matchesIndexKey( BSON( "" << BSON_ARRAY( 8 << 6 ) ), indexSpec ) ); + } + + TEST( GteOp, MatchesIndexKeyArray ) { + BSONObj operand = BSON( "$gte" << BSON_ARRAY( 4 << 5 ) ); + GteOp gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + gte.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + } + + TEST( GteOp, MatchesIndexKeyArrayValue ) { + BSONObj operand = BSON( "$gte" << 6 ); + GteOp gte; + ASSERT( gte.init( "a", operand[ "$gte" ] ).isOK() ); + IndexSpec indexSpec( BSON( "loc" << "mockarrayvalue" << "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + gte.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + gte.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + gte.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << 6 << 4 ) ), indexSpec ) ); + } + */ + + TEST( RegexMatchExpression, MatchesElementExact ) { + BSONObj match = BSON( "a" << "b" ); + BSONObj notMatch = BSON( "a" << "c" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "b", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, TooLargePattern ) { + string tooLargePattern( 50 * 1000, 'z' ); + RegexMatchExpression regex; + ASSERT( !regex.init( "a", tooLargePattern, "" ).isOK() ); + } + + TEST( RegexMatchExpression, MatchesElementSimplePrefix ) { + BSONObj match = BSON( "x" << "abc" ); + BSONObj notMatch = BSON( "x" << "adz" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "^ab", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementCaseSensitive ) { + BSONObj match = BSON( "x" << "abc" ); + BSONObj notMatch = BSON( "x" << "ABC" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "abc", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementCaseInsensitive ) { + BSONObj match = BSON( "x" << "abc" ); + BSONObj matchUppercase = BSON( "x" << "ABC" ); + BSONObj notMatch = BSON( "x" << "abz" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "abc", "i" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( regex.matchesSingleElement( matchUppercase.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementMultilineOff ) { + BSONObj match = BSON( "x" << "az" ); + BSONObj notMatch = BSON( "x" << "\naz" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "^a", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementMultilineOn ) { + BSONObj match = BSON( "x" << "az" ); + BSONObj matchMultiline = BSON( "x" << "\naz" ); + BSONObj notMatch = BSON( "x" << "\n\n" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "^a", "m" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( regex.matchesSingleElement( matchMultiline.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementExtendedOff ) { + BSONObj match = BSON( "x" << "a b" ); + BSONObj notMatch = BSON( "x" << "ab" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "a b", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementExtendedOn ) { + BSONObj match = BSON( "x" << "ab" ); + BSONObj notMatch = BSON( "x" << "a b" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "a b", "x" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementDotAllOff ) { + BSONObj match = BSON( "x" << "a b" ); + BSONObj notMatch = BSON( "x" << "a\nb" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "a.b", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementDotAllOn ) { + BSONObj match = BSON( "x" << "a b" ); + BSONObj matchDotAll = BSON( "x" << "a\nb" ); + BSONObj notMatch = BSON( "x" << "ab" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "a.b", "s" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( regex.matchesSingleElement( matchDotAll.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementMultipleFlags ) { + BSONObj matchMultilineDotAll = BSON( "x" << "\na\nb" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "^a.b", "ms" ).isOK() ); + ASSERT( regex.matchesSingleElement( matchMultilineDotAll.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementRegexType ) { + BSONObj match = BSONObjBuilder().appendRegex( "x", "yz", "i" ).obj(); + BSONObj notMatchPattern = BSONObjBuilder().appendRegex( "x", "r", "i" ).obj(); + BSONObj notMatchFlags = BSONObjBuilder().appendRegex( "x", "yz", "s" ).obj(); + RegexMatchExpression regex; + ASSERT( regex.init( "", "yz", "i" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatchPattern.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatchFlags.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementSymbolType ) { + BSONObj match = BSONObjBuilder().appendSymbol( "x", "yz" ).obj(); + BSONObj notMatch = BSONObjBuilder().appendSymbol( "x", "gg" ).obj(); + RegexMatchExpression regex; + ASSERT( regex.init( "", "yz", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( match.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatch.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementWrongType ) { + BSONObj notMatchInt = BSON( "x" << 1 ); + BSONObj notMatchBool = BSON( "x" << true ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "1", "" ).isOK() ); + ASSERT( !regex.matchesSingleElement( notMatchInt.firstElement() ) ); + ASSERT( !regex.matchesSingleElement( notMatchBool.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesElementUtf8 ) { + BSONObj multiByteCharacter = BSON( "x" << "\xc2\xa5" ); + RegexMatchExpression regex; + ASSERT( regex.init( "", "^.$", "" ).isOK() ); + ASSERT( regex.matchesSingleElement( multiByteCharacter.firstElement() ) ); + } + + TEST( RegexMatchExpression, MatchesScalar ) { + RegexMatchExpression regex; + ASSERT( regex.init( "a", "b", "" ).isOK() ); + ASSERT( regex.matchesBSON( BSON( "a" << "b" ), NULL ) ); + ASSERT( !regex.matchesBSON( BSON( "a" << "c" ), NULL ) ); + } + + TEST( RegexMatchExpression, MatchesArrayValue ) { + RegexMatchExpression regex; + ASSERT( regex.init( "a", "b", "" ).isOK() ); + ASSERT( regex.matchesBSON( BSON( "a" << BSON_ARRAY( "c" << "b" ) ), NULL ) ); + ASSERT( !regex.matchesBSON( BSON( "a" << BSON_ARRAY( "d" << "c" ) ), NULL ) ); + } + + TEST( RegexMatchExpression, MatchesNull ) { + RegexMatchExpression regex; + ASSERT( regex.init( "a", "b", "" ).isOK() ); + ASSERT( !regex.matchesBSON( BSONObj(), NULL ) ); + ASSERT( !regex.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + } + + TEST( RegexMatchExpression, ElemMatchKey ) { + RegexMatchExpression regex; + ASSERT( regex.init( "a", "b", "" ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !regex.matchesBSON( BSON( "a" << "c" ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( regex.matchesBSON( BSON( "a" << "b" ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( regex.matchesBSON( BSON( "a" << BSON_ARRAY( "c" << "b" ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + TEST( RegexMatchExpression, Equality1 ) { + RegexMatchExpression r1; + RegexMatchExpression r2; + RegexMatchExpression r3; + RegexMatchExpression r4; + ASSERT( r1.init( "a" , "b" ,"" ).isOK() ); + ASSERT( r2.init( "a" , "b" ,"x" ).isOK() ); + ASSERT( r3.init( "a" , "c" ,"" ).isOK() ); + ASSERT( r4.init( "b" , "b" ,"" ).isOK() ); + + ASSERT( r1.equivalent( &r1 ) ); + ASSERT( !r1.equivalent( &r2 ) ); + ASSERT( !r1.equivalent( &r3 ) ); + ASSERT( !r1.equivalent( &r4 ) ); + } + + /** + TEST( RegexMatchExpression, MatchesIndexKeyScalar ) { + RegexMatchExpression regex; + ASSERT( regex.init( "a", "xyz", "" ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + regex.matchesIndexKey( BSON( "" << "z xyz" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + regex.matchesIndexKey( BSON( "" << "xy" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + regex.matchesIndexKey( BSON( "" << BSON_ARRAY( "xyz" ) ), indexSpec ) ); + } + + TEST( RegexMatchExpression, MatchesIndexKeyMissing ) { + RegexMatchExpression regex; + ASSERT( regex.init( "a", "xyz", "" ).isOK() ); + IndexSpec indexSpec( BSON( "b" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + regex.matchesIndexKey( BSON( "" << "z xyz" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + regex.matchesIndexKey( BSON( "" << "xy" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + regex.matchesIndexKey( BSON( "" << BSON_ARRAY( 8 << "xyz" ) ), indexSpec ) ); + } + + TEST( RegexMatchExpression, MatchesIndexKeyArrayValue ) { + RegexMatchExpression regex; + ASSERT( regex.init( "a", "xyz", "" ).isOK() ); + IndexSpec indexSpec( BSON( "loc" << "mockarrayvalue" << "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + regex.matchesIndexKey( BSON( "" << "dummygeohash" << "" << "xyz" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + regex.matchesIndexKey( BSON( "" << "dummygeohash" << "" << "z" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + regex.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( "r" << 6 << "xyz" ) ), indexSpec ) ); + } + */ + + TEST( ModMatchExpression, MatchesElement ) { + BSONObj match = BSON( "a" << 1 ); + BSONObj largerMatch = BSON( "a" << 4.0 ); + BSONObj longLongMatch = BSON( "a" << 68719476736LL ); + BSONObj notMatch = BSON( "a" << 6 ); + BSONObj negativeNotMatch = BSON( "a" << -2 ); + ModMatchExpression mod; + ASSERT( mod.init( "", 3, 1 ).isOK() ); + ASSERT( mod.matchesSingleElement( match.firstElement() ) ); + ASSERT( mod.matchesSingleElement( largerMatch.firstElement() ) ); + ASSERT( mod.matchesSingleElement( longLongMatch.firstElement() ) ); + ASSERT( !mod.matchesSingleElement( notMatch.firstElement() ) ); + ASSERT( !mod.matchesSingleElement( negativeNotMatch.firstElement() ) ); + } + + TEST( ModMatchExpression, ZeroDivisor ) { + ModMatchExpression mod; + ASSERT( !mod.init( "", 0, 1 ).isOK() ); + } + + TEST( ModMatchExpression, MatchesScalar ) { + ModMatchExpression mod; + ASSERT( mod.init( "a", 5, 2 ).isOK() ); + ASSERT( mod.matchesBSON( BSON( "a" << 7.0 ), NULL ) ); + ASSERT( !mod.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( ModMatchExpression, MatchesArrayValue ) { + ModMatchExpression mod; + ASSERT( mod.init( "a", 5, 2 ).isOK() ); + ASSERT( mod.matchesBSON( BSON( "a" << BSON_ARRAY( 5 << 12LL ) ), NULL ) ); + ASSERT( !mod.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 8 ) ), NULL ) ); + } + + TEST( ModMatchExpression, MatchesNull ) { + ModMatchExpression mod; + ASSERT( mod.init( "a", 5, 2 ).isOK() ); + ASSERT( !mod.matchesBSON( BSONObj(), NULL ) ); + ASSERT( !mod.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + } + + TEST( ModMatchExpression, ElemMatchKey ) { + ModMatchExpression mod; + ASSERT( mod.init( "a", 5, 2 ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !mod.matchesBSON( BSON( "a" << 4 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( mod.matchesBSON( BSON( "a" << 2 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( mod.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 << 5 ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + TEST( ModMatchExpression, Equality1 ) { + ModMatchExpression m1; + ModMatchExpression m2; + ModMatchExpression m3; + ModMatchExpression m4; + + m1.init( "a" , 1 , 2 ); + m2.init( "a" , 2 , 2 ); + m3.init( "a" , 1 , 1 ); + m4.init( "b" , 1 , 2 ); + + ASSERT( m1.equivalent( &m1 ) ); + ASSERT( !m1.equivalent( &m2 ) ); + ASSERT( !m1.equivalent( &m3 ) ); + ASSERT( !m1.equivalent( &m4 ) ); + } + + /** + TEST( ModMatchExpression, MatchesIndexKey ) { + BSONObj operand = BSON( "$mod" << BSON_ARRAY( 2 << 1 ) ); + ModMatchExpression mod; + ASSERT( mod.init( "a", operand[ "$mod" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + BSONObj indexKey = BSON( "" << 1 ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + mod.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + + TEST( ExistsMatchExpression, MatchesElement ) { + BSONObj existsInt = BSON( "a" << 5 ); + BSONObj existsNull = BSON( "a" << BSONNULL ); + BSONObj doesntExist = BSONObj(); + ExistsMatchExpression exists; + ASSERT( exists.init( "" ).isOK() ); + ASSERT( exists.matchesSingleElement( existsInt.firstElement() ) ); + ASSERT( exists.matchesSingleElement( existsNull.firstElement() ) ); + ASSERT( !exists.matchesSingleElement( doesntExist.firstElement() ) ); + } + + TEST( ExistsMatchExpression, MatchesElementExistsTrueValue ) { + BSONObj exists = BSON( "a" << 5 ); + BSONObj missing = BSONObj(); + ExistsMatchExpression existsTrueValue; + ASSERT( existsTrueValue.init( "" ).isOK() ); + ASSERT( existsTrueValue.matchesSingleElement( exists.firstElement() ) ); + ASSERT( !existsTrueValue.matchesSingleElement( missing.firstElement() ) ); + } + + TEST( ExistsMatchExpression, MatchesScalar ) { + ExistsMatchExpression exists; + ASSERT( exists.init( "a" ).isOK() ); + ASSERT( exists.matchesBSON( BSON( "a" << 1 ), NULL ) ); + ASSERT( exists.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !exists.matchesBSON( BSON( "b" << 1 ), NULL ) ); + } + + TEST( ExistsMatchExpression, MatchesArray ) { + ExistsMatchExpression exists; + ASSERT( exists.init( "a" ).isOK() ); + ASSERT( exists.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5.5 ) ), NULL ) ); + } + + TEST( ExistsMatchExpression, ElemMatchKey ) { + ExistsMatchExpression exists; + ASSERT( exists.init( "a.b" ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !exists.matchesBSON( BSON( "a" << 1 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( exists.matchesBSON( BSON( "a" << BSON( "b" << 6 ) ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( exists.matchesBSON( BSON( "a" << BSON_ARRAY( 2 << BSON( "b" << 7 ) ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + TEST( ExistsMatchExpression, Equivalent ) { + ExistsMatchExpression e1; + ExistsMatchExpression e2; + e1.init( "a" ); + e2.init( "b" ); + + ASSERT( e1.equivalent( &e1 ) ); + ASSERT( !e1.equivalent( &e2 ) ); + } + + /** + TEST( ExistsMatchExpression, MatchesIndexKey ) { + BSONObj operand = BSON( "$exists" << true ); + ExistsMatchExpression exists; + ASSERT( exists.init( "a", operand[ "$exists" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + BSONObj indexKey = BSON( "" << 1 ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + exists.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + + + + TEST( TypeMatchExpression, MatchesElementStringType ) { + BSONObj match = BSON( "a" << "abc" ); + BSONObj notMatch = BSON( "a" << 5 ); + TypeMatchExpression type; + ASSERT( type.init( "", String ).isOK() ); + ASSERT( type.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !type.matchesSingleElement( notMatch[ "a" ] ) ); + } + + TEST( TypeMatchExpression, MatchesElementNullType ) { + BSONObj match = BSON( "a" << BSONNULL ); + BSONObj notMatch = BSON( "a" << "abc" ); + TypeMatchExpression type; + ASSERT( type.init( "", jstNULL ).isOK() ); + ASSERT( type.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !type.matchesSingleElement( notMatch[ "a" ] ) ); + } + + TEST( TypeMatchExpression, InvalidTypeMatchExpressionerand ) { + // If the provided type number is not a valid BSONType, it is not a parse error. The + // operator will simply not match anything. + BSONObj notMatch1 = BSON( "a" << BSONNULL ); + BSONObj notMatch2 = BSON( "a" << "abc" ); + TypeMatchExpression type; + ASSERT( type.init( "", JSTypeMax + 1 ).isOK() ); + ASSERT( !type.matchesSingleElement( notMatch1[ "a" ] ) ); + ASSERT( !type.matchesSingleElement( notMatch2[ "a" ] ) ); + } + + TEST( TypeMatchExpression, MatchesScalar ) { + TypeMatchExpression type; + ASSERT( type.init( "a", Bool ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << true ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << 1 ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesArray ) { + TypeMatchExpression type; + ASSERT( type.init( "a", NumberInt ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + ASSERT( type.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << "a" ) ), NULL ) ); + ASSERT( type.matchesBSON( BSON( "a" << BSON_ARRAY( "a" << 4 ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << BSON_ARRAY( "a" ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 4 ) ) ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesOuterArray ) { + TypeMatchExpression type; + ASSERT( type.init( "a", Array ).isOK() ); + // The outer array is not matched. + ASSERT( !type.matchesBSON( BSON( "a" << BSONArray() ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << "a" ) ), NULL ) ); + ASSERT( type.matchesBSON( BSON( "a" << BSON_ARRAY( BSONArray() << 2 ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << "bar" ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesObject ) { + TypeMatchExpression type; + ASSERT( type.init( "a", Object ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON( "b" << 1 ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << 1 ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesDotNotationFieldObject ) { + TypeMatchExpression type; + ASSERT( type.init( "a.b", Object ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON( "b" << BSON( "c" << 1 ) ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << BSON( "b" << 1 ) ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesDotNotationArrayElementArray ) { + TypeMatchExpression type; + ASSERT( type.init( "a.0", Array ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 1 ) ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << BSON_ARRAY( "b" ) ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesDotNotationArrayElementScalar ) { + TypeMatchExpression type; + ASSERT( type.init( "a.0", String ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON_ARRAY( "b" ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesDotNotationArrayElementObject ) { + TypeMatchExpression type; + ASSERT( type.init( "a.0", Object ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON_ARRAY( BSON( "b" << 1 ) ) ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) ), NULL ) ); + } + + TEST( TypeMatchExpression, MatchesNull ) { + TypeMatchExpression type; + ASSERT( type.init( "a", jstNULL ).isOK() ); + ASSERT( type.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !type.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( !type.matchesBSON( BSONObj(), NULL ) ); + } + + TEST( TypeMatchExpression, ElemMatchKey ) { + TypeMatchExpression type; + ASSERT( type.init( "a.b", String ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !type.matchesBSON( BSON( "a" << 1 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON( "b" << "string" ) ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( type.matchesBSON( BSON( "a" << BSON( "b" << BSON_ARRAY( "string" ) ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "0", details.elemMatchKey() ); + ASSERT( type.matchesBSON( BSON( "a" << + BSON_ARRAY( 2 << + BSON( "b" << BSON_ARRAY( "string" ) ) ) ), + &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + TEST( TypeMatchExpression, Equivalent ) { + TypeMatchExpression e1; + TypeMatchExpression e2; + TypeMatchExpression e3; + e1.init( "a", String ); + e2.init( "a", NumberDouble ); + e3.init( "b", String ); + + ASSERT( e1.equivalent( &e1 ) ); + ASSERT( !e1.equivalent( &e2 ) ); + ASSERT( !e1.equivalent( &e3 ) ); + } + + + /** + TEST( TypeMatchExpression, MatchesIndexKey ) { + BSONObj operand = BSON( "$type" << 2 ); + TypeMatchExpression type; + ASSERT( type.init( "a", operand[ "$type" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + BSONObj indexKey = BSON( "" << "q" ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + type.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + + + TEST( InMatchExpression, MatchesElementSingle ) { + BSONArray operand = BSON_ARRAY( 1 ); + BSONObj match = BSON( "a" << 1 ); + BSONObj notMatch = BSON( "a" << 2 ); + InMatchExpression in; + in.getArrayFilterEntries()->addEquality( operand.firstElement() ); + ASSERT( in.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !in.matchesSingleElement( notMatch[ "a" ] ) ); + } + + TEST( InMatchExpression, MatchesEmpty ) { + InMatchExpression in; + in.init( "a" ); + + BSONObj notMatch = BSON( "a" << 2 ); + ASSERT( !in.matchesSingleElement( notMatch[ "a" ] ) ); + ASSERT( !in.matchesBSON( BSON( "a" << 1 ), NULL ) ); + ASSERT( !in.matchesBSON( BSONObj(), NULL ) ); + } + + TEST( InMatchExpression, MatchesElementMultiple ) { + BSONObj operand = BSON_ARRAY( 1 << "r" << true << 1 ); + InMatchExpression in; + in.getArrayFilterEntries()->addEquality( operand[0] ); + in.getArrayFilterEntries()->addEquality( operand[1] ); + in.getArrayFilterEntries()->addEquality( operand[2] ); + in.getArrayFilterEntries()->addEquality( operand[3] ); + + BSONObj matchFirst = BSON( "a" << 1 ); + BSONObj matchSecond = BSON( "a" << "r" ); + BSONObj matchThird = BSON( "a" << true ); + BSONObj notMatch = BSON( "a" << false ); + ASSERT( in.matchesSingleElement( matchFirst[ "a" ] ) ); + ASSERT( in.matchesSingleElement( matchSecond[ "a" ] ) ); + ASSERT( in.matchesSingleElement( matchThird[ "a" ] ) ); + ASSERT( !in.matchesSingleElement( notMatch[ "a" ] ) ); + } + + + TEST( InMatchExpression, MatchesScalar ) { + BSONObj operand = BSON_ARRAY( 5 ); + InMatchExpression in; + in.init( "a" ); + in.getArrayFilterEntries()->addEquality( operand.firstElement() ); + + ASSERT( in.matchesBSON( BSON( "a" << 5.0 ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( InMatchExpression, MatchesArrayValue ) { + BSONObj operand = BSON_ARRAY( 5 ); + InMatchExpression in; + in.init( "a" ); + in.getArrayFilterEntries()->addEquality( operand.firstElement() ); + + ASSERT( in.matchesBSON( BSON( "a" << BSON_ARRAY( 5.0 << 6 ) ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << BSON_ARRAY( 6 << 7 ) ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << BSON_ARRAY( BSON_ARRAY( 5 ) ) ), NULL ) ); + } + + TEST( InMatchExpression, MatchesNull ) { + BSONObj operand = BSON_ARRAY( BSONNULL ); + + InMatchExpression in; + in.init( "a" ); + in.getArrayFilterEntries()->addEquality( operand.firstElement() ); + + ASSERT( in.matchesBSON( BSONObj(), NULL ) ); + ASSERT( in.matchesBSON( BSON( "a" << BSONNULL ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << 4 ), NULL ) ); + // A non-existent field is treated same way as an empty bson object + ASSERT( in.matchesBSON( BSON( "b" << 4 ), NULL ) ); + } + + TEST( InMatchExpression, MatchesUndefined ) { + BSONObj operand = BSON_ARRAY( BSONUndefined ); + + InMatchExpression in; + in.init( "a" ); + Status s = in.getArrayFilterEntries()->addEquality( operand.firstElement() ); + ASSERT_NOT_OK(s); + } + + TEST( InMatchExpression, MatchesMinKey ) { + BSONObj operand = BSON_ARRAY( MinKey ); + InMatchExpression in; + in.init( "a" ); + in.getArrayFilterEntries()->addEquality( operand.firstElement() ); + + ASSERT( in.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( InMatchExpression, MatchesMaxKey ) { + BSONObj operand = BSON_ARRAY( MaxKey ); + InMatchExpression in; + in.init( "a" ); + in.getArrayFilterEntries()->addEquality( operand.firstElement() ); + + ASSERT( in.matchesBSON( BSON( "a" << MaxKey ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << MinKey ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( InMatchExpression, MatchesFullArray ) { + BSONObj operand = BSON_ARRAY( BSON_ARRAY( 1 << 2 ) << 4 << 5 ); + InMatchExpression in; + in.init( "a" ); + in.getArrayFilterEntries()->addEquality( operand[0] ); + in.getArrayFilterEntries()->addEquality( operand[1] ); + in.getArrayFilterEntries()->addEquality( operand[2] ); + + ASSERT( in.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 ) ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 << 3 ) ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) ), NULL ) ); + ASSERT( !in.matchesBSON( BSON( "a" << 1 ), NULL ) ); + } + + TEST( InMatchExpression, ElemMatchKey ) { + BSONObj operand = BSON_ARRAY( 5 << 2 ); + InMatchExpression in; + in.init( "a" ); + in.getArrayFilterEntries()->addEquality( operand[0] ); + in.getArrayFilterEntries()->addEquality( operand[1] ); + + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !in.matchesBSON( BSON( "a" << 4 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( in.matchesBSON( BSON( "a" << 5 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( in.matchesBSON( BSON( "a" << BSON_ARRAY( 1 << 2 << 5 ) ), &details ) ); + ASSERT( details.hasElemMatchKey() ); + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + /** + TEST( InMatchExpression, MatchesIndexKeyScalar ) { + BSONObj operand = BSON( "$in" << BSON_ARRAY( 6 << 5 ) ); + InMatchExpression in; + ASSERT( in.init( "a", operand[ "$in" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + in.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + in.matchesIndexKey( BSON( "" << 5 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + in.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + in.matchesIndexKey( BSON( "" << BSON_ARRAY( 6 ) ), indexSpec ) ); + } + + TEST( InMatchExpression, MatchesIndexKeyMissing ) { + BSONObj operand = BSON( "$in" << BSON_ARRAY( 6 ) ); + ComparisonMatchExpression eq + ASSERT( eq.init( "a", operand[ "$in" ] ).isOK() ); + IndexSpec indexSpec( BSON( "b" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + eq.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + eq.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + eq.matchesIndexKey( BSON( "" << BSON_ARRAY( 8 << 6 ) ), indexSpec ) ); + } + + TEST( InMatchExpression, MatchesIndexKeyArray ) { + BSONObj operand = BSON( "$in" << BSON_ARRAY( 4 << BSON_ARRAY( 5 ) ) ); + InMatchExpression in; + ASSERT( in.init( "a", operand[ "$in" ] ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + in.matchesIndexKey( BSON( "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + in.matchesIndexKey( BSON( "" << 5 ), indexSpec ) ); + } + + TEST( InMatchExpression, MatchesIndexKeyArrayValue ) { + BSONObjBuilder inArray; + inArray.append( "0", 4 ).append( "1", 5 ).appendRegex( "2", "abc", "" ); + BSONObj operand = BSONObjBuilder().appendArray( "$in", inArray.obj() ).obj(); + InMatchExpression in; + ASSERT( in.init( "a", operand[ "$in" ] ).isOK() ); + IndexSpec indexSpec( BSON( "loc" << "mockarrayvalue" << "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + in.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 4 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + in.matchesIndexKey( BSON( "" << "dummygeohash" << "" << 6 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + in.matchesIndexKey( BSON( "" << "dummygeohash" << "" << "abcd" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + in.matchesIndexKey( BSONObjBuilder() + .append( "", "dummygeohash" ) + .appendRegex( "", "abc", "" ).obj(), + indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + in.matchesIndexKey( BSON( "" << "dummygeohash" << "" << "ab" ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + in.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << 5 ) ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + in.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << 9 ) ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + in.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << "abc" ) ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + in.matchesIndexKey( BSON( "" << "dummygeohash" << + "" << BSON_ARRAY( 8 << "ac" ) ), indexSpec ) ); + } + */ + +} diff --git a/src/mongo/db/matcher/expression_parser.cpp b/src/mongo/db/matcher/expression_parser.cpp new file mode 100644 index 00000000000..348745c7dc4 --- /dev/null +++ b/src/mongo/db/matcher/expression_parser.cpp @@ -0,0 +1,821 @@ +// expression_parser.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/bson/bson-inl.h" +#include "mongo/db/matcher/expression_array.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/matcher/expression_tree.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" + +namespace { + + using namespace mongo; + + /** + * Returns true if subtree contains MatchExpression 'type'. + */ + bool hasNode(const MatchExpression* root, MatchExpression::MatchType type) { + if (type == root->matchType()) { + return true; + } + for (size_t i = 0; i < root->numChildren(); ++i) { + if (hasNode(root->getChild(i), type)) { + return true; + } + } + return false; + } + +} // namespace + +namespace mongo { + + StatusWithMatchExpression MatchExpressionParser::_parseComparison( const char* name, + ComparisonMatchExpression* cmp, + const BSONElement& e ) { + std::auto_ptr temp(cmp); + + // Non-equality comparison match expressions cannot have + // a regular expression as the argument (e.g. {a: {$gt: /b/}} is illegal). + if (MatchExpression::EQ != cmp->matchType() && RegEx == e.type()) { + std::stringstream ss; + ss << "Can't have RegEx as arg to predicate over field '" << name << "'."; + return StatusWithMatchExpression(Status(ErrorCodes::BadValue, ss.str())); + } + + Status s = temp->init( name, e ); + if ( !s.isOK() ) + return StatusWithMatchExpression(s); + + return StatusWithMatchExpression( temp.release() ); + } + + StatusWithMatchExpression MatchExpressionParser::_parseSubField( const BSONObj& context, + const AndMatchExpression* andSoFar, + const char* name, + const BSONElement& e, + int level ) { + + // TODO: these should move to getGtLtOp, or its replacement + + if ( mongoutils::str::equals( "$eq", e.fieldName() ) ) + return _parseComparison( name, new EqualityMatchExpression(), e ); + + if ( mongoutils::str::equals( "$not", e.fieldName() ) ) { + return _parseNot( name, e, level ); + } + + int x = e.getGtLtOp(-1); + switch ( x ) { + case -1: + // $where cannot be a sub-expression because it works on top-level documents only. + if ( mongoutils::str::equals( "$where", e.fieldName() ) ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$where cannot be applied to a field" ); + } + + return StatusWithMatchExpression( ErrorCodes::BadValue, + mongoutils::str::stream() << "unknown operator: " + << e.fieldName() ); + case BSONObj::LT: + return _parseComparison( name, new LTMatchExpression(), e ); + case BSONObj::LTE: + return _parseComparison( name, new LTEMatchExpression(), e ); + case BSONObj::GT: + return _parseComparison( name, new GTMatchExpression(), e ); + case BSONObj::GTE: + return _parseComparison( name, new GTEMatchExpression(), e ); + case BSONObj::NE: { + if (RegEx == e.type()) { + // Just because $ne can be rewritten as the negation of an + // equality does not mean that $ne of a regex is allowed. See SERVER-1705. + return StatusWithMatchExpression(Status(ErrorCodes::BadValue, + "Can't have regex as arg to $ne.")); + } + StatusWithMatchExpression s = _parseComparison( name, new EqualityMatchExpression(), e ); + if ( !s.isOK() ) + return s; + std::auto_ptr n( new NotMatchExpression() ); + Status s2 = n->init( s.getValue() ); + if ( !s2.isOK() ) + return StatusWithMatchExpression( s2 ); + return StatusWithMatchExpression( n.release() ); + } + case BSONObj::Equality: + return _parseComparison( name, new EqualityMatchExpression(), e ); + + case BSONObj::opIN: { + if ( e.type() != Array ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$in needs an array" ); + std::auto_ptr temp( new InMatchExpression() ); + Status s = temp->init( name ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + s = _parseArrayFilterEntries( temp->getArrayFilterEntries(), e.Obj() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( temp.release() ); + } + + case BSONObj::NIN: { + if ( e.type() != Array ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$nin needs an array" ); + std::auto_ptr temp( new InMatchExpression() ); + Status s = temp->init( name ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + s = _parseArrayFilterEntries( temp->getArrayFilterEntries(), e.Obj() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + std::auto_ptr temp2( new NotMatchExpression() ); + s = temp2->init( temp.release() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + return StatusWithMatchExpression( temp2.release() ); + } + + case BSONObj::opSIZE: { + int size = 0; + if ( e.type() == String ) { + // matching old odd semantics + size = 0; + } + else if ( e.type() == NumberInt || e.type() == NumberLong ) { + if (e.numberLong() < 0) { + // SERVER-11952. Setting 'size' to -1 means that no documents + // should match this $size expression. + size = -1; + } + else { + size = e.numberInt(); + } + } + else if ( e.type() == NumberDouble ) { + if ( e.numberInt() == e.numberDouble() ) { + size = e.numberInt(); + } + else { + // old semantcs require exact numeric match + // so [1,2] != 1 or 2 + size = -1; + } + } + else { + return StatusWithMatchExpression( ErrorCodes::BadValue, "$size needs a number" ); + } + + std::auto_ptr temp( new SizeMatchExpression() ); + Status s = temp->init( name, size ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( temp.release() ); + } + + case BSONObj::opEXISTS: { + if ( e.eoo() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$exists can't be eoo" ); + std::auto_ptr temp( new ExistsMatchExpression() ); + Status s = temp->init( name ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + if ( e.trueValue() ) + return StatusWithMatchExpression( temp.release() ); + std::auto_ptr temp2( new NotMatchExpression() ); + s = temp2->init( temp.release() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( temp2.release() ); + } + + case BSONObj::opTYPE: { + if ( !e.isNumber() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$type has to be a number" ); + int type = e.numberInt(); + if ( e.type() != NumberInt && type != e.number() ) + type = -1; + std::auto_ptr temp( new TypeMatchExpression() ); + Status s = temp->init( name, type ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( temp.release() ); + } + + + case BSONObj::opMOD: + return _parseMOD( name, e ); + + case BSONObj::opOPTIONS: { + // TODO: try to optimize this + // we have to do this since $options can be before or after a $regex + // but we validate here + BSONObjIterator i( context ); + while ( i.more() ) { + BSONElement temp = i.next(); + if ( temp.getGtLtOp( -1 ) == BSONObj::opREGEX ) + return StatusWithMatchExpression( NULL ); + } + + return StatusWithMatchExpression( ErrorCodes::BadValue, "$options needs a $regex" ); + } + + case BSONObj::opREGEX: { + return _parseRegexDocument( name, context ); + } + + case BSONObj::opELEM_MATCH: + return _parseElemMatch( name, e, level ); + + case BSONObj::opALL: + return _parseAll( name, e, level ); + + case BSONObj::opWITHIN: + case BSONObj::opGEO_INTERSECTS: + return expressionParserGeoCallback( name, x, context ); + } + + return StatusWithMatchExpression( ErrorCodes::BadValue, + mongoutils::str::stream() << "not handled: " << e.fieldName() ); + } + + StatusWithMatchExpression MatchExpressionParser::_parse( const BSONObj& obj, int level ) { + if (level > kMaximumTreeDepth) { + mongoutils::str::stream ss; + ss << "exceeded maximum query tree depth of " << kMaximumTreeDepth + << " at " << obj.toString(); + return StatusWithMatchExpression( ErrorCodes::BadValue, ss ); + } + + std::auto_ptr root( new AndMatchExpression() ); + + bool topLevel = (level == 0); + level++; + + BSONObjIterator i( obj ); + while ( i.more() ){ + + BSONElement e = i.next(); + if ( e.fieldName()[0] == '$' ) { + const char * rest = e.fieldName() + 1; + + // TODO: optimize if block? + if ( mongoutils::str::equals( "or", rest ) ) { + if ( e.type() != Array ) + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$or needs an array" ); + std::auto_ptr temp( new OrMatchExpression() ); + Status s = _parseTreeList( e.Obj(), temp.get(), level ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + root->add( temp.release() ); + } + else if ( mongoutils::str::equals( "and", rest ) ) { + if ( e.type() != Array ) + return StatusWithMatchExpression( ErrorCodes::BadValue, + "and needs an array" ); + std::auto_ptr temp( new AndMatchExpression() ); + Status s = _parseTreeList( e.Obj(), temp.get(), level ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + root->add( temp.release() ); + } + else if ( mongoutils::str::equals( "nor", rest ) ) { + if ( e.type() != Array ) + return StatusWithMatchExpression( ErrorCodes::BadValue, + "and needs an array" ); + std::auto_ptr temp( new NorMatchExpression() ); + Status s = _parseTreeList( e.Obj(), temp.get(), level ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + root->add( temp.release() ); + } + else if ( mongoutils::str::equals( "atomic", rest ) || + mongoutils::str::equals( "isolated", rest ) ) { + if ( !topLevel ) + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$atomic/$isolated has to be at the top level" ); + if ( e.trueValue() ) + root->add( new AtomicMatchExpression() ); + } + else if ( mongoutils::str::equals( "where", rest ) ) { + /* + if ( !topLevel ) + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$where has to be at the top level" ); + */ + StatusWithMatchExpression s = expressionParserWhereCallback( e ); + if ( !s.isOK() ) + return s; + root->add( s.getValue() ); + } + else if ( mongoutils::str::equals( "text", rest ) ) { + if ( e.type() != Object ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$text expects an object" ); + } + StatusWithMatchExpression s = expressionParserTextCallback( e.Obj() ); + if ( !s.isOK() ) { + return s; + } + root->add( s.getValue() ); + } + else if ( mongoutils::str::equals( "comment", rest ) ) { + } + else if ( mongoutils::str::equals( "ref", rest ) || + mongoutils::str::equals( "id", rest ) || + mongoutils::str::equals( "db", rest ) ) { + // DBRef fields. + std::auto_ptr eq( new EqualityMatchExpression() ); + Status s = eq->init( e.fieldName(), e ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + root->add( eq.release() ); + } + else { + return StatusWithMatchExpression( ErrorCodes::BadValue, + mongoutils::str::stream() + << "unknown top level operator: " + << e.fieldName() ); + } + + continue; + } + + if ( _isExpressionDocument( e, false ) ) { + Status s = _parseSub( e.fieldName(), e.Obj(), root.get(), level ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + continue; + } + + if ( e.type() == RegEx ) { + StatusWithMatchExpression result = _parseRegexElement( e.fieldName(), e ); + if ( !result.isOK() ) + return result; + root->add( result.getValue() ); + continue; + } + + std::auto_ptr eq( new EqualityMatchExpression() ); + Status s = eq->init( e.fieldName(), e ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + root->add( eq.release() ); + } + + if ( root->numChildren() == 1 ) { + const MatchExpression* real = root->getChild(0); + root->clearAndRelease(); + return StatusWithMatchExpression( const_cast(real) ); + } + + return StatusWithMatchExpression( root.release() ); + } + + Status MatchExpressionParser::_parseSub( const char* name, + const BSONObj& sub, + AndMatchExpression* root, + int level ) { + // The one exception to {field : {fully contained argument} } is, of course, geo. Example: + // sub == { field : {$near[Sphere]: [0,0], $maxDistance: 1000, $minDistance: 10 } } + // We peek inside of 'sub' to see if it's possibly a $near. If so, we can't iterate over + // its subfields and parse them one at a time (there is no $maxDistance without $near), so + // we hand the entire object over to the geo parsing routines. + + if (level > kMaximumTreeDepth) { + mongoutils::str::stream ss; + ss << "exceeded maximum query tree depth of " << kMaximumTreeDepth + << " at " << sub.toString(); + return Status( ErrorCodes::BadValue, ss ); + } + + level++; + + BSONObjIterator geoIt(sub); + if (geoIt.more()) { + BSONElement firstElt = geoIt.next(); + if (firstElt.isABSONObj()) { + const char* fieldName = firstElt.fieldName(); + // TODO: Having these $fields here isn't ideal but we don't want to pull in anything + // from db/geo at this point, since it may not actually be linked in... + if (mongoutils::str::equals(fieldName, "$near") + || mongoutils::str::equals(fieldName, "$nearSphere") + || mongoutils::str::equals(fieldName, "$geoNear") + || mongoutils::str::equals(fieldName, "$maxDistance") + || mongoutils::str::equals(fieldName, "$minDistance")) { + + StatusWithMatchExpression s = expressionParserGeoCallback(name, + firstElt.getGtLtOp(), + sub); + if (s.isOK()) { + root->add(s.getValue()); + } + + // Propagate geo parsing result to caller. + return s.getStatus(); + } + } + } + + BSONObjIterator j( sub ); + while ( j.more() ) { + BSONElement deep = j.next(); + + StatusWithMatchExpression s = _parseSubField( sub, root, name, deep, level ); + if ( !s.isOK() ) + return s.getStatus(); + + if ( s.getValue() ) + root->add( s.getValue() ); + } + + return Status::OK(); + } + + bool MatchExpressionParser::_isExpressionDocument( const BSONElement& e, + bool allowIncompleteDBRef ) { + if ( e.type() != Object ) + return false; + + BSONObj o = e.Obj(); + if ( o.isEmpty() ) + return false; + + const char* name = o.firstElement().fieldName(); + if ( name[0] != '$' ) + return false; + + if ( _isDBRefDocument( o, allowIncompleteDBRef ) ) { + return false; + } + + return true; + } + + /** + * DBRef fields are ordered in the collection. + * In the query, we consider an embedded object a query on + * a DBRef as long as it contains $ref and $id. + * Required fields: $ref and $id (if incomplete DBRefs are not allowed) + * + * If incomplete DBRefs are allowed, we accept the BSON object as long as it + * contains $ref, $id or $db. + * + * Field names are checked but not field types. + */ + bool MatchExpressionParser::_isDBRefDocument( const BSONObj& obj, bool allowIncompleteDBRef ) { + bool hasRef = false; + bool hasID = false; + bool hasDB = false; + + BSONObjIterator i( obj ); + while ( i.more() && !( hasRef && hasID ) ) { + BSONElement element = i.next(); + const char *fieldName = element.fieldName(); + // $ref + if ( !hasRef && mongoutils::str::equals( "$ref", fieldName ) ) { + hasRef = true; + } + // $id + else if ( !hasID && mongoutils::str::equals( "$id", fieldName ) ) { + hasID = true; + } + // $db + else if ( !hasDB && mongoutils::str::equals( "$db", fieldName ) ) { + hasDB = true; + } + } + + if (allowIncompleteDBRef) { + return hasRef || hasID || hasDB; + } + + return hasRef && hasID; + } + + StatusWithMatchExpression MatchExpressionParser::_parseMOD( const char* name, + const BSONElement& e ) { + + if ( e.type() != Array ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "malformed mod, needs to be an array" ); + + BSONObjIterator i( e.Obj() ); + + if ( !i.more() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "malformed mod, not enough elements" ); + BSONElement d = i.next(); + if ( !d.isNumber() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "malformed mod, divisor not a number" ); + + if ( !i.more() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "malformed mod, not enough elements" ); + BSONElement r = i.next(); + if ( !d.isNumber() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "malformed mod, remainder not a number" ); + + if ( i.more() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "malformed mod, too many elements" ); + + std::auto_ptr temp( new ModMatchExpression() ); + Status s = temp->init( name, d.numberInt(), r.numberInt() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( temp.release() ); + } + + StatusWithMatchExpression MatchExpressionParser::_parseRegexElement( const char* name, + const BSONElement& e ) { + if ( e.type() != RegEx ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "not a regex" ); + + std::auto_ptr temp( new RegexMatchExpression() ); + Status s = temp->init( name, e.regex(), e.regexFlags() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( temp.release() ); + } + + StatusWithMatchExpression MatchExpressionParser::_parseRegexDocument( const char* name, + const BSONObj& doc ) { + string regex; + string regexOptions; + + BSONObjIterator i( doc ); + while ( i.more() ) { + BSONElement e = i.next(); + switch ( e.getGtLtOp() ) { + case BSONObj::opREGEX: + if ( e.type() == String ) { + regex = e.String(); + } + else if ( e.type() == RegEx ) { + regex = e.regex(); + regexOptions = e.regexFlags(); + } + else { + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$regex has to be a string" ); + } + + break; + case BSONObj::opOPTIONS: + if ( e.type() != String ) + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$options has to be a string" ); + regexOptions = e.String(); + break; + default: + break; + } + + } + + std::auto_ptr temp( new RegexMatchExpression() ); + Status s = temp->init( name, regex, regexOptions ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( temp.release() ); + + } + + Status MatchExpressionParser::_parseArrayFilterEntries( ArrayFilterEntries* entries, + const BSONObj& theArray ) { + + BSONObjIterator i( theArray ); + while ( i.more() ) { + BSONElement e = i.next(); + + // allow DBRefs but reject all fields with names starting wiht $ + if ( _isExpressionDocument( e, false ) ) { + return Status( ErrorCodes::BadValue, "cannot nest $ under $in" ); + } + + if ( e.type() == RegEx ) { + std::auto_ptr r( new RegexMatchExpression() ); + Status s = r->init( "", e ); + if ( !s.isOK() ) + return s; + s = entries->addRegex( r.release() ); + if ( !s.isOK() ) + return s; + } + else { + Status s = entries->addEquality( e ); + if ( !s.isOK() ) + return s; + } + } + return Status::OK(); + + } + + StatusWithMatchExpression MatchExpressionParser::_parseElemMatch( const char* name, + const BSONElement& e, + int level ) { + if ( e.type() != Object ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$elemMatch needs an Object" ); + + BSONObj obj = e.Obj(); + + // $elemMatch value case applies when the children all + // work on the field 'name'. + // This is the case when: + // 1) the argument is an expression document; and + // 2) expression is not a AND/NOR/OR logical operator. Children of + // these logical operators are initialized with field names. + // 3) expression is not a WHERE operator. WHERE works on objects instead + // of specific field. + bool isElemMatchValue = false; + if ( _isExpressionDocument( e, true ) ) { + BSONObj o = e.Obj(); + BSONElement elt = o.firstElement(); + invariant( !elt.eoo() ); + + isElemMatchValue = !mongoutils::str::equals( "$and", elt.fieldName() ) && + !mongoutils::str::equals( "$nor", elt.fieldName() ) && + !mongoutils::str::equals( "$or", elt.fieldName() ) && + !mongoutils::str::equals( "$where", elt.fieldName() ); + } + + if ( isElemMatchValue ) { + // value case + + AndMatchExpression theAnd; + Status s = _parseSub( "", obj, &theAnd, level ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + std::auto_ptr temp( new ElemMatchValueMatchExpression() ); + s = temp->init( name ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + for ( size_t i = 0; i < theAnd.numChildren(); i++ ) { + temp->add( theAnd.getChild( i ) ); + } + theAnd.clearAndRelease(); + + return StatusWithMatchExpression( temp.release() ); + } + + // DBRef value case + // A DBRef document under a $elemMatch should be treated as an object case + // because it may contain non-DBRef fields in addition to $ref, $id and $db. + + // object case + + StatusWithMatchExpression subRaw = _parse( obj, level ); + if ( !subRaw.isOK() ) + return subRaw; + std::auto_ptr sub( subRaw.getValue() ); + + // $where is not supported under $elemMatch because $where + // applies to top-level document, not array elements in a field. + if ( hasNode( sub.get(), MatchExpression::WHERE ) ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$elemMatch cannot contain $where expression" ); + } + + std::auto_ptr temp( new ElemMatchObjectMatchExpression() ); + Status status = temp->init( name, sub.release() ); + if ( !status.isOK() ) + return StatusWithMatchExpression( status ); + + return StatusWithMatchExpression( temp.release() ); + } + + StatusWithMatchExpression MatchExpressionParser::_parseAll( const char* name, + const BSONElement& e, + int level ) { + if ( e.type() != Array ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$all needs an array" ); + + BSONObj arr = e.Obj(); + std::auto_ptr myAnd( new AndMatchExpression() ); + BSONObjIterator i( arr ); + + if ( arr.firstElement().type() == Object && + mongoutils::str::equals( "$elemMatch", + arr.firstElement().Obj().firstElement().fieldName() ) ) { + // $all : [ { $elemMatch : {} } ... ] + + while ( i.more() ) { + BSONElement hopefullyElemMatchElement = i.next(); + + if ( hopefullyElemMatchElement.type() != Object ) { + // $all : [ { $elemMatch : ... }, 5 ] + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$all/$elemMatch has to be consistent" ); + } + + BSONObj hopefullyElemMatchObj = hopefullyElemMatchElement.Obj(); + if ( !mongoutils::str::equals( "$elemMatch", + hopefullyElemMatchObj.firstElement().fieldName() ) ) { + // $all : [ { $elemMatch : ... }, { x : 5 } ] + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$all/$elemMatch has to be consistent" ); + } + + StatusWithMatchExpression inner = + _parseElemMatch( name, hopefullyElemMatchObj.firstElement(), level ); + if ( !inner.isOK() ) + return inner; + myAnd->add( inner.getValue() ); + } + + return StatusWithMatchExpression( myAnd.release() ); + } + + while ( i.more() ) { + BSONElement e = i.next(); + + if ( e.type() == RegEx ) { + std::auto_ptr r( new RegexMatchExpression() ); + Status s = r->init( name, e ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + myAnd->add( r.release() ); + } + else if ( e.type() == Object && e.Obj().firstElement().getGtLtOp(-1) != -1 ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, "no $ expressions in $all" ); + } + else { + std::auto_ptr x( new EqualityMatchExpression() ); + Status s = x->init( name, e ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + myAnd->add( x.release() ); + } + } + + if ( myAnd->numChildren() == 0 ) { + return StatusWithMatchExpression( new FalseMatchExpression() ); + } + + return StatusWithMatchExpression( myAnd.release() ); + } + + // Geo + StatusWithMatchExpression expressionParserGeoCallbackDefault( const char* name, + int type, + const BSONObj& section ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, "geo not linked in" ); + } + + MatchExpressionParserGeoCallback expressionParserGeoCallback = + expressionParserGeoCallbackDefault; + + // Where + StatusWithMatchExpression expressionParserWhereCallbackDefault(const BSONElement& where) { + return StatusWithMatchExpression( ErrorCodes::BadValue, "$where not linked in" ); + } + + MatchExpressionParserWhereCallback expressionParserWhereCallback = + expressionParserWhereCallbackDefault; + + // Text + StatusWithMatchExpression expressionParserTextCallbackDefault( const BSONObj& queryObj ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, "$text not linked in" ); + } + + MatchExpressionParserTextCallback expressionParserTextCallback = + expressionParserTextCallbackDefault; + +} diff --git a/src/mongo/db/matcher/expression_parser.h b/src/mongo/db/matcher/expression_parser.h new file mode 100644 index 00000000000..b4c45e6cb81 --- /dev/null +++ b/src/mongo/db/matcher/expression_parser.h @@ -0,0 +1,157 @@ +// expression_parser.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/base/status_with.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/matcher/expression_tree.h" + +namespace mongo { + + typedef StatusWith StatusWithMatchExpression; + + class MatchExpressionParser { + public: + + /** + * caller has to maintain ownership obj + * the tree has views (BSONElement) into obj + */ + static StatusWithMatchExpression parse( const BSONObj& obj ) { + // The 0 initializes the match expression tree depth. + return _parse( obj, 0 ); + } + + private: + + /** + * 5 = false + * { a : 5 } = false + * { $lt : 5 } = true + * { $ref: "s", $id: "x" } = false + * { $ref: "s", $id: "x", $db: "mydb" } = false + * { $ref : "s" } = false (if incomplete DBRef is allowed) + * { $id : "x" } = false (if incomplete DBRef is allowed) + * { $db : "mydb" } = false (if incomplete DBRef is allowed) + */ + static bool _isExpressionDocument( const BSONElement& e, bool allowIncompleteDBRef ); + + /** + * { $ref: "s", $id: "x" } = true + * { $ref : "s" } = true (if incomplete DBRef is allowed) + * { $id : "x" } = true (if incomplete DBRef is allowed) + * { $db : "x" } = true (if incomplete DBRef is allowed) + */ + static bool _isDBRefDocument( const BSONObj& obj, bool allowIncompleteDBRef ); + + /** + * Parse 'obj' and return either a MatchExpression or an error. + * + * 'level' tracks the current depth of the tree across recursive calls to this + * function. Used in order to apply special logic at the top-level and to return an + * error if the tree exceeds the maximum allowed depth. + */ + static StatusWithMatchExpression _parse( const BSONObj& obj, int level ); + + /** + * parses a field in a sub expression + * if the query is { x : { $gt : 5, $lt : 8 } } + * e is { $gt : 5, $lt : 8 } + */ + static Status _parseSub( const char* name, + const BSONObj& obj, + AndMatchExpression* root, + int level ); + + /** + * parses a single field in a sub expression + * if the query is { x : { $gt : 5, $lt : 8 } } + * e is $gt : 5 + */ + static StatusWithMatchExpression _parseSubField( const BSONObj& context, + const AndMatchExpression* andSoFar, + const char* name, + const BSONElement& e, + int level ); + + static StatusWithMatchExpression _parseComparison( const char* name, + ComparisonMatchExpression* cmp, + const BSONElement& e ); + + static StatusWithMatchExpression _parseMOD( const char* name, + const BSONElement& e ); + + static StatusWithMatchExpression _parseRegexElement( const char* name, + const BSONElement& e ); + + static StatusWithMatchExpression _parseRegexDocument( const char* name, + const BSONObj& doc ); + + + static Status _parseArrayFilterEntries( ArrayFilterEntries* entries, + const BSONObj& theArray ); + + // arrays + + static StatusWithMatchExpression _parseElemMatch( const char* name, + const BSONElement& e, + int level ); + + static StatusWithMatchExpression _parseAll( const char* name, + const BSONElement& e, + int level ); + + // tree + + static Status _parseTreeList( const BSONObj& arr, ListOfMatchExpression* out, int level ); + + static StatusWithMatchExpression _parseNot( const char* name, + const BSONElement& e, + int level ); + + // The maximum allowed depth of a query tree. Just to guard against stack overflow. + static const int kMaximumTreeDepth; + }; + + typedef boost::function MatchExpressionParserGeoCallback; + extern MatchExpressionParserGeoCallback expressionParserGeoCallback; + + typedef boost::function MatchExpressionParserWhereCallback; + extern MatchExpressionParserWhereCallback expressionParserWhereCallback; + + typedef boost::function MatchExpressionParserTextCallback; + extern MatchExpressionParserTextCallback expressionParserTextCallback; + +} diff --git a/src/mongo/db/matcher/expression_parser_array_test.cpp b/src/mongo/db/matcher/expression_parser_array_test.cpp new file mode 100644 index 00000000000..e2818bb245b --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_array_test.cpp @@ -0,0 +1,647 @@ +// expression_parser_array_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_array.h" + +namespace mongo { + + TEST( MatchExpressionParserArrayTest, Size1 ) { + BSONObj query = BSON( "x" << BSON( "$size" << 2 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 << 3 ) ) ) ); + } + + TEST( MatchExpressionParserArrayTest, SizeAsString ) { + BSONObj query = BSON( "x" << BSON( "$size" << "a" ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSONArray() ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 ) ) ) ); + } + + TEST( MatchExpressionParserArrayTest, SizeWithDouble ) { + BSONObj query = BSON( "x" << BSON( "$size" << 2.5 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSONArray() ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 << 3 ) ) ) ); + } + + TEST( MatchExpressionParserArrayTest, SizeBad ) { + BSONObj query = BSON( "x" << BSON( "$size" << BSONNULL ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + // --------- + + TEST( MatchExpressionParserArrayTest, ElemMatchArr1 ) { + BSONObj query = BSON( "x" << BSON( "$elemMatch" << BSON( "x" << 1 << "y" << 2 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSON( "x" << 1 ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( BSON( "x" << 1 << "y" << 2 ) ) ) ) ); + + } + + TEST( MatchExpressionParserArrayTest, ElemMatchAnd ) { + BSONObj query = BSON( "x" << + BSON( "$elemMatch" << + BSON( "$and" << BSON_ARRAY( BSON( "x" << 1 << "y" << 2 ) ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSON( "x" << 1 ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( BSON( "x" << 1 << "y" << 2 ) ) ) ) ); + + } + + TEST( MatchExpressionParserArrayTest, ElemMatchNor ) { + BSONObj query = BSON( "x" << + BSON( "$elemMatch" << + BSON( "$nor" << BSON_ARRAY( BSON( "x" << 1 ) ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSON( "x" << 1 ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( BSON( "x" << 2 << "y" << 2 ) ) ) ) ); + + } + + TEST( MatchExpressionParserArrayTest, ElemMatchOr ) { + BSONObj query = BSON( "x" << + BSON( "$elemMatch" << + BSON( "$or" << BSON_ARRAY( BSON( "x" << 1 << "y" << 2 ) ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSON( "x" << 1 ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( BSON( "x" << 1 << "y" << 2 ) ) ) ) ); + + } + + TEST( MatchExpressionParserArrayTest, ElemMatchVal1 ) { + BSONObj query = BSON( "x" << BSON( "$elemMatch" << BSON( "$gt" << 5 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 4 ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 6 ) ) ) ); + } + + // with explicit $eq + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef1 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ); + OID oidx = OID::gen(); + BSONObj notMatch = BSON( "$ref" << "coll" << "$id" << oidx << "$db" << "db" ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << BSON( "$eq" << match ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + } + + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef2 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ); + OID oidx = OID::gen(); + BSONObj notMatch = BSON( "$ref" << "coll" << "$id" << oidx << "$db" << "db" ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << match ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + } + + // Additional fields after $ref and $id. + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef3 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 ); + OID oidx = OID::gen(); + BSONObj notMatch = BSON( "$ref" << "coll" << "$id" << oidx << "foo" << 12345 ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << match ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + + // Document contains fields not referred to in $elemMatch query. + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 << "bar" << 678 ) ) ) ) ); + } + + // Query with DBRef fields out of order. + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef4 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ); + BSONObj matchOutOfOrder = BSON( "$db" << "db" << "$id" << oid << "$ref" << "coll" ); + OID oidx = OID::gen(); + BSONObj notMatch = BSON( "$ref" << "coll" << "$id" << oidx << "$db" << "db" ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << matchOutOfOrder ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + } + + // Query with DBRef fields out of order. + // Additional fields besides $ref and $id. + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef5 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 ); + BSONObj matchOutOfOrder = BSON( "foo" << 12345 << "$id" << oid << "$ref" << "coll" ); + OID oidx = OID::gen(); + BSONObj notMatch = BSON( "$ref" << "coll" << "$id" << oidx << "foo" << 12345 ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << matchOutOfOrder ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + + // Document contains fields not referred to in $elemMatch query. + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 << "bar" << 678 ) ) ) ) ); + } + + // Incomplete DBRef - $id missing. + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef6 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 ); + BSONObj matchMissingID = BSON( "$ref" << "coll" << "foo" << 12345 ); + BSONObj notMatch = BSON( "$ref" << "collx" << "$id" << oid << "foo" << 12345 ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << matchMissingID ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + + // Document contains fields not referred to in $elemMatch query. + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 << "bar" << 678 ) ) ) ) ); + } + + // Incomplete DBRef - $ref missing. + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef7 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 ); + BSONObj matchMissingRef = BSON( "$id" << oid << "foo" << 12345 ); + OID oidx = OID::gen(); + BSONObj notMatch = BSON( "$ref" << "coll" << "$id" << oidx << "foo" << 12345 ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << matchMissingRef ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + + // Document contains fields not referred to in $elemMatch query. + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 << "bar" << 678 ) ) ) ) ); + } + + // Incomplete DBRef - $db only. + TEST( MatchExpressionParserArrayTest, ElemMatchDBRef8 ) { + OID oid = OID::gen(); + BSONObj match = BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" + << "foo" << 12345 ); + BSONObj matchDBOnly = BSON( "$db" << "db" << "foo" << 12345 ); + BSONObj notMatch = BSON( "$ref" << "coll" << "$id" << oid << "$db" << "dbx" + << "foo" << 12345 ); + + BSONObj query = BSON( "x" << BSON( "$elemMatch" << matchDBOnly ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( notMatch ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match ) ) ) ); + + // Document contains fields not referred to in $elemMatch query. + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" + << "foo" << 12345 << "bar" << 678 ) ) ) ) ); + } + + TEST( MatchExpressionParserArrayTest, All1 ) { + BSONObj query = BSON( "x" << BSON( "$all" << BSON_ARRAY( 1 << 2 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + // Verify that the $all got parsed to AND. + ASSERT_EQUALS( MatchExpression::AND, result.getValue()->matchType() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 2 ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 << 3 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 2 << 3 ) ) ) ); + } + + TEST( MatchExpressionParserArrayTest, AllNull ) { + BSONObj query = BSON( "x" << BSON( "$all" << BSON_ARRAY( BSONNULL ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + // Verify that the $all got parsed to AND. + ASSERT_EQUALS( MatchExpression::AND, result.getValue()->matchType() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSONNULL ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL ) ) ) ); + } + + TEST( MatchExpressionParserArrayTest, AllBadArg ) { + BSONObj query = BSON( "x" << BSON( "$all" << 1 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserArrayTest, AllBadRegexArg ) { + string tooLargePattern( 50 * 1000, 'z' ); + BSONObjBuilder allArray; + allArray.appendRegex( "0", tooLargePattern, "" ); + BSONObjBuilder operand; + operand.appendArray( "$all", allArray.obj() ); + + BSONObj query = BSON( "x" << operand.obj() ); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + + TEST( MatchExpressionParserArrayTest, AllRegex1 ) { + BSONObjBuilder allArray; + allArray.appendRegex( "0", "^a", "" ); + allArray.appendRegex( "1", "B", "i" ); + BSONObjBuilder all; + all.appendArray( "$all", allArray.obj() ); + BSONObj query = BSON( "a" << all.obj() ); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + // Verify that the $all got parsed to AND. + ASSERT_EQUALS( MatchExpression::AND, result.getValue()->matchType() ); + + BSONObj notMatchFirst = BSON( "a" << "ax" ); + BSONObj notMatchSecond = BSON( "a" << "qqb" ); + BSONObj matchesBoth = BSON( "a" << "ab" ); + + ASSERT( !result.getValue()->matchesSingleElement( notMatchFirst[ "a" ] ) ); + ASSERT( !result.getValue()->matchesSingleElement( notMatchSecond[ "a" ] ) ); + ASSERT( result.getValue()->matchesSingleElement( matchesBoth[ "a" ] ) ); + } + + TEST( MatchExpressionParserArrayTest, AllRegex2 ) { + BSONObjBuilder allArray; + allArray.appendRegex( "0", "^a", "" ); + allArray.append( "1", "abc" ); + BSONObjBuilder all; + all.appendArray( "$all", allArray.obj() ); + BSONObj query = BSON( "a" << all.obj() ); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + // Verify that the $all got parsed to AND. + ASSERT_EQUALS( MatchExpression::AND, result.getValue()->matchType() ); + + BSONObj notMatchFirst = BSON( "a" << "ax" ); + BSONObj matchesBoth = BSON( "a" << "abc" ); + + ASSERT( !result.getValue()->matchesSingleElement( notMatchFirst[ "a" ] ) ); + ASSERT( result.getValue()->matchesSingleElement( matchesBoth[ "a" ] ) ); + } + + TEST( MatchExpressionParserArrayTest, AllNonArray ) { + BSONObj query = BSON( "x" << BSON( "$all" << BSON_ARRAY( 5 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + // Verify that the $all got parsed to AND. + ASSERT_EQUALS( MatchExpression::AND, result.getValue()->matchType() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 5 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 4 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 4 ) ) ) ); + } + + + TEST( MatchExpressionParserArrayTest, AllElemMatch1 ) { + BSONObj internal = BSON( "x" << 1 << "y" << 2 ); + BSONObj query = BSON( "x" << BSON( "$all" << BSON_ARRAY( BSON( "$elemMatch" << internal ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + // Verify that the $all got parsed to an AND with a single ELEM_MATCH_OBJECT child. + ASSERT_EQUALS( MatchExpression::AND, result.getValue()->matchType() ); + ASSERT_EQUALS( 1U, result.getValue()->numChildren() ); + MatchExpression* child = result.getValue()->getChild( 0 ); + ASSERT_EQUALS( MatchExpression::ELEM_MATCH_OBJECT, child->matchType() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( 1 << 2 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSON( "x" << 1 ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( BSON( "x" << 1 << "y" << 2 ) ) ) ) ); + + } + + // $all and $elemMatch on dotted field. + // Top level field can be either document or array. + TEST( MatchExpressionParserArrayTest, AllElemMatch2 ) { + BSONObj internal = BSON( "z" << 1 ); + BSONObj query = BSON( "x.y" << BSON( "$all" << + BSON_ARRAY( BSON( "$elemMatch" << internal ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + // Verify that the $all got parsed to an AND with a single ELEM_MATCH_OBJECT child. + ASSERT_EQUALS( MatchExpression::AND, result.getValue()->matchType() ); + ASSERT_EQUALS( 1U, result.getValue()->numChildren() ); + MatchExpression* child = result.getValue()->getChild( 0 ); + ASSERT_EQUALS( MatchExpression::ELEM_MATCH_OBJECT, child->matchType() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON( "y" << 1 ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON( "y" << + BSON_ARRAY( 1 << 2 ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << + BSON( "y" << + BSON_ARRAY( BSON( "x" << 1 ) ) ) ) ) ); + // x is a document. Internal document does not contain z. + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << + BSON( "y" << + BSON_ARRAY( + BSON( "x" << 1 << "y" << 1 ) ) ) ) ) ); + // x is an array. Internal document does not contain z. + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( + BSON( "y" << + BSON_ARRAY( + BSON( "x" << 1 << "y" << 1 ) ) ) ) ) ) ); + // x is a document but y is not an array. + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << + BSON( "y" << + BSON( "x" << 1 << "z" << 1 ) ) ) ) ); + // x is an array but y is not an array. + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( + BSON( "y" << + BSON( "x" << 1 << "z" << 1 ) ) ) ) ) ); + // x is a document. + ASSERT( result.getValue()->matchesBSON( BSON( "x" << + BSON( "y" << + BSON_ARRAY( + BSON( "x" << 1 << "z" << 1 ) ) ) ) ) ); + // x is an array. + ASSERT( result.getValue()->matchesBSON( BSON( "x" << + BSON_ARRAY( + BSON( "y" << + BSON_ARRAY( + BSON( "x" << 1 << "z" << 1 ) ) ) ) ) ) ); + } + + // Check the structure of the resulting MatchExpression, and make sure that the paths + // are correct. + TEST( MatchExpressionParserArrayTest, AllElemMatch3 ) { + BSONObj query = fromjson( "{x: {$all: [{$elemMatch: {y: 1, z: 1}}]}}" ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + boost::scoped_ptr expr( result.getValue() ); + + // Root node should be an AND with one child. + ASSERT_EQUALS( MatchExpression::AND, expr->matchType() ); + ASSERT_EQUALS( 1U, expr->numChildren() ); + + // Child should be an ELEM_MATCH_OBJECT with one child and path "x". + MatchExpression* emObject = expr->getChild( 0 ); + ASSERT_EQUALS( MatchExpression::ELEM_MATCH_OBJECT, emObject->matchType() ); + ASSERT_EQUALS( 1U, emObject->numChildren() ); + ASSERT_EQUALS( "x", emObject->path().toString() ); + + // Child should be another AND with two children. + MatchExpression* and2 = emObject->getChild( 0 ); + ASSERT_EQUALS( MatchExpression::AND, and2->matchType() ); + ASSERT_EQUALS( 2U, and2->numChildren() ); + + // Both children should be equalites, with paths "y" and "z". + MatchExpression* leaf1 = and2->getChild( 0 ); + ASSERT_EQUALS( MatchExpression::EQ, leaf1->matchType() ); + ASSERT_EQUALS( 0U, leaf1->numChildren() ); + ASSERT_EQUALS( "y", leaf1->path().toString() ); + MatchExpression* leaf2 = and2->getChild( 1 ); + ASSERT_EQUALS( MatchExpression::EQ, leaf2->matchType() ); + ASSERT_EQUALS( 0U, leaf2->numChildren() ); + ASSERT_EQUALS( "z", leaf2->path().toString() ); + } + + TEST( MatchExpressionParserArrayTest, AllElemMatchBad ) { + BSONObj internal = BSON( "x" << 1 << "y" << 2 ); + + BSONObj query = BSON( "x" << BSON( "$all" << BSON_ARRAY( BSON( "$elemMatch" << internal ) << 5 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + query = BSON( "x" << BSON( "$all" << BSON_ARRAY( 5 << BSON( "$elemMatch" << internal ) ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + // You can't mix $elemMatch and regular equality inside $all. + TEST( MatchExpressionParserArrayTest, AllElemMatchBadMixed ) { + // $elemMatch first, equality second. + BSONObj bad1 = fromjson( "{x: {$all: [{$elemMatch: {y: 1}}, 3]}}" ); + StatusWithMatchExpression result1 = MatchExpressionParser::parse( bad1 ); + ASSERT_FALSE( result1.isOK() ); + + // equality first, $elemMatch second + BSONObj bad2 = fromjson( "{x: {$all: [3, {$elemMatch: {y: 1}}]}}" ); + StatusWithMatchExpression result2 = MatchExpressionParser::parse( bad2 ); + ASSERT_FALSE( result1.isOK() ); + + // $elemMatch first, object second + BSONObj bad3 = fromjson( "{x: {$all: [{$elemMatch: {y: 1}}, {z: 1}]}}" ); + StatusWithMatchExpression result3 = MatchExpressionParser::parse( bad3 ); + ASSERT_FALSE( result3.isOK() ); + + // object first, $elemMatch second + BSONObj bad4 = fromjson( "{x: {$all: [{z: 1}, {$elemMatch: {y: 1}}]}}" ); + StatusWithMatchExpression result4 = MatchExpressionParser::parse( bad4 ); + ASSERT_FALSE( result4.isOK() ); + } + + // $all with empty string. + TEST( MatchExpressionParserArrayTest, AllEmptyString ) { + BSONObj query = BSON( "x" << BSON( "$all" << BSON_ARRAY( "" ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "a" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << "a" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << "a" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSONArray() ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "" ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << "" ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << "" ) ) ) ); + } + + // $all with ISO date. + TEST( MatchExpressionParserArrayTest, AllISODate ) { + StatusWith matchResult = dateFromISOString("2014-12-31T00:00:00.000Z"); + ASSERT_TRUE( matchResult.isOK() ); + const Date_t& match = matchResult.getValue(); + StatusWith notMatchResult = dateFromISOString("2014-12-30T00:00:00.000Z"); + ASSERT_TRUE( notMatchResult.isOK() ); + const Date_t& notMatch = notMatchResult.getValue(); + + BSONObj query = BSON( "x" << BSON( "$all" << BSON_ARRAY( match ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << notMatch ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << + notMatch ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << + notMatch ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSONArray() ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << + match ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << + match ) ) ) ); + } + + // $all on array element with empty string. + TEST( MatchExpressionParserArrayTest, AllDottedEmptyString ) { + BSONObj query = BSON( "x.1" << BSON( "$all" << BSON_ARRAY( "" ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "a" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << "a" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << "a" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( "" << BSONNULL ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( "" << BSONObj() ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSONArray() ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "" ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << "" ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << "" ) ) ) ); + } + + // $all on array element with ISO date. + TEST( MatchExpressionParserArrayTest, AllDottedISODate ) { + StatusWith matchResult = dateFromISOString("2014-12-31T00:00:00.000Z"); + ASSERT_TRUE( matchResult.isOK() ); + const Date_t& match = matchResult.getValue(); + StatusWith notMatchResult = dateFromISOString("2014-12-30T00:00:00.000Z"); + ASSERT_TRUE( notMatchResult.isOK() ); + const Date_t& notMatch = notMatchResult.getValue(); + + BSONObj query = BSON( "x.1" << BSON( "$all" << BSON_ARRAY( match ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << notMatch ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << + notMatch ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << + notMatch ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match << + BSONNULL ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( match << + BSONObj() ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << BSONArray() ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << match ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONNULL << + match ) ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << BSON_ARRAY( BSONObj() << + match ) ) ) ); + } + +} diff --git a/src/mongo/db/matcher/expression_parser_geo.cpp b/src/mongo/db/matcher/expression_parser_geo.cpp new file mode 100644 index 00000000000..1e712a5e43e --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_geo.cpp @@ -0,0 +1,87 @@ +// expression_parser_geo.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/base/init.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + StatusWithMatchExpression expressionParserGeoCallbackReal( const char* name, + int type, + const BSONObj& section ) { + if (BSONObj::opWITHIN == type || BSONObj::opGEO_INTERSECTS == type) { + GeoQuery gq(name); + if ( !gq.parseFrom( section ) ) + return StatusWithMatchExpression(ErrorCodes::BadValue, + string("bad geo query: ") + section.toString()); + + auto_ptr e( new GeoMatchExpression() ); + + // Until the index layer accepts non-BSON predicates, or special indices are moved into + // stages, we have to clean up the raw object so it can be passed down to the index + // layer. + BSONObjBuilder bob; + bob.append(name, section); + Status s = e->init( name, gq, bob.obj() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( e.release() ); + } + else { + verify(BSONObj::opNEAR == type); + NearQuery nq(name); + Status s = nq.parseFrom( section ); + if ( !s.isOK() ) { + return StatusWithMatchExpression( s ); + } + auto_ptr e( new GeoNearMatchExpression() ); + // Until the index layer accepts non-BSON predicates, or special indices are moved into + // stages, we have to clean up the raw object so it can be passed down to the index + // layer. + BSONObjBuilder bob; + bob.append(name, section); + s = e->init( name, nq, bob.obj() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( e.release() ); + } + } + + MONGO_INITIALIZER( MatchExpressionParserGeo )( ::mongo::InitializerContext* context ) { + expressionParserGeoCallback = expressionParserGeoCallbackReal; + return Status::OK(); + } + +} diff --git a/src/mongo/db/matcher/expression_parser_geo_test.cpp b/src/mongo/db/matcher/expression_parser_geo_test.cpp new file mode 100644 index 00000000000..bb3bb158677 --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_geo_test.cpp @@ -0,0 +1,78 @@ +// expression_parser_geo_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_geo.h" + +namespace mongo { + + TEST( MatchExpressionParserGeo, WithinBox ) { + BSONObj query = fromjson("{a:{$within:{$box:[{x: 4, y:4},[6,6]]}}}"); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT(!result.getValue()->matchesBSON(fromjson("{a: [3,4]}"))); + ASSERT(result.getValue()->matchesBSON(fromjson("{a: [4,4]}"))); + ASSERT(result.getValue()->matchesBSON(fromjson("{a: [5,5]}"))); + ASSERT(result.getValue()->matchesBSON(fromjson("{a: [5,5.1]}"))); + ASSERT(result.getValue()->matchesBSON(fromjson("{a: {x: 5, y:5.1}}"))); + + } + + TEST( MatchExpressionParserGeoNear, ParseNear ) { + BSONObj query = fromjson("{loc:{$near:{$maxDistance:100, " + "$geometry:{type:\"Point\", coordinates:[0,0]}}}}"); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + MatchExpression* exp = result.getValue(); + ASSERT_EQUALS(MatchExpression::GEO_NEAR, exp->matchType()); + + GeoNearMatchExpression* gnexp = static_cast(exp); + ASSERT_EQUALS(gnexp->getData().maxDistance, 100); + } + + // $near must be the only field in the expression object. + TEST( MatchExpressionParserGeoNear, ParseNearExtraField ) { + BSONObj query = fromjson("{loc:{$near:{$maxDistance:100, " + "$geometry:{type:\"Point\", coordinates:[0,0]}}, foo: 1}}"); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } +} diff --git a/src/mongo/db/matcher/expression_parser_leaf_test.cpp b/src/mongo/db/matcher/expression_parser_leaf_test.cpp new file mode 100644 index 00000000000..594cd5d7309 --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_leaf_test.cpp @@ -0,0 +1,538 @@ +// expression_parser_leaf_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + TEST( MatchExpressionParserLeafTest, SimpleEQ2 ) { + BSONObj query = BSON( "x" << BSON( "$eq" << 2 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleEQUndefined ) { + BSONObj query = BSON( "x" << BSON( "$eq" << BSONUndefined ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, SimpleGT1 ) { + BSONObj query = BSON( "x" << BSON( "$gt" << 2 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleLT1 ) { + BSONObj query = BSON( "x" << BSON( "$lt" << 2 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleGTE1 ) { + BSONObj query = BSON( "x" << BSON( "$gte" << 2 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleLTE1 ) { + BSONObj query = BSON( "x" << BSON( "$lte" << 2 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleNE1 ) { + BSONObj query = BSON( "x" << BSON( "$ne" << 2 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleModBad1 ) { + BSONObj query = BSON( "x" << BSON( "$mod" << BSON_ARRAY( 3 << 2 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + query = BSON( "x" << BSON( "$mod" << BSON_ARRAY( 3 ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( !result.isOK() ); + + query = BSON( "x" << BSON( "$mod" << BSON_ARRAY( 3 << 2 << 4 ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( !result.isOK() ); + + query = BSON( "x" << BSON( "$mod" << BSON_ARRAY( "q" << 2 ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( !result.isOK() ); + + query = BSON( "x" << BSON( "$mod" << 3 ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( !result.isOK() ); + + query = BSON( "x" << BSON( "$mod" << BSON( "a" << 1 << "b" << 2 ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( !result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, SimpleMod1 ) { + BSONObj query = BSON( "x" << BSON( "$mod" << BSON_ARRAY( 3 << 2 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 4 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 8 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleModNotNumber ) { + BSONObj query = BSON( "x" << BSON( "$mod" << BSON_ARRAY( 2 << "r" ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 4 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "a" ) ) ); + } + + + TEST( MatchExpressionParserLeafTest, SimpleIN1 ) { + BSONObj query = BSON( "x" << BSON( "$in" << BSON_ARRAY( 2 << 3 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, INSingleDBRef ) { + OID oid = OID::gen(); + BSONObj query = + BSON( "x" << BSON( "$in" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + OID oidx = OID::gen(); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "coll" << "$id" << oidx << "$db" << "db" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$id" << oid << "$ref" << "coll" << "$db" << "db" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$id" << oid << "$ref" << "coll" << "$db" << "db" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$id" << oid << "$ref" << "coll" << "$db" << "db" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "coll" << "$id" << oid << "$db" << "dbx" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$db" << "db" << "$ref" << "coll" << "$id" << oid ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) << + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ) ); + } + + TEST( MatchExpressionParserLeafTest, INMultipleDBRef ) { + OID oid = OID::gen(); + OID oidy = OID::gen(); + BSONObj query = BSON( "x" << BSON( "$in" << BSON_ARRAY( + BSON( "$ref" << "colly" << "$id" << oidy << "$db" << "db" ) << + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + OID oidx = OID::gen(); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "coll" << "$id" << oidx << "$db" << "db" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$id" << oid << "$ref" << "coll" << "$db" << "db" ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oidy << "$db" << "db" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "colly" << "$id" << oid << "$db" << "db" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$id" << oid << "$ref" << "coll" << "$db" << "db" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "dbx" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$id" << oidy << "$ref" << "colly" << "$db" << "db" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) << + BSON( "$ref" << "coll" << "$id" << oidx << "$db" << "db" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) << + BSON( "$ref" << "colly" << "$id" << oidx << "$db" << "db" ) ) ) ) ); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) << + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "dbx" ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "colly" << "$id" << oidy << "$db" << "db" ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "colly" << "$id" << oidy << "$db" << "db" ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) << + BSON( "$ref" << "coll" << "$id" << oid << "$db" << "db" ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "collx" << "$id" << oidx << "$db" << "db" ) << + BSON( "$ref" << "colly" << "$id" << oidy << "$db" << "db" ) ) ) ) ); + } + + TEST( MatchExpressionParserLeafTest, INDBRefWithOptionalField1 ) { + OID oid = OID::gen(); + BSONObj query = + BSON( "x" << BSON( "$in" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + OID oidx = OID::gen(); + ASSERT( !result.getValue()->matchesBSON( + BSON( "x" << BSON( "$ref" << "coll" << "$id" << oidx << "$db" << "db" ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 ) ) ) ) ); + ASSERT( result.getValue()->matchesBSON( + BSON( "x" << BSON_ARRAY( + BSON( "$ref" << "collx" << "$id" << oidx << "foo" << 12345 ) << + BSON( "$ref" << "coll" << "$id" << oid << "foo" << 12345 ) ) ) ) ); + } + + TEST( MatchExpressionParserLeafTest, INInvalidDBRefs ) { + // missing $id + BSONObj query = BSON( "x" << BSON( "$in" << BSON_ARRAY( + BSON( "$ref" << "coll" ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + result = MatchExpressionParser::parse( query ); + + // second field is not $id + query = BSON( "x" << BSON( "$in" << BSON_ARRAY( + BSON( "$ref" << "coll" << + "$foo" << 1 ) ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + OID oid = OID::gen(); + + // missing $ref field + query = BSON( "x" << BSON( "$in" << BSON_ARRAY( + BSON( "$id" << oid << + "foo" << 3 ) ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + // missing $id and $ref field + query = BSON( "x" << BSON( "$in" << BSON_ARRAY( + BSON( "$db" << "test" << + "foo" << 3 ) ) ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + } + + TEST( MatchExpressionParserLeafTest, INExpressionDocument ) { + BSONObj query = BSON( "x" << BSON( "$in" << BSON_ARRAY( BSON( "$foo" << 1 ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, INNotArray ) { + BSONObj query = BSON( "x" << BSON( "$in" << 5 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, INUndefined ) { + BSONObj query = BSON( "x" << BSON( "$in" << BSON_ARRAY( BSONUndefined ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, INNotElemMatch ) { + BSONObj query = BSON( "x" << BSON( "$in" << BSON_ARRAY( BSON( "$elemMatch" << 1 ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, INRegexTooLong ) { + string tooLargePattern( 50 * 1000, 'z' ); + BSONObjBuilder inArray; + inArray.appendRegex( "0", tooLargePattern, "" ); + BSONObjBuilder operand; + operand.appendArray( "$in", inArray.obj() ); + BSONObj query = BSON( "x" << operand.obj() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, INRegexTooLong2 ) { + string tooLargePattern( 50 * 1000, 'z' ); + BSONObj query = BSON( "x" << BSON( "$in" << BSON_ARRAY( BSON( "$regex" << tooLargePattern ) ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, INRegexStuff ) { + BSONObjBuilder inArray; + inArray.appendRegex( "0", "^a", "" ); + inArray.appendRegex( "1", "B", "i" ); + inArray.append( "2", 4 ); + BSONObjBuilder operand; + operand.appendArray( "$in", inArray.obj() ); + + BSONObj query = BSON( "a" << operand.obj() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + BSONObj matchFirst = BSON( "a" << "ax" ); + BSONObj matchFirstRegex = BSONObjBuilder().appendRegex( "a", "^a", "" ).obj(); + BSONObj matchSecond = BSON( "a" << "qqb" ); + BSONObj matchSecondRegex = BSONObjBuilder().appendRegex( "a", "B", "i" ).obj(); + BSONObj matchThird = BSON( "a" << 4 ); + BSONObj notMatch = BSON( "a" << "l" ); + BSONObj notMatchRegex = BSONObjBuilder().appendRegex( "a", "B", "" ).obj(); + + ASSERT( result.getValue()->matchesBSON( matchFirst ) ); + ASSERT( result.getValue()->matchesBSON( matchFirstRegex ) ); + ASSERT( result.getValue()->matchesBSON( matchSecond ) ); + ASSERT( result.getValue()->matchesBSON( matchSecondRegex ) ); + ASSERT( result.getValue()->matchesBSON( matchThird ) ); + ASSERT( !result.getValue()->matchesBSON( notMatch ) ); + ASSERT( !result.getValue()->matchesBSON( notMatchRegex ) ); + } + + TEST( MatchExpressionParserLeafTest, SimpleNIN1 ) { + BSONObj query = BSON( "x" << BSON( "$nin" << BSON_ARRAY( 2 << 3 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, NINNotArray ) { + BSONObj query = BSON( "x" << BSON( "$nin" << 5 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + + TEST( MatchExpressionParserLeafTest, Regex1 ) { + BSONObjBuilder b; + b.appendRegex( "x", "abc", "i" ); + BSONObj query = b.obj(); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "abc" ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "ABC" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "AC" ) ) ); + } + + TEST( MatchExpressionParserLeafTest, Regex2 ) { + BSONObj query = BSON( "x" << BSON( "$regex" << "abc" << "$options" << "i" ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "abc" ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "ABC" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "AC" ) ) ); + } + + TEST( MatchExpressionParserLeafTest, Regex3 ) { + BSONObj query = BSON( "x" << BSON( "$options" << "i" << "$regex" << "abc" ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + log() << "result: " << result << endl; + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "abc" ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "ABC" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "AC" ) ) ); + } + + + TEST( MatchExpressionParserLeafTest, RegexBad ) { + BSONObj query = BSON( "x" << BSON( "$regex" << "abc" << "$optionas" << "i" ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + // $regex does not with numbers + query = BSON( "x" << BSON( "$regex" << 123 ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + query = BSON( "x" << BSON( "$regex" << BSON_ARRAY("abc") ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + query = BSON( "x" << BSON( "$optionas" << "i" ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + + query = BSON( "x" << BSON( "$options" << "i" ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, ExistsYes1 ) { + BSONObjBuilder b; + b.appendBool( "$exists", true ); + BSONObj query = BSON( "x" << b.obj() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "abc" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "y" << "AC" ) ) ); + } + + TEST( MatchExpressionParserLeafTest, ExistsNO1 ) { + BSONObjBuilder b; + b.appendBool( "$exists", false ); + BSONObj query = BSON( "x" << b.obj() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "abc" ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "y" << "AC" ) ) ); + } + + TEST( MatchExpressionParserLeafTest, Type1 ) { + BSONObj query = BSON( "x" << BSON( "$type" << String ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "abc" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, Type2 ) { + BSONObj query = BSON( "x" << BSON( "$type" << (double)NumberDouble ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 5.3 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, TypeDoubleOperator ) { + BSONObj query = BSON( "x" << BSON( "$type" << 1.5 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5.3 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, TypeNull ) { + BSONObj query = BSON( "x" << BSON( "$type" << jstNULL ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSONObj() ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + BSONObjBuilder b; + b.appendNull( "x" ); + ASSERT( result.getValue()->matchesBSON( b.obj() ) ); + } + + TEST( MatchExpressionParserLeafTest, TypeBadType ) { + BSONObjBuilder b; + b.append( "$type", ( JSTypeMax + 1 ) ); + BSONObj query = BSON( "x" << b.obj() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5.3 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 ) ) ); + } + + TEST( MatchExpressionParserLeafTest, TypeBad ) { + BSONObj query = BSON( "x" << BSON( "$type" << BSON( "x" << 1 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + +} diff --git a/src/mongo/db/matcher/expression_parser_test.cpp b/src/mongo/db/matcher/expression_parser_test.cpp new file mode 100644 index 00000000000..0512f83499b --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_test.cpp @@ -0,0 +1,109 @@ +// expression_parser_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + TEST( MatchExpressionParserTest, SimpleEQ1 ) { + BSONObj query = BSON( "x" << 2 ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + } + + TEST( MatchExpressionParserTest, Multiple1 ) { + BSONObj query = BSON( "x" << 5 << "y" << BSON( "$gt" << 5 << "$lt" << 8 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 5 << "y" << 7 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 5 << "y" << 6 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 6 << "y" << 7 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 << "y" << 9 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 5 << "y" << 4 ) ) ); + } + + TEST( AtomicMatchExpressionTest, Simple1 ) { + BSONObj query = BSON( "x" << 5 << "$atomic" << BSON( "$gt" << 5 << "$lt" << 8 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + query = BSON( "x" << 5 << "$isolated" << 1 ); + result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + query = BSON( "x" << 5 << "y" << BSON( "$isolated" << 1 ) ); + result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + StatusWith fib( int n ) { + if ( n < 0 ) return StatusWith( ErrorCodes::BadValue, "paramter to fib has to be >= 0" ); + if ( n <= 1 ) return StatusWith( 1 ); + StatusWith a = fib( n - 1 ); + StatusWith b = fib( n - 2 ); + if ( !a.isOK() ) return a; + if ( !b.isOK() ) return b; + return StatusWith( a.getValue() + b.getValue() ); + } + + TEST( StatusWithTest, Fib1 ) { + StatusWith x = fib( -2 ); + ASSERT( !x.isOK() ); + + x = fib(0); + ASSERT( x.isOK() ); + ASSERT( 1 == x.getValue() ); + + x = fib(1); + ASSERT( x.isOK() ); + ASSERT( 1 == x.getValue() ); + + x = fib(2); + ASSERT( x.isOK() ); + ASSERT( 2 == x.getValue() ); + + x = fib(3); + ASSERT( x.isOK() ); + ASSERT( 3 == x.getValue() ); + + + } +} diff --git a/src/mongo/db/matcher/expression_parser_text.cpp b/src/mongo/db/matcher/expression_parser_text.cpp new file mode 100644 index 00000000000..86b16e71130 --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_text.cpp @@ -0,0 +1,82 @@ +// expression_parser_text.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/base/init.h" +#include "mongo/db/fts/fts_language.h" +#include "mongo/db/fts/fts_spec.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/matcher/expression_text.h" + +namespace mongo { + + StatusWithMatchExpression expressionParserTextCallbackReal( const BSONObj& queryObj ) { + // Validate queryObj, but defer construction of FTSQuery (which requires access to the + // target namespace) until stage building time. + + if ( mongo::String != queryObj["$search"].type() ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, "$search needs a String" ); + } + + string language = ""; + BSONElement languageElt = queryObj["$language"]; + if ( !languageElt.eoo() ) { + if ( mongo::String != languageElt.type() ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$language needs a String" ); + } + language = languageElt.String(); + Status status = + fts::FTSLanguage::make( language, fts::TEXT_INDEX_VERSION_2 ).getStatus(); + if ( !status.isOK() ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, + "$language specifies unsupported language" ); + } + } + string query = queryObj["$search"].String(); + + if ( queryObj.nFields() != ( languageElt.eoo() ? 1 : 2 ) ) { + return StatusWithMatchExpression( ErrorCodes::BadValue, "extra fields in $text" ); + } + + auto_ptr e( new TextMatchExpression() ); + Status s = e->init( query, language ); + if ( !s.isOK() ) { + return StatusWithMatchExpression( s ); + } + return StatusWithMatchExpression( e.release() ); + } + + MONGO_INITIALIZER( MatchExpressionParserText )( ::mongo::InitializerContext* context ) { + expressionParserTextCallback = expressionParserTextCallbackReal; + return Status::OK(); + } + +} diff --git a/src/mongo/db/matcher/expression_parser_text_test.cpp b/src/mongo/db/matcher/expression_parser_text_test.cpp new file mode 100644 index 00000000000..b0d7166e5fe --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_text_test.cpp @@ -0,0 +1,62 @@ +// expression_parser_text_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_text.h" + +namespace mongo { + + TEST( MatchExpressionParserText, Parse1 ) { + BSONObj query = fromjson( "{$text:{$search:\"awesome\", $language:\"english\"}}" ); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + MatchExpression* exp = result.getValue(); + ASSERT_EQUALS( MatchExpression::TEXT, exp->matchType() ); + + TextMatchExpression* textExp = static_cast( exp ); + ASSERT_EQUALS( textExp->getQuery(), "awesome" ); + ASSERT_EQUALS( textExp->getLanguage(), "english" ); + } + + TEST( MatchExpressionParserText, Parse2 ) { + BSONObj query = fromjson( "{$text:{$search:\"awesome\", $language:\"spanglish\"}}" ); + + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } +} diff --git a/src/mongo/db/matcher/expression_parser_tree.cpp b/src/mongo/db/matcher/expression_parser_tree.cpp new file mode 100644 index 00000000000..04a25dac779 --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_tree.cpp @@ -0,0 +1,112 @@ +// expression_parser_tree.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/bson/bson-inl.h" +#include "mongo/db/matcher/expression_array.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/matcher/expression_tree.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + // static + const int MatchExpressionParser::kMaximumTreeDepth = 100; + + Status MatchExpressionParser::_parseTreeList( const BSONObj& arr, + ListOfMatchExpression* out, + int level ) { + if ( arr.isEmpty() ) + return Status( ErrorCodes::BadValue, + "$and/$or/$nor must be a nonempty array" ); + + BSONObjIterator i( arr ); + while ( i.more() ) { + BSONElement e = i.next(); + + if ( e.type() != Object ) + return Status( ErrorCodes::BadValue, + "$or/$and/$nor entries need to be full objects" ); + + StatusWithMatchExpression sub = _parse( e.Obj(), level ); + if ( !sub.isOK() ) + return sub.getStatus(); + + out->add( sub.getValue() ); + } + return Status::OK(); + } + + StatusWithMatchExpression MatchExpressionParser::_parseNot( const char* name, + const BSONElement& e, + int level ) { + if ( e.type() == RegEx ) { + StatusWithMatchExpression s = _parseRegexElement( name, e ); + if ( !s.isOK() ) + return s; + std::auto_ptr n( new NotMatchExpression() ); + Status s2 = n->init( s.getValue() ); + if ( !s2.isOK() ) + return StatusWithMatchExpression( s2 ); + return StatusWithMatchExpression( n.release() ); + } + + if ( e.type() != Object ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$not needs a regex or a document" ); + + BSONObj notObject = e.Obj(); + if ( notObject.isEmpty() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$not cannot be empty" ); + + std::auto_ptr theAnd( new AndMatchExpression() ); + Status s = _parseSub( name, notObject, theAnd.get(), level ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + // TODO: this seems arbitrary? + // tested in jstests/not2.js + for ( unsigned i = 0; i < theAnd->numChildren(); i++ ) + if ( theAnd->getChild(i)->matchType() == MatchExpression::REGEX ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "$not cannot have a regex" ); + + std::auto_ptr theNot( new NotMatchExpression() ); + s = theNot->init( theAnd.release() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + + return StatusWithMatchExpression( theNot.release() ); + } + +} diff --git a/src/mongo/db/matcher/expression_parser_tree_test.cpp b/src/mongo/db/matcher/expression_parser_tree_test.cpp new file mode 100644 index 00000000000..aba286fb4c3 --- /dev/null +++ b/src/mongo/db/matcher/expression_parser_tree_test.cpp @@ -0,0 +1,188 @@ +// expression_parser_tree_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/matcher/expression_parser.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + TEST( MatchExpressionParserTreeTest, OR1 ) { + BSONObj query = BSON( "$or" << BSON_ARRAY( BSON( "x" << 1 ) << + BSON( "y" << 2 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "y" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "y" << 1 ) ) ); + } + + TEST( MatchExpressionParserTreeTest, OREmbedded ) { + BSONObj query1 = BSON( "$or" << BSON_ARRAY( BSON( "x" << 1 ) << + BSON( "y" << 2 ) ) ); + BSONObj query2 = BSON( "$or" << BSON_ARRAY( query1 ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query2 ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "y" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "y" << 1 ) ) ); + } + + + TEST( MatchExpressionParserTreeTest, AND1 ) { + BSONObj query = BSON( "$and" << BSON_ARRAY( BSON( "x" << 1 ) << + BSON( "y" << 2 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "y" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "y" << 1 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 1 << "y" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 2 << "y" << 2 ) ) ); + } + + TEST( MatchExpressionParserTreeTest, NOREmbedded ) { + BSONObj query = BSON( "$nor" << BSON_ARRAY( BSON( "x" << 1 ) << + BSON( "y" << 2 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 1 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "y" << 2 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 3 ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "y" << 1 ) ) ); + } + + TEST( MatchExpressionParserTreeTest, NOT1 ) { + BSONObj query = BSON( "x" << BSON( "$not" << BSON( "$gt" << 5 ) ) ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( result.getValue()->matchesBSON( BSON( "x" << 2 ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << 8 ) ) ); + } + + // Test a deep match tree that is not deep enough to hit the maximum depth limit. + TEST( MatchExpressionParserTreeTest, MaximumTreeDepthNotExceed ) { + static const int depth = 60; + + std::stringstream ss; + for (int i = 0; i < depth/2; i++) { + ss << "{$and: [{a: 3}, {$or: [{b: 2},"; + } + ss << "{b: 4}"; + for (int i = 0; i < depth/2; i++) { + ss << "]}]}"; + } + + BSONObj query = fromjson( ss.str() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT( result.isOK() ); + } + + // Test a tree that exceeds the maximum depth limit. + TEST( MatchExpressionParserTreeTest, MaximumTreeDepthExceed ) { + static const int depth = 105; + + std::stringstream ss; + for (int i = 0; i < depth/2; i++) { + ss << "{$and: [{a: 3}, {$or: [{b: 2},"; + } + ss << "{b: 4}"; + for (int i = 0; i < depth/2; i++) { + ss << "]}]}"; + } + + BSONObj query = fromjson( ss.str() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + // We should also exceed the depth limit through deeply nested $not. + TEST( MatchExpressionParserTreeTest, MaximumTreeDepthExceededNestedNots ) { + static const int depth = 105; + + std::stringstream ss; + ss << "{a: "; + for (int i = 0; i < depth; i++) { + ss << "{$not: "; + } + ss << "{$eq: 5}"; + for (int i = 0; i < depth+1; i++) { + ss << "}"; + } + + BSONObj query = fromjson( ss.str() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + // Depth limit with nested $elemMatch object. + TEST( MatchExpressionParserTreeTest, MaximumTreeDepthExceededNestedElemMatch ) { + static const int depth = 105; + + std::stringstream ss; + for (int i = 0; i < depth; i++) { + ss << "{a: {$elemMatch: "; + } + ss << "{b: 5}"; + for (int i = 0; i < depth; i++) { + ss << "}}"; + } + + BSONObj query = fromjson( ss.str() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_FALSE( result.isOK() ); + } + + TEST( MatchExpressionParserLeafTest, NotRegex1 ) { + BSONObjBuilder b; + b.appendRegex( "$not", "abc", "i" ); + BSONObj query = BSON( "x" << b.obj() ); + StatusWithMatchExpression result = MatchExpressionParser::parse( query ); + ASSERT_TRUE( result.isOK() ); + + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "abc" ) ) ); + ASSERT( !result.getValue()->matchesBSON( BSON( "x" << "ABC" ) ) ); + ASSERT( result.getValue()->matchesBSON( BSON( "x" << "AC" ) ) ); + } + +} diff --git a/src/mongo/db/matcher/expression_test.cpp b/src/mongo/db/matcher/expression_test.cpp new file mode 100644 index 00000000000..62b4631a717 --- /dev/null +++ b/src/mongo/db/matcher/expression_test.cpp @@ -0,0 +1,107 @@ +// expression_test.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** Unit tests for MatchMatchExpression operator implementations in match_operators.{h,cpp}. */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + TEST( MatchExpressionTest, Parse1 ) { + //TreeMatchExpression* e = NULL; + //Status s = MatchExpression::parse( BSON( "x" << 1 ), &e ); + //ASSERT_TRUE( s.isOK() ); + } + + TEST( LeafMatchExpressionTest, Equal1 ) { + BSONObj temp = BSON( "x" << 5 ); + EqualityMatchExpression e; + e.init( "x", temp["x"] ); + + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : 5 }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : [5] }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : [1,5] }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : [1,5,2] }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : [5,2] }" ) ) ); + + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : null }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 6 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : [4,2] }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : [[5]] }" ) ) ); + } + + TEST( LeafMatchExpressionTest, Comp1 ) { + BSONObj temp = BSON( "x" << 5 ); + + { + LTEMatchExpression e; + e.init( "x", temp["x"] ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : 5 }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : 4 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 6 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 'eliot' }" ) ) ); + } + + { + LTMatchExpression e; + e.init( "x", temp["x"] ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 5 }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : 4 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 6 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 'eliot' }" ) ) ); + } + + { + GTEMatchExpression e; + e.init( "x", temp["x"] ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : 5 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 4 }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : 6 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 'eliot' }" ) ) ); + } + + { + GTMatchExpression e; + e.init( "x", temp["x"] ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 5 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 4 }" ) ) ); + ASSERT_TRUE( e.matchesBSON( fromjson( "{ x : 6 }" ) ) ); + ASSERT_FALSE( e.matchesBSON( fromjson( "{ x : 'eliot' }" ) ) ); + } + + + } + +} diff --git a/src/mongo/db/matcher/expression_text.cpp b/src/mongo/db/matcher/expression_text.cpp new file mode 100644 index 00000000000..f81c3dc1fa3 --- /dev/null +++ b/src/mongo/db/matcher/expression_text.cpp @@ -0,0 +1,88 @@ +// expression_text.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" +#include "mongo/db/matcher/expression_text.h" + +namespace mongo { + + Status TextMatchExpression::init( const string& query, const string& language ) { + _query = query; + _language = language; + return initPath( "_fts" ); + } + + bool TextMatchExpression::matchesSingleElement( const BSONElement& e ) const { + // See ops/update.cpp. + // This node is removed by the query planner. It's only ever called if we're getting an + // elemMatchKey. + return true; + } + + void TextMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace(debug, level); + debug << "TEXT : query=" << _query << ", language=" << _language << ", tag="; + MatchExpression::TagData* td = getTag(); + if ( NULL != td ) { + td->debugString( &debug ); + } + else { + debug << "NULL"; + } + debug << "\n"; + } + + bool TextMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) { + return false; + } + const TextMatchExpression* realOther = static_cast( other ); + + // TODO This is way too crude. It looks for string equality, but it should be looking for + // common parsed form + if ( realOther->getQuery() != _query ) { + return false; + } + if ( realOther->getLanguage() != _language ) { + return false; + } + return true; + } + + LeafMatchExpression* TextMatchExpression::shallowClone() const { + TextMatchExpression* next = new TextMatchExpression(); + next->init( _query, _language ); + if ( getTag() ) { + next->setTag( getTag()->clone() ); + } + return next; + } + +} diff --git a/src/mongo/db/matcher/expression_text.h b/src/mongo/db/matcher/expression_text.h new file mode 100644 index 00000000000..5842cdc3293 --- /dev/null +++ b/src/mongo/db/matcher/expression_text.h @@ -0,0 +1,61 @@ +// expression_text.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/fts/fts_query.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + class TextMatchExpression : public LeafMatchExpression { + public: + TextMatchExpression() : LeafMatchExpression( TEXT ) {} + virtual ~TextMatchExpression() {} + + Status init( const std::string& query, const std::string& language ); + + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + virtual bool equivalent( const MatchExpression* other ) const; + + virtual LeafMatchExpression* shallowClone() const; + + const string& getQuery() const { return _query; } + const string& getLanguage() const { return _language; } + private: + std::string _query; + std::string _language; + }; + +} // namespace mongo diff --git a/src/mongo/db/matcher/expression_tree.cpp b/src/mongo/db/matcher/expression_tree.cpp new file mode 100644 index 00000000000..6b7304b024a --- /dev/null +++ b/src/mongo/db/matcher/expression_tree.cpp @@ -0,0 +1,171 @@ +// expression_tree.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression_tree.h" + +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/util/log.h" + +namespace mongo { + + ListOfMatchExpression::~ListOfMatchExpression() { + for ( unsigned i = 0; i < _expressions.size(); i++ ) + delete _expressions[i]; + _expressions.clear(); + } + + void ListOfMatchExpression::add( MatchExpression* e ) { + verify( e ); + _expressions.push_back( e ); + } + + + void ListOfMatchExpression::_debugList( StringBuilder& debug, int level ) const { + for ( unsigned i = 0; i < _expressions.size(); i++ ) + _expressions[i]->debugString( debug, level + 1 ); + } + + bool ListOfMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + const ListOfMatchExpression* realOther = static_cast( other ); + + if ( _expressions.size() != realOther->_expressions.size() ) + return false; + + // TOOD: order doesn't matter + for ( unsigned i = 0; i < _expressions.size(); i++ ) + if ( !_expressions[i]->equivalent( realOther->_expressions[i] ) ) + return false; + + return true; + } + + // ----- + + bool AndMatchExpression::matches( const MatchableDocument* doc, MatchDetails* details ) const { + for ( size_t i = 0; i < numChildren(); i++ ) { + if ( !getChild(i)->matches( doc, details ) ) { + if ( details ) + details->resetOutput(); + return false; + } + } + return true; + } + + bool AndMatchExpression::matchesSingleElement( const BSONElement& e ) const { + for ( size_t i = 0; i < numChildren(); i++ ) { + if ( !getChild(i)->matchesSingleElement( e ) ) { + return false; + } + } + return true; + } + + + void AndMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "$and\n"; + _debugList( debug, level ); + } + + // ----- + + bool OrMatchExpression::matches( const MatchableDocument* doc, MatchDetails* details ) const { + for ( size_t i = 0; i < numChildren(); i++ ) { + if ( getChild(i)->matches( doc, NULL ) ) { + return true; + } + } + return false; + } + + bool OrMatchExpression::matchesSingleElement( const BSONElement& e ) const { + for ( size_t i = 0; i < numChildren(); i++ ) { + if ( getChild(i)->matchesSingleElement( e ) ) { + return true; + } + } + return false; + } + + + void OrMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "$or\n"; + _debugList( debug, level ); + } + + // ---- + + bool NorMatchExpression::matches( const MatchableDocument* doc, MatchDetails* details ) const { + for ( size_t i = 0; i < numChildren(); i++ ) { + if ( getChild(i)->matches( doc, NULL ) ) { + return false; + } + } + return true; + } + + bool NorMatchExpression::matchesSingleElement( const BSONElement& e ) const { + for ( size_t i = 0; i < numChildren(); i++ ) { + if ( getChild(i)->matchesSingleElement( e ) ) { + return false; + } + } + return true; + } + + void NorMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "$nor\n"; + _debugList( debug, level ); + } + + // ------- + + void NotMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "$not\n"; + _exp->debugString( debug, level + 1 ); + } + + bool NotMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + + return _exp->equivalent( other->getChild(0) ); + } + +} diff --git a/src/mongo/db/matcher/expression_tree.h b/src/mongo/db/matcher/expression_tree.h new file mode 100644 index 00000000000..9105b0d2b79 --- /dev/null +++ b/src/mongo/db/matcher/expression_tree.h @@ -0,0 +1,186 @@ +// expression_tree.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/matcher/expression.h" + +#include + +/** + * this contains all Expessions that define the structure of the tree + * they do not look at the structure of the documents themselves, just combine other things + */ +namespace mongo { + + class ListOfMatchExpression : public MatchExpression { + public: + ListOfMatchExpression( MatchType type ) : MatchExpression( type ){} + virtual ~ListOfMatchExpression(); + + /** + * @param e - I take ownership + */ + void add( MatchExpression* e ); + + /** + * clears all the thingsd we own, and does NOT delete + * someone else has taken ownership + */ + void clearAndRelease() { _expressions.clear(); } + + virtual size_t numChildren() const { return _expressions.size(); } + + virtual MatchExpression* getChild( size_t i ) const { return _expressions[i]; } + + virtual std::vector* getChildVector() { return &_expressions; } + + bool equivalent( const MatchExpression* other ) const; + + protected: + void _debugList( StringBuilder& debug, int level ) const; + + private: + std::vector< MatchExpression* > _expressions; + }; + + class AndMatchExpression : public ListOfMatchExpression { + public: + AndMatchExpression() : ListOfMatchExpression( AND ){} + virtual ~AndMatchExpression(){} + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const; + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual MatchExpression* shallowClone() const { + AndMatchExpression* self = new AndMatchExpression(); + for (size_t i = 0; i < numChildren(); ++i) { + self->add(getChild(i)->shallowClone()); + } + if ( getTag() ) { + self->setTag(getTag()->clone()); + } + return self; + } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + }; + + class OrMatchExpression : public ListOfMatchExpression { + public: + OrMatchExpression() : ListOfMatchExpression( OR ){} + virtual ~OrMatchExpression(){} + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const; + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual MatchExpression* shallowClone() const { + OrMatchExpression* self = new OrMatchExpression(); + for (size_t i = 0; i < numChildren(); ++i) { + self->add(getChild(i)->shallowClone()); + } + if ( getTag() ) { + self->setTag(getTag()->clone()); + } + return self; + } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + }; + + class NorMatchExpression : public ListOfMatchExpression { + public: + NorMatchExpression() : ListOfMatchExpression( NOR ){} + virtual ~NorMatchExpression(){} + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const; + virtual bool matchesSingleElement( const BSONElement& e ) const; + + virtual MatchExpression* shallowClone() const { + NorMatchExpression* self = new NorMatchExpression(); + for (size_t i = 0; i < numChildren(); ++i) { + self->add(getChild(i)->shallowClone()); + } + if ( getTag() ) { + self->setTag(getTag()->clone()); + } + return self; + } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + }; + + class NotMatchExpression : public MatchExpression { + public: + NotMatchExpression() : MatchExpression( NOT ){} + NotMatchExpression( MatchExpression* e ) : MatchExpression( NOT ), _exp( e ){} + /** + * @param exp - I own it, and will delete + */ + virtual Status init( MatchExpression* exp ) { + _exp.reset( exp ); + return Status::OK(); + } + + virtual MatchExpression* shallowClone() const { + NotMatchExpression* self = new NotMatchExpression(); + MatchExpression* child = _exp->shallowClone(); + self->init(child); + if ( getTag() ) { + self->setTag(getTag()->clone()); + } + return self; + } + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const { + return !_exp->matches( doc, NULL ); + } + + virtual bool matchesSingleElement( const BSONElement& e ) const { + return !_exp->matchesSingleElement( e ); + } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + bool equivalent( const MatchExpression* other ) const; + + virtual size_t numChildren() const { return 1; } + + virtual MatchExpression* getChild( size_t i ) const { return _exp.get(); } + + MatchExpression* releaseChild(void) { return _exp.release(); } + + void resetChild( MatchExpression* newChild) { _exp.reset(newChild); } + + private: + std::auto_ptr _exp; + }; + +} diff --git a/src/mongo/db/matcher/expression_tree_test.cpp b/src/mongo/db/matcher/expression_tree_test.cpp new file mode 100644 index 00000000000..c0ba4cead87 --- /dev/null +++ b/src/mongo/db/matcher/expression_tree_test.cpp @@ -0,0 +1,571 @@ +/** + * Copyright (C) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** Unit tests for MatchMatchExpression operator implementations in match_operators.{h,cpp}. */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_tree.h" +#include "mongo/db/matcher/expression_leaf.h" + +namespace mongo { + + TEST( NotMatchExpression, MatchesScalar ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + auto_ptr lt( new LTMatchExpression() ); + ASSERT( lt->init( "a", baseOperand[ "$lt" ] ).isOK() ); + NotMatchExpression notOp; + ASSERT( notOp.init( lt.release() ).isOK() ); + ASSERT( notOp.matchesBSON( BSON( "a" << 6 ), NULL ) ); + ASSERT( !notOp.matchesBSON( BSON( "a" << 4 ), NULL ) ); + } + + TEST( NotMatchExpression, MatchesArray ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + auto_ptr lt( new LTMatchExpression() ); + ASSERT( lt->init( "a", baseOperand[ "$lt" ] ).isOK() ); + NotMatchExpression notOp; + ASSERT( notOp.init( lt.release() ).isOK() ); + ASSERT( notOp.matchesBSON( BSON( "a" << BSON_ARRAY( 6 ) ), NULL ) ); + ASSERT( !notOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 ) ), NULL ) ); + // All array elements must match. + ASSERT( !notOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5 << 6 ) ), NULL ) ); + } + + TEST( NotMatchExpression, ElemMatchKey ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + auto_ptr lt( new LTMatchExpression() ); + ASSERT( lt->init( "a", baseOperand[ "$lt" ] ).isOK() ); + NotMatchExpression notOp; + ASSERT( notOp.init( lt.release() ).isOK() ); + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !notOp.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( notOp.matchesBSON( BSON( "a" << 6 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( notOp.matchesBSON( BSON( "a" << BSON_ARRAY( 6 ) ), &details ) ); + // elemMatchKey is not implemented for negative match operators. + ASSERT( !details.hasElemMatchKey() ); + } + /* + TEST( NotMatchExpression, MatchesIndexKey ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + auto_ptr lt( new ComparisonMatchExpression() ); + ASSERT( lt->init( "a", baseOperand[ "$lt" ] ).isOK() ); + NotMatchExpression notOp; + ASSERT( notOp.init( lt.release() ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + BSONObj indexKey = BSON( "" << "7" ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + notOp.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + + /** + TEST( AndOp, MatchesElementSingleClause ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + BSONObj match = BSON( "a" << 4 ); + BSONObj notMatch = BSON( "a" << 5 ); + auto_ptr lt( new ComparisonMatchExpression() ); + ASSERT( lt->init( "", baseOperand[ "$lt" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( lt.release() ); + AndOp andOp; + ASSERT( andOp.init( &subMatchExpressions ).isOK() ); + ASSERT( andOp.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !andOp.matchesSingleElement( notMatch[ "a" ] ) ); + } + */ + + TEST( AndOp, NoClauses ) { + AndMatchExpression andMatchExpression; + ASSERT( andMatchExpression.matchesBSON( BSONObj(), NULL ) ); + } + + TEST( AndOp, MatchesElementThreeClauses ) { + BSONObj baseOperand1 = BSON( "$lt" << "z1" ); + BSONObj baseOperand2 = BSON( "$gt" << "a1" ); + BSONObj match = BSON( "a" << "r1" ); + BSONObj notMatch1 = BSON( "a" << "z1" ); + BSONObj notMatch2 = BSON( "a" << "a1" ); + BSONObj notMatch3 = BSON( "a" << "r" ); + + auto_ptr sub1( new LTMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$lt" ] ).isOK() ); + auto_ptr sub2( new GTMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$gt" ] ).isOK() ); + auto_ptr sub3( new RegexMatchExpression() ); + ASSERT( sub3->init( "a", "1", "" ).isOK() ); + + AndMatchExpression andOp; + andOp.add( sub1.release() ); + andOp.add( sub2.release() ); + andOp.add( sub3.release() ); + + ASSERT( andOp.matchesBSON( match ) ); + ASSERT( !andOp.matchesBSON( notMatch1 ) ); + ASSERT( !andOp.matchesBSON( notMatch2 ) ); + ASSERT( !andOp.matchesBSON( notMatch3 ) ); + } + + TEST( AndOp, MatchesSingleClause ) { + BSONObj baseOperand = BSON( "$ne" << 5 ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "a", baseOperand[ "$ne" ] ).isOK() ); + auto_ptr ne( new NotMatchExpression() ); + ASSERT( ne->init( eq.release() ).isOK() ); + + AndMatchExpression andOp; + andOp.add( ne.release() ); + + ASSERT( andOp.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( andOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 6 ) ), NULL ) ); + ASSERT( !andOp.matchesBSON( BSON( "a" << 5 ), NULL ) ); + ASSERT( !andOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5 ) ), NULL ) ); + } + + TEST( AndOp, MatchesThreeClauses ) { + BSONObj baseOperand1 = BSON( "$gt" << 1 ); + BSONObj baseOperand2 = BSON( "$lt" << 10 ); + BSONObj baseOperand3 = BSON( "$lt" << 100 ); + + auto_ptr sub1( new GTMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$gt" ] ).isOK() ); + + auto_ptr sub2( new LTMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$lt" ] ).isOK() ); + + auto_ptr sub3( new LTMatchExpression() ); + ASSERT( sub3->init( "b", baseOperand3[ "$lt" ] ).isOK() ); + + AndMatchExpression andOp; + andOp.add( sub1.release() ); + andOp.add( sub2.release() ); + andOp.add( sub3.release() ); + + ASSERT( andOp.matchesBSON( BSON( "a" << 5 << "b" << 6 ), NULL ) ); + ASSERT( !andOp.matchesBSON( BSON( "a" << 5 ), NULL ) ); + ASSERT( !andOp.matchesBSON( BSON( "b" << 6 ), NULL ) ); + ASSERT( !andOp.matchesBSON( BSON( "a" << 1 << "b" << 6 ), NULL ) ); + ASSERT( !andOp.matchesBSON( BSON( "a" << 10 << "b" << 6 ), NULL ) ); + } + + TEST( AndOp, ElemMatchKey ) { + BSONObj baseOperand1 = BSON( "a" << 1 ); + BSONObj baseOperand2 = BSON( "b" << 2 ); + + auto_ptr sub1( new EqualityMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "a" ] ).isOK() ); + + auto_ptr sub2( new EqualityMatchExpression() ); + ASSERT( sub2->init( "b", baseOperand2[ "b" ] ).isOK() ); + + AndMatchExpression andOp; + andOp.add( sub1.release() ); + andOp.add( sub2.release() ); + + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !andOp.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( !andOp.matchesBSON( BSON( "b" << BSON_ARRAY( 2 ) ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( andOp.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) << "b" << BSON_ARRAY( 1 << 2 ) ), + &details ) ); + ASSERT( details.hasElemMatchKey() ); + // The elem match key for the second $and clause is recorded. + ASSERT_EQUALS( "1", details.elemMatchKey() ); + } + + /** + TEST( AndOp, MatchesIndexKeyWithoutUnknown ) { + BSONObj baseOperand1 = BSON( "$gt" << 1 ); + BSONObj baseOperand2 = BSON( "$lt" << 5 ); + auto_ptr sub1( new ComparisonMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$gt" ] ).isOK() ); + auto_ptr sub2( new ComparisonMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$lt" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( sub1.release() ); + subMatchExpressions.mutableVector().push_back( sub2.release() ); + AndOp andOp; + ASSERT( andOp.init( &subMatchExpressions ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + andOp.matchesIndexKey( BSON( "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + andOp.matchesIndexKey( BSON( "" << 0 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + andOp.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + } + + TEST( AndOp, MatchesIndexKeyWithUnknown ) { + BSONObj baseOperand1 = BSON( "$gt" << 1 ); + BSONObj baseOperand2 = BSON( "$lt" << 5 ); + // This part will return PartialMatchResult_Unknown. + BSONObj baseOperand3 = BSON( "$ne" << 5 ); + auto_ptr sub1( new ComparisonMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$gt" ] ).isOK() ); + auto_ptr sub2( new ComparisonMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$lt" ] ).isOK() ); + auto_ptr sub3( new NeOp() ); + ASSERT( sub3->init( "a", baseOperand3[ "$ne" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( sub1.release() ); + subMatchExpressions.mutableVector().push_back( sub2.release() ); + subMatchExpressions.mutableVector().push_back( sub3.release() ); + AndOp andOp; + ASSERT( andOp.init( &subMatchExpressions ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + andOp.matchesIndexKey( BSON( "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + andOp.matchesIndexKey( BSON( "" << 0 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + andOp.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + } + */ + + /** + TEST( OrOp, MatchesElementSingleClause ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + BSONObj match = BSON( "a" << 4 ); + BSONObj notMatch = BSON( "a" << 5 ); + auto_ptr lt( new ComparisonMatchExpression() ); + ASSERT( lt->init( "a", baseOperand[ "$lt" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( lt.release() ); + OrOp orOp; + ASSERT( orOp.init( &subMatchExpressions ).isOK() ); + ASSERT( orOp.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !orOp.matchesSingleElement( notMatch[ "a" ] ) ); + } + */ + + TEST( OrOp, NoClauses ) { + OrMatchExpression orOp; + ASSERT( !orOp.matchesBSON( BSONObj(), NULL ) ); + } + /* + TEST( OrOp, MatchesElementThreeClauses ) { + BSONObj baseOperand1 = BSON( "$lt" << 0 ); + BSONObj baseOperand2 = BSON( "$gt" << 10 ); + BSONObj baseOperand3 = BSON( "a" << 5 ); + BSONObj match1 = BSON( "a" << -1 ); + BSONObj match2 = BSON( "a" << 11 ); + BSONObj match3 = BSON( "a" << 5 ); + BSONObj notMatch = BSON( "a" << "6" ); + auto_ptr sub1( new ComparisonMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$lt" ] ).isOK() ); + auto_ptr sub2( new ComparisonMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$gt" ] ).isOK() ); + auto_ptr sub3( new ComparisonMatchExpression() ); + ASSERT( sub3->init( "a", baseOperand3[ "a" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( sub1.release() ); + subMatchExpressions.mutableVector().push_back( sub2.release() ); + subMatchExpressions.mutableVector().push_back( sub3.release() ); + OrOp orOp; + ASSERT( orOp.init( &subMatchExpressions ).isOK() ); + ASSERT( orOp.matchesSingleElement( match1[ "a" ] ) ); + ASSERT( orOp.matchesSingleElement( match2[ "a" ] ) ); + ASSERT( orOp.matchesSingleElement( match3[ "a" ] ) ); + ASSERT( !orOp.matchesSingleElement( notMatch[ "a" ] ) ); + } + */ + TEST( OrOp, MatchesSingleClause ) { + BSONObj baseOperand = BSON( "$ne" << 5 ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "a", baseOperand[ "$ne" ] ).isOK() ); + auto_ptr ne( new NotMatchExpression() ); + ASSERT( ne->init( eq.release() ).isOK() ); + + OrMatchExpression orOp; + orOp.add( ne.release() ); + + ASSERT( orOp.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( orOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 6 ) ), NULL ) ); + ASSERT( !orOp.matchesBSON( BSON( "a" << 5 ), NULL ) ); + ASSERT( !orOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5 ) ), NULL ) ); + } + + TEST( OrOp, MatchesThreeClauses ) { + BSONObj baseOperand1 = BSON( "$gt" << 10 ); + BSONObj baseOperand2 = BSON( "$lt" << 0 ); + BSONObj baseOperand3 = BSON( "b" << 100 ); + auto_ptr sub1( new GTMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$gt" ] ).isOK() ); + auto_ptr sub2( new LTMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$lt" ] ).isOK() ); + auto_ptr sub3( new EqualityMatchExpression() ); + ASSERT( sub3->init( "b", baseOperand3[ "b" ] ).isOK() ); + + OrMatchExpression orOp; + orOp.add( sub1.release() ); + orOp.add( sub2.release() ); + orOp.add( sub3.release() ); + + ASSERT( orOp.matchesBSON( BSON( "a" << -1 ), NULL ) ); + ASSERT( orOp.matchesBSON( BSON( "a" << 11 ), NULL ) ); + ASSERT( !orOp.matchesBSON( BSON( "a" << 5 ), NULL ) ); + ASSERT( orOp.matchesBSON( BSON( "b" << 100 ), NULL ) ); + ASSERT( !orOp.matchesBSON( BSON( "b" << 101 ), NULL ) ); + ASSERT( !orOp.matchesBSON( BSONObj(), NULL ) ); + ASSERT( orOp.matchesBSON( BSON( "a" << 11 << "b" << 100 ), NULL ) ); + } + + TEST( OrOp, ElemMatchKey ) { + BSONObj baseOperand1 = BSON( "a" << 1 ); + BSONObj baseOperand2 = BSON( "b" << 2 ); + auto_ptr sub1( new EqualityMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "a" ] ).isOK() ); + auto_ptr sub2( new EqualityMatchExpression() ); + ASSERT( sub2->init( "b", baseOperand2[ "b" ] ).isOK() ); + + OrMatchExpression orOp; + orOp.add( sub1.release() ); + orOp.add( sub2.release() ); + + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !orOp.matchesBSON( BSONObj(), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( !orOp.matchesBSON( BSON( "a" << BSON_ARRAY( 10 ) << "b" << BSON_ARRAY( 10 ) ), + &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( orOp.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) << "b" << BSON_ARRAY( 1 << 2 ) ), + &details ) ); + // The elem match key feature is not implemented for $or. + ASSERT( !details.hasElemMatchKey() ); + } + + /** + TEST( OrOp, MatchesIndexKeyWithoutUnknown ) { + BSONObj baseOperand1 = BSON( "$gt" << 5 ); + BSONObj baseOperand2 = BSON( "$lt" << 1 ); + auto_ptr sub1( new ComparisonMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$gt" ] ).isOK() ); + auto_ptr sub2( new ComparisonMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$lt" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( sub1.release() ); + subMatchExpressions.mutableVector().push_back( sub2.release() ); + OrOp orOp; + ASSERT( orOp.init( &subMatchExpressions ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_False == + orOp.matchesIndexKey( BSON( "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + orOp.matchesIndexKey( BSON( "" << 0 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + orOp.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + } + + TEST( OrOp, MatchesIndexKeyWithUnknown ) { + BSONObj baseOperand1 = BSON( "$gt" << 5 ); + BSONObj baseOperand2 = BSON( "$lt" << 1 ); + // This part will return PartialMatchResult_Unknown. + BSONObj baseOperand3 = BSON( "$ne" << 5 ); + auto_ptr sub1( new ComparisonMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$gt" ] ).isOK() ); + auto_ptr sub2( new ComparisonMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$lt" ] ).isOK() ); + auto_ptr sub3( new NeOp() ); + ASSERT( sub3->init( "a", baseOperand3[ "$ne" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( sub1.release() ); + subMatchExpressions.mutableVector().push_back( sub2.release() ); + subMatchExpressions.mutableVector().push_back( sub3.release() ); + OrOp orOp; + ASSERT( orOp.init( &subMatchExpressions ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + orOp.matchesIndexKey( BSON( "" << 3 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + orOp.matchesIndexKey( BSON( "" << 0 ), indexSpec ) ); + ASSERT( MatchMatchExpression::PartialMatchResult_True == + orOp.matchesIndexKey( BSON( "" << 6 ), indexSpec ) ); + } + */ + + /** + TEST( NorOp, MatchesElementSingleClause ) { + BSONObj baseOperand = BSON( "$lt" << 5 ); + BSONObj match = BSON( "a" << 5 ); + BSONObj notMatch = BSON( "a" << 4 ); + auto_ptr lt( new ComparisonMatchExpression() ); + ASSERT( lt->init( "a", baseOperand[ "$lt" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( lt.release() ); + NorOp norOp; + ASSERT( norOp.init( &subMatchExpressions ).isOK() ); + ASSERT( norOp.matchesSingleElement( match[ "a" ] ) ); + ASSERT( !norOp.matchesSingleElement( notMatch[ "a" ] ) ); + } + */ + + TEST( NorOp, NoClauses ) { + NorMatchExpression norOp; + ASSERT( norOp.matchesBSON( BSONObj(), NULL ) ); + } + /* + TEST( NorOp, MatchesElementThreeClauses ) { + BSONObj baseOperand1 = BSON( "$lt" << 0 ); + BSONObj baseOperand2 = BSON( "$gt" << 10 ); + BSONObj baseOperand3 = BSON( "a" << 5 ); + BSONObj notMatch1 = BSON( "a" << -1 ); + BSONObj notMatch2 = BSON( "a" << 11 ); + BSONObj notMatch3 = BSON( "a" << 5 ); + BSONObj match = BSON( "a" << "6" ); + auto_ptr sub1( new ComparisonMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$lt" ] ).isOK() ); + auto_ptr sub2( new ComparisonMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$gt" ] ).isOK() ); + auto_ptr sub3( new ComparisonMatchExpression() ); + ASSERT( sub3->init( "a", baseOperand3[ "a" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( sub1.release() ); + subMatchExpressions.mutableVector().push_back( sub2.release() ); + subMatchExpressions.mutableVector().push_back( sub3.release() ); + NorOp norOp; + ASSERT( norOp.init( &subMatchExpressions ).isOK() ); + ASSERT( !norOp.matchesSingleElement( notMatch1[ "a" ] ) ); + ASSERT( !norOp.matchesSingleElement( notMatch2[ "a" ] ) ); + ASSERT( !norOp.matchesSingleElement( notMatch3[ "a" ] ) ); + ASSERT( norOp.matchesSingleElement( match[ "a" ] ) ); + } + */ + + TEST( NorOp, MatchesSingleClause ) { + BSONObj baseOperand = BSON( "$ne" << 5 ); + auto_ptr eq( new EqualityMatchExpression() ); + ASSERT( eq->init( "a", baseOperand[ "$ne" ] ).isOK() ); + auto_ptr ne( new NotMatchExpression() ); + ASSERT( ne->init( eq.release() ).isOK() ); + + NorMatchExpression norOp; + norOp.add( ne.release() ); + + ASSERT( !norOp.matchesBSON( BSON( "a" << 4 ), NULL ) ); + ASSERT( !norOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 6 ) ), NULL ) ); + ASSERT( norOp.matchesBSON( BSON( "a" << 5 ), NULL ) ); + ASSERT( norOp.matchesBSON( BSON( "a" << BSON_ARRAY( 4 << 5 ) ), NULL ) ); + } + + TEST( NorOp, MatchesThreeClauses ) { + BSONObj baseOperand1 = BSON( "$gt" << 10 ); + BSONObj baseOperand2 = BSON( "$lt" << 0 ); + BSONObj baseOperand3 = BSON( "b" << 100 ); + + auto_ptr sub1( new GTMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "$gt" ] ).isOK() ); + auto_ptr sub2( new LTMatchExpression() ); + ASSERT( sub2->init( "a", baseOperand2[ "$lt" ] ).isOK() ); + auto_ptr sub3( new EqualityMatchExpression() ); + ASSERT( sub3->init( "b", baseOperand3[ "b" ] ).isOK() ); + + NorMatchExpression norOp; + norOp.add( sub1.release() ); + norOp.add( sub2.release() ); + norOp.add( sub3.release() ); + + ASSERT( !norOp.matchesBSON( BSON( "a" << -1 ), NULL ) ); + ASSERT( !norOp.matchesBSON( BSON( "a" << 11 ), NULL ) ); + ASSERT( norOp.matchesBSON( BSON( "a" << 5 ), NULL ) ); + ASSERT( !norOp.matchesBSON( BSON( "b" << 100 ), NULL ) ); + ASSERT( norOp.matchesBSON( BSON( "b" << 101 ), NULL ) ); + ASSERT( norOp.matchesBSON( BSONObj(), NULL ) ); + ASSERT( !norOp.matchesBSON( BSON( "a" << 11 << "b" << 100 ), NULL ) ); + } + + TEST( NorOp, ElemMatchKey ) { + BSONObj baseOperand1 = BSON( "a" << 1 ); + BSONObj baseOperand2 = BSON( "b" << 2 ); + auto_ptr sub1( new EqualityMatchExpression() ); + ASSERT( sub1->init( "a", baseOperand1[ "a" ] ).isOK() ); + auto_ptr sub2( new EqualityMatchExpression() ); + ASSERT( sub2->init( "b", baseOperand2[ "b" ] ).isOK() ); + + NorMatchExpression norOp; + norOp.add( sub1.release() ); + norOp.add( sub2.release() ); + + MatchDetails details; + details.requestElemMatchKey(); + ASSERT( !norOp.matchesBSON( BSON( "a" << 1 ), &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( !norOp.matchesBSON( BSON( "a" << BSON_ARRAY( 1 ) << "b" << BSON_ARRAY( 10 ) ), + &details ) ); + ASSERT( !details.hasElemMatchKey() ); + ASSERT( norOp.matchesBSON( BSON( "a" << BSON_ARRAY( 3 ) << "b" << BSON_ARRAY( 4 ) ), + &details ) ); + // The elem match key feature is not implemented for $nor. + ASSERT( !details.hasElemMatchKey() ); + } + + + TEST( NorOp, Equivalent ) { + BSONObj baseOperand1 = BSON( "a" << 1 ); + BSONObj baseOperand2 = BSON( "b" << 2 ); + EqualityMatchExpression sub1; + ASSERT( sub1.init( "a", baseOperand1[ "a" ] ).isOK() ); + EqualityMatchExpression sub2; + ASSERT( sub2.init( "b", baseOperand2[ "b" ] ).isOK() ); + + NorMatchExpression e1; + e1.add( sub1.shallowClone() ); + e1.add( sub2.shallowClone() ); + + NorMatchExpression e2; + e2.add( sub1.shallowClone() ); + + ASSERT( e1.equivalent( &e1 ) ); + ASSERT( !e1.equivalent( &e2 ) ); + } + + /** + TEST( NorOp, MatchesIndexKey ) { + BSONObj baseOperand = BSON( "a" << 5 ); + auto_ptr eq( new ComparisonMatchExpression() ); + ASSERT( eq->init( "a", baseOperand[ "a" ] ).isOK() ); + OwnedPointerVector subMatchExpressions; + subMatchExpressions.mutableVector().push_back( eq.release() ); + NorOp norOp; + ASSERT( norOp.init( &subMatchExpressions ).isOK() ); + IndexSpec indexSpec( BSON( "a" << 1 ) ); + BSONObj indexKey = BSON( "" << "7" ); + ASSERT( MatchMatchExpression::PartialMatchResult_Unknown == + norOp.matchesIndexKey( indexKey, indexSpec ) ); + } + */ + +} diff --git a/src/mongo/db/matcher/expression_where.cpp b/src/mongo/db/matcher/expression_where.cpp new file mode 100644 index 00000000000..8f02a748467 --- /dev/null +++ b/src/mongo/db/matcher/expression_where.cpp @@ -0,0 +1,202 @@ +// expression_where.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" +#include "mongo/base/init.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/client.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/scripting/engine.h" + +namespace mongo { + + class WhereMatchExpression : public MatchExpression { + public: + WhereMatchExpression() : MatchExpression( WHERE ){ _func = 0; } + virtual ~WhereMatchExpression(){} + + Status init( const StringData& ns, const StringData& theCode, const BSONObj& scope ); + + virtual bool matches( const MatchableDocument* doc, MatchDetails* details = 0 ) const; + + virtual bool matchesSingleElement( const BSONElement& e ) const { + return false; + } + + virtual MatchExpression* shallowClone() const { + WhereMatchExpression* e = new WhereMatchExpression(); + e->init(_ns, _code, _userScope); + if ( getTag() ) { + e->setTag(getTag()->clone()); + } + return e; + } + + virtual void debugString( StringBuilder& debug, int level = 0 ) const; + + virtual bool equivalent( const MatchExpression* other ) const ; + + virtual void resetTag() { setTag(NULL); } + + private: + string _ns; + string _code; + BSONObj _userScope; + + auto_ptr _scope; + ScriptingFunction _func; + }; + + Status WhereMatchExpression::init( const StringData& ns, + const StringData& theCode, + const BSONObj& scope ) { + + if ( ns.size() == 0 ) + return Status( ErrorCodes::BadValue, "ns for $where cannot be empty" ); + + if ( theCode.size() == 0 ) + return Status( ErrorCodes::BadValue, "code for $where cannot be empty" ); + + _ns = ns.toString(); + _code = theCode.toString(); + _userScope = scope.getOwned(); + + NamespaceString nswrapper( _ns ); + const string userToken = ClientBasic::getCurrent()->getAuthorizationSession() + ->getAuthenticatedUserNamesToken(); + _scope = globalScriptEngine->getPooledScope( nswrapper.db().toString(), + "where" + userToken ); + _func = _scope->createFunction( _code.c_str() ); + + if ( !_func ) + return Status( ErrorCodes::BadValue, "$where compile error" ); + + return Status::OK(); + } + + bool WhereMatchExpression::matches( const MatchableDocument* doc, MatchDetails* details ) const { + verify( _func ); + BSONObj obj = doc->toBSON(); + + if ( ! _userScope.isEmpty() ) { + _scope->init( &_userScope ); + } + _scope->setObject( "obj", const_cast< BSONObj & >( obj ) ); + _scope->setBoolean( "fullObject" , true ); // this is a hack b/c fullObject used to be relevant + + int err = _scope->invoke( _func, 0, &obj, 1000 * 60, false ); + if ( err == -3 ) { // INVOKE_ERROR + stringstream ss; + ss << "error on invocation of $where function:\n" + << _scope->getError(); + uassert( 16812, ss.str(), false); + } + else if ( err != 0 ) { // ! INVOKE_SUCCESS + uassert( 16813, "unknown error in invocation of $where function", false); + } + + return _scope->getBoolean( "__returnValue" ) != 0; + } + + void WhereMatchExpression::debugString( StringBuilder& debug, int level ) const { + _debugAddSpace( debug, level ); + debug << "$where\n"; + + _debugAddSpace( debug, level + 1 ); + debug << "ns: " << _ns << "\n"; + + _debugAddSpace( debug, level + 1 ); + debug << "code: " << _code << "\n"; + + _debugAddSpace( debug, level + 1 ); + debug << "scope: " << _userScope << "\n"; + } + + bool WhereMatchExpression::equivalent( const MatchExpression* other ) const { + if ( matchType() != other->matchType() ) + return false; + const WhereMatchExpression* realOther = static_cast(other); + return + _ns == realOther->_ns && + _code == realOther->_code && + _userScope == realOther->_userScope; + } + + + // ----------------- + + StatusWithMatchExpression expressionParserWhereCallbackReal(const BSONElement& where) { + if ( !haveClient() ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "no current client needed for $where" ); + + Client::Context* context = cc().getContext(); + if ( !context ) + return StatusWithMatchExpression( ErrorCodes::NoClientContext, + "no context in $where parsing" ); + + const char* ns = context->ns(); + if ( !ns ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "no ns in $where parsing" ); + + if ( !globalScriptEngine ) + return StatusWithMatchExpression( ErrorCodes::BadValue, "no globalScriptEngine in $where parsing" ); + + auto_ptr exp( new WhereMatchExpression() ); + if ( where.type() == String || where.type() == Code ) { + Status s = exp->init( ns, where.valuestr(), BSONObj() ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( exp.release() ); + } + + if ( where.type() == CodeWScope ) { + Status s = exp->init( ns, + where.codeWScopeCode(), + BSONObj( where.codeWScopeScopeDataUnsafe() ) ); + if ( !s.isOK() ) + return StatusWithMatchExpression( s ); + return StatusWithMatchExpression( exp.release() ); + } + + return StatusWithMatchExpression( ErrorCodes::BadValue, "$where got bad type" ); + } + + MONGO_INITIALIZER( MatchExpressionWhere )( ::mongo::InitializerContext* context ) { + expressionParserWhereCallback = expressionParserWhereCallbackReal; + return Status::OK(); + } + + + + +} diff --git a/src/mongo/db/matcher/match_details.cpp b/src/mongo/db/matcher/match_details.cpp new file mode 100644 index 00000000000..1c437129adc --- /dev/null +++ b/src/mongo/db/matcher/match_details.cpp @@ -0,0 +1,72 @@ +// match_details.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/matcher/match_details.h" + +#include + +#include "mongo/util/assert_util.h" + +namespace mongo { + + MatchDetails::MatchDetails() : + _elemMatchKeyRequested() { + resetOutput(); + } + + void MatchDetails::resetOutput() { + _loadedRecord = false; + _elemMatchKey.reset(); + } + + bool MatchDetails::hasElemMatchKey() const { + return _elemMatchKey.get(); + } + + std::string MatchDetails::elemMatchKey() const { + verify( hasElemMatchKey() ); + return *(_elemMatchKey.get()); + } + + void MatchDetails::setElemMatchKey( const std::string &elemMatchKey ) { + if ( _elemMatchKeyRequested ) { + _elemMatchKey.reset( new std::string( elemMatchKey ) ); + } + } + + string MatchDetails::toString() const { + std::stringstream ss; + ss << "loadedRecord: " << _loadedRecord << " "; + ss << "elemMatchKeyRequested: " << _elemMatchKeyRequested << " "; + ss << "elemMatchKey: " << ( _elemMatchKey ? _elemMatchKey->c_str() : "NONE" ) << " "; + return ss.str(); + } + +} diff --git a/src/mongo/db/matcher/match_details.h b/src/mongo/db/matcher/match_details.h new file mode 100644 index 00000000000..2209c4ac1ff --- /dev/null +++ b/src/mongo/db/matcher/match_details.h @@ -0,0 +1,74 @@ +// match_details.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include + +namespace mongo { + + /** Reports information about a match request. */ + class MatchDetails { + public: + MatchDetails(); + + void resetOutput(); + + // for debugging only + std::string toString() const; + + // relating to whether or not we had to load the full record + + void setLoadedRecord( bool loadedRecord ) { _loadedRecord = loadedRecord; } + + bool hasLoadedRecord() const { return _loadedRecord; } + + // this name is wrong + + bool needRecord() const { return _elemMatchKeyRequested; } + + // if we need to store the offset into an array where we found the match + + /** Request that an elemMatchKey be recorded. */ + void requestElemMatchKey() { _elemMatchKeyRequested = true; } + + bool hasElemMatchKey() const; + std::string elemMatchKey() const; + + void setElemMatchKey( const std::string &elemMatchKey ); + + private: + bool _loadedRecord; + bool _elemMatchKeyRequested; + boost::scoped_ptr _elemMatchKey; + }; +} diff --git a/src/mongo/db/matcher/matchable.cpp b/src/mongo/db/matcher/matchable.cpp new file mode 100644 index 00000000000..f2e9df701b4 --- /dev/null +++ b/src/mongo/db/matcher/matchable.cpp @@ -0,0 +1,45 @@ +// matchable.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/matchable.h" + +namespace mongo { + + BSONMatchableDocument::BSONMatchableDocument( const BSONObj& obj ) + : _obj( obj ) { + _iteratorUsed = false; + } + + BSONMatchableDocument::~BSONMatchableDocument() { + } + +} diff --git a/src/mongo/db/matcher/matchable.h b/src/mongo/db/matcher/matchable.h new file mode 100644 index 00000000000..a3506005901 --- /dev/null +++ b/src/mongo/db/matcher/matchable.h @@ -0,0 +1,105 @@ +// matchable.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/bsonobj.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/matcher/path.h" + +namespace mongo { + + class MatchableDocument { + public: + // Inlining to allow subclasses to see that this is a no-op and avoid a function call. + // Speeds up query execution measurably. + virtual ~MatchableDocument() {} + + virtual BSONObj toBSON() const = 0; + + /** + * The neewly returned ElementIterator is allowed to keep a pointer to path. + * So the caller of this function should make sure path is in scope until + * the ElementIterator is deallocated + */ + virtual ElementIterator* allocateIterator( const ElementPath* path ) const = 0; + + virtual void releaseIterator( ElementIterator* iterator ) const = 0; + + class IteratorHolder { + public: + IteratorHolder( const MatchableDocument* doc, const ElementPath* path ) { + _doc = doc; + _iterator = _doc->allocateIterator( path ); + } + + ~IteratorHolder() { + _doc->releaseIterator( _iterator ); + } + + ElementIterator* operator->() const { + return _iterator; + } + private: + const MatchableDocument* _doc; + ElementIterator* _iterator; + }; + }; + + class BSONMatchableDocument : public MatchableDocument { + public: + BSONMatchableDocument( const BSONObj& obj ); + virtual ~BSONMatchableDocument(); + + virtual BSONObj toBSON() const { return _obj; } + + virtual ElementIterator* allocateIterator( const ElementPath* path ) const { + if ( _iteratorUsed ) + return new BSONElementIterator( path, _obj ); + _iteratorUsed = true; + _iterator.reset( path, _obj ); + return &_iterator; + } + + virtual void releaseIterator( ElementIterator* iterator ) const { + if ( iterator == &_iterator ) { + _iteratorUsed = false; + } + else { + delete iterator; + } + } + + private: + BSONObj _obj; + mutable BSONElementIterator _iterator; + mutable bool _iteratorUsed; + }; +} diff --git a/src/mongo/db/matcher/matcher.cpp b/src/mongo/db/matcher/matcher.cpp new file mode 100644 index 00000000000..a1d76fd943d --- /dev/null +++ b/src/mongo/db/matcher/matcher.cpp @@ -0,0 +1,62 @@ +// matcher.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/pch.h" + +#include "mongo/base/init.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/matcher/matcher.h" +#include "mongo/db/matcher/path.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/stacktrace.h" + +namespace mongo { + + Matcher2::Matcher2( const BSONObj& pattern, bool nested ) + : _pattern( pattern ) { + + StatusWithMatchExpression result = MatchExpressionParser::parse( pattern ); + uassert( 16810, + mongoutils::str::stream() << "bad query: " << result.toString(), + result.isOK() ); + + _expression.reset( result.getValue() ); + } + + bool Matcher2::matches(const BSONObj& doc, MatchDetails* details ) const { + if ( !_expression ) + return true; + + return _expression->matchesBSON( doc, details ); + } + +} // namespace mongo diff --git a/src/mongo/db/matcher/matcher.h b/src/mongo/db/matcher/matcher.h new file mode 100644 index 00000000000..c09f79b279a --- /dev/null +++ b/src/mongo/db/matcher/matcher.h @@ -0,0 +1,64 @@ +// matcher.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/match_details.h" + +namespace mongo { + + /** + * Matcher is a simple wrapper around a BSONObj and the MatchExpression created from it. + */ + class Matcher2 { + MONGO_DISALLOW_COPYING( Matcher2 ); + + public: + explicit Matcher2( const BSONObj& pattern, bool nested=false /* do not use */ ); + + bool matches(const BSONObj& doc, MatchDetails* details = NULL ) const; + + const BSONObj* getQuery() const { return &_pattern; }; + + std::string toString() const { return _pattern.toString(); } + + private: + BSONObj _pattern; + + boost::scoped_ptr _expression; + }; + +} // namespace mongo diff --git a/src/mongo/db/matcher/path.cpp b/src/mongo/db/matcher/path.cpp new file mode 100644 index 00000000000..96355f2a1e8 --- /dev/null +++ b/src/mongo/db/matcher/path.cpp @@ -0,0 +1,317 @@ +// path.cpp + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/path_internal.h" +#include "mongo/db/matcher/path.h" + +namespace mongo { + + Status ElementPath::init( const StringData& path ) { + _shouldTraverseLeafArray = true; + _fieldRef.parse( path ); + return Status::OK(); + } + + // ----- + + ElementIterator::~ElementIterator(){ + } + + void ElementIterator::Context::reset() { + _element = BSONElement(); + } + + void ElementIterator::Context::reset( BSONElement element, + BSONElement arrayOffset, + bool outerArray ) { + _element = element; + _arrayOffset = arrayOffset; + _outerArray = outerArray; + } + + + // ------ + + SimpleArrayElementIterator::SimpleArrayElementIterator( const BSONElement& theArray, bool returnArrayLast ) + : _theArray( theArray ), _returnArrayLast( returnArrayLast ), _iterator( theArray.Obj() ) { + + } + + bool SimpleArrayElementIterator::more() { + return _iterator.more() || _returnArrayLast; + } + + ElementIterator::Context SimpleArrayElementIterator::next() { + if ( _iterator.more() ) { + Context e; + e.reset( _iterator.next(), BSONElement(), false ); + return e; + } + _returnArrayLast = false; + Context e; + e.reset( _theArray, BSONElement(), true ); + return e; + } + + + + // ------ + BSONElementIterator::BSONElementIterator() { + _path = NULL; + } + + BSONElementIterator::BSONElementIterator( const ElementPath* path, const BSONObj& context ) + : _path( path ), _context( context ) { + _state = BEGIN; + //log() << "path: " << path.fieldRef().dottedField() << " context: " << context << endl; + } + + BSONElementIterator::~BSONElementIterator() { + } + + void BSONElementIterator::reset( const ElementPath* path, const BSONObj& context ) { + _path = path; + _context = context; + _state = BEGIN; + _next.reset(); + + _subCursor.reset(); + _subCursorPath.reset(); + } + + + void BSONElementIterator::ArrayIterationState::reset( const FieldRef& ref, int start ) { + restOfPath = ref.dottedField( start ).toString(); + hasMore = restOfPath.size() > 0; + if ( hasMore ) { + nextPieceOfPath = ref.getPart( start ); + nextPieceOfPathIsNumber = isAllDigits( nextPieceOfPath ); + } + else { + nextPieceOfPathIsNumber = false; + } + } + + bool BSONElementIterator::ArrayIterationState::isArrayOffsetMatch( const StringData& fieldName ) const { + if ( !nextPieceOfPathIsNumber ) + return false; + return nextPieceOfPath == fieldName; + } + + + void BSONElementIterator::ArrayIterationState::startIterator( BSONElement e ) { + _theArray = e; + _iterator.reset( new BSONObjIterator( _theArray.Obj() ) ); + } + + bool BSONElementIterator::ArrayIterationState::more() { + return _iterator && _iterator->more(); + } + + BSONElement BSONElementIterator::ArrayIterationState::next() { + _current = _iterator->next(); + return _current; + } + + + bool BSONElementIterator::subCursorHasMore() { + // While we still are still finding arrays along the path, keep traversing deeper. + while ( _subCursor ) { + + if ( _subCursor->more() ) { + return true; + } + _subCursor.reset(); + + // If the subcursor doesn't have more, see if the current element is an array offset + // match (see comment in BSONElementIterator::more() for an example). If it is indeed + // an array offset match, create a new subcursor and examine it. + if ( _arrayIterationState.isArrayOffsetMatch( _arrayIterationState._current.fieldName() ) ) { + if ( _arrayIterationState.nextEntireRest() ) { + // Our path terminates at the array offset. _next should point at the current + // array element. + _next.reset( _arrayIterationState._current, + _arrayIterationState._current, + true ); + _arrayIterationState._current = BSONElement(); + return true; + } + + _subCursorPath.reset( new ElementPath() ); + _subCursorPath->init( _arrayIterationState.restOfPath.substr( _arrayIterationState.nextPieceOfPath.size() + 1 ) ); + _subCursorPath->setTraverseLeafArray( _path->shouldTraverseLeafArray() ); + _subCursor.reset( new BSONElementIterator( _subCursorPath.get(), + _arrayIterationState._current.Obj() ) ); + _arrayIterationState._current = BSONElement(); + } + + } + + return false; + } + + bool BSONElementIterator::more() { + if ( subCursorHasMore() ) { + return true; + } + + if ( !_next.element().eoo() ) { + return true; + } + + if ( _state == DONE ){ + return false; + } + + if ( _state == BEGIN ) { + size_t idxPath = 0; + BSONElement e = getFieldDottedOrArray( _context, _path->fieldRef(), &idxPath ); + + if ( e.type() != Array ) { + _next.reset( e, BSONElement(), false ); + _state = DONE; + return true; + } + + // It's an array. + + _arrayIterationState.reset( _path->fieldRef(), idxPath + 1 ); + + if ( !_arrayIterationState.hasMore && !_path->shouldTraverseLeafArray() ) { + _next.reset( e, BSONElement(), true ); + _state = DONE; + return true; + } + + _arrayIterationState.startIterator( e ); + _state = IN_ARRAY; + + invariant( _next.element().eoo() ); + } + + if ( _state == IN_ARRAY ) { + // We're traversing an array. Look at each array element. + + while ( _arrayIterationState.more() ) { + + BSONElement eltInArray = _arrayIterationState.next(); + if ( !_arrayIterationState.hasMore ) { + // Our path terminates at this array. _next should point at the current array + // element. + _next.reset( eltInArray, eltInArray, false ); + return true; + } + + // Our path does not terminate at this array; there's a subpath left over. Inspect + // the current array element to see if it could match the subpath. + + if ( eltInArray.type() == Object ) { + // The current array element is a subdocument. See if the subdocument generates + // any elements matching the remaining subpath. + _subCursorPath.reset( new ElementPath() ); + _subCursorPath->init( _arrayIterationState.restOfPath ); + _subCursorPath->setTraverseLeafArray( _path->shouldTraverseLeafArray() ); + + _subCursor.reset( new BSONElementIterator( _subCursorPath.get(), + eltInArray.Obj() ) ); + if ( subCursorHasMore() ) { + return true; + } + } + else if ( _arrayIterationState.isArrayOffsetMatch( eltInArray.fieldName() ) ) { + // The path we're traversing has an array offset component, and the current + // array element corresponds to the offset we're looking for (for example: our + // path has a ".0" component, and we're looking at the first element of the + // array, so we should look inside this element). + + if ( _arrayIterationState.nextEntireRest() ) { + // Our path terminates at the array offset. _next should point at the + // current array element. + _next.reset( eltInArray, eltInArray, false ); + return true; + } + + invariant( eltInArray.type() != Object ); // Handled above. + if ( eltInArray.type() == Array ) { + // The current array element is itself an array. See if the nested array + // has any elements matching the remainihng. + _subCursorPath.reset( new ElementPath() ); + _subCursorPath->init( _arrayIterationState.restOfPath.substr( _arrayIterationState.nextPieceOfPath.size() + 1 ) ); + _subCursorPath->setTraverseLeafArray( _path->shouldTraverseLeafArray() ); + BSONElementIterator* real = + new BSONElementIterator( _subCursorPath.get(), + _arrayIterationState._current.Obj() ); + _subCursor.reset( real ); + real->_arrayIterationState.reset( _subCursorPath->fieldRef(), 0 ); + real->_arrayIterationState.startIterator( eltInArray ); + real->_state = IN_ARRAY; + _arrayIterationState._current = BSONElement(); + if ( subCursorHasMore() ) { + return true; + } + } + } + + } + + if ( _arrayIterationState.hasMore ) { + return false; + } + + _next.reset( _arrayIterationState._theArray, BSONElement(), true ); + _state = DONE; + return true; + } + + return false; + } + + ElementIterator::Context BSONElementIterator::next() { + if ( _subCursor ) { + Context e = _subCursor->next(); + // Use our array offset if we have one, otherwise copy our subcursor's. This has the + // effect of preferring the outermost array offset, in the case where we are implicitly + // traversing nested arrays and have multiple candidate array offsets. For example, + // when we use the path "a.b" to generate elements from the document {a: [{b: [1, 2]}]}, + // the element with a value of 2 should be returned with an array offset of 0. + if ( !_arrayIterationState._current.eoo() ) { + e.setArrayOffset( _arrayIterationState._current ); + } + return e; + } + Context x = _next; + _next.reset(); + return x; + } + + +} diff --git a/src/mongo/db/matcher/path.h b/src/mongo/db/matcher/path.h new file mode 100644 index 00000000000..c3fec41ad47 --- /dev/null +++ b/src/mongo/db/matcher/path.h @@ -0,0 +1,168 @@ +// path.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjiterator.h" +#include "mongo/db/field_ref.h" + +namespace mongo { + + class ElementPath { + public: + Status init( const StringData& path ); + + void setTraverseLeafArray( bool b ) { _shouldTraverseLeafArray = b; } + + const FieldRef& fieldRef() const { return _fieldRef; } + bool shouldTraverseLeafArray() const { return _shouldTraverseLeafArray; } + + private: + FieldRef _fieldRef; + bool _shouldTraverseLeafArray; + }; + + class ElementIterator { + public: + class Context { + public: + + void reset(); + + void reset( BSONElement element, BSONElement arrayOffset, bool outerArray ); + + void setArrayOffset( BSONElement e ) { _arrayOffset = e; } + + BSONElement element() const { return _element; } + BSONElement arrayOffset() const { return _arrayOffset; } + bool outerArray() const { return _outerArray; } + + private: + BSONElement _element; + BSONElement _arrayOffset; + bool _outerArray; + }; + + virtual ~ElementIterator(); + + virtual bool more() = 0; + virtual Context next() = 0; + + }; + + // --------------------------------------------------------------- + + class SingleElementElementIterator : public ElementIterator { + public: + explicit SingleElementElementIterator( BSONElement e ) + : _seen( false ) { + _element.reset( e, BSONElement(), false ); + } + virtual ~SingleElementElementIterator(){} + + virtual bool more() { return !_seen; } + virtual Context next() { _seen = true; return _element; } + + private: + bool _seen; + ElementIterator::Context _element; + }; + + class SimpleArrayElementIterator : public ElementIterator { + public: + SimpleArrayElementIterator( const BSONElement& theArray, bool returnArrayLast ); + + virtual bool more(); + virtual Context next(); + + private: + BSONElement _theArray; + bool _returnArrayLast; + BSONObjIterator _iterator; + }; + + class BSONElementIterator : public ElementIterator { + public: + BSONElementIterator(); + BSONElementIterator( const ElementPath* path, const BSONObj& context ); + + virtual ~BSONElementIterator(); + + void reset( const ElementPath* path, const BSONObj& context ); + + bool more(); + Context next(); + + private: + /** + * Helper for more(). Recurs on _subCursor (which traverses the remainder of a path through + * subdocuments of an array). + */ + bool subCursorHasMore(); + + const ElementPath* _path; + BSONObj _context; + + enum State { BEGIN, IN_ARRAY, DONE } _state; + Context _next; + + struct ArrayIterationState { + + void reset( const FieldRef& ref, int start ); + void startIterator( BSONElement theArray ); + + bool more(); + BSONElement next(); + + bool isArrayOffsetMatch( const StringData& fieldName ) const; + bool nextEntireRest() const { return nextPieceOfPath.size() == restOfPath.size(); } + + string restOfPath; + bool hasMore; + StringData nextPieceOfPath; + bool nextPieceOfPathIsNumber; + + BSONElement _theArray; + BSONElement _current; + boost::scoped_ptr _iterator; + }; + + ArrayIterationState _arrayIterationState; + + boost::scoped_ptr _subCursor; + boost::scoped_ptr _subCursorPath; + }; + +} diff --git a/src/mongo/db/matcher/path_internal.cpp b/src/mongo/db/matcher/path_internal.cpp new file mode 100644 index 00000000000..f52841aec84 --- /dev/null +++ b/src/mongo/db/matcher/path_internal.cpp @@ -0,0 +1,87 @@ +// path_internal.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/path_internal.h" + +namespace mongo { + + bool isAllDigits( const StringData& str ) { + for ( unsigned i = 0; i < str.size(); i++ ) { + if ( !isdigit( str[i] ) ) + return false; + } + return true; + } + + BSONElement getFieldDottedOrArray( const BSONObj& doc, + const FieldRef& path, + size_t* idxPath ) { + if ( path.numParts() == 0 ) + return doc.getField( "" ); + + BSONElement res; + + BSONObj curr = doc; + bool stop = false; + size_t partNum = 0; + while ( partNum < path.numParts() && !stop ) { + + res = curr.getField( path.getPart( partNum ) ); + + switch ( res.type() ) { + + case EOO: + stop = true; + break; + + case Object: + curr = res.Obj(); + ++partNum; + break; + + case Array: + stop = true; + break; + + default: + if ( partNum+1 < path.numParts() ) { + res = BSONElement(); + } + stop = true; + + } + } + + *idxPath = partNum; + return res; + } + + +} // namespace mongo diff --git a/src/mongo/db/matcher/path_internal.h b/src/mongo/db/matcher/path_internal.h new file mode 100644 index 00000000000..bf0163f8d31 --- /dev/null +++ b/src/mongo/db/matcher/path_internal.h @@ -0,0 +1,48 @@ +// path_internal.h + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/string_data.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/cstdint.h" + +namespace mongo { + + bool isAllDigits( const StringData& str ); + + // XXX document me + // Replaces getFieldDottedOrArray without recursion nor string manipulation + BSONElement getFieldDottedOrArray( const BSONObj& doc, + const FieldRef& path, + size_t* idxPath ); + +} // namespace mongo diff --git a/src/mongo/db/matcher/path_test.cpp b/src/mongo/db/matcher/path_test.cpp new file mode 100644 index 00000000000..3bd3b0f4568 --- /dev/null +++ b/src/mongo/db/matcher/path_test.cpp @@ -0,0 +1,485 @@ +// path_test.cpp + + +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/path.h" + +namespace mongo { + + TEST( Path, Root1 ) { + ElementPath p; + ASSERT( p.init( "a" ).isOK() ); + + BSONObj doc = BSON( "x" << 4 << "a" << 5 ); + + BSONElementIterator cursor( &p, doc ); + ASSERT( cursor.more() ); + ElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( (string)"a", e.element().fieldName() ); + ASSERT_EQUALS( 5, e.element().numberInt() ); + ASSERT( !cursor.more() ); + } + + TEST( Path, RootArray1 ) { + ElementPath p; + ASSERT( p.init( "a" ).isOK() ); + + BSONObj doc = BSON( "x" << 4 << "a" << BSON_ARRAY( 5 << 6 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( 5, e.element().numberInt() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( 6, e.element().numberInt() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + + ASSERT( !cursor.more() ); + } + + TEST( Path, RootArray2 ) { + ElementPath p; + ASSERT( p.init( "a" ).isOK() ); + p.setTraverseLeafArray( false ); + + BSONObj doc = BSON( "x" << 4 << "a" << BSON_ARRAY( 5 << 6 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT( e.element().type() == Array ); + + ASSERT( !cursor.more() ); + } + + TEST( Path, Nested1 ) { + ElementPath p; + ASSERT( p.init( "a.b" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( BSON( "b" << 5 ) << + 3 << + BSONObj() << + BSON( "b" << BSON_ARRAY( 9 << 11 ) ) << + BSON( "b" << 7 ) ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( 5, e.element().numberInt() ); + ASSERT( !e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT( e.element().eoo() ); + ASSERT_EQUALS( (string)"2", e.arrayOffset().fieldName() ); + ASSERT( !e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( 9, e.element().numberInt() ); + ASSERT( !e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( 11, e.element().numberInt() ); + ASSERT( !e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + ASSERT_EQUALS( 2, e.element().Obj().nFields() ); + ASSERT( e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( 7, e.element().numberInt() ); + ASSERT( !e.outerArray() ); + + ASSERT( !cursor.more() ); + } + + TEST( Path, NestedPartialMatchScalar ) { + ElementPath p; + ASSERT( p.init( "a.b" ).isOK() ); + + BSONObj doc = BSON( "a" << 4 ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT( e.element().eoo() ); + ASSERT( e.arrayOffset().eoo() ); + ASSERT( !e.outerArray() ); + + ASSERT( !cursor.more() ); + } + + // When the path (partially or in its entirety) refers to an array, + // the iteration logic does not return an EOO. + // what we want ideally. + TEST( Path, NestedPartialMatchArray ) { + ElementPath p; + ASSERT( p.init( "a.b" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( 4 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( !cursor.more() ); + } + + // Note that this describes existing behavior and not necessarily + TEST( Path, NestedEmptyArray ) { + ElementPath p; + ASSERT( p.init( "a.b" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON( "b" << BSONArray() ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + ASSERT_EQUALS( 0, e.element().Obj().nFields() ); + ASSERT( e.outerArray() ); + + ASSERT( !cursor.more() ); + } + + TEST( Path, NestedNoLeaf1 ) { + ElementPath p; + ASSERT( p.init( "a.b" ).isOK() ); + p.setTraverseLeafArray( false ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( BSON( "b" << 5 ) << + 3 << + BSONObj() << + BSON( "b" << BSON_ARRAY( 9 << 11 ) ) << + BSON( "b" << 7 ) ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( 5, e.element().numberInt() ); + ASSERT( !e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT( e.element().eoo() ); + ASSERT_EQUALS( (string)"2", e.arrayOffset().fieldName() ); + ASSERT( !e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + ASSERT_EQUALS( 2, e.element().Obj().nFields() ); + ASSERT( e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( 7, e.element().numberInt() ); + ASSERT( !e.outerArray() ); + + ASSERT( !cursor.more() ); + } + + + TEST( Path, ArrayIndex1 ) { + ElementPath p; + ASSERT( p.init( "a.1" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( 5 << 7 << 3 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( 7, e.element().numberInt() ); + + ASSERT( !cursor.more() ); + } + + TEST( Path, ArrayIndex2 ) { + ElementPath p; + ASSERT( p.init( "a.1" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( 5 << BSON_ARRAY( 2 << 4 ) << 3 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + + ASSERT( !cursor.more() ); + } + + TEST( Path, ArrayIndex3 ) { + ElementPath p; + ASSERT( p.init( "a.1" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( 5 << BSON( "1" << 4 ) << 3 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( 4, e.element().numberInt() ); + ASSERT( !e.outerArray() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( BSON( "1" << 4 ), e.element().Obj() ); + ASSERT( e.outerArray() ); + + ASSERT( !cursor.more() ); + } + + TEST( Path, ArrayIndexNested1 ) { + ElementPath p; + ASSERT( p.init( "a.1.b" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( 5 << BSON( "b" << 4 ) << 3 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT( e.element().eoo() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( 4, e.element().numberInt() ); + + + ASSERT( !cursor.more() ); + } + + TEST( Path, ArrayIndexNested2 ) { + ElementPath p; + ASSERT( p.init( "a.1.b" ).isOK() ); + + BSONObj doc = BSON( "a" << BSON_ARRAY( 5 << BSON_ARRAY( BSON( "b" << 4 ) ) << 3 ) ); + + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + BSONElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( 4, e.element().numberInt() ); + + + ASSERT( !cursor.more() ); + } + + // SERVER-15899: test iteration using a path that generates no elements, but traverses a long + // array containing subdocuments with nested arrays. + TEST( Path, NonMatchingLongArrayOfSubdocumentsWithNestedArrays ) { + ElementPath p; + ASSERT( p.init( "a.b.x" ).isOK() ); + + // Build the document {a: [{b: []}, {b: []}, {b: []}, ...]}. + BSONObj subdoc = BSON( "b" << BSONArray() ); + BSONArrayBuilder builder; + for ( int i = 0; i < 100 * 1000; ++i ) { + builder.append( subdoc ); + } + BSONObj doc = BSON( "a" << builder.arr() ); + + BSONElementIterator cursor( &p, doc ); + + // The path "a.b.x" matches no elements. + ASSERT( !cursor.more() ); + } + + // When multiple arrays are traversed implicitly in the same path, + // ElementIterator::Context::arrayOffset() should always refer to the current offset of the + // outermost array that is implicitly traversed. + TEST( Path, NestedArrayImplicitTraversal ) { + ElementPath p; + ASSERT( p.init( "a.b" ).isOK() ); + BSONObj doc = fromjson("{a: [{b: [2, 3]}, {b: [4, 5]}]}"); + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + ElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( NumberInt, e.element().type() ); + ASSERT_EQUALS( 2, e.element().numberInt() ); + ASSERT_EQUALS( "0", e.arrayOffset().fieldNameStringData() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( NumberInt, e.element().type() ); + ASSERT_EQUALS( 3, e.element().numberInt() ); + ASSERT_EQUALS( "0", e.arrayOffset().fieldNameStringData() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + ASSERT_EQUALS( BSON( "0" << 2 << "1" << 3 ), e.element().Obj() ); + ASSERT_EQUALS( "0", e.arrayOffset().fieldNameStringData() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( NumberInt, e.element().type() ); + ASSERT_EQUALS( 4, e.element().numberInt() ); + ASSERT_EQUALS( "1", e.arrayOffset().fieldNameStringData() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( NumberInt, e.element().type() ); + ASSERT_EQUALS( 5, e.element().numberInt() ); + ASSERT_EQUALS( "1", e.arrayOffset().fieldNameStringData() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + ASSERT_EQUALS( BSON( "0" << 4 << "1" << 5 ), e.element().Obj() ); + ASSERT_EQUALS( "1", e.arrayOffset().fieldNameStringData() ); + + ASSERT( !cursor.more() ); + } + + // SERVER-14886: when an array is being traversed explictly at the same time that a nested array + // is being traversed implicitly, ElementIterator::Context::arrayOffset() should return the + // current offset of the array being implicitly traversed. + TEST( Path, ArrayOffsetWithImplicitAndExplicitTraversal ) { + ElementPath p; + ASSERT( p.init( "a.0.b" ).isOK() ); + BSONObj doc = fromjson("{a: [{b: [2, 3]}, {b: [4, 5]}]}"); + BSONElementIterator cursor( &p, doc ); + + ASSERT( cursor.more() ); + ElementIterator::Context e = cursor.next(); + ASSERT_EQUALS( EOO, e.element().type() ); + ASSERT_EQUALS( "0", e.arrayOffset().fieldNameStringData() ); // First elt of outer array. + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( NumberInt, e.element().type() ); + ASSERT_EQUALS( 2, e.element().numberInt() ); + ASSERT_EQUALS( "0", e.arrayOffset().fieldNameStringData() ); // First elt of inner array. + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( NumberInt, e.element().type() ); + ASSERT_EQUALS( 3, e.element().numberInt() ); + ASSERT_EQUALS( "1", e.arrayOffset().fieldNameStringData() ); // Second elt of inner array. + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( Array, e.element().type() ); + ASSERT_EQUALS( BSON( "0" << 2 << "1" << 3 ), e.element().Obj() ); + ASSERT( e.arrayOffset().eoo() ); + + ASSERT( cursor.more() ); + e = cursor.next(); + ASSERT_EQUALS( EOO, e.element().type() ); + ASSERT_EQUALS( "1", e.arrayOffset().fieldNameStringData() ); // Second elt of outer array. + + ASSERT( !cursor.more() ); + } + + TEST( SimpleArrayElementIterator, SimpleNoArrayLast1 ) { + BSONObj obj = BSON( "a" << BSON_ARRAY( 5 << BSON( "x" << 6 ) << BSON_ARRAY( 7 << 9 ) << 11 ) ); + SimpleArrayElementIterator i( obj["a"], false ); + + ASSERT( i.more() ); + ElementIterator::Context e = i.next(); + ASSERT_EQUALS( 5, e.element().numberInt() ); + + ASSERT( i.more() ); + e = i.next(); + ASSERT_EQUALS( 6, e.element().Obj()["x"].numberInt() ); + + ASSERT( i.more() ); + e = i.next(); + ASSERT_EQUALS( 7, e.element().Obj().firstElement().numberInt() ); + + ASSERT( i.more() ); + e = i.next(); + ASSERT_EQUALS( 11, e.element().numberInt() ); + + ASSERT( !i.more() ); + } + + TEST( SimpleArrayElementIterator, SimpleArrayLast1 ) { + BSONObj obj = BSON( "a" << BSON_ARRAY( 5 << BSON( "x" << 6 ) << BSON_ARRAY( 7 << 9 ) << 11 ) ); + SimpleArrayElementIterator i( obj["a"], true ); + + ASSERT( i.more() ); + ElementIterator::Context e = i.next(); + ASSERT_EQUALS( 5, e.element().numberInt() ); + + ASSERT( i.more() ); + e = i.next(); + ASSERT_EQUALS( 6, e.element().Obj()["x"].numberInt() ); + + ASSERT( i.more() ); + e = i.next(); + ASSERT_EQUALS( 7, e.element().Obj().firstElement().numberInt() ); + + ASSERT( i.more() ); + e = i.next(); + ASSERT_EQUALS( 11, e.element().numberInt() ); + + ASSERT( i.more() ); + e = i.next(); + ASSERT_EQUALS( Array, e.element().type() ); + + ASSERT( !i.more() ); + } + + TEST( SingleElementElementIterator, Simple1 ) { + BSONObj obj = BSON( "x" << 3 << "y" << 5 ); + SingleElementElementIterator i( obj["y"] ); + + ASSERT( i.more() ); + ElementIterator::Context e = i.next(); + ASSERT_EQUALS( 5, e.element().numberInt() ); + + ASSERT( !i.more() ); + + } + +} diff --git a/src/mongo/db/matcher_covered.cpp b/src/mongo/db/matcher_covered.cpp deleted file mode 100644 index 487c0b4603f..00000000000 --- a/src/mongo/db/matcher_covered.cpp +++ /dev/null @@ -1,132 +0,0 @@ -// matcher_covered.cpp - -/* Matcher is our boolean expression evaluator for "where" clauses */ - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" - -#include "mongo/db/matcher.h" - -#include "mongo/db/cursor.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/queryutil.h" - -namespace mongo { - - CoveredIndexMatcher::CoveredIndexMatcher( const BSONObj &jsobj, - const BSONObj &indexKeyPattern ) : - _docMatcher( new Matcher( jsobj ) ), - _keyMatcher( *_docMatcher, indexKeyPattern ) { - init(); - } - - CoveredIndexMatcher::CoveredIndexMatcher( const CoveredIndexMatcher &prevClauseMatcher, - const shared_ptr &prevClauseFrv, - const BSONObj &nextClauseIndexKeyPattern ) : - _docMatcher( prevClauseMatcher._docMatcher ), - _keyMatcher( *_docMatcher, nextClauseIndexKeyPattern ), - _orDedupConstraints( prevClauseMatcher._orDedupConstraints ) { - if ( prevClauseFrv ) { - _orDedupConstraints.push_back( prevClauseFrv ); - } - init(); - } - - void CoveredIndexMatcher::init() { - _needRecord = - !_keyMatcher.keyMatch( *_docMatcher ) || - !_orDedupConstraints.empty(); - } - - bool CoveredIndexMatcher::matchesCurrent( Cursor * cursor , MatchDetails * details ) const { - // bool keyUsable = ! cursor->isMultiKey() && check for $orish like conditions in matcher SERVER-1264 - - bool keyUsable = true; - if ( cursor->indexKeyPattern().isEmpty() ) { // unindexed cursor - keyUsable = false; - } - else if ( cursor->isMultiKey() ) { - keyUsable = - _keyMatcher.singleSimpleCriterion() && - ( ! _docMatcher || _docMatcher->singleSimpleCriterion() ); - } - return matches( cursor->currKey(), - cursor->currLoc(), - details, - keyUsable ); - } - - bool CoveredIndexMatcher::matches( const BSONObj& key, const DiskLoc& recLoc, - MatchDetails* details, bool keyUsable ) const { - - LOG(5) << "CoveredIndexMatcher::matches() " << key.toString() << ' ' << recLoc.toString() << ' ' << keyUsable << endl; - - dassert( key.isValid() ); - - if ( details ) - details->resetOutput(); - - if ( keyUsable ) { - if ( !_keyMatcher.matches(key, details ) ) { - return false; - } - bool needRecordForDetails = details && details->needRecord(); - if ( !_needRecord && !needRecordForDetails ) { - return true; - } - } - - if ( details ) - details->setLoadedRecord( true ); - - BSONObj obj = recLoc.obj(); - bool res = - _docMatcher->matches( obj, details ) && - !isOrClauseDup( obj ); - LOG(5) << "CoveredIndexMatcher _docMatcher->matches() returns " << res << endl; - return res; - } - - bool CoveredIndexMatcher::isOrClauseDup( const BSONObj &obj ) const { - for( vector >::const_iterator i = _orDedupConstraints.begin(); - i != _orDedupConstraints.end(); ++i ) { - if ( (*i)->matches( obj ) ) { - // If a document matches a prior $or clause index range, generally it would have - // been returned while scanning that range and so is reported as a dup. - return true; - } - } - return false; - } - - string CoveredIndexMatcher::toString() const { - StringBuilder buf; - buf << "(CoveredIndexMatcher "; - - if ( _needRecord ) - buf << "needRecord "; - - buf << "keyMatcher: " << _keyMatcher.toString() << " "; - - if ( _docMatcher ) - buf << "docMatcher: " << _docMatcher->toString() << " "; - - buf << ")"; - return buf.str(); - } -} diff --git a/src/mongo/db/max_time.h b/src/mongo/db/max_time.h new file mode 100644 index 00000000000..47ee245c2ba --- /dev/null +++ b/src/mongo/db/max_time.h @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { + + enum { + kMaxTimeCursorTimeLimitExpired = -1, + kMaxTimeCursorNoTimeLimit = 0 + }; + +} // namespace mongo diff --git a/src/mongo/db/memconcept.cpp b/src/mongo/db/memconcept.cpp deleted file mode 100644 index 00918691584..00000000000 --- a/src/mongo/db/memconcept.cpp +++ /dev/null @@ -1,126 +0,0 @@ -#include "pch.h" - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#if 1 - -#define DDD(x) - -#include -#include -#include "memconcept.h" -#include "../util/assert_util.h" -#include -using namespace std; -#include "../util/log.h" -#include "../util/startup_test.h" - -namespace mongo { - namespace memconcept { - - concept::concept(const char * desc) : c(desc) { } - - // these string pointers we use as unique identifiers - like enums. thus it is important - // you don't use another with the same literal name - concept concept::err("err"); - concept concept::something("something"); - concept concept::database("database"); - concept concept::other("other"); - concept concept::memorymappedfile("memorymappedfile"); - concept concept::nsdetails("nsdetails"); - concept concept::datafileheader("datafileheader"); - concept concept::extent("extent"); - concept concept::record("record"); - concept concept::deletedrecord("deletedrecord"); - concept concept::btreebucket("btreebucket"); - - class X : public StartupTest { - public: - virtual void run() { - } - } concepttest; - - struct C { - void *p; - unsigned len; - concept c; - char desc[16]; - string toString() const; - }; - - string C::toString() const { - stringstream ss; - ss << p << ' ' << c.toString() << ' ' << len << ' ' << desc; - return ss.str(); - } - - const int N = 100003; - - class map { - C nodes[N]; - boost::hash h; - public: - C& find(void *p) { - unsigned x = h(p); - return nodes[x % N]; - } - map() { - memset(this, 0, sizeof(*this)); - for( int i = 0; i < N; i++ ) - nodes[i].c = concept::err; - } - void dump(); - } map; - - void map::dump() { - // sort - std::map m; - for( int i = 0; i < N; i++ ) { - if( nodes[i].p ) { - m[ nodes[i].p ] = &nodes[i]; - } - } - // print - for( std::map::const_iterator i = m.begin(); i != m.end(); i++ ) { - log() << i->second->toString() << endl; - } - } - -#if 0 && defined(_DEBUG) - bool d = false; - void is(void *p, concept c, const StringData& description, unsigned len) { - DDD( log() << "is " << p << ' ' << c.toString() << ' ' << description.data() << ' ' << len << endl; ) - C &node = map.find(p); - node.p = p; - node.c = c; - node.len = len; - strncpy(node.desc, description.data(), 15); - } - - void invalidate(void *p, unsigned len) { - DDD( log() << "inv " << p << " invalidate" << endl; ) - C &node = map.find(p); - node.p = p; - node.c = concept::err; - // len is not used currently. hmmm. - } -#endif - - } -} - -#endif diff --git a/src/mongo/db/memconcept.h b/src/mongo/db/memconcept.h deleted file mode 100644 index 18ce1ea403a..00000000000 --- a/src/mongo/db/memconcept.h +++ /dev/null @@ -1,68 +0,0 @@ -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - - -/* The idea here is to 'name' memory pointers so that we can do diagnostics. - these diagnostics might involve concurrency or other things. mainly would - be for _DEBUG builds. Experimental we'll see how useful. -*/ - -#pragma once - -#include "mongo/base/string_data.h" - -namespace mongo { - namespace memconcept { - - /** these are like fancy enums - you can use them as "types" of things - and see if foo.concept == bar.concept. - copyable. - */ - class concept { - public: - concept() { *this = err; } - const char * toString() const { return c; } - static concept err; - static concept something; - static concept database; - static concept other; - static concept memorymappedfile; - static concept nsdetails; - static concept datafileheader; - static concept extent; - static concept record; - static concept deletedrecord; - static concept btreebucket; - private: - const char * c; - concept(const char *); - }; - - /** file was unmapped or something */ - void invalidate(void *p, unsigned len=0); - - /** note you can be more than one thing; a datafile header is also the starting pointer - for a file */ - void is(void *p, concept c, const StringData& desc = StringData( "", 0 ), unsigned len=0); - -#if 1 -//#if !defined(_DEBUG) - inline void invalidate(void *p, unsigned len) { } - inline void is(void *p, concept c, const StringData& desc, unsigned) { } -#endif - - } -} diff --git a/src/mongo/db/minilex.h b/src/mongo/db/minilex.h index f77bd9b464b..a3a157121be 100644 --- a/src/mongo/db/minilex.h +++ b/src/mongo/db/minilex.h @@ -15,6 +15,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #error does anything use this? diff --git a/src/mongo/db/module.cpp b/src/mongo/db/module.cpp deleted file mode 100644 index f3035b56dde..00000000000 --- a/src/mongo/db/module.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// module.cpp -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "mongo/pch.h" - -#include "mongo/db/module.h" - -namespace mongo { - - std::list * Module::_all; - - Module::Module( const string& name ) - : _name( name ) , _options( (string)"Module " + name + " options" ) { - if ( ! _all ) - _all = new list(); - _all->push_back( this ); - } - - Module::~Module() {} - - void Module::addOptions( boost::program_options::options_description& options ) { - if ( ! _all ) { - return; - } - for ( list::iterator i=_all->begin(); i!=_all->end(); i++ ) { - Module* m = *i; - options.add( m->_options ); - } - } - - void Module::configAll( boost::program_options::variables_map& params ) { - if ( ! _all ) { - return; - } - for ( list::iterator i=_all->begin(); i!=_all->end(); i++ ) { - Module* m = *i; - m->config( params ); - } - - } - - - void Module::initAll() { - if ( ! _all ) { - return; - } - for ( list::iterator i=_all->begin(); i!=_all->end(); i++ ) { - Module* m = *i; - m->init(); - } - - } - -} diff --git a/src/mongo/db/module.h b/src/mongo/db/module.h deleted file mode 100644 index 662624aba9e..00000000000 --- a/src/mongo/db/module.h +++ /dev/null @@ -1,70 +0,0 @@ -// module.h - -/** -* Copyright (C) 2008 10gen Inc.info -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include -#include -#include - -namespace mongo { - - /** - * Module is the base class for adding modules to MongoDB - * modules allow adding hooks and features to mongo - * the idea is to add hooks into the main code for module support where needed - * some ideas are: monitoring, indexes, full text search - */ - class Module { - public: - Module( const std::string& name ); - virtual ~Module(); - - boost::program_options::options_description_easy_init add_options() { - return _options.add_options(); - } - - /** - * read config from command line - */ - virtual void config( boost::program_options::variables_map& params ) = 0; - - /** - * called after configuration when the server is ready start - */ - virtual void init() = 0; - - /** - * called when the database is about to shutdown - */ - virtual void shutdown() = 0; - - const std::string& getName() { return _name; } - - // --- static things - - static void addOptions( boost::program_options::options_description& options ); - static void configAll( boost::program_options::variables_map& params ); - static void initAll(); - - private: - static std::list * _all; - std::string _name; - boost::program_options::options_description _options; - }; -} diff --git a/src/mongo/db/mongod.vcxproj b/src/mongo/db/mongod.vcxproj index 9b4811af659..4a642721e99 100644 --- a/src/mongo/db/mongod.vcxproj +++ b/src/mongo/db/mongod.vcxproj @@ -1909,7 +1909,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..\..\third_party\v8\src - + + true + true + true + true + true + true + true + true + @@ -1917,29 +1926,214 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + - - - - + + true + true + true + true + true + true + true + true + + + $(IntDir)bson_mutable_document.obj + $(IntDir)bson_mutable_document.obj + $(IntDir)bson_mutable_document.obj + $(IntDir)bson_mutable_document.obj + $(IntDir)bson_mutable_document.obj + $(IntDir)bson_mutable_document.obj + $(IntDir)bson_mutable_document.obj + $(IntDir)bson_mutable_document.obj + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + - + + + + + + + true + true + true + true + true + true + true + true + + + + + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + - - - - + + true true true @@ -1949,7 +2143,8 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true - + + true true true @@ -1959,7 +2154,9 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true - + + + true true true @@ -1969,8 +2166,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true - - + true true true @@ -1980,6 +2176,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true + true @@ -1993,7 +2190,17 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - + + + true + true + true + true + true + true + true + true + @@ -2021,110 +2228,890 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4355;4800;4267;4244;4351;4291 - 4355;4800;4267;4244;4351;4291 - 4355;4800;4267;4244;4351;4291 - 4355;4800;4267;4244;4351;4291 - 4355;4800;4267;4244;4351;4291 - 4355;4800;4267;4244;4351;4291 - 4355;4800;4267;4244;4351;4291 - 4355;4800;4267;4244;4351;4291 + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + $(IntDir)db_background.obj + $(IntDir)db_background.obj + $(IntDir)db_background.obj + $(IntDir)db_background.obj + $(IntDir)db_background.obj + $(IntDir)db_background.obj + $(IntDir)db_background.obj + $(IntDir)db_background.obj + + + + + + + + + true + true + true + true + true + true + true + true + + + + + + + + + + + + + + + + + + + + true + true + true + true + true + true + true + true + + + + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + $(IntDir)matcher_path.obj + $(IntDir)matcher_path.obj + $(IntDir)matcher_path.obj + $(IntDir)matcher_path.obj + $(IntDir)matcher_path.obj + $(IntDir)matcher_path.obj + $(IntDir)matcher_path.obj + $(IntDir)matcher_path.obj + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + + + true + true + true + true + true + true + true + true + + + + + + + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + + + + + + + + + + + + + + + + + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + ..;..\..;..\..\third_party\snappy;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost + + + + + + + + + + + + + + + + + + 4355;4800;4267;4244;4351;4291 + 4355;4800;4267;4244;4351;4291 + 4355;4800;4267;4244;4351;4291 + 4355;4800;4267;4244;4351;4291 + 4355;4800;4267;4244;4351;4291 + 4355;4800;4267;4244;4351;4291 + 4355;4800;4267;4244;4351;4291 + 4355;4800;4267;4244;4351;4291 + + + + + + + + + + + + true + true + true + true + true + true + true + true + + + + + + + + + + + + + + true + true + true + true + true + true + true + true + + + + + + + + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + + + + + + + + + + + + + + + + + + + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + + true + true + true + true + true + true + true + true + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + + true + true + true + true + true + true + true + true + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + true + true + true + true + true + true + true + true ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2134,7 +3121,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2144,7 +3131,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + true true true @@ -2162,7 +3149,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2172,7 +3159,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2182,7 +3169,15 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + true + true + true + true + true + true + true + true ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2192,7 +3187,17 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + true true true @@ -2210,7 +3215,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2220,7 +3225,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + true true true @@ -2238,7 +3243,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2248,7 +3253,17 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include + + true true true @@ -2266,7 +3281,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2276,7 +3291,15 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + true + true + true + true + true + true + true + true ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include @@ -2286,7 +3309,183 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + + + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + 4355;4800;4267;4244;4351;4146 + + + + + + + + + + + + + + + + + + + + + + + + + true + true + true + true + true + true + true + true + + + + + + + + $(IntDir)\expression1.obj + $(IntDir)\expression1.obj + $(IntDir)\expression1.obj + $(IntDir)\expression1.obj + $(IntDir)\expression1.obj + $(IntDir)\expression1.obj + $(IntDir)\expression1.obj + $(IntDir)\expression1.obj + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + true true true @@ -2295,26 +3494,8 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true true - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + true true true @@ -2323,26 +3504,9 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true true - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + true true true @@ -2351,36 +3515,22 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true true - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + + + $(IntDir)matcher2.obj + $(IntDir)matcher2.obj + $(IntDir)matcher2.obj + $(IntDir)matcher2.obj + $(IntDir)matcher2.obj + $(IntDir)matcher2.obj + $(IntDir)matcher2.obj + $(IntDir)matcher2.obj + + + + true true true @@ -2389,26 +3539,9 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true true - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - + + true true true @@ -2417,36 +3550,9 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true true - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..;..\..;..\..\third_party\v8\include;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - - - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - + + true true true @@ -2456,9 +3562,8 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true - - - + + true true true @@ -2468,54 +3573,29 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true - - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - - - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - - - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 + + + true + true + true + true + true + true + true + true - - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 - 4355;4800;4267;4244;4351;4146 + + + true + true + true + true + true + true + true + true - - - - - - - - + @@ -2529,7 +3609,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -2544,40 +3623,32 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - - - - - - - @@ -2586,18 +3657,81 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - + + + + + + + + + + + + + + + + + $(IntDir)repl_write_concern.obj + $(IntDir)repl_write_concern.obj + $(IntDir)repl_write_concern.obj + $(IntDir)repl_write_concern.obj + $(IntDir)repl_write_concern.obj + $(IntDir)repl_write_concern.obj + $(IntDir)repl_write_concern.obj + $(IntDir)repl_write_concern.obj + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + + + + + + + + true + true + true + true + true + true + true + true + + + @@ -2696,6 +3830,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + @@ -2995,7 +4130,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3010,32 +4144,92 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + + - - - - + + + + + + + + + + true + true + true + true + true + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + true + true + true + true + true + true + true + true + + + true + true + true + true + true + true + true + true + + + @@ -3044,17 +4238,97 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - true - true - true - true - true - true - true - true - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true true true @@ -3064,16 +4338,8 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj true true - - - - - - - - - - + + @@ -3100,7 +4366,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3108,8 +4373,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + + @@ -3117,7 +4384,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3130,13 +4396,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - - - + @@ -3148,7 +4411,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3164,7 +4426,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3175,19 +4436,40 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3198,16 +4480,33 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + + + + + + + + + + + - - + + + + + + + + @@ -3216,8 +4515,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - @@ -3225,7 +4522,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3247,20 +4543,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - - - @@ -3303,21 +4595,46 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - + + + + + + + + + + + + + + - + + + + + + + + + + + + + + @@ -3347,6 +4664,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + + + + @@ -3406,9 +4727,13 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + + + + @@ -3427,6 +4752,8 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj + + @@ -3443,7 +4770,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3470,13 +4796,11 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - @@ -3489,10 +4813,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - - - @@ -3501,24 +4821,20 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - - - @@ -3539,10 +4855,8 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - - @@ -3562,7 +4876,6 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - @@ -3597,4 +4910,4 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj - \ No newline at end of file + diff --git a/src/mongo/db/mongod.vcxproj.filters b/src/mongo/db/mongod.vcxproj.filters index 8e27c34a716..e7d7c998e19 100644 --- a/src/mongo/db/mongod.vcxproj.filters +++ b/src/mongo/db/mongod.vcxproj.filters @@ -1,9 +1,6 @@  - - third_party\snappy - third_party\snappy @@ -163,12 +160,6 @@ db\commands - - db\geo - - - db\pipeline\Source Files - db\pipeline\Source Files @@ -190,15 +181,9 @@ db\pipeline\Source Files - - db\pipeline\Source Files - db\pipeline\Source Files - - db\pipeline\Source Files - db\pipeline\Source Files @@ -211,12 +196,6 @@ db\pipeline\Source Files - - db\pipeline\Source Files - - - db\pipeline\Source Files - db\pipeline\Source Files @@ -268,9 +247,6 @@ db\stats - - s - s @@ -401,9 +377,6 @@ db\ops - - util\Source Files - db\repl @@ -425,9 +398,6 @@ client\Source Files - - client\Source Files - db\repl @@ -482,9 +452,6 @@ util\Source Files - - util\Source Files - client\Source Files @@ -620,9 +587,6 @@ db\Source Files\e to n - - db\Source Files\e to n - db\Source Files\e to n @@ -644,9 +608,6 @@ db\Source Files\e to n - - db\Source Files\e to n - db\Source Files\e to n @@ -656,18 +617,9 @@ db\Source Files\e to n - - db\Source Files\e to n - - - db\Source Files\e to n - db\Source Files\e to n - - db\Source Files\o to z - db\Source Files\o to z @@ -680,9 +632,6 @@ db\Source Files\o to z - - db\Source Files\o to z - db\Source Files\o to z @@ -692,15 +641,6 @@ db\Source Files\o to z - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - db\Source Files\o to z @@ -752,9 +692,6 @@ util\Source Files - - db\Source Files\e to n - db\Source Files\e to n @@ -773,9 +710,6 @@ third_party\MurmurHash3 - - db\Source Files\a to d - db\pipeline\Source Files @@ -791,9 +725,6 @@ util\Source Files - - base - base @@ -875,24 +806,6 @@ db\auth - - bson\mutable - - - bson\mutable - - - bson\mutable - - - bson\mutable - - - db\geo - - - db\geo - db\commands @@ -1364,12 +1277,6 @@ db\auth - - db\auth - - - s - s @@ -1379,9 +1286,6 @@ db\geo - - db\geo - s @@ -1403,15 +1307,6 @@ db\Source Files\o to z - - db\auth - - - db\auth - - - s - s @@ -1712,9 +1607,6 @@ db\fts - - db\fts - db\fts @@ -1769,9 +1661,6 @@ db\fts\test - - db\auth - db\stats @@ -1781,12 +1670,6 @@ scripting - - scripting - - - scripting - scripting @@ -1814,2037 +1697,3255 @@ db\Source Files\e to n - + + bson\mutable + + + bson\mutable + + + bson\mutable + + + bson\mutable + + + db\commands + + + db\repl + + + db\repl + + + db\repl + + + db\repl + + + db\repl + + + db\repl + + + db\repl + + + db\repl + + + db\repl + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\Source Files\o to z + + platform - + platform - + platform - - - - - third_party\snappy - - - third_party\snappy - - - third_party\pcre - - - third_party\pcre - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - bson\util - - - util\Header Files - - - util\Header Files - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson\util - - - util\mongoutils - - - db\repl - - - db\geo - - + + db\Source Files\o to z + + + db\index + + + db\index + + + db\index + + + db\index + + + db\index + + + db\index + + + db\Source Files\e to n + + + db\index + + + db\index + + + db\index + + + db\index + + db\ops - - - db\stats - - + + + db\index + + + db\index + + s - - - db\pipeline\Header Files - - - util\Header Files - - - db\ops - - - scripting - - - scripting - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - db\pipeline\Header Files - - + + + s + + + db\index + + + db\index + + + db\index + + + db\Source Files\e to n + + + db\Source Files\e to n + + + db\Source Files\e to n + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\Source Files\o to z + + + db\matcher + + + db\matcher + + + db\Source Files\e to n + + + db\ops + + + db\ops + + + db\ops + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\matcher + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\matcher + + + db\auth + + + db\auth + + + db\auth + + + base + + + base + + + base + + + base + + + base + + + base + + + base + + + base + + + bson + + + bson + + + bson + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\auth + + + db\auth + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + util\Source Files + + + db\matcher + + + db\matcher + + + db\matcher + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + db\ops + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + util\Source Files + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\auth + + + db\Source Files\e to n + + + util\Source Files + + + util\Source Files + + + platform + + + platform + + + db\auth + + + db\commands + + + platform + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + s + + + db\ops + + + db\ops + + + platform + + + platform + + + db\Source Files\a to d + + + db\Source Files\e to n + + + logger + + + logger + + + logger + + + logger + + + logger + + + logger + + + logger + + + logger + + + logger + + + logger + + + logger + + + logger + + + util\concurrency + + + db\storage + + + db\storage + + + db\storage + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\exec + + + db\sorter + + + db\commands + + + db\Source Files\o to z + + + db\Source Files\o to z + + + db\Source Files\e to n + + + db\Source Files\e to n + + + db\Source Files\e to n + + + db\Source Files\e to n + + + db\Source Files\e to n + + + db\Source Files\e to n + + + bson\mutable + + + db\auth + + + db\Source Files\a to d + + + db\commands\write_commands + + + db\commands\write_commands + + + db\commands\write_commands + + + db\commands\write_commands + + + db\repl + + + db\stats + + + db\Source Files\a to d + + + db\exec + + + db\exec + + + db\Source Files\a to d + + + db\storage + + + db\storage + + + db\storage + + + db\storage + + + bson + + + db\exec + + + db\exec + + + db\ops + + + db\ops + + + db\repl + + + db\query + + + db\query + + + db\auth + + + db\auth + + + db\query + + + db\query + + + db\query + + + db\storage + + + db\query + + + db\query + + + db\query + + + db\commands + + + db\pipeline\Source Files + + + + + third_party\snappy + + + third_party\snappy + + + third_party\pcre + + + third_party\pcre + + + db\pipeline\Header Files + + + db\pipeline\Header Files + + + db\pipeline\Header Files + + + util\Header Files + + + util\Header Files + + + util\Header Files + + + util\Header Files + + + bson\util + + + util\Header Files + + + util\Header Files + + + bson + + + bson + + + bson + + + bson + + + bson + + + bson + + + bson + + + bson + + + bson + + + bson\util + + + util\mongoutils + + + db\repl + + + db\geo + + + db\ops + + + db\stats + + + s + + + db\pipeline\Header Files + + + util\Header Files + + + db\ops + + + scripting + + + scripting + + + util\Header Files + + + util\Header Files + + + util\Header Files + + + db\pipeline\Header Files + + + db\pipeline\Header Files + + + db\pipeline\Header Files + + + util\Header Files + + + util\Header Files + + + db\repl + + + util\net + + + util\mongoutils + + + util\net + + + bson + + + util\Header Files + + + util\Header Files + + + util\concurrency + + + util\net + + + util\concurrency + + + util\Header Files + + + util\Header Files + + + util\Header Files + + + util\net + + + util\net + + + util\net + + + util\net + + + util\Header Files + + + util\Header Files + + + bson\util + + + util\Header Files + + + util\concurrency + + + db\repl + + + util\concurrency + + + util\concurrency + + + bson + + + util\Header Files + + + bson + + + + util\Header Files + + + util\Header Files + + + util\Header Files + + + util\concurrency + + + util\concurrency + + + util\Header Files + + + util\Header Files + + + client\Header Files + + + client\Header Files + + + client\Header Files + + + client\Header Files + + + client\Header Files + + + client\Header Files + + + client\Header Files + + + db\repl + + + db\repl + + + util\concurrency + + + util\concurrency + + + db\stats + + + util\net + + + util\mongoutils + + + base + + + util\Header Files + + + util\concurrency + + + client\Header Files + + + util\concurrency + + + util\Header Files + + + db\stats + + + util\Header Files + + + util\Header Files + + + util\Header Files + + + util\Header Files + + + db\ops + + + scripting + + + scripting + + + util\concurrency + + + client\Header Files + + + db\pipeline\Header Files + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\a to d + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\e to n + + + db\Header Files\o to z + + + db\Header Files\o to z + + + db\Header Files\o to z + + + db\Header Files\o to z + + + db\Header Files\o to z + + + db\Header Files\o to z + + + db\Header Files\o to z + + + util\Header Files + + + util\Header Files + + + db\stats + + + db\Header Files\e to n + + + db\Header Files\o to z + + + db\ops + + + db\ops + + + util\Header Files + + + util\Header Files + + + db\Header Files\e to n + + + s + + + scripting + + + s + + + db\Header Files\e to n + + + db\Header Files\o to z + + + util\concurrency + + + util\concurrency + + + platform + + + platform + + + platform + + + platform + + + platform + + + platform + + + platform + + + platform + + + third_party\MurmurHash3 + + db\pipeline\Header Files - + db\pipeline\Header Files - - util\Header Files + + scripting - + util\Header Files - - db\repl + + base - - util\net + + base - - util\mongoutils + + base - - util\net + + base - + + base + + + base + + + platform + + + base + + + base + + + base + + + base + + + base + + + base + + + base + + bson - + + bson + + + platform + + + platform + + + platform + + util\Header Files - + util\Header Files - - util\concurrency + + db\auth - - util\net + + db\auth - - util\concurrency + + db\auth - - util\Header Files + + db\Header Files\e to n - - util\Header Files + + db\Header Files\e to n - - util\Header Files + + platform - - util\net + + db\Header Files\e to n - - util\net + + platform - - util\net + + db\geo - + + db\geo + + util\net - - util\Header Files + + db\Header Files\e to n - - util\Header Files + + s - - bson\util + + s - + util\Header Files - - util\concurrency + + db\auth - - db\repl + + base - - util\concurrency + + db\auth - - util\concurrency + + db\auth - - bson + + db\Header Files\a to d - - util\Header Files + + db\commands - - bson + + db\commands - - - util\Header Files + + db\commands - - util\Header Files + + db\Header Files\a to d - - util\Header Files + + Boost - + + db\auth + + + db\geo + + util\Header Files - - util\concurrency + + bson\util - - util\concurrency + + third_party\v8\Consumer Include Files - - util\Header Files + + third_party\v8\Consumer Include Files - - util\Header Files + + third_party\v8\Consumer Include Files - - client\Header Files + + third_party\v8\Consumer Include Files - - client\Header Files + + third_party\v8\Consumer Include Files - - client\Header Files + + third_party\v8\Consumer Include Files - - client\Header Files + + third_party\v8\src\extensions - - client\Header Files + + third_party\v8\src\extensions - - client\Header Files + + third_party\v8\src\extensions - - client\Header Files + + third_party\v8\src\Header Files\a to d - - client\Header Files + + third_party\v8\src\Header Files\a to d - - db\repl + + third_party\v8\src\Header Files\a to d - - db\repl + + third_party\v8\src\Header Files\a to d - - db\repl + + third_party\v8\src\Header Files\a to d - - util\concurrency + + third_party\v8\src\Header Files\a to d - - util\concurrency + + third_party\v8\src\Header Files\a to d - - db\stats + + third_party\v8\src\Header Files\a to d - - util\net + + third_party\v8\src\Header Files\a to d - - util\mongoutils + + third_party\v8\src\Header Files\a to d - - base + + third_party\v8\src\Header Files\a to d - - util\Header Files + + third_party\v8\src\Header Files\a to d - - util\concurrency + + third_party\v8\src\Header Files\a to d - - util\Header Files + + third_party\v8\src\Header Files\a to d - - client\Header Files + + third_party\v8\src\Header Files\a to d - - util\concurrency + + third_party\v8\src\Header Files\a to d - - util\Header Files + + third_party\v8\src\Header Files\a to d - - db\stats + + third_party\v8\src\Header Files\a to d - - util\Header Files + + third_party\v8\src\Header Files\a to d - - util\Header Files + + third_party\v8\src\Header Files\a to d - - util\Header Files + + third_party\v8\src\Header Files\a to d - - util\Header Files + + third_party\v8\src\Header Files\a to d - - db\ops + + third_party\v8\src\Header Files\a to d - - scripting + + third_party\v8\src\Header Files\a to d - - scripting + + third_party\v8\src\Header Files\a to d - - util\concurrency + + third_party\v8\src\Header Files\a to d - - client\Header Files + + third_party\v8\src\Header Files\a to d - - db\pipeline\Header Files + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\a to d - - db\Header Files\a to d + + third_party\v8\src\Header Files\e to n - - db\Header Files\a to d + + third_party\v8\src\Header Files\e to n - - db\Header Files\a to d + + third_party\v8\src\Header Files\e to n - - db\Header Files\a to d + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - util\Header Files + + third_party\v8\src\Header Files\e to n - - util\Header Files + + third_party\v8\src\Header Files\e to n - - db\stats + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - db\ops + + third_party\v8\src\Header Files\e to n - - db\ops + + third_party\v8\src\Header Files\e to n - - util\Header Files + + third_party\v8\src\Header Files\e to n - - util\Header Files + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - s + + third_party\v8\src\Header Files\e to n - - scripting + + third_party\v8\src\Header Files\e to n - - s + + third_party\v8\src\Header Files\e to n - - db\Header Files\e to n + + third_party\v8\src\Header Files\e to n - - db\Header Files\o to z + + third_party\v8\src\Header Files\e to n - - util\concurrency + + third_party\v8\src\Header Files\e to n - - util\concurrency + + third_party\v8\src\Header Files\e to n - - platform + + third_party\v8\src\Header Files\e to n - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - third_party\MurmurHash3 + + third_party\v8\src\Header Files\o to z - - db\pipeline\Header Files + + third_party\v8\src\Header Files\o to z - - db\pipeline\Header Files + + third_party\v8\src\Header Files\o to z - - scripting + + third_party\v8\src\Header Files\o to z - - util\Header Files + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - bson + + third_party\v8\src\Header Files\o to z - - bson + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - util\Header Files + + third_party\v8\src\Header Files\o to z - - util\Header Files + + third_party\v8\src\Header Files\o to z - - db\auth + + third_party\v8\src\Header Files\o to z - - db\auth + + third_party\v8\src\Header Files\o to z - - db\auth + + third_party\v8\src\Header Files\o to z - - db\Header Files\e to n + + third_party\v8\src\Header Files\o to z - - db\Header Files\e to n + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - db\Header Files\e to n + + third_party\v8\src\Header Files\o to z - - platform + + third_party\v8\src\Header Files\o to z - - db\geo + + third_party\v8\src\Header Files\o to z - - db\geo + + third_party\v8\src\Header Files\o to z - - util\net + + third_party\v8\src\Header Files\o to z - - db\Header Files\e to n + + third_party\v8\src\Header Files\o to z - - s + + third_party\v8\src\Header Files\o to z - - s + + third_party\v8\src\Header Files\o to z - - s + + third_party\v8\src\Header Files\o to z - - util\Header Files + + third_party\v8\src\Header Files\o to z - - db\auth + + third_party\v8\src\Header Files\o to z - - base + + third_party\v8\src\Header Files\o to z - - db\auth + + third_party\v8\src\Header Files\o to z - - db\auth + + third_party\v8\src\Header Files\o to z - - db\Header Files\a to d + + third_party\v8\src\Header Files\o to z - - db\commands + + third_party\v8\src\Header Files\o to z - - db\commands + + third_party\v8\src\Header Files\o to z - - db\commands + + third_party\v8\src\Header Files\o to z - - db\Header Files\a to d + + third_party\v8\src\Header Files\o to z - - Boost + + third_party\v8\src\Header Files\o to z - - db\auth + + third_party\v8\src\Header Files\o to z - - bson\mutable + + third_party\v8\src\Header Files\o to z - - bson\mutable + + third_party\v8\src\Header Files\o to z - - bson\mutable + + third_party\v8\src\Header Files\o to z - - bson\mutable + + third_party\v8\src\Header Files\o to z - - db\geo + + third_party\v8\src\Header Files\o to z - - db\geo + + third_party\v8\src\Header Files\o to z - - util\Header Files + + third_party\v8\src\Header Files\o to z - - bson\util + + third_party\v8\src\Header Files\o to z - - third_party\v8\Consumer Include Files + + third_party\v8\src\Header Files\o to z - - third_party\v8\Consumer Include Files + + third_party\v8\src\Header Files\o to z - - third_party\v8\Consumer Include Files + + third_party\v8\src\Header Files\o to z - - third_party\v8\Consumer Include Files + + third_party\v8\src\ia32\Header Files - - third_party\v8\Consumer Include Files + + third_party\v8\src\ia32\Header Files - - third_party\v8\Consumer Include Files + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\extensions + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\extensions + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\extensions + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\ia32\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + third_party\v8\src\x64\Header Files - - third_party\v8\src\Header Files\a to d + + db\auth - - third_party\v8\src\Header Files\a to d + + db\auth - - third_party\v8\src\Header Files\a to d + + s - - third_party\v8\src\Header Files\a to d + + db\Header Files\e to n - - third_party\v8\src\Header Files\a to d + + db\Header Files\a to d + + + db\Header Files\a to d - - third_party\v8\src\Header Files\a to d + + db\Header Files\a to d - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + util\Header Files - - third_party\v8\src\Header Files\a to d + + db\pipeline\Header Files - - third_party\v8\src\Header Files\a to d + + db\pipeline\Header Files - - third_party\v8\src\Header Files\a to d + + s - - third_party\v8\src\Header Files\a to d + + s - - third_party\v8\src\Header Files\e to n + + db\Header Files\a to d - - third_party\v8\src\Header Files\e to n + + db\Header Files\e to n - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + db\commands - - third_party\v8\src\Header Files\e to n + + util\Header Files - - third_party\v8\src\Header Files\e to n + + base - - third_party\v8\src\Header Files\e to n + + base - - third_party\v8\src\Header Files\e to n + + db\Header Files\a to d - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\o to z - - third_party\v8\src\Header Files\e to n + + db\Header Files\e to n - - third_party\v8\src\Header Files\e to n + + db\Header Files\e to n - - third_party\v8\src\Header Files\e to n + + db\Header Files\e to n - - third_party\v8\src\Header Files\e to n + + db\Header Files\e to n - - third_party\v8\src\Header Files\e to n + + db\Header Files\e to n - - third_party\v8\src\Header Files\e to n + + db\Header Files\e to n - - third_party\v8\src\Header Files\e to n + + db\Header Files\a to d - - third_party\v8\src\Header Files\e to n + + db\Header Files\a to d - - third_party\v8\src\Header Files\e to n + + db\Header Files\a to d - - third_party\v8\src\Header Files\e to n + + db\Header Files\a to d - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + s - - third_party\v8\src\Header Files\e to n + + db\geo - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math\exactfloat - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\math - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\hash - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\endian - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\coding - - third_party\v8\src\Header Files\e to n + + third_party\s2\util\coding - - third_party\v8\src\Header Files\e to n + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\base - - third_party\v8\src\Header Files\o to z + + third_party\s2\strings - - third_party\v8\src\Header Files\o to z + + third_party\s2\strings - - third_party\v8\src\Header Files\o to z + + third_party\s2\strings - - third_party\v8\src\Header Files\o to z + + third_party\s2\strings - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + third_party\s2 - - third_party\v8\src\Header Files\o to z + + bson - - third_party\v8\src\Header Files\o to z + + db\commands - - third_party\v8\src\Header Files\o to z + + db\auth - - third_party\v8\src\Header Files\o to z + + client\Header Files - - third_party\v8\src\Header Files\o to z + + client\Header Files - - third_party\v8\src\Header Files\o to z + + client\Header Files - - third_party\v8\src\Header Files\o to z + + client\Header Files - - third_party\v8\src\Header Files\o to z + + db\auth - - third_party\v8\src\Header Files\o to z + + db\commands - - third_party\v8\src\Header Files\o to z + + db\repl - - third_party\v8\src\Header Files\o to z + + db\repl - - third_party\v8\src\Header Files\o to z + + db\repl - - third_party\v8\src\Header Files\o to z + + db\repl - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\include - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\libstemmer - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\libstemmer - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\runtime - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\runtime - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\Header Files\o to z + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\ia32\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - third_party\v8\src\x64\Header Files + + third_party\libstemmer_c\src_c - - db\auth + + third_party\libstemmer_c\src_c - - db\auth + + db\fts - - db\auth + + db\fts - - s + + db\fts - - s + + db\fts - - db\Header Files\e to n + + db\fts - - db\Header Files\a to d + + db\fts - - db\Header Files\a to d + + db\fts - - db\Header Files\a to d + + db\fts - - util\Header Files + + db\fts - - util\Header Files + + db\fts - - util\Header Files + + db\fts - - util\Header Files + + db\fts - - util\Header Files + + db\stats - - util\Header Files + + db\stats - - util\Header Files + + scripting - - util\Header Files + + db\geo - - util\Header Files + + scripting - - util\Header Files + + db\Header Files\e to n - - util\Header Files + + client\Header Files - - util\Header Files + + db\Header Files\e to n - - util\Header Files + + bson\mutable - - util\Header Files + + bson\mutable - - util\Header Files + + bson\mutable - - util\Header Files + + bson\mutable - - util\Header Files + + bson\mutable - - util\Header Files + + bson\mutable - - util\Header Files + + bson\mutable - - util\Header Files + + db\repl - - util\Header Files + + db\repl - - util\Header Files + + db\repl - - util\Header Files + + db\repl - - util\Header Files + + db\repl - - db\geo + + db\repl - - db\pipeline\Header Files + + db\repl - - db\pipeline\Header Files + + db\repl - - s + + db\repl - - s + + db\Header Files\o to z - - db\Header Files\a to d + + db\Header Files\o to z - - db\Header Files\e to n + + db\Header Files\o to z - - s + + db\Header Files\o to z - + db\Header Files\o to z - + + platform + + + platform + + + platform + + db\Header Files\o to z - - db\auth + + db\index - - db\auth + + db\index - - s + + db\index - - s + + db\index - - db\commands + + db\index - - util\Header Files + + db\index - - base + + db\index - - base + + db\index - - db\auth + + db\index - - db\Header Files\a to d + + db\index - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\Header Files\e to n - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\ops - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\index - - db\Header Files\o to z + + db\index - - db\Header Files\e to n + + third_party\snappy - - db\Header Files\e to n + + db\repl - - db\Header Files\e to n + + s - - db\Header Files\e to n + + db\index - - db\Header Files\e to n + + db\index - - db\Header Files\e to n + + db\index - - db\Header Files\a to d + + db\index - - db\Header Files\a to d + + util\Header Files - - db\Header Files\a to d + + + base - - db\Header Files\a to d + + db\Header Files\e to n - - db\Header Files\a to d + + db\Header Files\e to n - - s + + db\matcher - - s + + db\matcher - - s + + db\matcher - - s + + db\matcher - - s + + db\matcher - - s + + db\Header Files\o to z - - s + + db\matcher - - db\geo + + db\matcher - - db\geo + + db\Header Files\e to n - - db\geo + + db\ops - - third_party\s2\util\math\exactfloat + + db\matcher - - third_party\s2\util\math + + db\ops - - third_party\s2\util\math + + db\ops - - third_party\s2\util\math + + db\ops - - third_party\s2\util\math + + db\Header Files\e to n - - third_party\s2\util\math + + util\Header Files - - third_party\s2\util\math + + db\ops - - third_party\s2\util\math + + db\ops - - third_party\s2\util\math + + db\matcher - - third_party\s2\util\math + + base - - third_party\s2\util\math + + db\Header Files\o to z - - third_party\s2\util\hash + + db\Header Files\o to z - - third_party\s2\util\endian + + db\Header Files\o to z - - third_party\s2\util\coding + + db\auth - - third_party\s2\util\coding + + db\auth - - third_party\s2\base + + db\auth - - third_party\s2\base + + db\Header Files\o to z - - third_party\s2\base + + db\Header Files\o to z - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\auth - - third_party\s2\base + + db\auth - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\matcher - - third_party\s2\base + + db\matcher - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\ops - - third_party\s2\base + + db\auth - - third_party\s2\base + + db\auth - - third_party\s2\strings + + db\auth - - third_party\s2\strings + + db\auth - - third_party\s2\strings + + db\auth - - third_party\s2\strings + + db\auth - - third_party\s2 + + db\auth - - third_party\s2 + + db\auth - - third_party\s2 + + db\auth - - third_party\s2 + + db\Header Files\e to n - - third_party\s2 + + db\Header Files\e to n - - third_party\s2 + + platform - - third_party\s2 + + platform - - third_party\s2 + + s - - third_party\s2 + + db\ops - - third_party\s2 + + platform - - third_party\s2 + + platform - - third_party\s2 + + db\auth - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - third_party\s2 + + logger - - bson + + logger - - db\commands + + logger - - db\auth + + logger - - client\Header Files + + logger - - client\Header Files + + logger - - client\Header Files + + logger - - client\Header Files + + logger - - db\auth + + logger - - db\commands + + logger - - db\repl + + logger - - db\repl + + util\concurrency - - db\repl + + util\concurrency - - db\repl + + util\concurrency - - third_party\libstemmer_c\include + + util\concurrency - - third_party\libstemmer_c\libstemmer + + util\concurrency - - third_party\libstemmer_c\libstemmer + + util\concurrency - - third_party\libstemmer_c\runtime + + db\storage - - third_party\libstemmer_c\runtime + + db\storage - - third_party\libstemmer_c\src_c + + db\storage - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\sorter - - third_party\libstemmer_c\src_c + + db\commands - - third_party\libstemmer_c\src_c + + db\Header Files\o to z - - third_party\libstemmer_c\src_c + + db\Header Files\o to z - - third_party\libstemmer_c\src_c + + db\Header Files\e to n - - third_party\libstemmer_c\src_c + + bson\mutable - - third_party\libstemmer_c\src_c + + bson\mutable - - third_party\libstemmer_c\src_c + + db\auth - - third_party\libstemmer_c\src_c + + db\commands\write_commands - - third_party\libstemmer_c\src_c + + db\commands\write_commands - - third_party\libstemmer_c\src_c + + db\commands\write_commands - - third_party\libstemmer_c\src_c + + db\Header Files\e to n - - third_party\libstemmer_c\src_c + + db\Header Files\e to n - - third_party\libstemmer_c\src_c + + db\Header Files\a to d - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\exec - - third_party\libstemmer_c\src_c + + db\Header Files\a to d - - db\fts + + db\storage - - db\fts + + db\storage - - db\fts + + db\storage - - db\fts + + db\storage - - db\fts + + bson - - db\fts + + db\exec - - db\fts + + db\exec - - db\fts + + db\ops - - db\fts + + db\repl - - db\fts + + db\exec - - db\fts + + db\query - - db\fts + + db\query - - db\fts + + db\query - - db\ops + + db\query - - db\stats + + db\auth - - db\stats + + db\query - - scripting + + db\query - - scripting + + db\query - - scripting + + db\query - - db\geo + + db\query - - scripting + + db\query - - db\Header Files\e to n + + db\query - - client\Header Files + + db\query - - db\Header Files\e to n + + db\query - - platform + + db\storage - - platform + + db\structure - - platform + + db\exec @@ -4037,6 +5138,36 @@ JavaScript source files\Included in shell only + + platform + + + db\ops + + + db\auth + + + base + + + bson + + + logger + + + util\concurrency + + + util\concurrency + + + db\exec + + + db\query + @@ -4285,6 +5416,33 @@ {023400bc-92fc-4d80-849e-ee5c8c6ca478} + + {222760f0-8bef-42ff-933e-cb765e706d9f} + + + {3191e905-ed7c-4a1e-a905-b29bb06aa1dd} + + + {c9be6b22-4457-4519-b275-72eb6c221f95} + + + {f350e4e4-c3a2-4021-9774-303674f2bb2a} + + + {b330728f-d98a-4694-892b-812645420c17} + + + {1619768e-2e75-4a58-a97e-7441d9dfe40a} + + + {2e34064a-aa9c-469d-ab06-3e4786e7113d} + + + {9d9e9477-2823-4e23-b417-a6f767231cca} + + + {425b220f-4691-4320-8f6a-22673acf99d6} + @@ -4296,4 +5454,4 @@ db\Resource Files - \ No newline at end of file + diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp new file mode 100644 index 00000000000..ac3b7c27fb3 --- /dev/null +++ b/src/mongo/db/mongod_options.cpp @@ -0,0 +1,1124 @@ +/* + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/mongod_options.h" + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/bson/util/builder.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/instance.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/repl/replication_server_status.h" +#include "mongo/db/server_options.h" +#include "mongo/db/server_options_helpers.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/net/ssl_options.h" +#include "mongo/util/options_parser/startup_options.h" +#include "mongo/util/version.h" +#include "mongo/util/version_reporting.h" + +namespace mongo { + + MongodGlobalParams mongodGlobalParams; + + extern DiagLog _diaglog; + + Status addMongodOptions(moe::OptionSection* options) { + + moe::OptionSection general_options("General options"); + + Status ret = addGeneralServerOptions(&general_options); + if (!ret.isOK()) { + return ret; + } + +#if defined(_WIN32) + moe::OptionSection windows_scm_options("Windows Service Control Manager options"); + + ret = addWindowsServerOptions(&windows_scm_options); + if (!ret.isOK()) { + return ret; + } +#endif + +#ifdef MONGO_SSL + moe::OptionSection ssl_options("SSL options"); + + ret = addSSLServerOptions(&ssl_options); + if (!ret.isOK()) { + return ret; + } +#endif + + moe::OptionSection ms_options("Master/slave options (old; use replica sets instead)"); + moe::OptionSection rs_options("Replica set options"); + moe::OptionSection replication_options("Replication options"); + moe::OptionSection sharding_options("Sharding options"); + + // Authentication Options + + // Way to enable or disable auth on command line and in Legacy config file + general_options.addOptionChaining("auth", "auth", moe::Switch, "run with security") + .setSources(moe::SourceAllLegacy) + .incompatibleWith("noauth"); + + general_options.addOptionChaining("noauth", "noauth", moe::Switch, "run without security") + .setSources(moe::SourceAllLegacy) + .incompatibleWith("auth"); + + // Way to enable or disable auth in JSON Config + general_options.addOptionChaining("security.authorization", "", moe::String, + "How the database behaves with respect to authorization of clients. " + "Options are \"disabled\", which means that authorization checks are not " + "performed, and \"enabled\" which means that a client cannot perform actions it is " + "not authorized to do.") + .setSources(moe::SourceYAMLConfig) + .format("(:?disabled)|(:?enabled)", + "(disabled/enabled)"); + + // setParameter parameters that we want as config file options + // TODO: Actually read these into our environment. Currently they have no effect + general_options.addOptionChaining("security.authSchemaVersion", "", moe::String, "TODO") + .setSources(moe::SourceYAMLConfig); + + general_options.addOptionChaining("security.enableLocalhostAuthBypass", "", moe::String, + "TODO") + .setSources(moe::SourceYAMLConfig); + + general_options.addOptionChaining("security.supportCompatibilityFormPrivilegeDocuments", "", + moe::String, "TODO") + .setSources(moe::SourceYAMLConfig); + + // Network Options + + general_options.addOptionChaining("net.ipv6", "ipv6", moe::Switch, + "enable IPv6 support (disabled by default)"); + + general_options.addOptionChaining("net.http.JSONPEnabled", "jsonp", moe::Switch, + "allow JSONP access via http (has security implications)"); + + general_options.addOptionChaining("net.http.RESTInterfaceEnabled", "rest", moe::Switch, + "turn on simple rest api"); + + // Diagnostic Options + + general_options.addOptionChaining("diaglog", "diaglog", moe::Int, + "DEPRECATED: 0=off 1=W 2=R 3=both 7=W+some reads") + .hidden() + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("operationProfiling.slowOpThresholdMs", "slowms", + moe::Int, "value of slow for profile and console log") + .setDefault(moe::Value(100)); + + general_options.addOptionChaining("profile", "profile", moe::Int, + "0=off 1=slow, 2=all") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("operationProfiling.mode", "", moe::String, + "(off/slowOp/all)") + .setSources(moe::SourceYAMLConfig) + .format("(:?off)|(:?slowOp)|(:?all)", "(off/slowOp/all)"); + + general_options.addOptionChaining("cpu", "cpu", moe::Switch, + "periodically show cpu and iowait utilization") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("sysinfo", "sysinfo", moe::Switch, + "print some diagnostic system information") + .setSources(moe::SourceAllLegacy); + + // Storage Options + +#ifdef _WIN32 + general_options.addOptionChaining("storage.dbPath", "dbpath", moe::String, + std::string("directory for datafiles - defaults to ") + + storageGlobalParams.kDefaultDbPath); + +#else + general_options.addOptionChaining("storage.dbPath", "dbpath", moe::String, + std::string("directory for datafiles - defaults to ") + + storageGlobalParams.kDefaultDbPath); + +#endif + general_options.addOptionChaining("storage.directoryPerDB", "directoryperdb", moe::Switch, + "each database will be stored in a separate directory"); + + general_options.addOptionChaining("noIndexBuildRetry", "noIndexBuildRetry", moe::Switch, + "don't retry any index builds that were interrupted by shutdown") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("storage.indexBuildRetry", "", moe::Bool, + "don't retry any index builds that were interrupted by shutdown") + .setSources(moe::SourceYAMLConfig); + + general_options.addOptionChaining("noprealloc", "noprealloc", moe::Switch, + "disable data file preallocation - will often hurt performance") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("storage.preallocDataFiles", "", moe::Bool, + "disable data file preallocation - will often hurt performance") + .setSources(moe::SourceYAMLConfig); + + general_options.addOptionChaining("storage.nsSize", "nssize", moe::Int, + ".ns file size (in MB) for new databases") + .setDefault(moe::Value(16)); + + general_options.addOptionChaining("storage.quota.enforced", "quota", moe::Switch, + "limits each database to a certain number of files (8 default)") + .incompatibleWith("keyFile"); + + general_options.addOptionChaining("storage.quota.maxFilesPerDB", "quotaFiles", moe::Int, + "number of files allowed per db, implies --quota"); + + general_options.addOptionChaining("storage.smallFiles", "smallfiles", moe::Switch, + "use a smaller default file size"); + + general_options.addOptionChaining("storage.syncPeriodSecs", "syncdelay", moe::Double, + "seconds between disk syncs (0=never, but not recommended)") + .setDefault(moe::Value(60.0)); + + // Upgrade and repair are disallowed in JSON configs since they trigger very heavyweight + // actions rather than specify configuration data + general_options.addOptionChaining("upgrade", "upgrade", moe::Switch, + "upgrade db if needed") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("repair", "repair", moe::Switch, + "run repair on all dbs") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("storage.repairPath", "repairpath", moe::String, + "root directory for repair files - defaults to dbpath"); + + // Javascript Options + + general_options.addOptionChaining("noscripting", "noscripting", moe::Switch, + "disable scripting engine") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("security.javascriptEnabled", "", moe::Bool, + "Enable javascript execution") + .setSources(moe::SourceYAMLConfig); + + // Query Options + + general_options.addOptionChaining("notablescan", "notablescan", moe::Switch, + "do not allow table scans") + .setSources(moe::SourceAllLegacy); + + // Journaling Options + + // Way to enable or disable journaling on command line and in Legacy config file + general_options.addOptionChaining("journal", "journal", moe::Switch, "enable journaling") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("nojournal", "nojournal", moe::Switch, + "disable journaling (journaling is on by default for 64 bit)") + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("dur", "dur", moe::Switch, "enable journaling") + .hidden() + .setSources(moe::SourceAllLegacy); + + general_options.addOptionChaining("nodur", "nodur", moe::Switch, "disable journaling") + .hidden() + .setSources(moe::SourceAllLegacy); + + // Way to enable or disable journaling in JSON Config + general_options.addOptionChaining("storage.journal.enabled", "", moe::Bool, + "enable journaling") + .setSources(moe::SourceYAMLConfig); + + // Two ways to set durability diagnostic options. durOptions is deprecated + general_options.addOptionChaining("storage.journal.debugFlags", "journalOptions", moe::Int, + "journal diagnostic options") + .incompatibleWith("durOptions"); + + general_options.addOptionChaining("durOptions", "durOptions", moe::Int, + "durability diagnostic options") + .hidden() + .setSources(moe::SourceAllLegacy) + .incompatibleWith("storage.journal.debugFlags"); + + general_options.addOptionChaining("storage.journal.commitIntervalMs", + "journalCommitInterval", moe::Unsigned, "how often to group/batch commit (ms)"); + + // Deprecated option that we don't want people to use for performance reasons + options->addOptionChaining("nopreallocj", "nopreallocj", moe::Switch, + "don't preallocate journal files") + .hidden() + .setSources(moe::SourceAllLegacy); + +#if defined(__linux__) + general_options.addOptionChaining("shutdown", "shutdown", moe::Switch, + "kill a running server (for init scripts)"); + +#endif + + // Master Slave Options + + ms_options.addOptionChaining("master", "master", moe::Switch, "master mode") + .incompatibleWith("replication.replSet") + .incompatibleWith("replication.replSetName") + .setSources(moe::SourceAllLegacy); + + ms_options.addOptionChaining("slave", "slave", moe::Switch, "slave mode") + .incompatibleWith("replication.replSet") + .incompatibleWith("replication.replSetName") + .setSources(moe::SourceAllLegacy); + + ms_options.addOptionChaining("source", "source", moe::String, + "when slave: specify master as ") + .incompatibleWith("replication.replSet") + .incompatibleWith("replication.replSetName") + .setSources(moe::SourceAllLegacy); + + ms_options.addOptionChaining("only", "only", moe::String, + "when slave: specify a single database to replicate") + .incompatibleWith("replication.replSet") + .incompatibleWith("replication.replSetName") + .setSources(moe::SourceAllLegacy); + + ms_options.addOptionChaining("slavedelay", "slavedelay", moe::Int, + "specify delay (in seconds) to be used when applying master ops to slave") + .incompatibleWith("replication.replSet") + .incompatibleWith("replication.replSetName") + .setSources(moe::SourceAllLegacy); + + ms_options.addOptionChaining("autoresync", "autoresync", moe::Switch, + "automatically resync if slave data is stale") + .incompatibleWith("replication.replSet") + .incompatibleWith("replication.replSetName") + .setSources(moe::SourceAllLegacy); + + // Replication Options + + replication_options.addOptionChaining("replication.oplogSizeMB", "oplogSize", moe::Int, + "size to use (in MB) for replication op log. default is 5% of disk space " + "(i.e. large is good)"); + + rs_options.addOptionChaining("replication.replSet", "replSet", moe::String, + "arg is [/]") + .setSources(moe::SourceAllLegacy); + + rs_options.addOptionChaining("replication.replSetName", "", moe::String, "arg is ") + .setSources(moe::SourceYAMLConfig) + .format("[^/]+", "[replica set name with no \"/\"]"); + + rs_options.addOptionChaining("replication.secondaryIndexPrefetch", "replIndexPrefetch", moe::String, + "specify index prefetching behavior (if secondary) [none|_id_only|all]") + .format("(:?none)|(:?_id_only)|(:?all)", + "(none/_id_only/all)"); + + // Sharding Options + + sharding_options.addOptionChaining("configsvr", "configsvr", moe::Switch, + "declare this is a config db of a cluster; default port 27019; " + "default dir /data/configdb") + .setSources(moe::SourceAllLegacy) + .incompatibleWith("shardsvr"); + + sharding_options.addOptionChaining("shardsvr", "shardsvr", moe::Switch, + "declare this is a shard db of a cluster; default port 27018") + .setSources(moe::SourceAllLegacy) + .incompatibleWith("configsvr"); + + sharding_options.addOptionChaining("sharding.clusterRole", "", moe::String, + "Choose what role this mongod has in a sharded cluster. Possible values are:\n" + " \"configsvr\": Start this node as a config server. Starts on port 27019 by " + "default." + " \"shardsvr\": Start this node as a shard server. Starts on port 27018 by " + "default.") + .setSources(moe::SourceYAMLConfig) + .format("(:?configsvr)|(:?shardsvr)", + "(configsvr/shardsvr)"); + + sharding_options.addOptionChaining("noMoveParanoia", "noMoveParanoia", moe::Switch, + "turn off paranoid saving of data for the moveChunk command; default") + .hidden() + .setSources(moe::SourceAllLegacy) + .incompatibleWith("moveParanoia"); + + sharding_options.addOptionChaining("moveParanoia", "moveParanoia", + moe::Switch, "turn on paranoid saving of data during the moveChunk command " + "(used for internal system diagnostics)") + .hidden() + .setSources(moe::SourceAllLegacy) + .incompatibleWith("noMoveParanoia"); + + sharding_options.addOptionChaining("sharding.archiveMovedChunks", "", + moe::Bool, "config file option to turn on paranoid saving of data during the " + "moveChunk command (used for internal system diagnostics)") + .hidden() + .setSources(moe::SourceYAMLConfig); + + + options->addSection(general_options); +#if defined(_WIN32) + options->addSection(windows_scm_options); +#endif + options->addSection(replication_options); + options->addSection(ms_options); + options->addSection(rs_options); + options->addSection(sharding_options); +#ifdef MONGO_SSL + options->addSection(ssl_options); +#endif + + // The following are legacy options that are disallowed in the JSON config file + + options->addOptionChaining("fastsync", "fastsync", moe::Switch, + "indicate that this instance is starting from a dbpath snapshot of the repl peer") + .hidden() + .setSources(moe::SourceAllLegacy); + + options->addOptionChaining("pretouch", "pretouch", moe::Int, + "n pretouch threads for applying master/slave operations") + .hidden() + .setSources(moe::SourceAllLegacy); + + // This is a deprecated option that we are supporting for backwards compatibility + // The first value for this option can be either 'dbpath' or 'run'. + // If it is 'dbpath', mongod prints the dbpath and exits. Any extra values are ignored. + // If it is 'run', mongod runs normally. Providing extra values is an error. + options->addOptionChaining("command", "command", moe::StringVector, "command") + .hidden() + .positional(1, 3) + .setSources(moe::SourceAllLegacy); + + options->addOptionChaining("cacheSize", "cacheSize", moe::Long, + "cache size (in MB) for rec store") + .hidden() + .setSources(moe::SourceAllLegacy); + + // things we don't want people to use + options->addOptionChaining("nohints", "nohints", moe::Switch, "ignore query hints") + .hidden() + .setSources(moe::SourceAllLegacy); + + // deprecated pairing command line options + options->addOptionChaining("pairwith", "pairwith", moe::Switch, "DEPRECATED") + .hidden() + .setSources(moe::SourceAllLegacy); + + options->addOptionChaining("arbiter", "arbiter", moe::Switch, "DEPRECATED") + .hidden() + .setSources(moe::SourceAllLegacy); + + options->addOptionChaining("opIdMem", "opIdMem", moe::Switch, "DEPRECATED") + .hidden() + .setSources(moe::SourceAllLegacy); + + return Status::OK(); + } + + void printMongodHelp(const moe::OptionSection& options) { + std::cout << options.helpString() << std::endl; + }; + + namespace { + void sysRuntimeInfo() { + out() << "sysinfo:" << endl; +#if defined(_SC_PAGE_SIZE) + out() << " page size: " << (int) sysconf(_SC_PAGE_SIZE) << endl; +#endif +#if defined(_SC_PHYS_PAGES) + out() << " _SC_PHYS_PAGES: " << sysconf(_SC_PHYS_PAGES) << endl; +#endif +#if defined(_SC_AVPHYS_PAGES) + out() << " _SC_AVPHYS_PAGES: " << sysconf(_SC_AVPHYS_PAGES) << endl; +#endif + } + } // namespace + + bool handlePreValidationMongodOptions(const moe::Environment& params, + const std::vector& args) { + if (params.count("help")) { + printMongodHelp(moe::startupOptions); + return false; + } + if (params.count("version")) { + cout << mongodVersion() << endl; + printGitVersion(); + printOpenSSLVersion(); + return false; + } + if (params.count("sysinfo")) { + sysRuntimeInfo(); + return false; + } + + return true; + } + + Status validateMongodOptions(const moe::Environment& params) { + + Status ret = validateServerOptions(params); + if (!ret.isOK()) { + return ret; + } + + if ((params.count("nodur") || params.count("nojournal")) && + (params.count("dur") || params.count("journal"))) { + return Status(ErrorCodes::BadValue, + "Can't specify both --journal and --nojournal options."); + } + + // SERVER-10019 Enabling rest/jsonp without --httpinterface should break in all cases in the + // future + if (params.count("net.http.RESTInterfaceEnabled")) { + + // If we are explicitly setting httpinterface to false in the config file (the source of + // "net.http.enabled") and not overriding it on the command line (the source of + // "httpinterface"), then we can fail with an error message without breaking backwards + // compatibility. + if (!params.count("httpinterface") && + params.count("net.http.enabled") && + params["net.http.enabled"].as() == false) { + return Status(ErrorCodes::BadValue, + "httpinterface must be enabled to use the rest api"); + } + } + + if (params.count("net.http.JSONPEnabled")) { + + // If we are explicitly setting httpinterface to false in the config file (the source of + // "net.http.enabled") and not overriding it on the command line (the source of + // "httpinterface"), then we can fail with an error message without breaking backwards + // compatibility. + if (!params.count("httpinterface") && + params.count("net.http.enabled") && + params["net.http.enabled"].as() == false) { + return Status(ErrorCodes::BadValue, + "httpinterface must be enabled to use jsonp"); + } + } + + return Status::OK(); + } + + Status canonicalizeMongodOptions(moe::Environment* params) { + + // Need to handle this before canonicalizing the general "server options", since + // httpinterface and nohttpinterface are shared between mongos and mongod, but mongod has + // extra validation required. + if (params->count("net.http.RESTInterfaceEnabled")) { + bool httpEnabled = false; + if (params->count("net.http.enabled")) { + Status ret = params->get("net.http.enabled", &httpEnabled); + if (!ret.isOK()) { + return ret; + } + } + if (params->count("nohttpinterface")) { + log() << "** WARNING: Should not specify both --rest and --nohttpinterface" << + startupWarningsLog; + } + else if (!(params->count("httpinterface") || + (params->count("net.http.enabled") && httpEnabled == true))) { + log() << "** WARNING: --rest is specified without --httpinterface," << + startupWarningsLog; + log() << "** enabling http interface" << startupWarningsLog; + Status ret = params->set("httpinterface", moe::Value(true)); + if (!ret.isOK()) { + return ret; + } + } + } + + if (params->count("net.http.JSONPEnabled")) { + if (params->count("nohttpinterface")) { + log() << "** WARNING: Should not specify both --jsonp and --nohttpinterface" << + startupWarningsLog; + } + else if (!params->count("httpinterface")) { + log() << "** WARNING --jsonp is specified without --httpinterface," << + startupWarningsLog; + log() << "** enabling http interface" << startupWarningsLog; + Status ret = params->set("httpinterface", moe::Value(true)); + if (!ret.isOK()) { + return ret; + } + } + } + + Status ret = canonicalizeServerOptions(params); + if (!ret.isOK()) { + return ret; + } + +#ifdef MONGO_SSL + ret = canonicalizeSSLServerOptions(params); + if (!ret.isOK()) { + return ret; + } +#endif + + // "storage.journal.enabled" comes from the config file, so override it if any of "journal", + // "nojournal", "dur", and "nodur" are set, since those come from the command line. + if (params->count("nodur") || params->count("nojournal")) { + Status ret = params->set("storage.journal.enabled", moe::Value(false)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("nodur"); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("nojournal"); + if (!ret.isOK()) { + return ret; + } + } + + if (params->count("dur") || params->count("journal")) { + Status ret = params->set("storage.journal.enabled", moe::Value(true)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("dur"); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("journal"); + if (!ret.isOK()) { + return ret; + } + } + + // "storage.journal.durOptions" comes from the config file, so override it if "durOptions" + // is set since that comes from the command line. + if (params->count("durOptions")) { + int durOptions; + Status ret = params->get("durOptions", &durOptions); + if (!ret.isOK()) { + return ret; + } + ret = params->set("storage.journal.debugFlags", moe::Value(durOptions)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("durOptions"); + if (!ret.isOK()) { + return ret; + } + } + + // "security.authorization" comes from the config file, so override it if "noauth" or + // "auth" are set since those come from the command line. + if (params->count("noauth")) { + Status ret = params->set("security.authorization", + moe::Value(std::string("disabled"))); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("noauth"); + if (!ret.isOK()) { + return ret; + } + } + if (params->count("auth")) { + Status ret = params->set("security.authorization", + moe::Value(std::string("enabled"))); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("auth"); + if (!ret.isOK()) { + return ret; + } + } + + // "storage.preallocDataFiles" comes from the config file, so override it if "noprealloc" is + // set since that comes from the command line. + if (params->count("noprealloc")) { + Status ret = params->set("storage.preallocDataFiles", moe::Value(false)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("noprealloc"); + if (!ret.isOK()) { + return ret; + } + } + + // "sharding.archiveMovedChunks" comes from the config file, so override it if + // "noMoveParanoia" or "moveParanoia" are set since those come from the command line. + if (params->count("noMoveParanoia")) { + Status ret = params->set("sharding.archiveMovedChunks", moe::Value(false)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("noMoveParanoia"); + if (!ret.isOK()) { + return ret; + } + } + if (params->count("moveParanoia")) { + Status ret = params->set("sharding.archiveMovedChunks", moe::Value(true)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("moveParanoia"); + if (!ret.isOK()) { + return ret; + } + } + + // "sharding.clusterRole" comes from the config file, so override it if "configsvr" or + // "shardsvr" are set since those come from the command line. + if (params->count("configsvr")) { + Status ret = params->set("sharding.clusterRole", moe::Value(std::string("configsvr"))); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("configsvr"); + if (!ret.isOK()) { + return ret; + } + } + if (params->count("shardsvr")) { + Status ret = params->set("sharding.clusterRole", moe::Value(std::string("shardsvr"))); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("shardsvr"); + if (!ret.isOK()) { + return ret; + } + } + + if (params->count("profile")) { + int profilingMode; + Status ret = params->get("profile", &profilingMode); + if (!ret.isOK()) { + return ret; + } + std::string profilingModeString; + if (profilingMode == 0) { + profilingModeString = "off"; + } + else if (profilingMode == 1) { + profilingModeString = "slowOp"; + } + else if (profilingMode == 2) { + profilingModeString = "all"; + } + else { + StringBuilder sb; + sb << "Bad value for profile: " << profilingMode + << ". Supported modes are: (0=off|1=slowOp|2=all)"; + return Status(ErrorCodes::BadValue, sb.str()); + } + ret = params->set("operationProfiling.mode", moe::Value(profilingModeString)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("profile"); + if (!ret.isOK()) { + return ret; + } + } + + // "storage.indexBuildRetry" comes from the config file, so override it if + // "noIndexBuildRetry" is set since that comes from the command line. + if (params->count("noIndexBuildRetry")) { + Status ret = params->set("storage.indexBuildRetry", moe::Value(false)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("noIndexBuildRetry"); + if (!ret.isOK()) { + return ret; + } + } + + // Ensure that "replication.replSet" logically overrides "replication.replSetName". We + // can't canonicalize them as the same option, because they mean slightly different things. + // "replication.replSet" can include a seed list, while "replication.replSetName" just has + // the replica set name. + if (params->count("replication.replSet") && params->count("replication.replSetName")) { + ret = params->remove("replication.replSetName"); + if (!ret.isOK()) { + return ret; + } + } + + // "security.javascriptEnabled" comes from the config file, so override it if "noscripting" + // is set since that comes from the command line. + if (params->count("noscripting")) { + Status ret = params->set("security.javascriptEnabled", moe::Value(false)); + if (!ret.isOK()) { + return ret; + } + ret = params->remove("noscripting"); + if (!ret.isOK()) { + return ret; + } + } + + return Status::OK(); + } + + Status storeMongodOptions(const moe::Environment& params, + const std::vector& args) { + + Status ret = storeServerOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + // TODO: Integrate these options with their setParameter counterparts + if (params.count("security.authSchemaVersion")) { + return Status(ErrorCodes::BadValue, + "security.authSchemaVersion is currently not supported in config files"); + } + + if (params.count("security.enableLocalhostAuthBypass")) { + return Status(ErrorCodes::BadValue, + "security.enableLocalhostAuthBypass is currently not supported in config " + "files"); + } + + if (params.count("security.supportCompatibilityFormPrivilegeDocuments")) { + return Status(ErrorCodes::BadValue, + "security.supportCompatibilityFormPrivilegeDocuments is currently not " + "supported in config files"); + } + + if (params.count("storage.dbPath")) { + storageGlobalParams.dbpath = params["storage.dbPath"].as(); + if (params.count("processManagement.fork") && storageGlobalParams.dbpath[0] != '/') { + // we need to change dbpath if we fork since we change + // cwd to "/" + // fork only exists on *nix + // so '/' is safe + storageGlobalParams.dbpath = serverGlobalParams.cwd + "/" + + storageGlobalParams.dbpath; + } + } +#ifdef _WIN32 + if (storageGlobalParams.dbpath.size() > 1 && + storageGlobalParams.dbpath[storageGlobalParams.dbpath.size()-1] == '/') { + // size() check is for the unlikely possibility of --dbpath "/" + storageGlobalParams.dbpath = + storageGlobalParams.dbpath.erase(storageGlobalParams.dbpath.size()-1); + } +#endif + + if (params.count("operationProfiling.mode")) { + std::string profilingMode = params["operationProfiling.mode"].as(); + if (profilingMode == "off") { + serverGlobalParams.defaultProfile = 0; + } + else if (profilingMode == "slowOp") { + serverGlobalParams.defaultProfile = 1; + } + else if (profilingMode == "all") { + serverGlobalParams.defaultProfile = 2; + } + else { + StringBuilder sb; + sb << "Bad value for operationProfiling.mode: " << profilingMode + << ". Supported modes are: (off|slowOp|all)"; + return Status(ErrorCodes::BadValue, sb.str()); + } + } + + if ( params.count("operationProfiling.slowOpThresholdMs")) { + serverGlobalParams.slowMS = params["operationProfiling.slowOpThresholdMs"].as(); + } + + if ( params.count("storage.syncPeriodSecs")) { + storageGlobalParams.syncdelay = params["storage.syncPeriodSecs"].as(); + } + + if (params.count("storage.directoryPerDB")) { + storageGlobalParams.directoryperdb = true; + } + if (params.count("cpu")) { + serverGlobalParams.cpu = true; + } + if (params.count("security.authorization") && + params["security.authorization"].as() == "disabled") { + getGlobalAuthorizationManager()->setAuthEnabled(false); + } + if (params.count("security.authorization") && + params["security.authorization"].as() == "enabled") { + getGlobalAuthorizationManager()->setAuthEnabled(true); + } + if (params.count("storage.quota.enforced")) { + storageGlobalParams.quota = true; + } + if (params.count("storage.quota.maxFilesPerDB")) { + storageGlobalParams.quota = true; + storageGlobalParams.quotaFiles = params["storage.quota.maxFilesPerDB"].as() - 1; + } + + if (params.count("storage.journal.enabled")) { + storageGlobalParams.dur = params["storage.journal.enabled"].as(); + } + + if (params.count("storage.journal.commitIntervalMs")) { + // don't check if dur is false here as many will just use the default, and will default + // to off on win32. ie no point making life a little more complex by giving an error on + // a dev environment. + storageGlobalParams.journalCommitInterval = + params["storage.journal.commitIntervalMs"].as(); + if (storageGlobalParams.journalCommitInterval <= 1 || + storageGlobalParams.journalCommitInterval > 300) { + return Status(ErrorCodes::BadValue, + "--journalCommitInterval out of allowed range (0-300ms)"); + } + } + if (params.count("storage.journal.debugFlags")) { + storageGlobalParams.durOptions = params["storage.journal.debugFlags"].as(); + } + if (params.count("nohints")) { + storageGlobalParams.useHints = false; + } + if (params.count("nopreallocj")) { + storageGlobalParams.preallocj = false; + } + + if (params.count("net.http.RESTInterfaceEnabled")) { + serverGlobalParams.rest = true; + } + if (params.count("net.http.JSONPEnabled")) { + serverGlobalParams.jsonp = true; + } + if (params.count("security.javascriptEnabled")) { + mongodGlobalParams.scriptingEnabled = params["security.javascriptEnabled"].as(); + } + if (params.count("storage.preallocDataFiles")) { + storageGlobalParams.prealloc = params["storage.preallocDataFiles"].as(); + cout << "note: noprealloc may hurt performance in many applications" << endl; + } + if (params.count("storage.smallFiles")) { + storageGlobalParams.smallfiles = true; + } + if (params.count("diaglog")) { + warning() << "--diaglog is deprecated and will be removed in a future release"; + int x = params["diaglog"].as(); + if ( x < 0 || x > 7 ) { + return Status(ErrorCodes::BadValue, "can't interpret --diaglog setting"); + } + _diaglog.setLevel(x); + } + + if ((params.count("storage.journal.enabled") && + params["storage.journal.enabled"].as() == true) && params.count("repair")) { + return Status(ErrorCodes::BadValue, + "Can't have journaling enabled when using --repair option."); + } + + if (params.count("repair")) { + Record::MemoryTrackingEnabled = false; + mongodGlobalParams.upgrade = 1; // --repair implies --upgrade + mongodGlobalParams.repair = 1; + storageGlobalParams.dur = false; + } + if (params.count("upgrade")) { + Record::MemoryTrackingEnabled = false; + mongodGlobalParams.upgrade = 1; + } + if (params.count("notablescan")) { + storageGlobalParams.noTableScan = true; + } + if (params.count("master")) { + replSettings.master = true; + } + if (params.count("slave")) { + replSettings.slave = SimpleSlave; + } + if (params.count("slavedelay")) { + replSettings.slavedelay = params["slavedelay"].as(); + } + if (params.count("fastsync")) { + replSettings.fastsync = true; + } + if (params.count("autoresync")) { + replSettings.autoresync = true; + } + if (params.count("source")) { + /* specifies what the source in local.sources should be */ + replSettings.source = params["source"].as().c_str(); + } + if( params.count("pretouch") ) { + replSettings.pretouch = params["pretouch"].as(); + } + if (params.count("replication.replSetName")) { + replSettings.replSet = params["replication.replSetName"].as().c_str(); + } + if (params.count("replication.replSet")) { + /* seed list of hosts for the repl set */ + replSettings.replSet = params["replication.replSet"].as().c_str(); + } + if (params.count("replication.secondaryIndexPrefetch")) { + replSettings.rsIndexPrefetch = + params["replication.secondaryIndexPrefetch"].as(); + } + + if (params.count("storage.indexBuildRetry")) { + serverGlobalParams.indexBuildRetry = params["storage.indexBuildRetry"].as(); + } + + if (params.count("only")) { + replSettings.only = params["only"].as().c_str(); + } + if( params.count("storage.nsSize") ) { + int x = params["storage.nsSize"].as(); + if (x <= 0 || x > (0x7fffffff/1024/1024)) { + return Status(ErrorCodes::BadValue, "bad --nssize arg"); + } + storageGlobalParams.lenForNewNsFiles = x * 1024 * 1024; + verify(storageGlobalParams.lenForNewNsFiles > 0); + } + if (params.count("replication.oplogSizeMB")) { + long long x = params["replication.oplogSizeMB"].as(); + if (x <= 0) { + return Status(ErrorCodes::BadValue, "bad --oplogSize arg"); + } + // note a small size such as x==1 is ok for an arbiter. + if( x > 1000 && sizeof(void*) == 4 ) { + StringBuilder sb; + sb << "--oplogSize of " << x + << "MB is too big for 32 bit version. Use 64 bit build instead."; + return Status(ErrorCodes::BadValue, sb.str()); + } + replSettings.oplogSize = x * 1024 * 1024; + verify(replSettings.oplogSize > 0); + } + if (params.count("cacheSize")) { + long x = params["cacheSize"].as(); + if (x <= 0) { + return Status(ErrorCodes::BadValue, "bad --cacheSize arg"); + } + return Status(ErrorCodes::BadValue, "--cacheSize option not currently supported"); + } + if (!params.count("net.port")) { + if (params.count("sharding.clusterRole")) { + std::string clusterRole = params["sharding.clusterRole"].as(); + if (clusterRole == "configsvr") { + serverGlobalParams.port = ServerGlobalParams::ConfigServerPort; + } + else if (clusterRole == "shardsvr") { + serverGlobalParams.port = ServerGlobalParams::ShardServerPort; + } + else { + StringBuilder sb; + sb << "Bad value for sharding.clusterRole: " << clusterRole + << ". Supported modes are: (configsvr|shardsvr)"; + return Status(ErrorCodes::BadValue, sb.str()); + } + } + } + else { + if (serverGlobalParams.port <= 0 || serverGlobalParams.port > 65535) { + return Status(ErrorCodes::BadValue, "bad --port number"); + } + } + if (params.count("sharding.clusterRole") && + params["sharding.clusterRole"].as() == "configsvr") { + serverGlobalParams.configsvr = true; + storageGlobalParams.smallfiles = true; // config server implies small files + if (replSettings.usingReplSets() || replSettings.master || replSettings.slave) { + return Status(ErrorCodes::BadValue, + "replication should not be enabled on a config server"); + } + + // If we haven't explicitly specified a journal option, default journaling to true for + // the config server role + if (!params.count("storage.journal.enabled")) { + storageGlobalParams.dur = true; + } + + if (!params.count("storage.dbPath")) { + storageGlobalParams.dbpath = storageGlobalParams.kDefaultConfigDbPath; + } + replSettings.master = true; + if (!params.count("replication.oplogSizeMB")) + replSettings.oplogSize = 5 * 1024 * 1024; + } + if (params.count("net.ipv6")) { + enableIPv6(); + } + + if (params.count("sharding.archiveMovedChunks")) { + serverGlobalParams.moveParanoia = params["sharding.archiveMovedChunks"].as(); + } + + if (params.count("pairwith") || params.count("arbiter") || params.count("opIdMem")) { + return Status(ErrorCodes::BadValue, + "****\n" + "Replica Pairs have been deprecated. Invalid options: " + "--pairwith, --arbiter, and/or --opIdMem\n" + "\n" + "****"); + } + + // needs to be after things like --configsvr parsing, thus here. + if (params.count("storage.repairPath")) { + storageGlobalParams.repairpath = params["storage.repairPath"].as(); + if (!storageGlobalParams.repairpath.size()) { + return Status(ErrorCodes::BadValue, "repairpath is empty"); + } + + if (storageGlobalParams.dur && + !str::startsWith(storageGlobalParams.repairpath, + storageGlobalParams.dbpath)) { + return Status(ErrorCodes::BadValue, + "You must use a --repairpath that is a subdirectory of --dbpath when " + "using journaling"); + } + } + else { + storageGlobalParams.repairpath = storageGlobalParams.dbpath; + } + + if (replSettings.pretouch) + log() << "--pretouch " << replSettings.pretouch << endl; + + // Check if we are 32 bit and have not explicitly specified any journaling options + if (sizeof(void*) == 4 && !params.count("storage.journal.enabled")) { + // trying to make this stand out more like startup warnings + log() << endl; + warning() << "32-bit servers don't have journaling enabled by default. " + << "Please use --journal if you want durability." << endl; + log() << endl; + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/mongod_options.h b/src/mongo/db/mongod_options.h new file mode 100644 index 00000000000..61112f54f3d --- /dev/null +++ b/src/mongo/db/mongod_options.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/server_options.h" +#include "mongo/db/storage_options.h" +#include "mongo/util/options_parser/environment.h" +#include "mongo/util/options_parser/option_section.h" + +namespace mongo { + + namespace optionenvironment { + class OptionSection; + class Environment; + } // namespace optionenvironment + + namespace moe = mongo::optionenvironment; + + struct MongodGlobalParams { + bool upgrade; + bool repair; + bool scriptingEnabled; // --noscripting + + MongodGlobalParams() : + upgrade(0), + repair(0), + scriptingEnabled(true) + { } + }; + + extern MongodGlobalParams mongodGlobalParams; + + Status addMongodOptions(moe::OptionSection* options); + + void printMongodHelp(const moe::OptionSection& options); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongodOptions(const moe::Environment& params, + const std::vector& args); + + /** + * Handle custom validation of mongod options that can not currently be done by using + * Constraints in the Environment. See the "validate" function in the Environment class for + * more details. + */ + Status validateMongodOptions(const moe::Environment& params); + + /** + * Canonicalize mongod options for the given environment. + * + * For example, the options "dur", "nodur", "journal", "nojournal", and + * "storage.journaling.enabled" should all be merged into "storage.journaling.enabled". + */ + Status canonicalizeMongodOptions(moe::Environment* params); + + Status storeMongodOptions(const moe::Environment& params, const std::vector& args); +} diff --git a/src/mongo/db/mongod_options_init.cpp b/src/mongo/db/mongod_options_init.cpp new file mode 100644 index 00000000000..ece2805abf6 --- /dev/null +++ b/src/mongo/db/mongod_options_init.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/mongod_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongodOptions)(InitializerContext* context) { + return addMongodOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongodOptions)(InitializerContext* context) { + if (!handlePreValidationMongodOptions(moe::startupOptionsParsed, context->args())) { + ::_exit(EXIT_SUCCESS); + } + // Run validation, but tell the Environment that we don't want it to be set as "valid", + // since we may be making it invalid in the canonicalization process. + Status ret = moe::startupOptionsParsed.validate(false/*setValid*/); + if (!ret.isOK()) { + return ret; + } + ret = validateMongodOptions(moe::startupOptionsParsed); + if (!ret.isOK()) { + return ret; + } + ret = canonicalizeMongodOptions(&moe::startupOptionsParsed); + if (!ret.isOK()) { + return ret; + } + ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_INITIALIZER_GENERAL(MongodOptions_Store, + ("BeginStartupOptionStorage", + "CreateAuthorizationManager"), // Requried to call + // getGlobalAuthorizationManager(). + ("EndStartupOptionStorage")) + (InitializerContext* context) { + Status ret = storeMongodOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/mongod_sm.sln b/src/mongo/db/mongod_sm.sln deleted file mode 100644 index 3c06c0cd1d0..00000000000 --- a/src/mongo/db/mongod_sm.sln +++ /dev/null @@ -1,58 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 11.00 -# Visual Studio 2010 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongod_sm", "mongod_sm.vcxproj", "{215B2D68-0A70-4D10-8E75-B31010C62A91}" - ProjectSection(ProjectDependencies) = postProject - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} = {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SpiderMonkey pre-build step", "..\..\third_party\js-1.7\SpiderMonkey-prebuild.vcxproj", "{7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - Win2008PlusDebug|Win32 = Win2008PlusDebug|Win32 - Win2008PlusDebug|x64 = Win2008PlusDebug|x64 - Win2008PlusRelease|Win32 = Win2008PlusRelease|Win32 - Win2008PlusRelease|x64 = Win2008PlusRelease|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.ActiveCfg = Debug|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.Build.0 = Debug|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|x64.ActiveCfg = Debug|x64 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|x64.Build.0 = Debug|x64 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.ActiveCfg = Release|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.Build.0 = Release|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|x64.ActiveCfg = Release|x64 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|x64.Build.0 = Release|x64 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusDebug|Win32.ActiveCfg = Win2008PlusDebug|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusDebug|Win32.Build.0 = Win2008PlusDebug|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusDebug|x64.ActiveCfg = Win2008PlusDebug|x64 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusDebug|x64.Build.0 = Win2008PlusDebug|x64 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusRelease|Win32.ActiveCfg = Win2008PlusRelease|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusRelease|Win32.Build.0 = Win2008PlusRelease|Win32 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusRelease|x64.ActiveCfg = Win2008PlusRelease|x64 - {215B2D68-0A70-4D10-8E75-B31010C62A91}.Win2008PlusRelease|x64.Build.0 = Win2008PlusRelease|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|Win32.ActiveCfg = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|Win32.Build.0 = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|x64.ActiveCfg = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Debug|x64.Build.0 = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|Win32.ActiveCfg = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|Win32.Build.0 = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Release|x64.ActiveCfg = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|Win32.ActiveCfg = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|Win32.Build.0 = Debug|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|x64.ActiveCfg = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusDebug|x64.Build.0 = Debug|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|Win32.ActiveCfg = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|Win32.Build.0 = Release|Win32 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|x64.ActiveCfg = Release|x64 - {7FA15DF8-14C8-4FD5-897B-6FACE8FDCF2A}.Win2008PlusRelease|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/src/mongo/db/mongod_sm.vcxproj b/src/mongo/db/mongod_sm.vcxproj deleted file mode 100644 index b7add373e92..00000000000 --- a/src/mongo/db/mongod_sm.vcxproj +++ /dev/null @@ -1,2959 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - Win2008PlusDebug - Win32 - - - Win2008PlusDebug - x64 - - - Win2008PlusRelease - Win32 - - - Win2008PlusRelease - x64 - - - - mongod_sm - {215B2D68-0A70-4D10-8E75-B31010C62A91} - Win32Proj - mongod_sm - - - - Application - Unicode - true - - - Application - Unicode - true - - - Application - Unicode - true - - - Application - Unicode - true - - - Application - false - false - Unicode - - - Application - false - false - Unicode - - - Application - false - false - Unicode - - - Application - false - false - Unicode - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - true - true - true - true - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - $(ProjectDir)$(Platform)\$(Configuration)_sm\ - false - false - false - false - AllRules.ruleset - AllRules.ruleset - AllRules.ruleset - AllRules.ruleset - - - - - - - - - AllRules.ruleset - AllRules.ruleset - AllRules.ruleset - AllRules.ruleset - - - - - - - - - ..\..\third_party\js-1.7;..;$(IncludePath) - ..\..\third_party\js-1.7;..;$(IncludePath) - ..\..\third_party\js-1.7;..;$(IncludePath) - ..\..\third_party\js-1.7;..;$(IncludePath) - ..\..\third_party\js-1.7;..;$(IncludePath) - ..\..\third_party\js-1.7;..;$(IncludePath) - ..\..\third_party\js-1.7;..;$(IncludePath) - ..\..\third_party\js-1.7;..;$(IncludePath) - - - - Disabled - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - No - EnableFastChecks - MultiThreadedDebug - pch.h - Level3 - EditAndContinue - 4355;4800;4267;4244 - true - true - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - true - Console - MachineX86 - msvcrtd;%(IgnoreSpecificDefaultLibraries) - false - true - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - Disabled - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - No - EnableFastChecks - MultiThreadedDebug - pch.h - Level3 - EditAndContinue - 4355;4800;4267;4244 - true - true - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - false - %(IgnoreSpecificDefaultLibraries) - true - Console - MachineX86 - true - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - Disabled - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - MONGOD_CONCURRENCY_LEVEL=1;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - EnableFastChecks - MultiThreadedDebug - pch.h - Level3 - ProgramDatabase - 4355;4800;4267;4244 - true - No - true - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - false - %(IgnoreSpecificDefaultLibraries) - true - Console - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - Disabled - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - EnableFastChecks - MultiThreadedDebug - pch.h - Level3 - ProgramDatabase - 4355;4800;4267;4244 - true - No - true - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - false - %(IgnoreSpecificDefaultLibraries) - true - Console - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - MaxSpeed - true - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - MultiThreaded - true - pch.h - Level3 - ProgramDatabase - 4355;4800;4267;4244 - true - No - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - true - Console - MachineX86 - false - true - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - MaxSpeed - true - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - MultiThreaded - true - pch.h - Level3 - ProgramDatabase - 4355;4800;4267;4244 - true - No - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - true - Console - MachineX86 - false - true - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - MaxSpeed - true - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - MultiThreaded - true - pch.h - Level3 - ProgramDatabase - 4355;4800;4267;4244 - true - No - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - true - Console - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - MaxSpeed - true - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost - BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - MultiThreaded - true - pch.h - Level3 - ProgramDatabase - 4355;4800;4267;4244 - true - No - - - ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies) - - - true - Console - - - cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\js-1.7" /input:"jskwgen.exe" /output:"jsautokw.h" /command:"jskwgen jsautokw.h" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp" -cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp" -cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).." - - Run pre-build commands - - - - - - - - - - - - - - $(IntDir)filesystem_utf8_codecvt_facet.obj - $(IntDir)filesystem_utf8_codecvt_facet.obj - $(IntDir)filesystem_utf8_codecvt_facet.obj - $(IntDir)filesystem_utf8_codecvt_facet.obj - $(IntDir)filesystem_utf8_codecvt_facet.obj - $(IntDir)filesystem_utf8_codecvt_facet.obj - $(IntDir)filesystem_utf8_codecvt_facet.obj - $(IntDir)filesystem_utf8_codecvt_facet.obj - - - - - - - - - - - - - - - - BOOST_THREAD_BUILD_LIB;BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - - - BOOST_THREAD_BUILD_LIB;BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - - - BOOST_THREAD_BUILD_LIB;BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - BOOST_THREAD_BUILD_LIB;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - $(IntDir)\$(InputName)1.obj - $(IntDir)\$(InputName)1.xml - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - 4047;4146 - 4047;4146 - 4047;4146 - 4047;4146 - 4047;4146;4267;4244 - 4047;4146;4267;4244 - 4047;4146;4267;4244 - 4047;4146;4267;4244 - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - JSFILE;EXPORT_JS_API;JS_C_STRINGS_ARE_UTF8;XP_WIN;_CRT_SECURE_NO_WARNINGS - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - 4355;4800;4267;4244;4334 - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4355;4800;4267;4244;4141 - 4355;4800;4267;4244;4141 - 4355;4800;4267;4244;4141 - 4355;4800;4267;4244;4141 - 4355;4800;4267;4244;4141 - 4355;4800;4267;4244;4141 - 4355;4800;4267;4244;4141 - 4355;4800;4267;4244;4141 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - $(IntDir)base_stringprintf.obj - $(IntDir)base_stringprintf.obj - $(IntDir)base_stringprintf.obj - $(IntDir)base_stringprintf.obj - $(IntDir)base_stringprintf.obj - $(IntDir)base_stringprintf.obj - $(IntDir)base_stringprintf.obj - $(IntDir)base_stringprintf.obj - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - DEBUG_MODE=false;MONGOD_CONCURRENCY_LEVEL=1;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - DEBUG_MODE=false;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - DEBUG_MODE=false;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - DEBUG_MODE=false;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - DEBUG_MODE=false;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - DEBUG_MODE=false;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - DEBUG_MODE=false;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;;%(PreprocessorDefinitions) - DEBUG_MODE=false;BOOST_ALL_NO_LIB;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - - - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - - - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - - - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - $(IntDir)s2_split.obj - $(IntDir)s2_split.obj - $(IntDir)s2_split.obj - $(IntDir)s2_split.obj - $(IntDir)s2_split.obj - $(IntDir)s2_split.obj - $(IntDir)s2_split.obj - $(IntDir)s2_split.obj - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - 4355;4800;4267;4244;4146;4996 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - true - true - true - true - true - true - true - true - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - true - true - true - true - true - true - true - true - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - ..\..;..\..\third_party\s2;..\..\third_party\boost - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4355;4800;4267;4244;4291 - 4355;4800;4267;4244;4291 - 4355;4800;4267;4244;4291 - 4355;4800;4267;4244;4291 - 4355;4800;4267;4244;4291 - 4355;4800;4267;4244;4291 - 4355;4800;4267;4244;4291 - 4355;4800;4267;4244;4291 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4355;4800;4267;4244;4351 - 4355;4800;4267;4244;4351 - 4355;4800;4267;4244;4351 - 4355;4800;4267;4244;4351 - 4355;4800;4267;4244;4351 - 4355;4800;4267;4244;4351 - 4355;4800;4267;4244;4351 - 4355;4800;4267;4244;4351 - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - ..\..;..\..\third_party\libstemmer_c\include;..\..\third_party\boost - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - - true - true - true - true - true - true - true - true - - - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - ..\..;..\..\third_party\pcre-8.30;..\..\third_party\boost;..\..\third_party\libstemmer_c\include - - - true - true - true - true - true - true - true - true - - - 4355;4800;4267;4244;4146;4200 - 4355;4800;4267;4244;4146;4200 - 4355;4800;4267;4244;4146;4200 - 4355;4800;4267;4244;4146;4200 - 4355;4800;4267;4244;4146;4200 - 4355;4800;4267;4244;4146;4200 - 4355;4800;4267;4244;4146;4200 - 4355;4800;4267;4244;4146;4200 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - true - true - true - true - true - true - true - true - - - - - true - true - true - true - true - true - true - true - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - 4355;4800;4267;4244;4146 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(IntDir)mongo_cmdline.obj - $(IntDir)mongo_cmdline.obj - $(IntDir)mongo_cmdline.obj - $(IntDir)mongo_cmdline.obj - $(IntDir)mongo_cmdline.obj - $(IntDir)mongo_cmdline.obj - $(IntDir)mongo_cmdline.obj - $(IntDir)mongo_cmdline.obj - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - true - true - true - true - true - true - true - true - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/mongo/db/mongod_sm.vcxproj.filters b/src/mongo/db/mongod_sm.vcxproj.filters deleted file mode 100644 index ee4a208c28b..00000000000 --- a/src/mongo/db/mongod_sm.vcxproj.filters +++ /dev/null @@ -1,3378 +0,0 @@ - - - - - third_party\snappy - - - third_party\snappy - - - third_party\snappy - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - third_party\pcre - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - Boost - - - db\commands - - - util\Source Files - - - db\commands - - - db\geo - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - util\Source Files - - - util\Source Files - - - util\Source Files - - - util\Source Files - - - scripting - - - db\pipeline\Source Files - - - s - - - s - - - db\ops - - - db\stats - - - s - - - s - - - s - - - s - - - db\repl - - - s - - - s - - - s - - - db\ops - - - scripting - - - scripting - - - util\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\commands - - - s - - - db\commands - - - db\geo - - - db\repl - - - db\repl - - - util\net - - - util\Source Files - - - db\commands - - - db\repl - - - util\net - - - util\Source Files - - - util\Source Files - - - util\net - - - util\net - - - util\net - - - util\net - - - util\Source Files - - - util\Source Files - - - util\Source Files - - - util\Source Files - - - Generated from JavaScript source - - - db\commands - - - bson - - - util\Source Files - - - - db\commands - - - util\Source Files - - - util\Source Files - - - util\Source Files - - - db\ops - - - util\Source Files - - - db\repl - - - client\Source Files - - - client\Source Files - - - client\Source Files - - - client\Source Files - - - client\Source Files - - - client\Source Files - - - client\Source Files - - - db\repl - - - db\repl - - - db\repl - - - db\repl - - - db\repl - - - db\repl - - - util\concurrency - - - s - - - s - - - s - - - db\stats - - - util\net - - - util\concurrency - - - util\Source Files - - - util\Source Files - - - util\concurrency - - - util\concurrency - - - util\Source Files - - - util\Source Files - - - client\Source Files - - - util\concurrency - - - util\Source Files - - - db\stats - - - db\ops - - - util\Source Files - - - util\Source Files - - - scripting - - - db\pipeline\Source Files - - - util\Source Files - - - util\Source Files - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\a to d - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - db\Source Files\o to z - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - third_party\SpiderMonkey\Source Files - - - db\Source Files\a to d - - - util\Source Files - - - client\Source Files - - - db\Source Files\a to d - - - util\Source Files - - - db\stats - - - db\Source Files\e to n - - - db\Source Files\e to n - - - util\Source Files - - - db\Source Files\o to z - - - db\ops - - - db\repl - - - util\Source Files - - - db\commands - - - util\Source Files - - - db\Source Files\e to n - - - db\Source Files\e to n - - - db\commands - - - scripting - - - db\Source Files\e to n - - - db\Source Files\o to z - - - util\concurrency - - - third_party\MurmurHash3 - - - db\Source Files\a to d - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - db\pipeline\Source Files - - - scripting - - - util\Source Files - - - base - - - base - - - base - - - base - - - base - - - base - - - base - - - base - - - base - - - base - - - util\Source Files - - - util\Source Files - - - db\auth - - - db\auth - - - db\auth - - - db\Source Files\e to n - - - db\Source Files\e to n - - - platform - - - db\Source Files\e to n - - - db\geo - - - db\geo - - - util\net - - - util\Source Files - - - db\auth - - - db\auth - - - db\Source Files\a to d - - - db\commands - - - db\auth - - - bson\mutable - - - bson\mutable - - - bson\mutable - - - bson\mutable - - - db\geo - - - db\geo - - - db\commands - - - util\Source Files - - - base - - - bson\util - - - db\auth - - - db\auth - - - db\auth - - - s - - - s - - - db\Source Files\e to n - - - db\geo - - - db\geo - - - s - - - s - - - db\Source Files\a to d - - - db\Source Files\e to n - - - s - - - db\commands - - - db\Source Files\o to z - - - db\auth - - - db\auth - - - s - - - s - - - db\commands - - - db\commands - - - util\Source Files - - - base - - - util\Source Files - - - db\pipeline\Source Files - - - util\Source Files - - - db\geo - - - db\geo - - - s - - - db\Source Files\a to d - - - db\Source Files\o to z - - - s - - - s - - - s - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\strings - - - third_party\s2\strings - - - third_party\s2\strings - - - third_party\s2\util\coding - - - third_party\s2\util\coding - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math\exactfloat - - - s - - - s - - - s - - - third_party\s2\strings - - - db\commands - - - db\commands - - - db\commands - - - bson - - - db\commands - - - db\auth - - - db\auth - - - db\commands - - - third_party\libstemmer_c\examples - - - third_party\libstemmer_c\libstemmer - - - third_party\libstemmer_c\libstemmer - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\runtime - - - third_party\libstemmer_c\runtime - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts\test - - - db\fts\test - - - db\fts\test - - - db\fts\test - - - db\fts\test - - - db\fts\test - - - db\fts\test - - - db\fts\test - - - db\fts - - - db\auth - - - db\stats - - - scripting - - - scripting - - - scripting - - - scripting - - - scripting - - - scripting - - - db\geo - - - db\geo - - - db\geo - - - scripting - - - db\Source Files\e to n - - - client\Source Files - - - db\Source Files\e to n - - - platform - - - platform - - - platform - - - - - - third_party\snappy - - - third_party\snappy - - - third_party\pcre - - - third_party\pcre - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - bson\util - - - util\Header Files - - - util\Header Files - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson - - - bson\util - - - util\mongoutils - - - db\repl - - - db\geo - - - db\ops - - - db\stats - - - s - - - db\pipeline\Header Files - - - util\Header Files - - - db\ops - - - scripting - - - scripting - - - scripting - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - util\Header Files - - - util\Header Files - - - db\repl - - - util\net - - - util\mongoutils - - - util\net - - - bson - - - util\Header Files - - - util\Header Files - - - util\concurrency - - - util\net - - - util\concurrency - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\net - - - util\net - - - util\net - - - util\net - - - util\Header Files - - - util\Header Files - - - bson\util - - - util\Header Files - - - util\concurrency - - - db\repl - - - util\concurrency - - - util\concurrency - - - bson - - - util\Header Files - - - bson - - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\concurrency - - - util\concurrency - - - util\Header Files - - - util\Header Files - - - client\Header Files - - - client\Header Files - - - client\Header Files - - - client\Header Files - - - client\Header Files - - - client\Header Files - - - client\Header Files - - - client\Header Files - - - db\repl - - - db\repl - - - db\repl - - - util\concurrency - - - util\concurrency - - - db\stats - - - util\net - - - util\mongoutils - - - base - - - util\Header Files - - - util\concurrency - - - util\Header Files - - - client\Header Files - - - util\concurrency - - - util\Header Files - - - db\stats - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - db\ops - - - scripting - - - scripting - - - util\concurrency - - - client\Header Files - - - db\pipeline\Header Files - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - util\Header Files - - - util\Header Files - - - db\stats - - - db\Header Files\e to n - - - db\Header Files\o to z - - - db\ops - - - db\ops - - - util\Header Files - - - util\Header Files - - - db\Header Files\e to n - - - db\Header Files\e to n - - - s - - - scripting - - - s - - - db\Header Files\e to n - - - db\Header Files\o to z - - - util\concurrency - - - util\concurrency - - - platform - - - platform - - - platform - - - platform - - - platform - - - platform - - - platform - - - platform - - - third_party\MurmurHash3 - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - scripting - - - util\Header Files - - - base - - - base - - - base - - - base - - - base - - - base - - - base - - - platform - - - base - - - base - - - base - - - base - - - base - - - base - - - base - - - bson - - - bson - - - platform - - - platform - - - platform - - - util\Header Files - - - util\Header Files - - - db\auth - - - db\auth - - - db\auth - - - db\Header Files\e to n - - - db\Header Files\e to n - - - platform - - - db\Header Files\e to n - - - platform - - - db\geo - - - db\geo - - - util\net - - - db\Header Files\e to n - - - s - - - s - - - s - - - util\Header Files - - - db\auth - - - base - - - db\auth - - - db\auth - - - db\Header Files\a to d - - - db\commands - - - db\commands - - - db\commands - - - db\Header Files\a to d - - - Boost - - - db\auth - - - bson\mutable - - - bson\mutable - - - bson\mutable - - - bson\mutable - - - db\geo - - - db\geo - - - util\Header Files - - - bson\util - - - db\auth - - - db\auth - - - db\auth - - - s - - - s - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\e to n - - - db\geo - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - util\Header Files - - - db\pipeline\Header Files - - - db\pipeline\Header Files - - - client\Header Files - - - s - - - s - - - db\Header Files\a to d - - - db\Header Files\e to n - - - s - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\auth - - - db\auth - - - s - - - s - - - db\commands - - - util\Header Files - - - base - - - base - - - db\auth - - - db\Header Files\o to z - - - db\geo - - - db\geo - - - db\geo - - - s - - - s - - - s - - - s - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2 - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\base - - - third_party\s2\strings - - - third_party\s2\strings - - - third_party\s2\strings - - - third_party\s2\strings - - - third_party\s2\util\coding - - - third_party\s2\util\coding - - - third_party\s2\util\endian - - - third_party\s2\util\hash - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math - - - third_party\s2\util\math\exactfloat - - - db\commands - - - bson - - - db\auth - - - db\repl - - - db\repl - - - db\repl - - - db\repl - - - db\auth - - - db\commands - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\a to d - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\e to n - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - db\Header Files\o to z - - - third_party\libstemmer_c\include - - - third_party\libstemmer_c\libstemmer - - - third_party\libstemmer_c\libstemmer - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\src_c - - - third_party\libstemmer_c\runtime - - - third_party\libstemmer_c\runtime - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\fts - - - db\ops - - - scripting - - - db\stats - - - db\stats - - - scripting - - - db\geo - - - scripting - - - db\Header Files\e to n - - - client\Header Files - - - db\Header Files\e to n - - - platform - - - platform - - - platform - - - - - db\Resource Files - - - util\mongoutils - - - third_party\SpiderMonkey\Header Files - - - third_party\SpiderMonkey\Header Files - - - db\auth - - - db\auth - - - base - - - base - - - third_party - - - third_party\s2 - - - third_party\libstemmer_c\libstemmer - - - third_party\libstemmer_c\libstemmer - - - third_party\libstemmer_c\libstemmer - - - third_party\libstemmer_c - - - db\fts - - - db\fts - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db\fts\stop_words - - - db - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in all executables - - - JavaScript source files\Included in shell only - - - JavaScript source files\Included in shell only - - - JavaScript source files\Included in shell only - - - JavaScript source files\Included in shell only - - - JavaScript source files\Included in shell only - - - - - {c666c392-87aa-4438-8e2a-e3b31442889f} - - - {ecca6cd2-cb88-4e36-9bf4-21ca22bbc3e6} - - - {3358f459-c103-4796-8950-e786af3dd61f} - - - {00003152-6207-482e-90ed-5db13bd67552} - - - {1f6de053-9963-48b3-9ae5-c6c3cfd42b6e} - - - {108f083d-3e32-4e36-9780-36b1ee587860} - - - {bb99c086-7926-4f50-838d-f5f0c18397c0} - - - {95efa43e-738b-4f26-9022-e26bda6cc39d} - - - {62421f8a-fd9e-4367-bce5-21ed840098d6} - - - {a54110ab-ac5b-4254-8354-a422876822f1} - - - {0bd38ec7-9c71-4c7e-b6a5-d40f393edc32} - - - {9d88cce6-3e5a-4c09-b946-2d1cb7718457} - - - {084cc105-eb1d-4434-9007-34aa13380e51} - - - {2f8edb1a-8b8d-49a2-ad2b-54728ded90fd} - - - {01ea671b-3105-4231-ab31-c7a7ece3e894} - - - {871be4c3-6d49-41de-8fab-8859419afb04} - - - {d61e8b6c-a9dc-428e-89dc-3de4f8135b2a} - - - {32666365-62a8-428a-98ca-bffed31913b4} - - - {d5f10c04-aa00-4646-8163-b7244fc0f4e3} - - - {e5097a87-cb9a-4b3a-b109-7584110729a4} - - - {7717e6b1-8b8f-4551-a1b2-6f73266b37b9} - - - {56d52247-c819-4e2b-a12f-c7da24879860} - - - {da58c1f7-ec48-4d5a-a63e-822a1f6f3900} - - - {d3338818-34d0-4575-96a7-917f44ba2f91} - - - {8ebae71f-a9ac-40e9-bd91-49b05795f805} - - - {74bf499c-8890-436e-a1f9-8bbaf9d5799a} - - - {97d24b2e-60a3-41d4-a9d5-97c05002e0a5} - - - {95737c65-b228-499d-b975-f4113dae1d0d} - - - {df1e6bfc-df51-47f2-ba76-a0d29ce7868e} - - - {c9802e7a-e40a-40cf-8a1d-6a2b946adc1a} - - - {399d9b2d-32aa-4e16-b475-2e95c42c00ea} - - - {83ca51e9-49da-4cb7-adcd-76fb8c14b314} - - - {1b44ff17-fd93-40b9-aad2-c98f75486000} - - - {d98fd563-5b9d-40a4-a45c-ef256f6abc5f} - - - {ae462979-1397-4617-acd3-e25345d23225} - - - {840d7ff5-8b79-4564-941e-867a2d69bbae} - - - {7ba395b0-8909-45a0-b3eb-27be826cc3d6} - - - {7fe0aa15-cf1b-47e0-9043-9fdcbe59d253} - - - {e29a0f06-8d7b-426f-b055-3c36af62b8a8} - - - {7034e006-ad70-4939-9b1b-cf2b448eec6b} - - - {54fa01f6-b02c-4626-9752-9b4ac1c73994} - - - {9431025a-8834-44a1-b94a-8a20c616206d} - - - {e1d472aa-0af3-48c4-9f3a-b06ee57d2db4} - - - {fcf379fd-83ef-44b4-b382-f15fb5fc3fec} - - - {df60ef6a-2ce0-4423-a9a7-759d05f690ff} - - - {03508fec-3cea-45a4-861f-7dd6990852a3} - - - {a7beaf14-1c40-48d2-832d-b0886e77da91} - - - {b5f90eeb-6f5d-457e-9d87-024af0c1332e} - - - {1786ad44-ef64-4aa9-8ebb-4107b1c9eefa} - - - {43f20e5d-d0cd-441a-84a0-ae367722fb31} - - - {88e5cd26-90ee-41af-81e5-7bbae49e86c7} - - - {b1afa496-4d7e-42ab-a37e-68409d21f2b4} - - - {1725ebab-8d00-4103-8937-59541a620407} - - - {016bbeaf-e0b8-4aba-93c2-529392fae2da} - - - {c38eff83-5451-4ed9-b552-4e71bd6cc21f} - - - {3d2e6175-acf6-4f67-896e-f93a5cbbacd4} - - - {577a305b-b943-4eba-85a6-2ffd2b4c64af} - - - {918ec912-403b-4139-9770-0ceb5f223a24} - - - {42a6fb8f-f3b9-455c-b68e-692222790ffc} - - - {b55142e9-7e40-4ad4-900d-a6cfe453a232} - - - {05758f9d-1e84-47c7-9819-8ddc7ad18174} - - - {0a44d577-b4b1-4497-8540-c894b0de4b8b} - - - {f508089b-9fc8-4818-af31-fe55d04b06f7} - - - {7d596946-ce52-4241-9d51-c433f0c529fa} - - - - - {bdc5ddb3-807b-4ded-962d-45b196271d00} - - - - - db\Resource Files - - - \ No newline at end of file diff --git a/src/mongo/db/mongommf.cpp b/src/mongo/db/mongommf.cpp deleted file mode 100644 index c68dcfb61b0..00000000000 --- a/src/mongo/db/mongommf.cpp +++ /dev/null @@ -1,244 +0,0 @@ -// @file mongommf.cpp - -/** -* Copyright (C) 2010 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -/* this module adds some of our layers atop memory mapped files - specifically our handling of private views & such - if you don't care about journaling/durability (temp sort files & such) use MemoryMappedFile class, not this. -*/ - -#include "pch.h" -#include "cmdline.h" -#include "mongommf.h" -#include "dur.h" -#include "dur_journalformat.h" -#include "../util/mongoutils/str.h" -#include "d_globals.h" -#include "memconcept.h" -#include "d_concurrency.h" - -using namespace mongoutils; - -namespace mongo { - - void MongoMMF::remapThePrivateView() { - verify( cmdLine.dur ); - - // todo 1.9 : it turns out we require that we always remap to the same address. - // so the remove / add isn't necessary and can be removed? - void *old = _view_private; - //privateViews.remove(_view_private); - _view_private = remapPrivateView(_view_private); - //privateViews.add(_view_private, this); - fassert( 16112, _view_private == old ); - } - - /** register view. threadsafe */ - void PointerToMMF::add(void *view, MongoMMF *f) { - verify(view); - verify(f); - mutex::scoped_lock lk(_m); - _views.insert( pair(view,f) ); - } - - /** de-register view. threadsafe */ - void PointerToMMF::remove(void *view) { - if( view ) { - mutex::scoped_lock lk(_m); - _views.erase(view); - } - } - - PointerToMMF::PointerToMMF() : _m("PointerToMMF") { -#if defined(SIZE_MAX) - size_t max = SIZE_MAX; -#else - size_t max = ~((size_t)0); -#endif - verify( max > (size_t) this ); // just checking that no one redef'd SIZE_MAX and that it is sane - - // this way we don't need any boundary checking in _find() - _views.insert( pair((void*)0,(MongoMMF*)0) ); - _views.insert( pair((void*)max,(MongoMMF*)0) ); - } - - /** underscore version of find is for when you are already locked - @param ofs out return our offset in the view - @return the MongoMMF to which this pointer belongs - */ - MongoMMF* PointerToMMF::find_inlock(void *p, /*out*/ size_t& ofs) { - // - // .................memory.......................... - // v1 p v2 - // [--------------------] [-------] - // - // e.g., _find(p) == v1 - // - const pair x = *(--_views.upper_bound(p)); - MongoMMF *mmf = x.second; - if( mmf ) { - size_t o = ((char *)p) - ((char*)x.first); - if( o < mmf->length() ) { - ofs = o; - return mmf; - } - } - return 0; - } - - /** find associated MMF object for a given pointer. - threadsafe - @param ofs out returns offset into the view of the pointer, if found. - @return the MongoMMF to which this pointer belongs. null if not found. - */ - MongoMMF* PointerToMMF::find(void *p, /*out*/ size_t& ofs) { - mutex::scoped_lock lk(_m); - return find_inlock(p, ofs); - } - - PointerToMMF privateViews; - - /* void* MongoMMF::switchToPrivateView(void *readonly_ptr) { - verify( cmdLine.dur ); - verify( testIntent ); - - void *p = readonly_ptr; - - { - size_t ofs=0; - MongoMMF *mmf = ourReadViews.find(p, ofs); - if( mmf ) { - void *res = ((char *)mmf->_view_private) + ofs; - return res; - } - } - - { - size_t ofs=0; - MongoMMF *mmf = privateViews.find(p, ofs); - if( mmf ) { - log() << "dur: perf warning p=" << p << " is already in the writable view of " << mmf->filename() << endl; - return p; - } - } - - // did you call writing() with a pointer that isn't into a datafile? - log() << "dur error switchToPrivateView " << p << endl; - return p; - }*/ - - /* switch to _view_write. normally, this is a bad idea since your changes will not - show up in _view_private if there have been changes there; thus the leading underscore - as a tad of a "warning". but useful when done with some care, such as during - initialization. - */ - void* MongoMMF::_switchToWritableView(void *p) { - size_t ofs; - MongoMMF *f = privateViews.find(p, ofs); - verify( f ); - return (((char *)f->_view_write)+ofs); - } - - extern string dbpath; - - // here so that it is precomputed... - void MongoMMF::setPath(const std::string& f) { - string suffix; - string prefix; - bool ok = str::rSplitOn(f, '.', prefix, suffix); - uassert(13520, str::stream() << "MongoMMF only supports filenames in a certain format " << f, ok); - if( suffix == "ns" ) - _fileSuffixNo = dur::JEntry::DotNsSuffix; - else - _fileSuffixNo = (int) str::toUnsigned(suffix); - - _p = RelativePath::fromFullPath(prefix); - } - - bool MongoMMF::open(const std::string& fname, bool sequentialHint) { - LOG(3) << "mmf open " << fname << endl; - setPath(fname); - _view_write = mapWithOptions(fname.c_str(), sequentialHint ? SEQUENTIAL : 0); - return finishOpening(); - } - - bool MongoMMF::create(const std::string& fname, unsigned long long& len, bool sequentialHint) { - LOG(3) << "mmf create " << fname << endl; - setPath(fname); - _view_write = map(fname.c_str(), len, sequentialHint ? SEQUENTIAL : 0); - return finishOpening(); - } - - bool MongoMMF::finishOpening() { - LOG(3) << "mmf finishOpening " << (void*) _view_write << ' ' << filename() << " len:" << length() << endl; - if( _view_write ) { - if( cmdLine.dur ) { - _view_private = createPrivateMap(); - if( _view_private == 0 ) { - msgasserted(13636, str::stream() << "file " << filename() << " open/create failed in createPrivateMap (look in log for more information)"); - } - privateViews.add(_view_private, this); // note that testIntent builds use this, even though it points to view_write then... - } - else { - _view_private = _view_write; - } - return true; - } - return false; - } - - MongoMMF::MongoMMF() : _willNeedRemap(false) { - _view_write = _view_private = 0; - } - - MongoMMF::~MongoMMF() { - try { - close(); - } - catch(...) { error() << "exception in ~MongoMMF" << endl; } - } - - namespace dur { - void closingFileNotification(); - } - - /*virtual*/ void MongoMMF::close() { - LOG(3) << "mmf close " << filename() << endl; - - if( view_write() /*actually was opened*/ ) { - if( cmdLine.dur ) { - dur::closingFileNotification(); - } - /* todo: is it ok to close files if we are not globally locked exclusively? - probably, but need to review. also note the lock assert below is - rather vague and not checking if the right database is locked - */ - if( !Lock::somethingWriteLocked() ) { - verify( inShutdown() ); - DEV { - log() << "is it really ok to close a mongommf outside a write lock? file:" << filename() << endl; - } - } - } - - LockMongoFilesExclusive lk; - privateViews.remove(_view_private); - memconcept::invalidate(_view_private); - _view_write = _view_private = 0; - MemoryMappedFile::close(); - } - -} diff --git a/src/mongo/db/mongommf.h b/src/mongo/db/mongommf.h deleted file mode 100644 index 3d6324443c6..00000000000 --- a/src/mongo/db/mongommf.h +++ /dev/null @@ -1,146 +0,0 @@ -/** @file mongommf.h -* -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "../util/mmap.h" -#include "../util/paths.h" - -namespace mongo { - - /** MongoMMF adds some layers atop memory mapped files - specifically our handling of private views & such. - if you don't care about journaling/durability (temp sort files & such) use MemoryMappedFile class, - not this. - */ - class MongoMMF : private MemoryMappedFile { - protected: - virtual void* viewForFlushing() { return _view_write; } - - public: - MongoMMF(); - virtual ~MongoMMF(); - virtual void close(); - - /** @return true if opened ok. */ - bool open(const std::string& fname, bool sequentialHint /*typically we open with this false*/); - - /** @return file length */ - unsigned long long length() const { return MemoryMappedFile::length(); } - - string filename() const { return MemoryMappedFile::filename(); } - - void flush(bool sync) { MemoryMappedFile::flush(sync); } - - /* Creates with length if DNE, otherwise uses existing file length, - passed length. - @param sequentialHint if true will be sequentially accessed - @return true for ok - */ - bool create(const std::string& fname, unsigned long long& len, bool sequentialHint); - - /* Get the "standard" view (which is the private one). - @return the private view. - */ - void* getView() const { return _view_private; } - - /* Get the "write" view (which is required for writing). - @return the write view. - */ - void* view_write() const { return _view_write; } - - - /* switch to _view_write. normally, this is a bad idea since your changes will not - show up in _view_private if there have been changes there; thus the leading underscore - as a tad of a "warning". but useful when done with some care, such as during - initialization. - */ - static void* _switchToWritableView(void *private_ptr); - - /** for a filename a/b/c.3 - filePath() is "a/b/c" - fileSuffixNo() is 3 - if the suffix is "ns", fileSuffixNo -1 - */ - const RelativePath& relativePath() const { - DEV verify( !_p._p.empty() ); - return _p; - } - - int fileSuffixNo() const { return _fileSuffixNo; } - HANDLE getFd() { return MemoryMappedFile::getFd(); } - - /** true if we have written. - set in PREPLOGBUFFER, it is NOT set immediately on write intent declaration. - reset to false in REMAPPRIVATEVIEW - */ - bool& willNeedRemap() { return _willNeedRemap; } - - void remapThePrivateView(); - - virtual bool isMongoMMF() { return true; } - - private: - - void *_view_write; - void *_view_private; - bool _willNeedRemap; - RelativePath _p; // e.g. "somepath/dbname" - int _fileSuffixNo; // e.g. 3. -1="ns" - - void setPath(const std::string& pathAndFileName); - bool finishOpening(); - }; - - /** for durability support we want to be able to map pointers to specific MongoMMF objects. - */ - class PointerToMMF : boost::noncopyable { - public: - PointerToMMF(); - - /** register view. - threadsafe - */ - void add(void *view, MongoMMF *f); - - /** de-register view. - threadsafe - */ - void remove(void *view); - - /** find associated MMF object for a given pointer. - threadsafe - @param ofs out returns offset into the view of the pointer, if found. - @return the MongoMMF to which this pointer belongs. null if not found. - */ - MongoMMF* find(void *p, /*out*/ size_t& ofs); - - /** for doing many finds in a row with one lock operation */ - mutex& _mutex() { return _m; } - MongoMMF* find_inlock(void *p, /*out*/ size_t& ofs); - - map::iterator finditer_inlock(void *p) { return _views.upper_bound(p); } - - unsigned numberOfViews_inlock() const { return _views.size(); } - - private: - mutex _m; - map _views; - }; - - // allows a pointer into any private view of a MongoMMF to be resolved to the MongoMMF object - extern PointerToMMF privateViews; -} diff --git a/src/mongo/db/namespace-inl.h b/src/mongo/db/namespace-inl.h deleted file mode 100644 index 04411870124..00000000000 --- a/src/mongo/db/namespace-inl.h +++ /dev/null @@ -1,72 +0,0 @@ -// @file namespace-inl.h - -/** -* Copyright (C) 2009 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/db/namespace.h" - -namespace mongo { - - inline Namespace& Namespace::operator=(const StringData& ns) { - // we fill the remaining space with all zeroes here. as the full Namespace struct is in - // the datafiles (the .ns files specifically), that is helpful as then they are deterministic - // in the bytes they have for a given sequence of operations. that makes testing and debugging - // the data files easier. - // - // if profiling indicates this method is a significant bottleneck, we could have a version we - // use for reads which does not fill with zeroes, and keep the zeroing behavior on writes. - // - memset( buf, 0, MaxNsLen ); - uassert( 10080 , "ns name too long, max size is 128", ns.size() < MaxNsLen - 1); - ns.copyTo( buf, true ); - return *this; - } - - inline string Namespace::extraName(int i) const { - char ex[] = "$extra"; - ex[5] += i; - string s = string(buf) + ex; - massert( 10348 , "$extra: ns name too long", s.size() < MaxNsLen); - return s; - } - - inline bool Namespace::isExtra() const { - const char *p = strstr(buf, "$extr"); - return p && p[5] && p[6] == 0; //==0 important in case an index uses name "$extra_1" for example - } - - inline int Namespace::hash() const { - unsigned x = 0; - const char *p = buf; - while ( *p ) { - x = x * 131 + *p; - p++; - } - return (x & 0x7fffffff) | 0x8000000; // must be > 0 - } - - /* future : this doesn't need to be an inline. */ - inline string Namespace::getSisterNS( const char * local ) const { - verify( local && local[0] != '.' ); - string old(buf); - if ( old.find( "." ) != string::npos ) - old = old.substr( 0 , old.find( "." ) ); - return old + "." + local; - } - -} // namespace mongo diff --git a/src/mongo/db/namespace.cpp b/src/mongo/db/namespace.cpp deleted file mode 100644 index f0bb9c1f59c..00000000000 --- a/src/mongo/db/namespace.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// namespace.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" - -#include "mongo/db/namespace.h" - -#include - -#include "mongo/db/namespacestring.h" - -namespace mongo { -namespace { -BOOST_STATIC_ASSERT( sizeof(Namespace) == 128 ); -BOOST_STATIC_ASSERT( Namespace::MaxNsLen == MaxDatabaseNameLen ); -} // namespace -} // namespace mongo - diff --git a/src/mongo/db/namespace.h b/src/mongo/db/namespace.h deleted file mode 100644 index 9d011f684e8..00000000000 --- a/src/mongo/db/namespace.h +++ /dev/null @@ -1,62 +0,0 @@ -// namespace.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/pch.h" - -#include -#include - -namespace mongo { - -#pragma pack(1) - class Namespace { - public: - Namespace(const StringData& ns) { *this = ns; } - Namespace& operator=(const StringData& ns); - - bool hasDollarSign() const { return strchr( buf , '$' ) > 0; } - void kill() { buf[0] = 0x7f; } - bool operator==(const char *r) const { return strcmp(buf, r) == 0; } - bool operator==(const Namespace& r) const { return strcmp(buf, r.buf) == 0; } - int hash() const; // value returned is always > 0 - - size_t size() const { return strlen( buf ); } - - string toString() const { return buf; } - operator string() const { return buf; } - - /* NamespaceDetails::Extra was added after fact to allow chaining of data blocks to support more than 10 indexes - (more than 10 IndexDetails). It's a bit hacky because of this late addition with backward - file support. */ - string extraName(int i) const; - bool isExtra() const; /* ends with $extr... -- when true an extra block not a normal NamespaceDetails block */ - - /** ( foo.bar ).getSisterNS( "blah" ) == foo.blah - perhaps this should move to the NamespaceString helper? - */ - string getSisterNS( const char * local ) const; - - enum MaxNsLenValue { MaxNsLen = 128 }; - private: - char buf[MaxNsLen]; - }; -#pragma pack() - -} // namespace mongo diff --git a/src/mongo/db/namespace_details-inl.h b/src/mongo/db/namespace_details-inl.h deleted file mode 100644 index 9e8ccbb37e6..00000000000 --- a/src/mongo/db/namespace_details-inl.h +++ /dev/null @@ -1,104 +0,0 @@ -// @file namespace-inl.h - -/** -* Copyright (C) 2009 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/db/namespace_details.h" - -namespace mongo { - - inline IndexDetails& NamespaceDetails::idx(int idxNo, bool missingExpected ) { - if( idxNo < NIndexesBase ) { - IndexDetails& id = _indexes[idxNo]; - return id; - } - Extra *e = extra(); - if ( ! e ) { - if ( missingExpected ) - throw MsgAssertionException( 13283 , "Missing Extra" ); - massert(14045, "missing Extra", e); - } - int i = idxNo - NIndexesBase; - if( i >= NIndexesExtra ) { - e = e->next(this); - if ( ! e ) { - if ( missingExpected ) - throw MsgAssertionException( 14823 , "missing extra" ); - massert(14824, "missing Extra", e); - } - i -= NIndexesExtra; - } - return e->details[i]; - } - - inline int NamespaceDetails::idxNo(const IndexDetails& idx) { - IndexIterator i = ii(); - while( i.more() ) { - if( &i.next() == &idx ) - return i.pos()-1; - } - massert( 10349 , "E12000 idxNo fails", false); - return -1; - } - - inline int NamespaceDetails::findIndexByKeyPattern(const BSONObj& keyPattern, - bool includeBackgroundInProgress) { - IndexIterator i = ii(includeBackgroundInProgress); - while( i.more() ) { - if( i.next().keyPattern() == keyPattern ) - return i.pos()-1; - } - return -1; - } - - inline const IndexDetails* NamespaceDetails::findIndexByPrefix( const BSONObj &keyPattern , - bool requireSingleKey ) { - const IndexDetails* bestMultiKeyIndex = NULL; - IndexIterator i = ii(); - while( i.more() ) { - const IndexDetails& currentIndex = i.next(); - if( keyPattern.isPrefixOf( currentIndex.keyPattern() ) ){ - if( ! isMultikey( i.pos()-1 ) ){ - return ¤tIndex; - } else { - bestMultiKeyIndex = ¤tIndex; - } - } - } - return requireSingleKey ? NULL : bestMultiKeyIndex; - } - - // @return offset in indexes[] - inline int NamespaceDetails::findIndexByName(const char *name, - bool includeBackgroundInProgress) { - IndexIterator i = ii(includeBackgroundInProgress); - while( i.more() ) { - if ( strcmp(i.next().info.obj().getStringField("name"),name) == 0 ) - return i.pos()-1; - } - return -1; - } - - inline NamespaceDetails::IndexIterator::IndexIterator(NamespaceDetails *_d, - bool includeBackgroundInProgress) { - d = _d; - i = 0; - n = includeBackgroundInProgress ? d->getTotalIndexCount() : d->nIndexes; - } - -} diff --git a/src/mongo/db/namespace_details.cpp b/src/mongo/db/namespace_details.cpp deleted file mode 100644 index 94adac276a6..00000000000 --- a/src/mongo/db/namespace_details.cpp +++ /dev/null @@ -1,992 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" - -#include "mongo/db/namespace_details.h" - -#include -#include - -#include - -#include "mongo/db/db.h" -#include "mongo/db/json.h" -#include "mongo/db/mongommf.h" -#include "mongo/db/ops/delete.h" -#include "mongo/db/ops/update.h" -#include "mongo/db/pdfile.h" -#include "mongo/scripting/engine.h" -#include "mongo/util/hashtab.h" - -namespace mongo { - - BSONObj idKeyPattern = fromjson("{\"_id\":1}"); - - /* Deleted list buckets are used to quickly locate free space based on size. Each bucket - contains records up to that size. All records > 4mb are placed into the 16mb bucket. - */ - int bucketSizes[] = { - 0x20, 0x40, 0x80, 0x100, - 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000, - 0x20000, 0x40000, 0x80000, 0x100000, - 0x200000, 0x400000, 0x1000000, - }; - - NamespaceDetails::NamespaceDetails( const DiskLoc &loc, bool capped ) { - /* be sure to initialize new fields here -- doesn't default to zeroes the way we use it */ - firstExtent = lastExtent = capExtent = loc; - stats.datasize = stats.nrecords = 0; - lastExtentSize = 0; - nIndexes = 0; - _isCapped = capped; - _maxDocsInCapped = 0x7fffffff; // no limit (value is for pre-v2.3.2 compatability) - _paddingFactor = 1.0; - _systemFlags = 0; - _userFlags = 0; - capFirstNewRecord = DiskLoc(); - // Signal that we are on first allocation iteration through extents. - capFirstNewRecord.setInvalid(); - // For capped case, signal that we are doing initial extent allocation. - if ( capped ) - cappedLastDelRecLastExtent().setInvalid(); - verify( sizeof(dataFileVersion) == 2 ); - dataFileVersion = 0; - indexFileVersion = 0; - multiKeyIndexBits = 0; - reservedA = 0; - extraOffset = 0; - indexBuildsInProgress = 0; - memset(reserved, 0, sizeof(reserved)); - } - - bool NamespaceIndex::exists() const { - return !boost::filesystem::exists(path()); - } - - boost::filesystem::path NamespaceIndex::path() const { - boost::filesystem::path ret( dir_ ); - if ( directoryperdb ) - ret /= database_; - ret /= ( database_ + ".ns" ); - return ret; - } - - void NamespaceIndex::maybeMkdir() const { - if ( !directoryperdb ) - return; - boost::filesystem::path dir( dir_ ); - dir /= database_; - if ( !boost::filesystem::exists( dir ) ) - MONGO_ASSERT_ON_EXCEPTION_WITH_MSG( boost::filesystem::create_directory( dir ), "create dir for db " ); - } - - unsigned lenForNewNsFiles = 16 * 1024 * 1024; - -#if defined(_DEBUG) - void NamespaceDetails::dump(const Namespace& k) { - if( !cmdLine.dur ) - cout << "ns offsets which follow will not display correctly with --journal disabled" << endl; - - size_t ofs = 1; // 1 is sentinel that the find call below failed - privateViews.find(this, /*out*/ofs); - - cout << "ns" << hex << setw(8) << ofs << ' '; - cout << k.toString() << '\n'; - - if( k.isExtra() ) { - cout << "ns\t extra" << endl; - return; - } - - cout << "ns " << firstExtent.toString() << ' ' << lastExtent.toString() << " nidx:" << nIndexes << '\n'; - cout << "ns " << stats.datasize << ' ' << stats.nrecords << ' ' << nIndexes << '\n'; - cout << "ns " << isCapped() << ' ' << _paddingFactor << ' ' << _systemFlags << ' ' << _userFlags << ' ' << dataFileVersion << '\n'; - cout << "ns " << multiKeyIndexBits << ' ' << indexBuildsInProgress << '\n'; - cout << "ns " << (int) reserved[0] << ' ' << (int) reserved[59]; - cout << endl; - } -#endif - - NOINLINE_DECL void NamespaceIndex::_init() { - verify( !ht ); - - Lock::assertWriteLocked(database_); - - /* if someone manually deleted the datafiles for a database, - we need to be sure to clear any cached info for the database in - local.*. - */ - /* - if ( "local" != database_ ) { - DBInfo i(database_.c_str()); - i.dbDropped(); - } - */ - - unsigned long long len = 0; - boost::filesystem::path nsPath = path(); - string pathString = nsPath.string(); - void *p = 0; - if( boost::filesystem::exists(nsPath) ) { - if( f.open(pathString, true) ) { - len = f.length(); - if ( len % (1024*1024) != 0 ) { - log() << "bad .ns file: " << pathString << endl; - uassert( 10079 , "bad .ns file length, cannot open database", len % (1024*1024) == 0 ); - } - p = f.getView(); - } - } - else { - // use lenForNewNsFiles, we are making a new database - massert( 10343, "bad lenForNewNsFiles", lenForNewNsFiles >= 1024*1024 ); - maybeMkdir(); - unsigned long long l = lenForNewNsFiles; - if( f.create(pathString, l, true) ) { - getDur().createdFile(pathString, l); // always a new file - len = l; - verify( len == lenForNewNsFiles ); - p = f.getView(); - } - } - - if ( p == 0 ) { - /** TODO: this shouldn't terminate? */ - log() << "error couldn't open file " << pathString << " terminating" << endl; - dbexit( EXIT_FS ); - } - - - verify( len <= 0x7fffffff ); - ht = new HashTable(p, (int) len, "namespace index"); - } - - static void namespaceGetNamespacesCallback( const Namespace& k , NamespaceDetails& v , void * extra ) { - list * l = (list*)extra; - if ( ! k.hasDollarSign() ) - l->push_back( (string)k ); - } - void NamespaceIndex::getNamespaces( list& tofill , bool onlyCollections ) const { - verify( onlyCollections ); // TODO: need to implement this - // need boost::bind or something to make this less ugly - - if ( ht ) - ht->iterAll( namespaceGetNamespacesCallback , (void*)&tofill ); - } - - void NamespaceDetails::addDeletedRec(DeletedRecord *d, DiskLoc dloc) { - BOOST_STATIC_ASSERT( sizeof(NamespaceDetails::Extra) <= sizeof(NamespaceDetails) ); - - { - Record *r = (Record *) getDur().writingPtr(d, sizeof(Record)); - d = &r->asDeleted(); - // defensive code: try to make us notice if we reference a deleted record - reinterpret_cast( r->data() )[0] = 0xeeeeeeee; - } - DEBUGGING log() << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs() << endl; - if ( isCapped() ) { - if ( !cappedLastDelRecLastExtent().isValid() ) { - // Initial extent allocation. Insert at end. - d->nextDeleted() = DiskLoc(); - if ( cappedListOfAllDeletedRecords().isNull() ) - getDur().writingDiskLoc( cappedListOfAllDeletedRecords() ) = dloc; - else { - DiskLoc i = cappedListOfAllDeletedRecords(); - for (; !i.drec()->nextDeleted().isNull(); i = i.drec()->nextDeleted() ) - ; - i.drec()->nextDeleted().writing() = dloc; - } - } - else { - d->nextDeleted() = cappedFirstDeletedInCurExtent(); - getDur().writingDiskLoc( cappedFirstDeletedInCurExtent() ) = dloc; - // always compact() after this so order doesn't matter - } - } - else { - int b = bucket(d->lengthWithHeaders()); - DiskLoc& list = deletedList[b]; - DiskLoc oldHead = list; - getDur().writingDiskLoc(list) = dloc; - d->nextDeleted() = oldHead; - } - } - - /* @return the size for an allocated record quantized to 1/16th of the BucketSize - @param allocSize requested size to allocate - */ - int NamespaceDetails::quantizeAllocationSpace(int allocSize) { - const int bucketIdx = bucket(allocSize); - int bucketSize = bucketSizes[bucketIdx]; - int quantizeUnit = bucketSize / 16; - if (allocSize >= (1 << 22)) // 4mb - // all allocatons > 4mb result in 4mb/16 quantization units, even if allocated in - // the 8mb+ bucket. idea is to reduce quantization overhead of large records at - // the cost of increasing the DeletedRecord size distribution in the largest bucket - // by factor of 4. - quantizeUnit = (1 << 18); // 256k - if (allocSize % quantizeUnit == 0) - // size is already quantized - return allocSize; - const int quantizedSpace = (allocSize | (quantizeUnit - 1)) + 1; - fassert(16484, quantizedSpace >= allocSize); - return quantizedSpace; - } - - /* predetermine location of the next alloc without actually doing it. - if cannot predetermine returns null (so still call alloc() then) - */ - DiskLoc NamespaceDetails::allocWillBeAt(const char *ns, int lenToAlloc) { - if ( ! isCapped() ) { - lenToAlloc = (lenToAlloc + 3) & 0xfffffffc; - return __stdAlloc(lenToAlloc, true); - } - return DiskLoc(); - } - - /** allocate space for a new record from deleted lists. - @param lenToAlloc is WITH header - @return null diskloc if no room - allocate a new extent then - */ - DiskLoc NamespaceDetails::alloc(const char* ns, int lenToAlloc) { - { - // align very slightly. - lenToAlloc = (lenToAlloc + 3) & 0xfffffffc; - } - - DiskLoc loc = _alloc(ns, lenToAlloc); - if ( loc.isNull() ) - return loc; - - DeletedRecord *r = loc.drec(); - //r = getDur().writing(r); - - /* note we want to grab from the front so our next pointers on disk tend - to go in a forward direction which is important for performance. */ - int regionlen = r->lengthWithHeaders(); - verify( r->extentOfs() < loc.getOfs() ); - - DEBUGGING out() << "TEMP: alloc() returns " << loc.toString() << ' ' << ns << " lentoalloc:" << lenToAlloc << endl; - - int left = regionlen - lenToAlloc; - if ( ! isCapped() ) { - if ( left < 24 || left < (lenToAlloc >> 3) ) { - // you get the whole thing. - return loc; - } - } - - // don't quantize: - // - capped collections: just wastes space - // - $ collections (indexes) as we already have those alligned the way we want SERVER-8425 - if ( !isCapped() && NamespaceString::normal( ns ) ) { - // we quantize here so that it only impacts newly sized records - // this prevents oddities with older records and space re-use SERVER-8435 - lenToAlloc = std::min( r->lengthWithHeaders(), - NamespaceDetails::quantizeAllocationSpace( lenToAlloc ) ); - left = regionlen - lenToAlloc; - - if ( left < 24 ) { - // you get the whole thing. - return loc; - } - } - - /* split off some for further use. */ - getDur().writingInt(r->lengthWithHeaders()) = lenToAlloc; - DiskLoc newDelLoc = loc; - newDelLoc.inc(lenToAlloc); - DeletedRecord* newDel = DataFileMgr::getDeletedRecord(newDelLoc); - DeletedRecord* newDelW = getDur().writing(newDel); - newDelW->extentOfs() = r->extentOfs(); - newDelW->lengthWithHeaders() = left; - newDelW->nextDeleted().Null(); - - addDeletedRec(newDel, newDelLoc); - - return loc; - } - - /* for non-capped collections. - @param peekOnly just look up where and don't reserve - returned item is out of the deleted list upon return - */ - DiskLoc NamespaceDetails::__stdAlloc(int len, bool peekOnly) { - DiskLoc *prev; - DiskLoc *bestprev = 0; - DiskLoc bestmatch; - int bestmatchlen = 0x7fffffff; - int b = bucket(len); - DiskLoc cur = deletedList[b]; - prev = &deletedList[b]; - int extra = 5; // look for a better fit, a little. - int chain = 0; - while ( 1 ) { - { // defensive check - int fileNumber = cur.a(); - int fileOffset = cur.getOfs(); - if (fileNumber < -1 || fileNumber >= 100000 || fileOffset < 0) { - StringBuilder sb; - sb << "Deleted record list corrupted in bucket " << b - << ", link number " << chain - << ", invalid link is " << cur.toString() - << ", throwing Fatal Assertion"; - problem() << sb.str() << endl; - fassertFailed(16469); - } - } - if ( cur.isNull() ) { - // move to next bucket. if we were doing "extra", just break - if ( bestmatchlen < 0x7fffffff ) - break; - b++; - if ( b > MaxBucket ) { - // out of space. alloc a new extent. - return DiskLoc(); - } - cur = deletedList[b]; - prev = &deletedList[b]; - continue; - } - DeletedRecord *r = cur.drec(); - if ( r->lengthWithHeaders() >= len && - r->lengthWithHeaders() < bestmatchlen ) { - bestmatchlen = r->lengthWithHeaders(); - bestmatch = cur; - bestprev = prev; - if (r->lengthWithHeaders() == len) - // exact match, stop searching - break; - } - if ( bestmatchlen < 0x7fffffff && --extra <= 0 ) - break; - if ( ++chain > 30 && b < MaxBucket ) { - // too slow, force move to next bucket to grab a big chunk - //b++; - chain = 0; - cur.Null(); - } - else { - /*this defensive check only made sense for the mmap storage engine: - if ( r->nextDeleted.getOfs() == 0 ) { - problem() << "~~ Assertion - bad nextDeleted " << r->nextDeleted.toString() << - " b:" << b << " chain:" << chain << ", fixing.\n"; - r->nextDeleted.Null(); - }*/ - cur = r->nextDeleted(); - prev = &r->nextDeleted(); - } - } - - /* unlink ourself from the deleted list */ - if( !peekOnly ) { - DeletedRecord *bmr = bestmatch.drec(); - *getDur().writing(bestprev) = bmr->nextDeleted(); - bmr->nextDeleted().writing().setInvalid(); // defensive. - verify(bmr->extentOfs() < bestmatch.getOfs()); - } - - return bestmatch; - } - - void NamespaceDetails::dumpDeleted(set *extents) { - for ( int i = 0; i < Buckets; i++ ) { - DiskLoc dl = deletedList[i]; - while ( !dl.isNull() ) { - DeletedRecord *r = dl.drec(); - DiskLoc extLoc(dl.a(), r->extentOfs()); - if ( extents == 0 || extents->count(extLoc) <= 0 ) { - out() << " bucket " << i << endl; - out() << " " << dl.toString() << " ext:" << extLoc.toString(); - if ( extents && extents->count(extLoc) <= 0 ) - out() << '?'; - out() << " len:" << r->lengthWithHeaders() << endl; - } - dl = r->nextDeleted(); - } - } - } - - DiskLoc NamespaceDetails::firstRecord( const DiskLoc &startExtent ) const { - for (DiskLoc i = startExtent.isNull() ? firstExtent : startExtent; - !i.isNull(); i = i.ext()->xnext ) { - if ( !i.ext()->firstRecord.isNull() ) - return i.ext()->firstRecord; - } - return DiskLoc(); - } - - DiskLoc NamespaceDetails::lastRecord( const DiskLoc &startExtent ) const { - for (DiskLoc i = startExtent.isNull() ? lastExtent : startExtent; - !i.isNull(); i = i.ext()->xprev ) { - if ( !i.ext()->lastRecord.isNull() ) - return i.ext()->lastRecord; - } - return DiskLoc(); - } - - int n_complaints_cap = 0; - void NamespaceDetails::maybeComplain( const char *ns, int len ) const { - if ( ++n_complaints_cap < 8 ) { - out() << "couldn't make room for new record (len: " << len << ") in capped ns " << ns << '\n'; - int i = 0; - for ( DiskLoc e = firstExtent; !e.isNull(); e = e.ext()->xnext, ++i ) { - out() << " Extent " << i; - if ( e == capExtent ) - out() << " (capExtent)"; - out() << '\n'; - out() << " magic: " << hex << e.ext()->magic << dec << " extent->ns: " << e.ext()->nsDiagnostic.toString() << '\n'; - out() << " fr: " << e.ext()->firstRecord.toString() << - " lr: " << e.ext()->lastRecord.toString() << " extent->len: " << e.ext()->length << '\n'; - } - verify( len * 5 > lastExtentSize ); // assume it is unusually large record; if not, something is broken - } - } - - /* alloc with capped table handling. */ - DiskLoc NamespaceDetails::_alloc(const char *ns, int len) { - if ( ! isCapped() ) - return __stdAlloc(len, false); - - return cappedAlloc(ns,len); - } - - void NamespaceIndex::kill_ns(const char *ns) { - Lock::assertWriteLocked(ns); - if ( !ht ) - return; - Namespace n(ns); - ht->kill(n); - - for( int i = 0; i<=1; i++ ) { - try { - Namespace extra(n.extraName(i).c_str()); - ht->kill(extra); - } - catch(DBException&) { - dlog(3) << "caught exception in kill_ns" << endl; - } - } - } - - void NamespaceIndex::add_ns(const char *ns, DiskLoc& loc, bool capped) { - NamespaceDetails details( loc, capped ); - add_ns( ns, details ); - } - void NamespaceIndex::add_ns( const char *ns, const NamespaceDetails &details ) { - Lock::assertWriteLocked(ns); - init(); - Namespace n(ns); - uassert( 10081 , "too many namespaces/collections", ht->put(n, details)); - } - - /* extra space for indexes when more than 10 */ - NamespaceDetails::Extra* NamespaceIndex::newExtra(const char *ns, int i, NamespaceDetails *d) { - Lock::assertWriteLocked(ns); - verify( i >= 0 && i <= 1 ); - Namespace n(ns); - Namespace extra(n.extraName(i).c_str()); // throws userexception if ns name too long - - massert( 10350 , "allocExtra: base ns missing?", d ); - massert( 10351 , "allocExtra: extra already exists", ht->get(extra) == 0 ); - - NamespaceDetails::Extra temp; - temp.init(); - uassert( 10082 , "allocExtra: too many namespaces/collections", ht->put(extra, (NamespaceDetails&) temp)); - NamespaceDetails::Extra *e = (NamespaceDetails::Extra *) ht->get(extra); - return e; - } - NamespaceDetails::Extra* NamespaceDetails::allocExtra(const char *ns, int nindexessofar) { - NamespaceIndex *ni = nsindex(ns); - int i = (nindexessofar - NIndexesBase) / NIndexesExtra; - Extra *e = ni->newExtra(ns, i, this); - long ofs = e->ofsFrom(this); - if( i == 0 ) { - verify( extraOffset == 0 ); - *getDur().writing(&extraOffset) = ofs; - verify( extra() == e ); - } - else { - Extra *hd = extra(); - verify( hd->next(this) == 0 ); - hd->setNext(ofs); - } - return e; - } - - void NamespaceDetails::setIndexIsMultikey(const char *thisns, int i, bool multikey) { - massert(16577, "index number greater than NIndexesMax", i < NIndexesMax ); - - unsigned long long mask = 1ULL << i; - - if (multikey) { - // Shortcut if the bit is already set correctly - if (multiKeyIndexBits & mask) { - return; - } - - *getDur().writing(&multiKeyIndexBits) |= mask; - } - else { - // Shortcut if the bit is already set correctly - if (!(multiKeyIndexBits & mask)) { - return; - } - - // Invert mask: all 1's except a 0 at the ith bit - mask = ~mask; - *getDur().writing(&multiKeyIndexBits) &= mask; - } - - NamespaceDetailsTransient::get(thisns).clearQueryCache(); - } - - IndexDetails& NamespaceDetails::getNextIndexDetails(const char* thisns) { - IndexDetails *id; - try { - id = &idx(getTotalIndexCount(), true); - } - catch(DBException&) { - allocExtra(thisns, getTotalIndexCount()); - id = &idx(getTotalIndexCount(), false); - } - return *id; - } - - /* you MUST call when adding an index. see pdfile.cpp */ - void NamespaceDetails::addIndex(const char* thisns) { - (*getDur().writing(&nIndexes))++; - NamespaceDetailsTransient::get(thisns).addedIndex(); - } - - // must be called when renaming a NS to fix up extra - void NamespaceDetails::copyingFrom(const char *thisns, NamespaceDetails *src) { - extraOffset = 0; // we are a copy -- the old value is wrong. fixing it up below. - Extra *se = src->extra(); - int n = NIndexesBase; - if( se ) { - Extra *e = allocExtra(thisns, n); - while( 1 ) { - n += NIndexesExtra; - e->copy(this, *se); - se = se->next(src); - if( se == 0 ) break; - Extra *nxt = allocExtra(thisns, n); - e->setNext( nxt->ofsFrom(this) ); - e = nxt; - } - verify( extraOffset ); - } - } - - /* returns index of the first index in which the field is present. -1 if not present. - (aug08 - this method not currently used) - */ - int NamespaceDetails::fieldIsIndexed(const char *fieldName) { - massert( 10346 , "not implemented", false); - /* - for ( int i = 0; i < nIndexes; i++ ) { - IndexDetails& idx = indexes[i]; - BSONObj idxKey = idx.info.obj().getObjectField("key"); // e.g., { ts : -1 } - if ( !idxKey.getField(fieldName).eoo() ) - return i; - }*/ - return -1; - } - - long long NamespaceDetails::storageSize( int * numExtents , BSONArrayBuilder * extentInfo ) const { - Extent * e = firstExtent.ext(); - verify( e ); - - long long total = 0; - int n = 0; - while ( e ) { - total += e->length; - n++; - - if ( extentInfo ) { - extentInfo->append( BSON( "len" << e->length << "loc: " << e->myLoc.toBSONObj() ) ); - } - - e = e->getNextExtent(); - } - - if ( numExtents ) - *numExtents = n; - - return total; - } - - NamespaceDetails *NamespaceDetails::writingWithExtra() { - vector< pair< long long, unsigned > > writeRanges; - writeRanges.push_back( make_pair( 0, sizeof( NamespaceDetails ) ) ); - for( Extra *e = extra(); e; e = e->next( this ) ) { - writeRanges.push_back( make_pair( (char*)e - (char*)this, sizeof( Extra ) ) ); - } - return reinterpret_cast< NamespaceDetails* >( getDur().writingRangesAtOffsets( this, writeRanges ) ); - } - - void NamespaceDetails::setMaxCappedDocs( long long max ) { - massert( 16499, - "max in a capped collection has to be < 2^31 or -1", - validMaxCappedDocs( &max ) ); - _maxDocsInCapped = max; - } - - bool NamespaceDetails::validMaxCappedDocs( long long* max ) { - if ( *max <= 0 || - *max == numeric_limits::max() ) { - *max = 0x7fffffff; - return true; - } - - if ( *max < ( 0x1LL << 31 ) ) { - return true; - } - - return false; - } - - long long NamespaceDetails::maxCappedDocs() const { - verify( isCapped() ); - if ( _maxDocsInCapped == 0x7fffffff ) - return numeric_limits::max(); - return _maxDocsInCapped; - } - - /* ------------------------------------------------------------------------- */ - - SimpleMutex NamespaceDetailsTransient::_qcMutex("qc"); - SimpleMutex NamespaceDetailsTransient::_isMutex("is"); - NamespaceDetailsTransient::DMap NamespaceDetailsTransient::_nsdMap; - - void NamespaceDetailsTransient::reset() { - Lock::assertWriteLocked(_ns); - clearQueryCache(); - _keysComputed = false; - _indexSpecs.clear(); - } - - NamespaceDetailsTransient::CMap& NamespaceDetailsTransient::get_cmap_inlock(const string& ns) { - CMap*& m = _nsdMap[ns]; - if ( ! m ) - m = new CMap(); - return *m; - } - - /*static*/ NOINLINE_DECL NamespaceDetailsTransient& NamespaceDetailsTransient::make_inlock(const string& ns) { - shared_ptr< NamespaceDetailsTransient > &t = get_cmap_inlock(ns)[ ns ]; - verify( t.get() == 0 ); - Database *database = cc().database(); - verify( database ); - if( _nsdMap.size() % 20000 == 10000 ) { - // so we notice if insanely large #s - log() << "opening namespace " << ns << endl; - log() << _nsdMap.size() << " namespaces in nsdMap" << endl; - } - t.reset( new NamespaceDetailsTransient(database, ns) ); - return *t; - } - - // note with repair there could be two databases with the same ns name. - // that is NOT handled here yet! TODO - // repair may not use nsdt though not sure. anyway, requires work. - NamespaceDetailsTransient::NamespaceDetailsTransient(Database *db, const string& ns) : - _ns(ns), _keysComputed(false), _qcWriteCount() - { - dassert(db); - } - - NamespaceDetailsTransient::~NamespaceDetailsTransient() { - } - - void NamespaceDetailsTransient::resetCollection(const string& ns ) { - SimpleMutex::scoped_lock lk(_qcMutex); - Lock::assertWriteLocked(ns); - get_cmap_inlock(ns)[ns].reset(); - } - - void NamespaceDetailsTransient::eraseDB(const string& db) { - SimpleMutex::scoped_lock lk(_qcMutex); - Lock::assertWriteLocked(db); - - DMap::iterator i = _nsdMap.find( db ); - if ( i != _nsdMap.end() ) { - delete i->second; - _nsdMap.erase( i ); - } - } - - void NamespaceDetailsTransient::eraseCollection(const string& ns) { - SimpleMutex::scoped_lock lk(_qcMutex); - Lock::assertWriteLocked(ns); - get_cmap_inlock(ns).erase(ns); - } - - - void NamespaceDetailsTransient::computeIndexKeys() { - _indexedPaths.clear(); - - NamespaceDetails *d = nsdetails(_ns); - if ( ! d ) - return; - - NamespaceDetails::IndexIterator i = d->ii( true ); - while( i.more() ) { - BSONObj key = i.next().keyPattern(); - BSONObjIterator j( key ); - while ( j.more() ) { - BSONElement e = j.next(); - _indexedPaths.addPath( e.fieldName() ); - } - } - - _keysComputed = true; - } - - void NamespaceDetails::updateTTLIndex( int idxNo , const BSONElement& newExpireSecs ) { - // Need to get the actual DiskLoc of the index to update. This is embedded in the 'info' - // object inside the IndexDetails. - IndexDetails idetails = idx( idxNo ); - BSONElement oldExpireSecs = idetails.info.obj().getField("expireAfterSeconds"); - - // Important that we set the new value in-place. We are writing directly to the - // object here so must be careful not to overwrite with a longer numeric type. - massert( 16630, "new 'expireAfterSeconds' must be a number", newExpireSecs.isNumber() ); - BSONElementManipulator manip( oldExpireSecs ); - switch( oldExpireSecs.type() ) { - case EOO: - massert( 16631, "index does not have an 'expireAfterSeconds' field", false ); - break; - case NumberInt: - case NumberDouble: - manip.SetNumber( newExpireSecs.numberDouble() ); - break; - case NumberLong: - manip.SetLong( newExpireSecs.numberLong() ); - break; - default: - massert( 16632, "current 'expireAfterSeconds' is not a number", false ); - } - } - - void NamespaceDetails::setSystemFlag( int flag ) { - getDur().writingInt(_systemFlags) |= flag; - } - - void NamespaceDetails::clearSystemFlag( int flag ) { - getDur().writingInt(_systemFlags) &= ~flag; - } - - /** - * keeping things in sync this way is a bit of a hack - * and the fact that we have to pass in ns again - * should be changed, just not sure to what - */ - void NamespaceDetails::syncUserFlags( const string& ns ) { - Lock::assertWriteLocked( ns ); - - string system_namespaces = NamespaceString( ns ).db + ".system.namespaces"; - - BSONObj oldEntry; - verify( Helpers::findOne( system_namespaces , BSON( "name" << ns ) , oldEntry ) ); - BSONObj newEntry = applyUpdateOperators( oldEntry , BSON( "$set" << BSON( "options.flags" << userFlags() ) ) ); - - verify( 1 == deleteObjects( system_namespaces.c_str() , oldEntry , true , false , true ) ); - theDataFileMgr.insert( system_namespaces.c_str(), - newEntry.objdata(), - newEntry.objsize(), - false, - true ); - } - - bool NamespaceDetails::setUserFlag( int flags ) { - if ( ( _userFlags & flags ) == flags ) - return false; - - getDur().writingInt(_userFlags) |= flags; - return true; - } - - bool NamespaceDetails::clearUserFlag( int flags ) { - if ( ( _userFlags & flags ) == 0 ) - return false; - - getDur().writingInt(_userFlags) &= ~flags; - return true; - } - - bool NamespaceDetails::replaceUserFlags( int flags ) { - if ( flags == _userFlags ) - return false; - - getDur().writingInt(_userFlags) = flags; - return true; - } - - - - int NamespaceDetails::getRecordAllocationSize( int minRecordSize ) { - - if ( isCapped() ) - return minRecordSize; - - if ( _paddingFactor == 0 ) { - warning() << "implicit updgrade of paddingFactor of very old collection" << endl; - setPaddingFactor(1.0); - } - verify( _paddingFactor >= 1 ); - - - if ( isUserFlagSet( Flag_UsePowerOf2Sizes ) ) { - int allocationSize = bucketSizes[ bucket( minRecordSize ) ]; - if ( allocationSize == bucketSizes[MaxBucket] ) { - // if we get here, it means we're allocating more than 4mb, so round - // to the nearest megabyte - allocationSize = 1 + ( minRecordSize | ( ( 1 << 20 ) - 1 ) ); - } - return allocationSize; - } - - // adjust for padding factor - return static_cast(minRecordSize * _paddingFactor); - } - - /* ------------------------------------------------------------------------- */ - - /* add a new namespace to the system catalog (.system.namespaces). - options: { capped : ..., size : ... } - */ - void addNewNamespaceToCatalog(const char *ns, const BSONObj *options = 0) { - LOG(1) << "New namespace: " << ns << endl; - if ( strstr(ns, "system.namespaces") ) { - // system.namespaces holds all the others, so it is not explicitly listed in the catalog. - // TODO: fix above should not be strstr! - return; - } - - BSONObjBuilder b; - b.append("name", ns); - if ( options ) - b.append("options", *options); - BSONObj j = b.done(); - char database[256]; - nsToDatabase(ns, database); - string s = string(database) + ".system.namespaces"; - theDataFileMgr.insert(s.c_str(), j.objdata(), j.objsize(), false, true); - } - - void renameNamespace( const char *from, const char *to, bool stayTemp) { - NamespaceIndex *ni = nsindex( from ); - verify( ni ); - verify( ni->details( from ) ); - verify( ! ni->details( to ) ); - - // Our namespace and index details will move to a different - // memory location. The only references to namespace and - // index details across commands are in cursors and nsd - // transient (including query cache) so clear these. - ClientCursor::invalidate( from ); - NamespaceDetailsTransient::eraseCollection( from ); - - NamespaceDetails *details = ni->details( from ); - ni->add_ns( to, *details ); - NamespaceDetails *todetails = ni->details( to ); - try { - todetails->copyingFrom(to, details); // fixes extraOffset - } - catch( DBException& ) { - // could end up here if .ns is full - if so try to clean up / roll back a little - ni->kill_ns(to); - throw; - } - ni->kill_ns( from ); - details = todetails; - - BSONObj oldSpec; - char database[MaxDatabaseNameLen]; - nsToDatabase(from, database); - string s = database; - s += ".system.namespaces"; - verify( Helpers::findOne( s.c_str(), BSON( "name" << from ), oldSpec ) ); - - BSONObjBuilder newSpecB; - BSONObjIterator i( oldSpec.getObjectField( "options" ) ); - while( i.more() ) { - BSONElement e = i.next(); - if ( strcmp( e.fieldName(), "create" ) != 0 ) { - if (stayTemp || (strcmp(e.fieldName(), "temp") != 0)) - newSpecB.append( e ); - } - else { - newSpecB << "create" << to; - } - } - BSONObj newSpec = newSpecB.done(); - addNewNamespaceToCatalog( to, newSpec.isEmpty() ? 0 : &newSpec ); - - deleteObjects( s.c_str(), BSON( "name" << from ), false, false, true ); - // oldSpec variable no longer valid memory - - BSONObj oldIndexSpec; - s = database; - s += ".system.indexes"; - while( Helpers::findOne( s.c_str(), BSON( "ns" << from ), oldIndexSpec ) ) { - BSONObjBuilder newIndexSpecB; - BSONObjIterator i( oldIndexSpec ); - while( i.more() ) { - BSONElement e = i.next(); - if ( strcmp( e.fieldName(), "ns" ) != 0 ) - newIndexSpecB.append( e ); - else - newIndexSpecB << "ns" << to; - } - BSONObj newIndexSpec = newIndexSpecB.done(); - DiskLoc newIndexSpecLoc = theDataFileMgr.insert( s.c_str(), - newIndexSpec.objdata(), - newIndexSpec.objsize(), - false, - true, - false ); - int indexI = details->findIndexByName( oldIndexSpec.getStringField( "name" ) ); - IndexDetails &indexDetails = details->idx(indexI); - string oldIndexNs = indexDetails.indexNamespace(); - indexDetails.info = newIndexSpecLoc; - string newIndexNs = indexDetails.indexNamespace(); - - renameNamespace( oldIndexNs.c_str(), newIndexNs.c_str(), false ); - deleteObjects( s.c_str(), oldIndexSpec.getOwned(), true, false, true ); - } - } - - bool legalClientSystemNS( const string& ns , bool write ) { - if( ns == "local.system.replset" ) return true; - - if ( ns.find( ".system.users" ) != string::npos ) - return true; - - if ( ns.find( ".system.js" ) != string::npos ) { - if ( write ) - Scope::storedFuncMod(); - return true; - } - - return false; - } - -} // namespace mongo diff --git a/src/mongo/db/namespace_details.h b/src/mongo/db/namespace_details.h deleted file mode 100644 index 0e7e324da67..00000000000 --- a/src/mongo/db/namespace_details.h +++ /dev/null @@ -1,692 +0,0 @@ -// namespace_details.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/pch.h" -#include "mongo/db/d_concurrency.h" -#include "mongo/db/diskloc.h" -#include "mongo/db/index.h" -#include "mongo/db/index_set.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/mongommf.h" -#include "mongo/db/namespace.h" -#include "mongo/db/namespacestring.h" -#include "mongo/db/queryoptimizercursor.h" -#include "mongo/db/querypattern.h" -#include "mongo/platform/unordered_map.h" -#include "mongo/util/hashtab.h" - -namespace mongo { - class Database; - - /** @return true if a client can modify this namespace even though it is under ".system." - For example .system.users is ok for regular clients to update. - @param write used when .system.js - */ - bool legalClientSystemNS( const string& ns , bool write ); - - /* deleted lists -- linked lists of deleted records -- are placed in 'buckets' of various sizes - so you can look for a deleterecord about the right size. - */ - const int Buckets = 19; - const int MaxBucket = 18; - - extern int bucketSizes[]; - -#pragma pack(1) - /* NamespaceDetails : this is the "header" for a collection that has all its details. - It's in the .ns file and this is a memory mapped region (thus the pack pragma above). - */ - class NamespaceDetails { - public: - enum { NIndexesMax = 64, NIndexesExtra = 30, NIndexesBase = 10 }; - - /*-------- data fields, as present on disk : */ - DiskLoc firstExtent; - DiskLoc lastExtent; - /* NOTE: capped collections v1 override the meaning of deletedList. - deletedList[0] points to a list of free records (DeletedRecord's) for all extents in - the capped namespace. - deletedList[1] points to the last record in the prev extent. When the "current extent" - changes, this value is updated. !deletedList[1].isValid() when this value is not - yet computed. - */ - DiskLoc deletedList[Buckets]; - // ofs 168 (8 byte aligned) - struct Stats { - // datasize and nrecords MUST Be adjacent code assumes! - long long datasize; // this includes padding, but not record headers - long long nrecords; - } stats; - int lastExtentSize; - int nIndexes; - private: - // ofs 192 - IndexDetails _indexes[NIndexesBase]; - - // ofs 352 (16 byte aligned) - int _isCapped; // there is wasted space here if I'm right (ERH) - int _maxDocsInCapped; // max # of objects for a capped table, -1 for inf. - - double _paddingFactor; // 1.0 = no padding. - // ofs 386 (16) - int _systemFlags; // things that the system sets/cares about - public: - DiskLoc capExtent; // the "current" extent we're writing too for a capped collection - DiskLoc capFirstNewRecord; - unsigned short dataFileVersion; // NamespaceDetails version. So we can do backward compatibility in the future. See filever.h - unsigned short indexFileVersion; - unsigned long long multiKeyIndexBits; - private: - // ofs 400 (16) - unsigned long long reservedA; - long long extraOffset; // where the $extra info is located (bytes relative to this) - public: - int indexBuildsInProgress; // Number of indexes currently being built - private: - int _userFlags; - char reserved[72]; - /*-------- end data 496 bytes */ - public: - explicit NamespaceDetails( const DiskLoc &loc, bool _capped ); - - class Extra { - long long _next; - public: - IndexDetails details[NIndexesExtra]; - private: - unsigned reserved2; - unsigned reserved3; - Extra(const Extra&) { verify(false); } - Extra& operator=(const Extra& r) { verify(false); return *this; } - public: - Extra() { } - long ofsFrom(NamespaceDetails *d) { - return ((char *) this) - ((char *) d); - } - void init() { memset(this, 0, sizeof(Extra)); } - Extra* next(NamespaceDetails *d) { - if( _next == 0 ) return 0; - return (Extra*) (((char *) d) + _next); - } - void setNext(long ofs) { *getDur().writing(&_next) = ofs; } - void copy(NamespaceDetails *d, const Extra& e) { - memcpy(this, &e, sizeof(Extra)); - _next = 0; - } - }; - Extra* extra() { - if( extraOffset == 0 ) return 0; - return (Extra *) (((char *) this) + extraOffset); - } - /* add extra space for indexes when more than 10 */ - Extra* allocExtra(const char *ns, int nindexessofar); - void copyingFrom(const char *thisns, NamespaceDetails *src); // must be called when renaming a NS to fix up extra - - /* dump info on this namespace. for debugging. */ - void dump(const Namespace& k); - - /* dump info on all extents for this namespace. for debugging. */ - void dumpExtents(); - - private: - Extent *theCapExtent() const { return capExtent.ext(); } - void advanceCapExtent( const char *ns ); - DiskLoc __capAlloc(int len); - DiskLoc cappedAlloc(const char *ns, int len); - DiskLoc &cappedFirstDeletedInCurExtent(); - bool nextIsInCapExtent( const DiskLoc &dl ) const; - - public: - - bool isCapped() const { return _isCapped; } - long long maxCappedDocs() const; - void setMaxCappedDocs( long long max ); - /** - * @param max in and out, will be adjusted - * @return if the value is valid at all - */ - static bool validMaxCappedDocs( long long* max ); - - DiskLoc& cappedListOfAllDeletedRecords() { return deletedList[0]; } - DiskLoc& cappedLastDelRecLastExtent() { return deletedList[1]; } - void cappedDumpDelInfo(); - bool capLooped() const { return _isCapped && capFirstNewRecord.isValid(); } - bool inCapExtent( const DiskLoc &dl ) const; - void cappedCheckMigrate(); - /** - * Truncate documents newer than the document at 'end' from the capped - * collection. The collection cannot be completely emptied using this - * function. An assertion will be thrown if that is attempted. - * @param inclusive - Truncate 'end' as well iff true - */ - void cappedTruncateAfter(const char *ns, DiskLoc end, bool inclusive); - /** Remove all documents from the capped collection */ - void emptyCappedCollection(const char *ns); - - /* when a background index build is in progress, we don't count the index in nIndexes until - complete, yet need to still use it in _indexRecord() - thus we use this function for that. - */ - int getTotalIndexCount() const { return nIndexes + indexBuildsInProgress; } - - /* NOTE: be careful with flags. are we manipulating them in read locks? if so, - this isn't thread safe. TODO - */ - enum SystemFlags { - Flag_HaveIdIndex = 1 << 0 // set when we have _id index (ONLY if ensureIdIndex was called -- 0 if that has never been called) - }; - - enum UserFlags { - Flag_UsePowerOf2Sizes = 1 << 0 - }; - - IndexDetails& idx(int idxNo, bool missingExpected = false ); - - class IndexIterator { - public: - int pos() { return i; } // note this is the next one to come - bool more() { return i < n; } - IndexDetails& next() { return d->idx(i++); } - private: - friend class NamespaceDetails; - int i, n; - NamespaceDetails *d; - IndexIterator(NamespaceDetails *_d, bool includeBackgroundInProgress); - }; - - IndexIterator ii( bool includeBackgroundInProgress = false ) { - return IndexIterator(this, includeBackgroundInProgress); - } - - /* hackish - find our index # in the indexes array */ - int idxNo(const IndexDetails& idx); - - /* multikey indexes are indexes where there are more than one key in the index - for a single document. see multikey in docs. - for these, we have to do some dedup work on queries. - */ - bool isMultikey(int i) const { return (multiKeyIndexBits & (((unsigned long long) 1) << i)) != 0; } - void setIndexIsMultikey(const char *thisns, int i, bool multikey = true); - - /** - * This fetches the IndexDetails for the next empty index slot. The caller must populate - * returned object. This handles allocating extra index space, if necessary. - */ - IndexDetails& getNextIndexDetails(const char* thisns); - - /** - * Add a new index. This does not add it to system.indexes etc. - just to NamespaceDetails. - * This resets the transient namespace details. - */ - void addIndex(const char* thisns); - - void aboutToDeleteAnIndex() { - clearSystemFlag( Flag_HaveIdIndex ); - } - - /* returns index of the first index in which the field is present. -1 if not present. */ - int fieldIsIndexed(const char *fieldName); - - /** - * @return the actual size to create - * will be >= oldRecordSize - * based on padding and any other flags - */ - int getRecordAllocationSize( int minRecordSize ); - - double paddingFactor() const { return _paddingFactor; } - - void setPaddingFactor( double paddingFactor ) { - *getDur().writing(&_paddingFactor) = paddingFactor; - } - - /* called to indicate that an update fit in place. - fits also called on an insert -- idea there is that if you had some mix and then went to - pure inserts it would adapt and PF would trend to 1.0. note update calls insert on a move - so there is a double count there that must be adjusted for below. - - todo: greater sophistication could be helpful and added later. for example the absolute - size of documents might be considered -- in some cases smaller ones are more likely - to grow than larger ones in the same collection? (not always) - */ - void paddingFits() { - MONGO_SOMETIMES(sometimes, 4) { // do this on a sampled basis to journal less - double x = _paddingFactor - 0.001; - if ( x >= 1.0 ) { - setPaddingFactor( x ); - } - } - } - void paddingTooSmall() { - MONGO_SOMETIMES(sometimes, 4) { // do this on a sampled basis to journal less - /* the more indexes we have, the higher the cost of a move. so we take that into - account herein. note on a move that insert() calls paddingFits(), thus - here for example with no inserts and nIndexes = 1 we have - .001*4-.001 or a 3:1 ratio to non moves -> 75% nonmoves. insert heavy - can pushes this down considerably. further tweaking will be a good idea but - this should be an adequate starting point. - */ - double N = min(nIndexes,7) + 3; - double x = _paddingFactor + (0.001 * N); - if ( x <= 2.0 ) { - setPaddingFactor( x ); - } - } - } - - // @return offset in indexes[] - int findIndexByName(const char *name, bool includeBackgroundInProgress = false); - - // @return offset in indexes[] - int findIndexByKeyPattern(const BSONObj& keyPattern, - bool includeBackgroundInProgress = false); - - void findIndexByType( const string& name , vector& matches ) { - IndexIterator i = ii(); - while ( i.more() ) { - if ( i.next().getSpec().getTypeName() == name ) - matches.push_back( i.pos() - 1 ); - } - } - - /* Returns the index entry for the first index whose prefix contains - * 'keyPattern'. If 'requireSingleKey' is true, skip indices that contain - * array attributes. Otherwise, returns NULL. - */ - const IndexDetails* findIndexByPrefix( const BSONObj &keyPattern , - bool requireSingleKey ); - - /* Updates the expireAfterSeconds field of the given index to the value in newExpireSecs. - * The specified index must already contain an expireAfterSeconds field, and the value in - * that field and newExpireSecs must both be numeric. - */ - void updateTTLIndex( int idxNo , const BSONElement& newExpireSecs ); - - - const int systemFlags() const { return _systemFlags; } - bool isSystemFlagSet( int flag ) const { return _systemFlags & flag; } - void setSystemFlag( int flag ); - void clearSystemFlag( int flag ); - - const int userFlags() const { return _userFlags; } - bool isUserFlagSet( int flag ) const { return _userFlags & flag; } - - - /** - * these methods only modify NamespaceDetails and do not - * sync changes back to system.namespaces - * a typical call might - if ( nsd->setUserFlag( 4 ) ) { - nsd->syncUserFlags(); - } - * these methods all return true iff only something was modified - */ - - bool setUserFlag( int flag ); - bool clearUserFlag( int flag ); - bool replaceUserFlags( int flags ); - - void syncUserFlags( const string& ns ); - - /* @return -1 = not found - generally id is first index, so not that expensive an operation (assuming present). - */ - int findIdIndex() { - IndexIterator i = ii(); - while( i.more() ) { - if( i.next().isIdIndex() ) - return i.pos()-1; - } - return -1; - } - - bool haveIdIndex() { - return isSystemFlagSet( NamespaceDetails::Flag_HaveIdIndex ) || findIdIndex() >= 0; - } - - /* return which "deleted bucket" for this size object */ - static int bucket(int n) { - for ( int i = 0; i < Buckets; i++ ) - if ( bucketSizes[i] > n ) - return i; - return Buckets-1; - } - - /* @return the size for an allocated record quantized to 1/16th of the BucketSize - @param allocSize requested size to allocate - */ - static int quantizeAllocationSpace(int allocSize); - - /* predetermine location of the next alloc without actually doing it. - if cannot predetermine returns null (so still call alloc() then) - */ - DiskLoc allocWillBeAt(const char *ns, int lenToAlloc); - - /** allocate space for a new record from deleted lists. - @param lenToAlloc is WITH header - @return null diskloc if no room - allocate a new extent then - */ - DiskLoc alloc(const char* ns, int lenToAlloc); - - /* add a given record to the deleted chains for this NS */ - void addDeletedRec(DeletedRecord *d, DiskLoc dloc); - void dumpDeleted(set *extents = 0); - // Start from firstExtent by default. - DiskLoc firstRecord( const DiskLoc &startExtent = DiskLoc() ) const; - // Start from lastExtent by default. - DiskLoc lastRecord( const DiskLoc &startExtent = DiskLoc() ) const; - long long storageSize( int * numExtents = 0 , BSONArrayBuilder * extentInfo = 0 ) const; - - int averageObjectSize() { - if ( stats.nrecords == 0 ) - return 5; - return (int) (stats.datasize / stats.nrecords); - } - - NamespaceDetails *writingWithoutExtra() { - return ( NamespaceDetails* ) getDur().writingPtr( this, sizeof( NamespaceDetails ) ); - } - /** Make all linked Extra objects writeable as well */ - NamespaceDetails *writingWithExtra(); - - private: - DiskLoc _alloc(const char *ns, int len); - void maybeComplain( const char *ns, int len ) const; - DiskLoc __stdAlloc(int len, bool willBeAt); - void compact(); // combine adjacent deleted records - friend class NamespaceIndex; - struct ExtraOld { - // note we could use this field for more chaining later, so don't waste it: - unsigned long long reserved1; - IndexDetails details[NIndexesExtra]; - unsigned reserved2; - unsigned reserved3; - }; - /** Update cappedLastDelRecLastExtent() after capExtent changed in cappedTruncateAfter() */ - void cappedTruncateLastDelUpdate(); - BOOST_STATIC_ASSERT( NIndexesMax <= NIndexesBase + NIndexesExtra*2 ); - BOOST_STATIC_ASSERT( NIndexesMax <= 64 ); // multiKey bits - BOOST_STATIC_ASSERT( sizeof(NamespaceDetails::ExtraOld) == 496 ); - BOOST_STATIC_ASSERT( sizeof(NamespaceDetails::Extra) == 496 ); - }; // NamespaceDetails -#pragma pack() - - class ParsedQuery; - class QueryPlanSummary; - - /* NamespaceDetailsTransient - - these are things we know / compute about a namespace that are transient -- things - we don't actually store in the .ns file. so mainly caching of frequently used - information. - - CAUTION: Are you maintaining this properly on a collection drop()? A dropdatabase()? Be careful. - The current field "allIndexKeys" may have too many keys in it on such an occurrence; - as currently used that does not cause anything terrible to happen. - - todo: cleanup code, need abstractions and separation - */ - // todo: multiple db's with the same name (repairDatabase) is not handled herein. that may be - // the way to go, if not used by repair, but need some sort of enforcement / asserts. - class NamespaceDetailsTransient : boost::noncopyable { - BOOST_STATIC_ASSERT( sizeof(NamespaceDetails) == 496 ); - - //Database *database; - const string _ns; - void reset(); - - // < db -> < fullns -> NDT > > - typedef unordered_map< string, shared_ptr > CMap; - typedef unordered_map< string, CMap*, NamespaceDBHash, NamespaceDBEquals > DMap; - static DMap _nsdMap; - - NamespaceDetailsTransient(Database*,const string& ns); - public: - ~NamespaceDetailsTransient(); - void addedIndex() { reset(); } - void deletedIndex() { reset(); } - - /** - * reset stats for a given collection - */ - static void resetCollection(const string& ns ); - - /** - * remove entry for a collection - */ - static void eraseCollection(const string& ns); - - /** - * remove all entries for db - */ - static void eraseDB(const string& db); - - - /** - * @return a cursor interface to the query optimizer. The implementation may utilize a - * single query plan or interleave results from multiple query plans before settling on a - * single query plan. Note that the schema of currKey() documents, indexKeyPattern(), the - * matcher(), and the isMultiKey() nature of the cursor may change over the course of - * iteration. - * - * @param query - Query used to select indexes and populate matchers; not copied if unowned - * (see bsonobj.h). - * - * @param order - Required ordering spec for documents produced by this cursor, empty object - * default indicates no order requirement. If no index exists that satisfies the required - * sort order, an empty shared_ptr is returned unless parsedQuery is also provided. This is - * not copied if unowned. - * - * @param planPolicy - A policy for selecting query plans - see queryoptimizercursor.h - * - * @param parsedQuery - Additional query parameters, as from a client query request. - * - * @param requireOrder - If false, the resulting cursor may return results in an order - * inconsistent with the @param order spec. See queryoptimizercursor.h for information on - * handling these results properly. - * - * @param singlePlanSummary - Query plan summary information that may be provided when a - * cursor running a single plan is returned. - * - * The returned cursor may @throw inside of advance() or recoverFromYield() in certain error - * cases, for example if a capped overrun occurred during a yield. This indicates that the - * cursor was unable to perform a complete scan. - * - * This is a work in progress. Partial list of features not yet implemented through this - * interface: - * - * - covered indexes - * - in memory sorting - */ - static shared_ptr getCursor( const StringData& ns, - const BSONObj& query, - const BSONObj& order = BSONObj(), - const QueryPlanSelectionPolicy& planPolicy = - QueryPlanSelectionPolicy::any(), - const shared_ptr& parsedQuery = - shared_ptr(), - bool requireOrder = true, - QueryPlanSummary* singlePlanSummary = NULL ); - - /** - * @return a single cursor that may work well for the given query. A $or style query will - * produce a single cursor, not a MultiCursor. - * It is possible no cursor is returned if the sort is not supported by an index. Clients are responsible - * for checking this if they are not sure an index for a sort exists, and defaulting to a non-sort if - * no suitable indices exist. - */ - static shared_ptr bestGuessCursor( const char *ns, const BSONObj &query, const BSONObj &sort ); - - /* indexKeys() cache ---------------------------------------------------- */ - /* assumed to be in write lock for this */ - private: - bool _keysComputed; - IndexPathSet _indexedPaths; - void computeIndexKeys(); - public: - /* get set of index keys for this namespace. handy to quickly check if a given - field is indexed (Note it might be a secondary component of a compound index.) - */ - const IndexPathSet& indexKeys() { - DEV Lock::assertWriteLocked(_ns); - if ( !_keysComputed ) - computeIndexKeys(); - return _indexedPaths; - } - - /* IndexSpec caching */ - private: - map _indexSpecs; - static SimpleMutex _isMutex; - public: - const IndexSpec& getIndexSpec( const IndexDetails * details ) { - IndexSpec& spec = _indexSpecs[details]; - if ( ! spec._finishedInit ) { - SimpleMutex::scoped_lock lk(_isMutex); - if ( ! spec._finishedInit ) { - spec.reset( details ); - verify( spec._finishedInit ); - } - } - return spec; - } - - /* query cache (for query optimizer) ------------------------------------- */ - private: - int _qcWriteCount; - map _qcCache; - static NamespaceDetailsTransient& make_inlock(const string& ns); - static CMap& get_cmap_inlock(const string& ns); - public: - static SimpleMutex _qcMutex; - - /* you must be in the qcMutex when calling this. - A NamespaceDetailsTransient object will not go out of scope on you if you are - d.dbMutex.atLeastReadLocked(), so you do't have to stay locked. - Creates a NamespaceDetailsTransient before returning if one DNE. - todo: avoid creating too many on erroneous ns queries. - */ - static NamespaceDetailsTransient& get_inlock(const string& ns); - - static NamespaceDetailsTransient& get(const char *ns) { - // todo : _qcMutex will create bottlenecks in our parallelism - SimpleMutex::scoped_lock lk(_qcMutex); - return get_inlock(ns); - } - - void clearQueryCache() { - _qcCache.clear(); - _qcWriteCount = 0; - } - /* you must notify the cache if you are doing writes, as query plan utility will change */ - void notifyOfWriteOp() { - if ( _qcCache.empty() ) - return; - if ( ++_qcWriteCount >= 100 ) - clearQueryCache(); - } - CachedQueryPlan cachedQueryPlanForPattern( const QueryPattern &pattern ) { - return _qcCache[ pattern ]; - } - void registerCachedQueryPlanForPattern( const QueryPattern &pattern, - const CachedQueryPlan &cachedQueryPlan ) { - _qcCache[ pattern ] = cachedQueryPlan; - } - - }; /* NamespaceDetailsTransient */ - - inline NamespaceDetailsTransient& NamespaceDetailsTransient::get_inlock(const string& ns) { - CMap& m = get_cmap_inlock(ns); - CMap::iterator i = m.find( ns ); - if ( i != m.end() && - i->second.get() ) { // could be null ptr from clearForPrefix - return *i->second; - } - return make_inlock(ns); - } - - /* NamespaceIndex is the ".ns" file you see in the data directory. It is the "system catalog" - if you will: at least the core parts. (Additional info in system.* collections.) - */ - class NamespaceIndex { - public: - NamespaceIndex(const string &dir, const string &database) : - ht( 0 ), dir_( dir ), database_( database ) {} - - /* returns true if new db will be created if we init lazily */ - bool exists() const; - - void init() { - if( !ht ) - _init(); - } - - void add_ns(const char *ns, DiskLoc& loc, bool capped); - void add_ns( const char *ns, const NamespaceDetails &details ); - - NamespaceDetails* details(const StringData& ns) { - if ( !ht ) - return 0; - Namespace n(ns); - NamespaceDetails *d = ht->get(n); - if ( d && d->isCapped() ) - d->cappedCheckMigrate(); - return d; - } - - void kill_ns(const char *ns); - - bool find(const char *ns, DiskLoc& loc) { - NamespaceDetails *l = details(ns); - if ( l ) { - loc = l->firstExtent; - return true; - } - return false; - } - - bool allocated() const { return ht != 0; } - - void getNamespaces( list& tofill , bool onlyCollections = true ) const; - - NamespaceDetails::Extra* newExtra(const char *ns, int n, NamespaceDetails *d); - - boost::filesystem::path path() const; - - unsigned long long fileLength() const { return f.length(); } - - private: - void _init(); - void maybeMkdir() const; - - MongoMMF f; - HashTable *ht; - string dir_; - string database_; - }; - - extern string dbpath; // --dbpath parm - extern bool directoryperdb; - - // Rename a namespace within current 'client' db. - // (Arguments should include db name) - void renameNamespace( const char *from, const char *to, bool stayTemp); - - -} // namespace mongo diff --git a/src/mongo/db/namespace_string-inl.h b/src/mongo/db/namespace_string-inl.h new file mode 100644 index 00000000000..597d9a93159 --- /dev/null +++ b/src/mongo/db/namespace_string-inl.h @@ -0,0 +1,200 @@ +// namespacestring-inl.h + + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +namespace mongo { + + inline StringData NamespaceString::db() const { + return _dotIndex == std::string::npos ? + StringData() : + StringData( _ns.c_str(), _dotIndex ); + } + + inline StringData NamespaceString::coll() const { + return _dotIndex == std::string::npos ? + StringData() : + StringData( _ns.c_str() + _dotIndex + 1, _ns.size() - 1 - _dotIndex ); + } + + inline bool NamespaceString::normal(const StringData& ns) { + if ( ns.find( '$' ) == std::string::npos ) + return true; + return oplog(ns); + } + + inline bool NamespaceString::oplog(const StringData& ns) { + return ns == "local.oplog.rs" || ns == "local.oplog.$main"; + } + + inline bool NamespaceString::special(const StringData& ns) { + return !normal(ns) || ns.substr(ns.find('.')).startsWith(".system."); + } + + inline bool NamespaceString::validDBName( const StringData& db ) { + if ( db.size() == 0 || db.size() > 64 ) + return false; + + for (StringData::const_iterator iter = db.begin(), end = db.end(); iter != end; ++iter) { + switch (*iter) { + case '\0': + case '/': + case '\\': + case '.': + case ' ': + case '"': + return false; +#ifdef _WIN32 + // We prohibit all FAT32-disallowed characters on Windows + case '*': + case '<': + case '>': + case ':': + case '|': + case '?': + return false; +#endif + default: + continue; + } + } + return true; + } + + inline bool NamespaceString::validCollectionComponent(const StringData& ns){ + size_t idx = ns.find( '.' ); + if ( idx == std::string::npos ) + return false; + + return validCollectionName(ns.substr(idx + 1)) || oplog(ns); + } + + inline bool NamespaceString::validCollectionName(const StringData& coll){ + if (coll.empty()) + return false; + + for (StringData::const_iterator iter = coll.begin(), end = coll.end(); + iter != end; ++iter) { + switch (*iter) { + case '\0': + case '$': + return false; + default: + continue; + } + } + + return true; + } + + inline NamespaceString::NamespaceString() : _ns(), _dotIndex(0) {} + inline NamespaceString::NamespaceString( const StringData& nsIn ) { + _ns = nsIn.toString(); // copy to our buffer + _dotIndex = _ns.find( '.' ); + } + + inline NamespaceString::NamespaceString( const StringData& dbName, + const StringData& collectionName ) + : _ns(dbName.size() + collectionName.size() + 1, '\0') { + + uassert(17235, + "'.' is an invalid character in a database name", + dbName.find('.') == std::string::npos); + uassert(17246, + "Collection names cannot start with '.'", + collectionName.empty() || collectionName[0] != '.'); + std::string::iterator it = std::copy(dbName.begin(), dbName.end(), _ns.begin()); + *it = '.'; + ++it; + it = std::copy(collectionName.begin(), collectionName.end(), it); + _dotIndex = dbName.size(); + dassert(it == _ns.end()); + dassert(_ns[_dotIndex] == '.'); + uassert(17295, "namespaces cannot have embedded null characters", + _ns.find('\0') == std::string::npos); + } + + inline int nsDBHash( const std::string& ns ) { + int hash = 7; + for ( size_t i = 0; i < ns.size(); i++ ) { + if ( ns[i] == '.' ) + break; + hash += 11 * ( ns[i] ); + hash *= 3; + } + return hash; + } + + inline bool nsDBEquals( const std::string& a, const std::string& b ) { + for ( size_t i = 0; i < a.size(); i++ ) { + + if ( a[i] == '.' ) { + // b has to either be done or a '.' + + if ( b.size() == i ) + return true; + + if ( b[i] == '.' ) + return true; + + return false; + } + + // a is another character + if ( b.size() == i ) + return false; + + if ( b[i] != a[i] ) + return false; + } + + // a is done + // make sure b is done + if ( b.size() == a.size() || + b[a.size()] == '.' ) + return true; + + return false; + } + + /* future : this doesn't need to be an inline. */ + inline std::string NamespaceString::getSisterNS( const StringData& local ) const { + verify( local.size() && local[0] != '.' ); + return db().toString() + "." + local.toString(); + } + + inline std::string NamespaceString::getSystemIndexesCollection() const { + return db().toString() + ".system.indexes"; + } + + inline std::string NamespaceString::getCommandNS() const { + return db().toString() + ".$cmd"; + } + +} diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h new file mode 100644 index 00000000000..d1679eb01a1 --- /dev/null +++ b/src/mongo/db/namespace_string.h @@ -0,0 +1,231 @@ +// @file namespacestring.h + +/** +* Copyright (C) 2008 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include + +#include "mongo/base/string_data.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + /* in the mongo source code, "client" means "database". */ + + const size_t MaxDatabaseNameLen = 128; // max str len for the db name, including null char + + /* e.g. + NamespaceString ns("acme.orders"); + cout << ns.coll; // "orders" + */ + class NamespaceString { + public: + /** + * Constructs an empty NamespaceString. + */ + NamespaceString(); + + /** + * Constructs a NamespaceString from the fully qualified namespace named in "ns". + */ + NamespaceString( const StringData& ns ); + + /** + * Constructs a NamespaceString for the given database and collection names. + * "dbName" must not contain a ".", and "collectionName" must not start with one. + */ + NamespaceString( const StringData& dbName, const StringData& collectionName ); + + StringData db() const; + StringData coll() const; + + const std::string& ns() const { return _ns; } + + operator std::string() const { return _ns; } + std::string toString() const { return _ns; } + + size_t size() const { return _ns.size(); } + + bool isSystem() const { return coll().startsWith( "system." ); } + bool isSystemDotIndexes() const { return coll() == "system.indexes"; } + bool isConfigDB() const { return db() == "config"; } + bool isCommand() const { return coll() == "$cmd"; } + bool isOplog() const { return oplog( _ns ); } + bool isSpecialCommand() const { return coll().startsWith("$cmd.sys"); } + bool isSpecial() const { return special( _ns ); } + bool isNormal() const { return normal( _ns ); } + + /** + * @return true if the namespace is valid. Special namespaces for internal use are considered as valid. + */ + bool isValid() const { return validDBName( db() ) && !coll().empty(); } + + bool operator==( const std::string& nsIn ) const { return nsIn == _ns; } + bool operator==( const NamespaceString& nsIn ) const { return nsIn._ns == _ns; } + + bool operator!=( const std::string& nsIn ) const { return nsIn != _ns; } + bool operator!=( const NamespaceString& nsIn ) const { return nsIn._ns != _ns; } + + bool operator<( const NamespaceString& rhs ) const { return _ns < rhs._ns; } + + /** ( foo.bar ).getSisterNS( "blah" ) == foo.blah + */ + std::string getSisterNS( const StringData& local ) const; + + // @return db() + ".system.indexes" + std::string getSystemIndexesCollection() const; + + // @return db() + ".$cmd" + std::string getCommandNS() const; + + /** + * @return true if ns is 'normal'. A "$" is used for namespaces holding index data, + * which do not contain BSON objects in their records. ("oplog.$main" is the exception) + */ + static bool normal(const StringData& ns); + + /** + * @return true if the ns is an oplog one, otherwise false. + */ + static bool oplog(const StringData& ns); + + static bool special(const StringData& ns); + + /** + * samples: + * good + * foo + * bar + * foo-bar + * bad: + * foo bar + * foo.bar + * foo"bar + * + * @param db - a possible database name + * @return if db is an allowed database name + */ + static bool validDBName( const StringData& dbin ); + + /** + * Takes a fully qualified namespace (ie dbname.collectionName), and returns true if + * the collection name component of the namespace is valid. + * samples: + * good: + * foo.bar + * bad: + * foo. + * + * @param ns - a full namespace (a.b) + * @return if db.coll is an allowed collection name + */ + static bool validCollectionComponent(const StringData& ns); + + /** + * Takes a collection name and returns true if it is a valid collection name. + * samples: + * good: + * foo + * system.indexes + * bad: + * $foo + * @param coll - a collection name component of a namespace + * @return if the input is a valid collection name + */ + static bool validCollectionName(const StringData& coll); + + private: + + std::string _ns; + size_t _dotIndex; + }; + + + // "database.a.b.c" -> "database" + inline StringData nsToDatabaseSubstring( const StringData& ns ) { + size_t i = ns.find( '.' ); + if ( i == std::string::npos ) { + massert(10078, "nsToDatabase: db too long", ns.size() < MaxDatabaseNameLen ); + return ns; + } + massert(10088, "nsToDatabase: db too long", i < static_cast(MaxDatabaseNameLen)); + return ns.substr( 0, i ); + } + + // "database.a.b.c" -> "database" + inline void nsToDatabase(const StringData& ns, char *database) { + StringData db = nsToDatabaseSubstring( ns ); + db.copyTo( database, true ); + } + + // TODO: make this return a StringData + inline std::string nsToDatabase(const StringData& ns) { + return nsToDatabaseSubstring( ns ).toString(); + } + + // "database.a.b.c" -> "a.b.c" + inline StringData nsToCollectionSubstring( const StringData& ns ) { + size_t i = ns.find( '.' ); + massert(16886, "nsToCollectionSubstring: no .", i != std::string::npos ); + return ns.substr( i + 1 ); + } + + + /** + * NamespaceDBHash and NamespaceDBEquals allow you to do something like + * unordered_map + * and use the full namespace for the string + * but comparisons are done only on the db piece + */ + + /** + * this can change, do not store on disk + */ + int nsDBHash( const std::string& ns ); + + bool nsDBEquals( const std::string& a, const std::string& b ); + + struct NamespaceDBHash { + int operator()( const std::string& ns ) const { + return nsDBHash( ns ); + } + }; + + struct NamespaceDBEquals { + bool operator()( const std::string& a, const std::string& b ) const { + return nsDBEquals( a, b ); + } + }; + +} + + +#include "mongo/db/namespace_string-inl.h" diff --git a/src/mongo/db/namespace_string_test.cpp b/src/mongo/db/namespace_string_test.cpp new file mode 100644 index 00000000000..b3154de2588 --- /dev/null +++ b/src/mongo/db/namespace_string_test.cpp @@ -0,0 +1,187 @@ +// namespacestring_test.cpp + +/* Copyright 2012 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/unittest/unittest.h" + +#include "mongo/db/namespace_string.h" + +namespace mongo { + + TEST( NamespaceStringTest, Normal ) { + ASSERT( NamespaceString::normal( "a" ) ); + ASSERT( NamespaceString::normal( "a.b" ) ); + ASSERT( NamespaceString::normal( "a.b.c" ) ); + + ASSERT( !NamespaceString::normal( "a.b.$c" ) ); + ASSERT( !NamespaceString::normal( "a.b.$.c" ) ); + + ASSERT( NamespaceString::normal( "local.oplog.$main" ) ); + ASSERT( NamespaceString::normal( "local.oplog.rs" ) ); + } + + TEST( NamespaceStringTest, Oplog ) { + ASSERT( !NamespaceString::oplog( "a" ) ); + ASSERT( !NamespaceString::oplog( "a.b" ) ); + + ASSERT( NamespaceString::oplog( "local.oplog.rs" ) ); + ASSERT( !NamespaceString::oplog( "local.oplog.foo" ) ); + ASSERT( NamespaceString::oplog( "local.oplog.$main" ) ); + ASSERT( !NamespaceString::oplog( "local.oplog.$foo" ) ); + } + + TEST( NamespaceStringTest, Special ) { + ASSERT( NamespaceString::special( "a.$.b" ) ); + ASSERT( NamespaceString::special( "a.system.foo" ) ); + ASSERT( !NamespaceString::special( "a.foo" ) ); + ASSERT( !NamespaceString::special( "a.foo.system.bar" ) ); + ASSERT( !NamespaceString::special( "a.systemfoo" ) ); + } + + TEST( NamespaceStringTest, DatabaseValidNames ) { + ASSERT( NamespaceString::validDBName( "foo" ) ); + ASSERT( !NamespaceString::validDBName( "foo/bar" ) ); + ASSERT( !NamespaceString::validDBName( "foo bar" ) ); + ASSERT( !NamespaceString::validDBName( "foo.bar" ) ); + ASSERT( !NamespaceString::validDBName( "foo.bar" ) ); + ASSERT( !NamespaceString::validDBName( "foo\\bar" ) ); + ASSERT( !NamespaceString::validDBName( "foo\"bar" ) ); + ASSERT( !NamespaceString::validDBName( StringData( "a\0b", StringData::LiteralTag() ) ) ); +#ifdef _WIN32 + ASSERT( !NamespaceString::validDBName( "foo*bar" ) ); + ASSERT( !NamespaceString::validDBName( "foobar" ) ); + ASSERT( !NamespaceString::validDBName( "foo:bar" ) ); + ASSERT( !NamespaceString::validDBName( "foo|bar" ) ); + ASSERT( !NamespaceString::validDBName( "foo?bar" ) ); +#endif + + ASSERT( NamespaceString::normal( "asdads" ) ); + ASSERT( !NamespaceString::normal( "asda$ds" ) ); + ASSERT( NamespaceString::normal( "local.oplog.$main" ) ); + } + + TEST( NamespaceStringTest, CollectionComponentValidNames ) { + ASSERT( NamespaceString::validCollectionComponent( "a.b" ) ); + ASSERT( NamespaceString::validCollectionComponent( "a.b" ) ); + ASSERT( !NamespaceString::validCollectionComponent( "a." ) ); + ASSERT( NamespaceString::validCollectionComponent( "a.b." ) ); // TODO: should this change? + } + + TEST( NamespaceStringTest, CollectionValidNames ) { + ASSERT( NamespaceString::validCollectionName( "a" ) ); + ASSERT( NamespaceString::validCollectionName( "a.b" ) ); + ASSERT( NamespaceString::validCollectionName( "a." ) ); // TODO: should this change? + ASSERT( NamespaceString::validCollectionName( "a.b." ) ); // TODO: should this change? + ASSERT( !NamespaceString::validCollectionName( "$a" ) ); + ASSERT( !NamespaceString::validCollectionName( "a$b" ) ); + ASSERT( !NamespaceString::validCollectionName( "" ) ); + ASSERT( !NamespaceString::validCollectionName( + StringData( "a\0b", StringData::LiteralTag() ) ) ); + } + + TEST( NamespaceStringTest, DBHash ) { + ASSERT_EQUALS( nsDBHash( "foo" ), nsDBHash( "foo" ) ); + ASSERT_EQUALS( nsDBHash( "foo" ), nsDBHash( "foo.a" ) ); + ASSERT_EQUALS( nsDBHash( "foo" ), nsDBHash( "foo." ) ); + + ASSERT_EQUALS( nsDBHash( "" ), nsDBHash( "" ) ); + ASSERT_EQUALS( nsDBHash( "" ), nsDBHash( ".a" ) ); + ASSERT_EQUALS( nsDBHash( "" ), nsDBHash( "." ) ); + + ASSERT_NOT_EQUALS( nsDBHash( "foo" ), nsDBHash( "food" ) ); + ASSERT_NOT_EQUALS( nsDBHash( "foo." ), nsDBHash( "food" ) ); + ASSERT_NOT_EQUALS( nsDBHash( "foo.d" ), nsDBHash( "food" ) ); + } + +#define testEqualsBothWays(X,Y) ASSERT_TRUE( nsDBEquals( (X), (Y) ) ); ASSERT_TRUE( nsDBEquals( (Y), (X) ) ); +#define testNotEqualsBothWays(X,Y) ASSERT_FALSE( nsDBEquals( (X), (Y) ) ); ASSERT_FALSE( nsDBEquals( (Y), (X) ) ); + + TEST( NamespaceStringTest, DBEquals ) { + testEqualsBothWays( "foo" , "foo" ); + testEqualsBothWays( "foo" , "foo.a" ); + testEqualsBothWays( "foo.a" , "foo.a" ); + testEqualsBothWays( "foo.a" , "foo.b" ); + + testEqualsBothWays( "" , "" ); + testEqualsBothWays( "" , "." ); + testEqualsBothWays( "" , ".x" ); + + testNotEqualsBothWays( "foo" , "bar" ); + testNotEqualsBothWays( "foo" , "food" ); + testNotEqualsBothWays( "foo." , "food" ); + + testNotEqualsBothWays( "" , "x" ); + testNotEqualsBothWays( "" , "x." ); + testNotEqualsBothWays( "" , "x.y" ); + testNotEqualsBothWays( "." , "x" ); + testNotEqualsBothWays( "." , "x." ); + testNotEqualsBothWays( "." , "x.y" ); + } + + TEST( NamespaceStringTest, nsToDatabase1 ) { + ASSERT_EQUALS( "foo", nsToDatabaseSubstring( "foo.bar" ) ); + ASSERT_EQUALS( "foo", nsToDatabaseSubstring( "foo" ) ); + ASSERT_EQUALS( "foo", nsToDatabase( "foo.bar" ) ); + ASSERT_EQUALS( "foo", nsToDatabase( "foo" ) ); + ASSERT_EQUALS( "foo", nsToDatabase( string("foo.bar") ) ); + ASSERT_EQUALS( "foo", nsToDatabase( string("foo") ) ); + } + + TEST( NamespaceStringTest, nsToDatabase2 ) { + char buf[MaxDatabaseNameLen]; + + nsToDatabase( "foo.bar", buf ); + ASSERT_EQUALS( 'f', buf[0] ); + ASSERT_EQUALS( 'o', buf[1] ); + ASSERT_EQUALS( 'o', buf[2] ); + ASSERT_EQUALS( 0, buf[3] ); + + nsToDatabase( "bar", buf ); + ASSERT_EQUALS( 'b', buf[0] ); + ASSERT_EQUALS( 'a', buf[1] ); + ASSERT_EQUALS( 'r', buf[2] ); + ASSERT_EQUALS( 0, buf[3] ); + + + } + + TEST( NamespaceStringTest, NamespaceStringParse1 ) { + NamespaceString ns( "a.b" ); + ASSERT_EQUALS( (string)"a", ns.db() ); + ASSERT_EQUALS( (string)"b", ns.coll() ); + } + + TEST( NamespaceStringTest, NamespaceStringParse2 ) { + NamespaceString ns( "a.b.c" ); + ASSERT_EQUALS( (string)"a", ns.db() ); + ASSERT_EQUALS( (string)"b.c", ns.coll() ); + } + + TEST( NamespaceStringTest, NamespaceStringParse3 ) { + NamespaceString ns( "abc" ); + ASSERT_EQUALS( (string)"", ns.db() ); + ASSERT_EQUALS( (string)"", ns.coll() ); + } + + TEST( NamespaceStringTest, NamespaceStringParse4 ) { + NamespaceString ns( "abc." ); + ASSERT_EQUALS( (string)"abc", ns.db() ); + ASSERT_EQUALS( (string)"", ns.coll() ); + } + +} + diff --git a/src/mongo/db/namespacestring.h b/src/mongo/db/namespacestring.h deleted file mode 100644 index d108bbaac72..00000000000 --- a/src/mongo/db/namespacestring.h +++ /dev/null @@ -1,231 +0,0 @@ -// @file namespacestring.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include - -#include "mongo/util/assert_util.h" - -namespace mongo { - - using std::string; - - /* in the mongo source code, "client" means "database". */ - - const size_t MaxDatabaseNameLen = 128; // max str len for the db name, including null char - - /* e.g. - NamespaceString ns("acme.orders"); - cout << ns.coll; // "orders" - */ - class NamespaceString { - public: - string db; - string coll; // note collection names can have periods in them for organizing purposes (e.g. "system.indexes") - - NamespaceString( const char * ns ) { init(ns); } - NamespaceString( const string& ns ) { init(ns.c_str()); } - - string ns() const { return db + '.' + coll; } - - bool isSystem() const { return strncmp(coll.c_str(), "system.", 7) == 0; } - bool isCommand() const { return coll == "$cmd"; } - - /** - * @return true if the namespace is valid. Special namespaces for internal use are considered as valid. - */ - bool isValid() const { - return validDBName( db ) && !coll.empty(); - } - - operator string() const { return ns(); } - - bool operator==( const string& nsIn ) const { return nsIn == ns(); } - bool operator==( const char* nsIn ) const { return (string)nsIn == ns(); } - bool operator==( const NamespaceString& nsIn ) const { return nsIn.db == db && nsIn.coll == coll; } - - bool operator!=( const string& nsIn ) const { return nsIn != ns(); } - bool operator!=( const char* nsIn ) const { return (string)nsIn != ns(); } - bool operator!=( const NamespaceString& nsIn ) const { return nsIn.db != db || nsIn.coll != coll; } - - size_t size() const { return ns().size(); } - - string toString() const { return ns(); } - - /** - * @return true if ns is 'normal'. A "$" is used for namespaces holding index data, - * which do not contain BSON objects in their records. ("oplog.$main" is the exception) - */ - static bool normal(const char* ns) { - const char *p = strchr(ns, '$'); - if( p == 0 ) - return true; - return oplog(ns); - } - - /** - * @return true if the ns is an oplog one, otherwise false. - */ - static bool oplog(const char* ns) { - return StringData(ns) == StringData("local.oplog.rs") || StringData(ns) == StringData("local.oplog.$main"); - } - - static bool special(const char *ns) { - return !normal(ns) || strstr(ns, ".system."); - } - - /** - * samples: - * good: - * foo - * bar - * foo-bar - * bad: - * foo bar - * foo.bar - * foo"bar - * - * @param db - a possible database name - * @return if db is an allowed database name - */ - static bool validDBName( const string& db ) { - if ( db.size() == 0 || db.size() > 64 ) - return false; -#ifdef _WIN32 - // We prohibit all FAT32-disallowed characters on Windows - size_t good = strcspn( db.c_str() , "/\\. \"*<>:|?" ); -#else - // For non-Windows platforms we are much more lenient - size_t good = strcspn( db.c_str() , "/\\. \"" ); -#endif - return good == db.size(); - } - - /** - * samples: - * good: - * foo.bar - * bad: - * foo. - * - * @param dbcoll - a possible collection name of the form db.coll - * @return if db.coll is an allowed collection name - */ - static bool validCollectionName(const char* dbcoll){ - const char *c = strchr( dbcoll, '.' ); - return (c != NULL) && (c[1] != '\0') && normal(dbcoll); - } - - private: - void init(const char *ns) { - const char *p = strchr(ns, '.'); - if( p == 0 ) return; - db = string(ns, p - ns); - coll = p + 1; - } - }; - - // "database.a.b.c" -> "database" - inline StringData nsToDatabaseSubstring( const StringData& ns ) { - size_t i = ns.find( '.' ); - if ( i == string::npos ) { - massert(10078, "nsToDatabase: ns too long", ns.size() < MaxDatabaseNameLen ); - return ns; - } - massert(10088, "nsToDatabase: ns too long", i < static_cast(MaxDatabaseNameLen)); - return ns.substr( 0, i ); - } - - // "database.a.b.c" -> "database" - inline void nsToDatabase(const StringData& ns, char *database) { - StringData db = nsToDatabaseSubstring( ns ); - db.copyTo( database, true ); - } - - // TODO: make this return a StringData - inline string nsToDatabase(const StringData& ns) { - return nsToDatabaseSubstring( ns ).toString(); - } - - /** - * NamespaceDBHash and NamespaceDBEquals allow you to do something like - * unordered_map - * and use the full namespace for the string - * but comparisons are done only on the db piece - */ - - /** - * this can change, do not store on disk - */ - inline int nsDBHash( const string& ns ) { - int hash = 7; - for ( size_t i = 0; i < ns.size(); i++ ) { - if ( ns[i] == '.' ) - break; - hash += 11 * ( ns[i] ); - hash *= 3; - } - return hash; - } - - inline bool nsDBEquals( const string& a, const string& b ) { - for ( size_t i = 0; i < a.size(); i++ ) { - - if ( a[i] == '.' ) { - // b has to either be done or a '.' - - if ( b.size() == i ) - return true; - - if ( b[i] == '.' ) - return true; - - return false; - } - - // a is another character - if ( b.size() == i ) - return false; - - if ( b[i] != a[i] ) - return false; - } - - // a is done - // make sure b is done - if ( b.size() == a.size() || - b[a.size()] == '.' ) - return true; - - return false; - } - - struct NamespaceDBHash { - int operator()( const string& ns ) const { - return nsDBHash( ns ); - } - }; - - struct NamespaceDBEquals { - bool operator()( const string& a, const string& b ) const { - return nsDBEquals( a, b ); - } - }; - -} diff --git a/src/mongo/db/namespacestring_test.cpp b/src/mongo/db/namespacestring_test.cpp deleted file mode 100644 index 817cf797e64..00000000000 --- a/src/mongo/db/namespacestring_test.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// namespacestring_test.cpp - -/* Copyright 2012 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongo/unittest/unittest.h" - -#include "mongo/db/namespacestring.h" - -namespace mongo { - - - TEST( NamespaceStringTest, DatabaseValidNames ) { - ASSERT( NamespaceString::validDBName( "foo" ) ); - ASSERT( !NamespaceString::validDBName( "foo/bar" ) ); - ASSERT( !NamespaceString::validDBName( "foo bar" ) ); - ASSERT( !NamespaceString::validDBName( "foo.bar" ) ); - - ASSERT( NamespaceString::normal( "asdads" ) ); - ASSERT( !NamespaceString::normal( "asda$ds" ) ); - ASSERT( NamespaceString::normal( "local.oplog.$main" ) ); - } - - TEST( NamespaceStringTest, DBHash ) { - ASSERT_EQUALS( nsDBHash( "foo" ), nsDBHash( "foo" ) ); - ASSERT_EQUALS( nsDBHash( "foo" ), nsDBHash( "foo.a" ) ); - ASSERT_EQUALS( nsDBHash( "foo" ), nsDBHash( "foo." ) ); - - ASSERT_EQUALS( nsDBHash( "" ), nsDBHash( "" ) ); - ASSERT_EQUALS( nsDBHash( "" ), nsDBHash( ".a" ) ); - ASSERT_EQUALS( nsDBHash( "" ), nsDBHash( "." ) ); - - ASSERT_NOT_EQUALS( nsDBHash( "foo" ), nsDBHash( "food" ) ); - ASSERT_NOT_EQUALS( nsDBHash( "foo." ), nsDBHash( "food" ) ); - ASSERT_NOT_EQUALS( nsDBHash( "foo.d" ), nsDBHash( "food" ) ); - } - -#define testEqualsBothWays(X,Y) ASSERT_TRUE( nsDBEquals( (X), (Y) ) ); ASSERT_TRUE( nsDBEquals( (Y), (X) ) ); -#define testNotEqualsBothWays(X,Y) ASSERT_FALSE( nsDBEquals( (X), (Y) ) ); ASSERT_FALSE( nsDBEquals( (Y), (X) ) ); - - TEST( NamespaceStringTest, DBEquals ) { - testEqualsBothWays( "foo" , "foo" ); - testEqualsBothWays( "foo" , "foo.a" ); - testEqualsBothWays( "foo.a" , "foo.a" ); - testEqualsBothWays( "foo.a" , "foo.b" ); - - testEqualsBothWays( "" , "" ); - testEqualsBothWays( "" , "." ); - testEqualsBothWays( "" , ".x" ); - - testNotEqualsBothWays( "foo" , "bar" ); - testNotEqualsBothWays( "foo" , "food" ); - testNotEqualsBothWays( "foo." , "food" ); - - testNotEqualsBothWays( "" , "x" ); - testNotEqualsBothWays( "" , "x." ); - testNotEqualsBothWays( "" , "x.y" ); - testNotEqualsBothWays( "." , "x" ); - testNotEqualsBothWays( "." , "x." ); - testNotEqualsBothWays( "." , "x.y" ); - } - - TEST( NamespaceStringTest, nsToDatabase1 ) { - ASSERT_EQUALS( "foo", nsToDatabaseSubstring( "foo.bar" ) ); - ASSERT_EQUALS( "foo", nsToDatabaseSubstring( "foo" ) ); - ASSERT_EQUALS( "foo", nsToDatabase( "foo.bar" ) ); - ASSERT_EQUALS( "foo", nsToDatabase( "foo" ) ); - ASSERT_EQUALS( "foo", nsToDatabase( string("foo.bar") ) ); - ASSERT_EQUALS( "foo", nsToDatabase( string("foo") ) ); - } - - TEST( NamespaceStringTest, nsToDatabase2 ) { - char buf[128]; - - nsToDatabase( "foo.bar", buf ); - ASSERT_EQUALS( 'f', buf[0] ); - ASSERT_EQUALS( 'o', buf[1] ); - ASSERT_EQUALS( 'o', buf[2] ); - ASSERT_EQUALS( 0, buf[3] ); - - nsToDatabase( "bar", buf ); - ASSERT_EQUALS( 'b', buf[0] ); - ASSERT_EQUALS( 'a', buf[1] ); - ASSERT_EQUALS( 'r', buf[2] ); - ASSERT_EQUALS( 0, buf[3] ); - - - } -} - diff --git a/src/mongo/db/oplog.cpp b/src/mongo/db/oplog.cpp deleted file mode 100644 index 12f4742c256..00000000000 --- a/src/mongo/db/oplog.cpp +++ /dev/null @@ -1,1020 +0,0 @@ -// @file oplog.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/oplog.h" - -#include - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/privilege.h" -#include "mongo/db/commands.h" -#include "mongo/db/index_update.h" -#include "mongo/db/instance.h" -#include "mongo/db/namespacestring.h" -#include "mongo/db/ops/update.h" -#include "mongo/db/ops/delete.h" -#include "mongo/db/queryoptimizer.h" -#include "mongo/db/repl.h" -#include "mongo/db/repl_block.h" -#include "mongo/db/repl/bgsync.h" -#include "mongo/db/repl/rs.h" -#include "mongo/db/stats/counters.h" -#include "mongo/util/elapsed_tracker.h" -#include "mongo/util/file.h" -#include "mongo/util/startup_test.h" - -namespace mongo { - - // from d_migrate.cpp - void logOpForSharding( const char * opstr , const char * ns , const BSONObj& obj , BSONObj * patt ); - - // Configurable for testing. - int FindingStartCursor::_initialTimeout = 5; - - // cached copies of these...so don't rename them, drop them, etc.!!! - static NamespaceDetails *localOplogMainDetails = 0; - static Database *localDB = 0; - static NamespaceDetails *rsOplogDetails = 0; - void oplogCheckCloseDatabase( Database * db ) { - verify( Lock::isW() ); - localDB = 0; - localOplogMainDetails = 0; - rsOplogDetails = 0; - resetSlaveCache(); - } - - static void _logOpUninitialized(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb, bool fromMigrate ) { - uassert(13288, "replSet error write op to db before replSet initialized", str::startsWith(ns, "local.") || *opstr == 'n'); - } - - /** write an op to the oplog that is already built. - todo : make _logOpRS() call this so we don't repeat ourself? - */ - void _logOpObjRS(const BSONObj& op) { - Lock::DBWrite lk("local"); - - const OpTime ts = op["ts"]._opTime(); - long long h = op["h"].numberLong(); - - { - const char *logns = rsoplog; - if ( rsOplogDetails == 0 ) { - Client::Context ctx(logns , dbpath); - localDB = ctx.db(); - verify( localDB ); - rsOplogDetails = nsdetails(logns); - massert(13389, "local.oplog.rs missing. did you drop it? if so restart server", rsOplogDetails); - } - Client::Context ctx(logns , localDB); - { - int len = op.objsize(); - Record *r = theDataFileMgr.fast_oplog_insert(rsOplogDetails, logns, len); - memcpy(getDur().writingPtr(r->data(), len), op.objdata(), len); - } - /* todo: now() has code to handle clock skew. but if the skew server to server is large it will get unhappy. - this code (or code in now() maybe) should be improved. - */ - if( theReplSet ) { - if( !(theReplSet->lastOpTimeWrittenlastOpTimeWritten.toString() << ' ' << endl; - } - theReplSet->lastOpTimeWritten = ts; - theReplSet->lastH = h; - ctx.getClient()->setLastOp( ts ); - } - } - - OpTime::setLast( ts ); - } - - /** given a BSON object, create a new one at dst which is the existing (partial) object - with a new object element appended at the end with fieldname "o". - - @param partial already build object with everything except the o member. e.g. something like: - { ts:..., ns:..., os2:... } - @param o a bson object to be added with fieldname "o" - @dst where to put the newly built combined object. e.g. ends up as something like: - { ts:..., ns:..., os2:..., o:... } - */ - void append_O_Obj(char *dst, const BSONObj& partial, const BSONObj& o) { - const int size1 = partial.objsize() - 1; // less the EOO char - const int oOfs = size1+3; // 3 = byte BSONOBJTYPE + byte 'o' + byte \0 - - void *p = getDur().writingPtr(dst, oOfs+o.objsize()+1); - - memcpy(p, partial.objdata(), size1); - - // adjust overall bson object size for the o: field - *(static_cast(p)) += o.objsize() + 1/*fieldtype byte*/ + 2/*"o" fieldname*/; - - char *b = static_cast(p); - b += size1; - *b++ = (char) Object; - *b++ = 'o'; // { o : ... } - *b++ = 0; // null terminate "o" fieldname - memcpy(b, o.objdata(), o.objsize()); - b += o.objsize(); - *b = EOO; - } - - /* we write to local.oplog.rs: - { ts : ..., h: ..., v: ..., op: ..., etc } - ts: an OpTime timestamp - h: hash - v: version - op: - "i" insert - "u" update - "d" delete - "c" db cmd - "db" declares presence of a database (ns is set to the db name + '.') - "n" no op - - bb param: - if not null, specifies a boolean to pass along to the other side as b: param. - used for "justOne" or "upsert" flags on 'd', 'u' - - */ - - // global is safe as we are in write lock. we put the static outside the function to avoid the implicit mutex - // the compiler would use if inside the function. the reason this is static is to avoid a malloc/free for this - // on every logop call. - static BufBuilder logopbufbuilder(8*1024); - static const int OPLOG_VERSION = 2; - static void _logOpRS(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb, bool fromMigrate ) { - Lock::DBWrite lk1("local"); - - if ( strncmp(ns, "local.", 6) == 0 ) { - if ( strncmp(ns, "local.slaves", 12) == 0 ) - resetSlaveCache(); - return; - } - - mutex::scoped_lock lk2(OpTime::m); - - const OpTime ts = OpTime::now(lk2); - long long hashNew; - if( theReplSet ) { - massert(13312, "replSet error : logOp() but not primary?", theReplSet->box.getState().primary()); - hashNew = (theReplSet->lastH * 131 + ts.asLL()) * 17 + theReplSet->selfId(); - } - else { - // must be initiation - verify( *ns == 0 ); - hashNew = 0; - } - - /* we jump through a bunch of hoops here to avoid copying the obj buffer twice -- - instead we do a single copy to the destination position in the memory mapped file. - */ - - logopbufbuilder.reset(); - BSONObjBuilder b(logopbufbuilder); - b.appendTimestamp("ts", ts.asDate()); - b.append("h", hashNew); - b.append("v", OPLOG_VERSION); - b.append("op", opstr); - b.append("ns", ns); - if (fromMigrate) - b.appendBool("fromMigrate", true); - if ( bb ) - b.appendBool("b", *bb); - if ( o2 ) - b.append("o2", *o2); - BSONObj partial = b.done(); - int posz = partial.objsize(); - int len = posz + obj.objsize() + 1 + 2 /*o:*/; - - Record *r; - DEV verify( logNS == 0 ); - { - const char *logns = rsoplog; - if ( rsOplogDetails == 0 ) { - Client::Context ctx(logns , dbpath); - localDB = ctx.db(); - verify( localDB ); - rsOplogDetails = nsdetails(logns); - massert(13347, "local.oplog.rs missing. did you drop it? if so restart server", rsOplogDetails); - } - Client::Context ctx(logns , localDB); - r = theDataFileMgr.fast_oplog_insert(rsOplogDetails, logns, len); - /* todo: now() has code to handle clock skew. but if the skew server to server is large it will get unhappy. - this code (or code in now() maybe) should be improved. - */ - if( theReplSet ) { - if( !(theReplSet->lastOpTimeWrittenlastOpTimeWritten << ' ' << ts << rsLog; - log() << "replSet " << theReplSet->isPrimary() << rsLog; - } - theReplSet->lastOpTimeWritten = ts; - theReplSet->lastH = hashNew; - ctx.getClient()->setLastOp( ts ); - } - } - - append_O_Obj(r->data(), partial, obj); - - if ( logLevel >= 6 ) { - LOG( 6 ) << "logOp:" << BSONObj::make(r) << endl; - } - } - - static void _logOpOld(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb, bool fromMigrate ) { - Lock::DBWrite lk("local"); - static BufBuilder bufbuilder(8*1024); // todo there is likely a mutex on this constructor - - if ( strncmp(ns, "local.", 6) == 0 ) { - if ( strncmp(ns, "local.slaves", 12) == 0 ) { - resetSlaveCache(); - } - return; - } - - mutex::scoped_lock lk2(OpTime::m); - - const OpTime ts = OpTime::now(lk2); - Client::Context context("", 0); - - /* we jump through a bunch of hoops here to avoid copying the obj buffer twice -- - instead we do a single copy to the destination position in the memory mapped file. - */ - - bufbuilder.reset(); - BSONObjBuilder b(bufbuilder); - b.appendTimestamp("ts", ts.asDate()); - b.append("op", opstr); - b.append("ns", ns); - if (fromMigrate) - b.appendBool("fromMigrate", true); - if ( bb ) - b.appendBool("b", *bb); - if ( o2 ) - b.append("o2", *o2); - BSONObj partial = b.done(); // partial is everything except the o:... part. - - int po_sz = partial.objsize(); - int len = po_sz + obj.objsize() + 1 + 2 /*o:*/; - - Record *r; - if( logNS == 0 ) { - logNS = "local.oplog.$main"; - if ( localOplogMainDetails == 0 ) { - Client::Context ctx(logNS , dbpath); - localDB = ctx.db(); - verify( localDB ); - localOplogMainDetails = nsdetails(logNS); - verify( localOplogMainDetails ); - } - Client::Context ctx(logNS , localDB); - r = theDataFileMgr.fast_oplog_insert(localOplogMainDetails, logNS, len); - } - else { - Client::Context ctx(logNS, dbpath); - verify( nsdetails( logNS ) ); - // first we allocate the space, then we fill it below. - r = theDataFileMgr.fast_oplog_insert( nsdetails( logNS ), logNS, len); - } - - append_O_Obj(r->data(), partial, obj); - - context.getClient()->setLastOp( ts ); - - LOG( 6 ) << "logging op:" << BSONObj::make(r) << endl; - } - - static void (*_logOp)(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb, bool fromMigrate ) = _logOpOld; - void newReplUp() { - replSettings.master = true; - _logOp = _logOpRS; - } - void newRepl() { - replSettings.master = true; - _logOp = _logOpUninitialized; - } - void oldRepl() { _logOp = _logOpOld; } - - void logKeepalive() { - _logOp("n", "", 0, BSONObj(), 0, 0, false); - } - void logOpComment(const BSONObj& obj) { - _logOp("n", "", 0, obj, 0, 0, false); - } - void logOpInitiate(const BSONObj& obj) { - _logOpRS("n", "", 0, obj, 0, 0, false); - } - - /*@ @param opstr: - c userCreateNS - i insert - n no-op / keepalive - d delete / remove - u update - */ - void logOp(const char *opstr, const char *ns, const BSONObj& obj, BSONObj *patt, bool *b, bool fromMigrate) { - if ( replSettings.master ) { - _logOp(opstr, ns, 0, obj, patt, b, fromMigrate); - } - - logOpForSharding( opstr , ns , obj , patt ); - } - - void createOplog() { - Lock::GlobalWrite lk; - - const char * ns = "local.oplog.$main"; - - bool rs = !cmdLine._replSet.empty(); - if( rs ) - ns = rsoplog; - - Client::Context ctx(ns); - - NamespaceDetails * nsd = nsdetails( ns ); - - if ( nsd ) { - - if ( cmdLine.oplogSize != 0 ) { - int o = (int)(nsd->storageSize() / ( 1024 * 1024 ) ); - int n = (int)(cmdLine.oplogSize / ( 1024 * 1024 ) ); - if ( n != o ) { - stringstream ss; - ss << "cmdline oplogsize (" << n << ") different than existing (" << o << ") see: http://dochub.mongodb.org/core/increase-oplog"; - log() << ss.str() << endl; - throw UserException( 13257 , ss.str() ); - } - } - - if( rs ) return; - - DBDirectClient c; - BSONObj lastOp = c.findOne( ns, Query().sort(reverseNaturalObj) ); - if ( !lastOp.isEmpty() ) { - OpTime::setLast( lastOp[ "ts" ].date() ); - } - return; - } - - /* create an oplog collection, if it doesn't yet exist. */ - BSONObjBuilder b; - double sz; - if ( cmdLine.oplogSize != 0 ) - sz = (double)cmdLine.oplogSize; - else { - /* not specified. pick a default size */ - sz = 50.0 * 1024 * 1024; - if ( sizeof(int *) >= 8 ) { -#if defined(__APPLE__) - // typically these are desktops (dev machines), so keep it smallish - sz = (256-64) * 1024 * 1024; -#else - sz = 990.0 * 1024 * 1024; - boost::intmax_t free = File::freeSpace(dbpath); //-1 if call not supported. - double fivePct = free * 0.05; - if ( fivePct > sz ) - sz = fivePct; -#endif - } - } - - log() << "******" << endl; - log() << "creating replication oplog of size: " << (int)( sz / ( 1024 * 1024 ) ) << "MB..." << endl; - - b.append("size", sz); - b.appendBool("capped", 1); - b.appendBool("autoIndexId", false); - - string err; - BSONObj o = b.done(); - userCreateNS(ns, o, err, false); - if( !rs ) - logOp( "n", "", BSONObj() ); - - /* sync here so we don't get any surprising lag later when we try to sync */ - MemoryMappedFile::flushAll(true); - log() << "******" << endl; - } - - // ------------------------------------- - - FindingStartCursor *FindingStartCursor::make( const QueryPlan &qp ) { - auto_ptr ret( new FindingStartCursor( qp ) ); - ret->init(); - return ret.release(); - } - - FindingStartCursor::FindingStartCursor( const QueryPlan &qp ) : - _qp( qp ), - _findingStart( true ), - _findingStartMode() { - } - - void FindingStartCursor::next() { - if ( !_findingStartCursor || !_findingStartCursor->ok() ) { - _findingStart = false; - _c = _qp.newCursor(); // on error, start from beginning - destroyClientCursor(); - return; - } - switch( _findingStartMode ) { - // Initial mode: scan backwards from end of collection - case Initial: { - if ( !_matcher->matchesCurrent( _findingStartCursor->c() ) ) { - _findingStart = false; // found first record out of query range, so scan normally - _c = _qp.newCursor( _findingStartCursor->currLoc() ); - destroyClientCursor(); - return; - } - _findingStartCursor->advance(); - RARELY { - if ( _findingStartTimer.seconds() >= _initialTimeout ) { - // If we've scanned enough, switch to find extent mode. - createClientCursor( extentFirstLoc( _findingStartCursor->currLoc() ) ); - _findingStartMode = FindExtent; - return; - } - } - return; - } - // FindExtent mode: moving backwards through extents, check first - // document of each extent. - case FindExtent: { - if ( !_matcher->matchesCurrent( _findingStartCursor->c() ) ) { - _findingStartMode = InExtent; - return; - } - DiskLoc prev = prevExtentFirstLoc( _findingStartCursor->currLoc() ); - if ( prev.isNull() ) { // hit beginning, so start scanning from here - createClientCursor(); - _findingStartMode = InExtent; - return; - } - // There might be a more efficient implementation than creating new cursor & client cursor each time, - // not worrying about that for now - createClientCursor( prev ); - return; - } - // InExtent mode: once an extent is chosen, find starting doc in the extent. - case InExtent: { - if ( _matcher->matchesCurrent( _findingStartCursor->c() ) ) { - _findingStart = false; // found first record in query range, so scan normally - _c = _qp.newCursor( _findingStartCursor->currLoc() ); - destroyClientCursor(); - return; - } - _findingStartCursor->advance(); - return; - } - default: { - massert( 14038, "invalid _findingStartMode", false ); - } - } - } - - DiskLoc FindingStartCursor::extentFirstLoc( const DiskLoc &rec ) { - Extent *e = rec.rec()->myExtent( rec ); - if ( !_qp.nsd()->capLooped() || ( e->myLoc != _qp.nsd()->capExtent ) ) - return e->firstRecord; - // Likely we are on the fresh side of capExtent, so return first fresh record. - // If we are on the stale side of capExtent, then the collection is small and it - // doesn't matter if we start the extent scan with capFirstNewRecord. - return _qp.nsd()->capFirstNewRecord; - } - - DiskLoc FindingStartCursor::prevExtentFirstLoc( const DiskLoc& rec ) const { - Extent *e = rec.rec()->myExtent( rec ); - if ( _qp.nsd()->capLooped() ) { - while( true ) { - // Advance e to preceding extent (looping to lastExtent if necessary). - if ( e->xprev.isNull() ) { - e = _qp.nsd()->lastExtent.ext(); - } - else { - e = e->xprev.ext(); - } - if ( e->myLoc == _qp.nsd()->capExtent ) { - // Reached the extent containing the oldest data in the collection. - return DiskLoc(); - } - if ( !e->firstRecord.isNull() ) { - // Return the first record of the first non empty extent encountered. - return e->firstRecord; - } - } - } - else { - while( true ) { - if ( e->xprev.isNull() ) { - // Reached the beginning of the collection. - return DiskLoc(); - } - e = e->xprev.ext(); - if ( !e->firstRecord.isNull() ) { - // Return the first record of the first non empty extent encountered. - return e->firstRecord; - } - } - } - } - - void FindingStartCursor::createClientCursor( const DiskLoc &startLoc ) { - shared_ptr c = _qp.newCursor( startLoc ); - _findingStartCursor.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns()) ); - } - - bool FindingStartCursor::firstDocMatchesOrEmpty() const { - shared_ptr c = _qp.newCursor(); - return !c->ok() || _matcher->matchesCurrent( c.get() ); - } - - void FindingStartCursor::init() { - BSONElement tsElt = _qp.originalQuery()[ "ts" ]; - massert( 13044, "no ts field in query", !tsElt.eoo() ); - BSONObjBuilder b; - b.append( tsElt ); - BSONObj tsQuery = b.obj(); - _matcher.reset(new CoveredIndexMatcher(tsQuery, _qp.indexKey())); - if ( firstDocMatchesOrEmpty() ) { - _c = _qp.newCursor(); - _findingStart = false; - return; - } - // Use a ClientCursor here so we can release db mutex while scanning - // oplog (can take quite a while with large oplogs). - shared_ptr c = _qp.newReverseCursor(); - _findingStartCursor.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns(), BSONObj()) ); - _findingStartTimer.reset(); - _findingStartMode = Initial; - } - - shared_ptr FindingStartCursor::getCursor( const char *ns, const BSONObj &query, const BSONObj &order ) { - NamespaceDetails *d = nsdetails(ns); - if ( !d ) { - return shared_ptr( new BasicCursor( DiskLoc() ) ); - } - FieldRangeSetPair frsp( ns, query ); - scoped_ptr oplogPlan( QueryPlan::make( d, -1, frsp, 0, query, order ) ); - scoped_ptr finder( FindingStartCursor::make( *oplogPlan ) ); - ElapsedTracker yieldCondition( 256, 20 ); - while( !finder->done() ) { - if ( yieldCondition.intervalHasElapsed() ) { - if ( finder->prepareToYield() ) { - ClientCursor::staticYield( 0, ns, 0 ); - finder->recoverFromYield(); - } - } - finder->next(); - } - shared_ptr ret = finder->cursor(); - shared_ptr matcher( new CoveredIndexMatcher( query, BSONObj() ) ); - ret->setMatcher( matcher ); - return ret; - } - - // ------------------------------------- - - struct TestOpTime : public StartupTest { - void run() { - OpTime t; - for ( int i = 0; i < 10; i++ ) { - OpTime s = OpTime::_now(); - verify( s != t ); - t = s; - } - OpTime q = t; - verify( q == t ); - verify( !(q != t) ); - } - } testoptime; - - int _dummy_z; - - void pretouchN(vector& v, unsigned a, unsigned b) { - DEV verify( ! Lock::isW() ); - - Client *c = currentClient.get(); - if( c == 0 ) { - Client::initThread("pretouchN"); - c = &cc(); - } - - Lock::GlobalRead lk; - for( unsigned i = a; i <= b; i++ ) { - const BSONObj& op = v[i]; - const char *which = "o"; - const char *opType = op.getStringField("op"); - if ( *opType == 'i' ) - ; - else if( *opType == 'u' ) - which = "o2"; - else - continue; - /* todo : other operations */ - - try { - BSONObj o = op.getObjectField(which); - BSONElement _id; - if( o.getObjectID(_id) ) { - const char *ns = op.getStringField("ns"); - BSONObjBuilder b; - b.append(_id); - BSONObj result; - Client::Context ctx( ns ); - if( Helpers::findById(cc(), ns, b.done(), result) ) - _dummy_z += result.objsize(); // touch - } - } - catch( DBException& e ) { - log() << "ignoring assertion in pretouchN() " << a << ' ' << b << ' ' << i << ' ' << e.toString() << endl; - } - } - } - - void pretouchOperation(const BSONObj& op) { - - if( Lock::somethingWriteLocked() ) - return; // no point pretouching if write locked. not sure if this will ever fire, but just in case. - - const char *which = "o"; - const char *opType = op.getStringField("op"); - if ( *opType == 'i' ) - ; - else if( *opType == 'u' ) - which = "o2"; - else - return; - /* todo : other operations */ - - try { - BSONObj o = op.getObjectField(which); - BSONElement _id; - if( o.getObjectID(_id) ) { - const char *ns = op.getStringField("ns"); - BSONObjBuilder b; - b.append(_id); - BSONObj result; - Client::ReadContext ctx( ns ); - if( Helpers::findById(cc(), ns, b.done(), result) ) - _dummy_z += result.objsize(); // touch - } - } - catch( DBException& ) { - log() << "ignoring assertion in pretouchOperation()" << endl; - } - } - - void Sync::setHostname(const string& hostname) { - hn = hostname; - } - - BSONObj Sync::getMissingDoc(const BSONObj& o) { - OplogReader missingObjReader; - const char *ns = o.getStringField("ns"); - - // capped collections - NamespaceDetails *nsd = nsdetails(ns); - if ( nsd && nsd->isCapped() ) { - log() << "replication missing doc, but this is okay for a capped collection (" << ns << ")" << endl; - return BSONObj(); - } - - const int retryMax = 3; - for (int retryCount = 1; retryCount <= retryMax; ++retryCount) { - if (retryCount != 1) { - // if we are retrying, sleep a bit to let the network possibly recover - sleepsecs(retryCount * retryCount); - } - try { - bool ok = missingObjReader.connect(hn); - if (!ok) { - warning() << "network problem detected while connecting to the " - << "sync source, attempt " << retryCount << " of " - << retryMax << endl; - continue; // try again - } - } - catch (const SocketException& exc) { - warning() << "network problem detected while connecting to the " - << "sync source, attempt " << retryCount << " of " - << retryMax << endl; - continue; // try again - } - - // might be more than just _id in the update criteria - BSONObj query = BSONObjBuilder().append(o.getObjectField("o2")["_id"]).obj(); - BSONObj missingObj; - try { - missingObj = missingObjReader.findOne(ns, query); - } - catch (const SocketException& exc) { - warning() << "network problem detected while fetching a missing document from the " - << "sync source, attempt " << retryCount << " of " - << retryMax << endl; - continue; // try again - } - catch (DBException& e) { - log() << "replication assertion fetching missing object: " << e.what() << endl; - throw; - } - - // success! - return missingObj; - } - // retry count exceeded - msgasserted(15916, - str::stream() << "Can no longer connect to initial sync source: " << hn); - } - - bool Sync::shouldRetry(const BSONObj& o) { - // should already have write lock - const char *ns = o.getStringField("ns"); - Client::Context ctx(ns); - - // we don't have the object yet, which is possible on initial sync. get it. - log() << "replication info adding missing object" << endl; // rare enough we can log - - BSONObj missingObj = getMissingDoc(o); - - if( missingObj.isEmpty() ) { - log() << "replication missing object not found on source. presumably deleted later in oplog" << endl; - log() << "replication o2: " << o.getObjectField("o2").toString() << endl; - log() << "replication o firstfield: " << o.getObjectField("o").firstElementFieldName() << endl; - - return false; - } - else { - DiskLoc d = theDataFileMgr.insert(ns, (void*) missingObj.objdata(), missingObj.objsize()); - uassert(15917, "Got bad disk location when attempting to insert", !d.isNull()); - - LOG(1) << "replication inserted missing doc: " << missingObj.toString() << endl; - return true; - } - } - - /** @param fromRepl false if from ApplyOpsCmd - @return true if was and update should have happened and the document DNE. see replset initial sync code. - */ - bool applyOperation_inlock(const BSONObj& op, bool fromRepl, bool convertUpdateToUpsert) { - LOG(3) << "applying op: " << op << endl; - bool failedUpdate = false; - - OpCounters * opCounters = fromRepl ? &replOpCounters : &globalOpCounters; - - const char *names[] = { "o", "ns", "op", "b" }; - BSONElement fields[4]; - op.getFields(4, names, fields); - - BSONObj o; - if( fields[0].isABSONObj() ) - o = fields[0].embeddedObject(); - - const char *ns = fields[1].valuestrsafe(); - - Lock::assertWriteLocked(ns); - - NamespaceDetails *nsd = nsdetails(ns); - - // operation type -- see logOp() comments for types - const char *opType = fields[2].valuestrsafe(); - - if ( *opType == 'i' ) { - opCounters->gotInsert(); - - if (NamespaceString(ns).coll == "system.indexes") { - // updates aren't allowed for indexes -- so we will do a regular insert. if index already - // exists, that is ok. - theDataFileMgr.insert(ns, (void*) o.objdata(), o.objsize()); - } - else { - // do upserts for inserts as we might get replayed more than once - OpDebug debug; - BSONElement _id; - if( !o.getObjectID(_id) ) { - /* No _id. This will be very slow. */ - Timer t; - updateObjectsForReplication(ns, o, o, true, false, false, debug, false, - QueryPlanSelectionPolicy::idElseNatural() ); - if( t.millis() >= 2 ) { - RARELY OCCASIONALLY log() << "warning, repl doing slow updates (no _id field) for " << ns << endl; - } - } - else { - // probably don't need this since all replicated colls have _id indexes now - // but keep it just in case - RARELY if ( nsd && !nsd->isCapped() ) { ensureHaveIdIndex(ns, false); } - - /* todo : it may be better to do an insert here, and then catch the dup key exception and do update - then. very few upserts will not be inserts... - */ - BSONObjBuilder b; - b.append(_id); - updateObjectsForReplication(ns, o, b.done(), true, false, false , debug, false, - QueryPlanSelectionPolicy::idElseNatural() ); - } - } - } - else if ( *opType == 'u' ) { - opCounters->gotUpdate(); - - // probably don't need this since all replicated colls have _id indexes now - // but keep it just in case - RARELY if ( nsd && !nsd->isCapped() ) { ensureHaveIdIndex(ns, false); } - - OpDebug debug; - BSONObj updateCriteria = op.getObjectField("o2"); - bool upsert = fields[3].booleanSafe() || convertUpdateToUpsert; - UpdateResult ur = - updateObjectsForReplication(ns, - o, - updateCriteria, - upsert, - /*multi*/ false, - /*logop*/ false, - debug, - /*fromMigrate*/ false, - QueryPlanSelectionPolicy::idElseNatural() ); - - if( ur.num == 0 ) { - if( ur.mod ) { - if( updateCriteria.nFields() == 1 ) { - // was a simple { _id : ... } update criteria - failedUpdate = true; - log() << "replication failed to apply update: " << op.toString() << endl; - } - // need to check to see if it isn't present so we can set failedUpdate correctly. - // note that adds some overhead for this extra check in some cases, such as an updateCriteria - // of the form - // { _id:..., { x : {$size:...} } - // thus this is not ideal. - else { - if (nsd == NULL || - (nsd->findIdIndex() >= 0 && Helpers::findById(nsd, updateCriteria).isNull()) || - // capped collections won't have an _id index - (nsd->findIdIndex() < 0 && Helpers::findOne(ns, updateCriteria, false).isNull())) { - failedUpdate = true; - log() << "replication couldn't find doc: " << op.toString() << endl; - } - - // Otherwise, it's present; zero objects were updated because of additional specifiers - // in the query for idempotence - } - } - else { - // this could happen benignly on an oplog duplicate replay of an upsert - // (because we are idempotent), - // if an regular non-mod update fails the item is (presumably) missing. - if( !upsert ) { - failedUpdate = true; - log() << "replication update of non-mod failed: " << op.toString() << endl; - } - } - } - } - else if ( *opType == 'd' ) { - opCounters->gotDelete(); - if ( opType[1] == 0 ) - deleteObjects(ns, o, /*justOne*/ fields[3].booleanSafe()); - else - verify( opType[1] == 'b' ); // "db" advertisement - } - else if ( *opType == 'c' ) { - opCounters->gotCommand(); - BufBuilder bb; - BSONObjBuilder ob; - _runCommands(ns, o, bb, ob, true, 0); - } - else if ( *opType == 'n' ) { - // no op - } - else { - throw MsgAssertionException( 14825 , ErrorMsg("error in applyOperation : unknown opType ", *opType) ); - } - return failedUpdate; - } - - class ApplyOpsCmd : public Command { - public: - virtual bool slaveOk() const { return false; } - virtual LockType locktype() const { return WRITE; } - virtual bool lockGlobally() const { return true; } // SERVER-4328 todo : is global ok or does this take a long time? i believe multiple ns used so locking individually requires more analysis - ApplyOpsCmd() : Command( "applyOps" ) {} - virtual void help( stringstream &help ) const { - help << "internal (sharding)\n{ applyOps : [ ] , preCondition : [ { ns : ... , q : ... , res : ... } ] }"; - } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - // applyOps can do pretty much anything, so require all privileges. - out->push_back(Privilege(PrivilegeSet::WILDCARD_RESOURCE, - AuthorizationManager::getAllUserActions())); - } - virtual bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - - if ( cmdObj.firstElement().type() != Array ) { - errmsg = "ops has to be an array"; - return false; - } - - BSONObj ops = cmdObj.firstElement().Obj(); - - { - // check input - BSONObjIterator i( ops ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( e.type() == Object ) - continue; - errmsg = "op not an object: "; - errmsg += e.fieldName(); - return false; - } - } - - if ( cmdObj["preCondition"].type() == Array ) { - BSONObjIterator i( cmdObj["preCondition"].Obj() ); - while ( i.more() ) { - BSONObj f = i.next().Obj(); - - BSONObj realres = db.findOne( f["ns"].String() , f["q"].Obj() ); - - Matcher m( f["res"].Obj() ); - if ( ! m.matches( realres ) ) { - result.append( "got" , realres ); - result.append( "whatFailed" , f ); - errmsg = "pre-condition failed"; - return false; - } - } - } - - // apply - int num = 0; - int errors = 0; - - BSONObjIterator i( ops ); - BSONArrayBuilder ab; - const bool alwaysUpsert = cmdObj.hasField("alwaysUpsert") ? - cmdObj["alwaysUpsert"].trueValue() : true; - - while ( i.more() ) { - BSONElement e = i.next(); - const BSONObj& temp = e.Obj(); - - Client::Context ctx(temp["ns"].String()); - bool failed = applyOperation_inlock(temp, false, alwaysUpsert); - ab.append(!failed); - if ( failed ) - errors++; - - num++; - } - - result.append( "applied" , num ); - result.append( "results" , ab.arr() ); - - if ( ! fromRepl ) { - // We want this applied atomically on slaves - // so we re-wrap without the pre-condition for speed - - string tempNS = str::stream() << dbname << ".$cmd"; - - // TODO: possibly use mutable BSON to remove preCondition field - // once it is available - BSONObjIterator iter(cmdObj); - BSONObjBuilder cmdBuilder; - - while (iter.more()) { - BSONElement elem(iter.next()); - if (strcmp(elem.fieldName(), "preCondition") != 0) { - cmdBuilder.append(elem); - } - } - - logOp("c", tempNS.c_str(), cmdBuilder.done()); - } - - return errors == 0; - } - - DBDirectClient db; - - } applyOpsCmd; - -} diff --git a/src/mongo/db/oplog.h b/src/mongo/db/oplog.h deleted file mode 100644 index 1b004d2c6e0..00000000000 --- a/src/mongo/db/oplog.h +++ /dev/null @@ -1,173 +0,0 @@ -// oplog.h - writing to and reading from oplog - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -/* - - local.oplog.$main is the default -*/ - -#pragma once - -#include "pdfile.h" -#include "db.h" -#include "dbhelpers.h" -#include "clientcursor.h" -#include "../util/optime.h" -#include "../util/timer.h" - -namespace mongo { - - void createOplog(); - - void _logOpObjRS(const BSONObj& op); - - /** Write operation to the log (local.oplog.$main) - - @param opstr - "i" insert - "u" update - "d" delete - "c" db cmd - "n" no-op - "db" declares presence of a database (ns is set to the db name + '.') - - See _logOp() in oplog.cpp for more details. - */ - void logOp( const char *opstr, const char *ns, const BSONObj& obj, BSONObj *patt = 0, bool *b = 0, bool fromMigrate = false ); - - void logKeepalive(); - - /** puts obj in the oplog as a comment (a no-op). Just for diags. - convention is - { msg : "text", ... } - */ - void logOpComment(const BSONObj& obj); - - void oplogCheckCloseDatabase( Database * db ); - - class QueryPlan; - - /** Implements an optimized procedure for finding the first op in the oplog. */ - class FindingStartCursor { - public: - - /** - * The cursor will attempt to find the first op in the oplog matching the - * 'ts' field of the qp's query. - */ - static FindingStartCursor *make( const QueryPlan &qp ); - - /** @return true if the first matching op in the oplog has been found. */ - bool done() const { return !_findingStart; } - - /** @return cursor pointing to the first matching op, if done(). */ - shared_ptr cursor() { verify( done() ); return _c; } - - /** Iterate the cursor, to continue trying to find matching op. */ - void next(); - - /** Yield cursor, if not done(). */ - bool prepareToYield() { - if ( _findingStartCursor ) { - return _findingStartCursor->prepareToYield( _yieldData ); - } - return false; - } - - /** Recover from cursor yield. */ - void recoverFromYield() { - if ( _findingStartCursor ) { - if ( !ClientCursor::recoverFromYield( _yieldData ) ) { - _findingStartCursor.reset( 0 ); - msgassertedNoTrace( 15889, "FindingStartCursor::recoverFromYield() failed to recover" ); - } - } - } - - /** - * @return a BasicCursor constructed using a FindingStartCursor with the provided query and - * order parameters. - * @yields the db lock. - * @asserts on yield recovery failure. - */ - static shared_ptr getCursor( const char *ns, const BSONObj &query, const BSONObj &order ); - - /** - * @return the first record of the first nonempty extent preceding the extent containing - * @param rec, or DiskLoc() if there is no such record or the beginning of the - * collection is reached. - * public for testing - */ - DiskLoc prevExtentFirstLoc( const DiskLoc& rec ) const; - - /** For testing only. */ - - static int getInitialTimeout() { return _initialTimeout; } - static void setInitialTimeout( int timeout ) { _initialTimeout = timeout; } - - private: - FindingStartCursor( const QueryPlan &qp ); - void init(); - - enum FindingStartMode { Initial, FindExtent, InExtent }; - const QueryPlan &_qp; - bool _findingStart; - FindingStartMode _findingStartMode; - auto_ptr< CoveredIndexMatcher > _matcher; - Timer _findingStartTimer; - ClientCursor::Holder _findingStartCursor; - shared_ptr _c; - ClientCursor::YieldData _yieldData; - static int _initialTimeout; - - /** @return the first record of the extent containing @param rec. */ - DiskLoc extentFirstLoc( const DiskLoc &rec ); - - void createClientCursor( const DiskLoc &startLoc = DiskLoc() ); - void destroyClientCursor() { - _findingStartCursor.reset( 0 ); - } - bool firstDocMatchesOrEmpty() const; - }; - - class Sync { - protected: - string hn; - public: - Sync(const string& hostname) : hn(hostname) {} - virtual ~Sync() {} - virtual BSONObj getMissingDoc(const BSONObj& o); - - /** - * If applyOperation_inlock should be called again after an update fails. - */ - virtual bool shouldRetry(const BSONObj& o); - void setHostname(const string& hostname); - }; - - void pretouchOperation(const BSONObj& op); - void pretouchN(vector&, unsigned a, unsigned b); - - /** - * take an op and apply locally - * used for applying from an oplog - * @param fromRepl really from replication or for testing/internal/command/etc... - * Returns if the op was an update that could not be applied (true on failure) - */ - bool applyOperation_inlock(const BSONObj& op, bool fromRepl = true, bool convertUpdateToUpsert = false); -} diff --git a/src/mongo/db/oplogreader.h b/src/mongo/db/oplogreader.h deleted file mode 100644 index d5fd5169550..00000000000 --- a/src/mongo/db/oplogreader.h +++ /dev/null @@ -1,137 +0,0 @@ -/** @file oplogreader.h */ - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - - -#pragma once - -#include "../client/constants.h" -#include "dbhelpers.h" -#include "mongo/client/dbclientcursor.h" - -namespace mongo { - - /* started abstracting out the querying of the primary/master's oplog - still fairly awkward but a start. - */ - - class OplogReader { - shared_ptr _conn; - shared_ptr cursor; - bool _doHandshake; - int _tailingQueryOptions; - public: - OplogReader( bool doHandshake = true ); - ~OplogReader() { } - void resetCursor() { cursor.reset(); } - void resetConnection() { - cursor.reset(); - _conn.reset(); - } - DBClientConnection* conn() { return _conn.get(); } - BSONObj findOne(const char *ns, const Query& q) { - return conn()->findOne(ns, q, 0, QueryOption_SlaveOk); - } - BSONObj getLastOp(const char *ns) { - return findOne(ns, Query().sort(reverseNaturalObj)); - } - - /* ok to call if already connected */ - bool connect(const std::string& hostname); - - bool connect(const BSONObj& rid, const int from, const string& to); - - void tailCheck() { - if( cursor.get() && cursor->isDead() ) { - log() << "repl: old cursor isDead, will initiate a new one" << endl; - resetCursor(); - } - } - - bool haveCursor() { return cursor.get() != 0; } - - /** this is ok but commented out as when used one should consider if QueryOption_OplogReplay - is needed; if not fine, but if so, need to change. - *//* - void query(const char *ns, const BSONObj& query) { - verify( !haveCursor() ); - cursor.reset( _conn->query(ns, query, 0, 0, 0, QueryOption_SlaveOk).release() ); - }*/ - - /** this can be used; it is commented out as it does not indicate - QueryOption_OplogReplay and that is likely important. could be uncommented - just need to add that. - */ - /* - void queryGTE(const char *ns, OpTime t) { - BSONObjBuilder q; - q.appendDate("$gte", t.asDate()); - BSONObjBuilder q2; - q2.append("ts", q.done()); - query(ns, q2.done()); - } - */ - - void tailingQuery(const char *ns, const BSONObj& query, const BSONObj* fields=0); - - void tailingQueryGTE(const char *ns, OpTime t, const BSONObj* fields=0); - - /* Do a tailing query, but only send the ts field back. */ - void ghostQueryGTE(const char *ns, OpTime t) { - const BSONObj fields = BSON("ts" << 1 << "_id" << 0); - return tailingQueryGTE(ns, t, &fields); - } - - bool more() { - uassert( 15910, "Doesn't have cursor for reading oplog", cursor.get() ); - return cursor->more(); - } - - bool moreInCurrentBatch() { - uassert( 15911, "Doesn't have cursor for reading oplog", cursor.get() ); - return cursor->moreInCurrentBatch(); - } - - int currentBatchMessageSize() { - if( NULL == cursor->getMessage() ) - return 0; - return cursor->getMessage()->size(); - } - - /* old mongod's can't do the await flag... */ - bool awaitCapable() { - return cursor->hasResultFlag(ResultFlag_AwaitCapable); - } - - int getTailingQueryOptions() const { return _tailingQueryOptions; } - void setTailingQueryOptions( int tailingQueryOptions ) { _tailingQueryOptions = tailingQueryOptions; } - - void peek(vector& v, int n) { - if( cursor.get() ) - cursor->peek(v,n); - } - BSONObj nextSafe() { return cursor->nextSafe(); } - BSONObj next() { return cursor->next(); } - void putBack(BSONObj op) { cursor->putBack(op); } - - private: - /** @return true iff connection was successful */ - bool commonConnect(const string& hostName); - bool passthroughHandshake(const BSONObj& rid, const int f); - }; - -} diff --git a/src/mongo/db/ops/SConscript b/src/mongo/db/ops/SConscript new file mode 100644 index 00000000000..12add29b644 --- /dev/null +++ b/src/mongo/db/ops/SConscript @@ -0,0 +1,230 @@ +# -*- mode: python -*- + +Import("env") + +env.Library( + target='update_common', + source=[ + 'field_checker.cpp', + 'log_builder.cpp', + 'path_support.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/bson', + '$BUILD_DIR/mongo/db/common', + '$BUILD_DIR/mongo/foundation', + ], +) + +env.CppUnitTest( + target='field_checker_test', + source=[ + 'field_checker_test.cpp', + ], + LIBDEPS=[ + 'update_common', + ], +) + +env.CppUnitTest( + target='log_builder_test', + source=[ + 'log_builder_test.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update_common', + ], +) + +env.CppUnitTest( + target='path_support_test', + source=[ + 'path_support_test.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update_common', + ], +) + +env.Library( + target='update', + source=[ + 'modifier_add_to_set.cpp', + 'modifier_bit.cpp', + 'modifier_compare.cpp', + 'modifier_current_date.cpp', + 'modifier_inc.cpp', + 'modifier_object_replace.cpp', + 'modifier_pop.cpp', + 'modifier_pull.cpp', + 'modifier_pull_all.cpp', + 'modifier_push.cpp', + 'modifier_rename.cpp', + 'modifier_set.cpp', + 'modifier_unset.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/expressions', + 'update_common', + ], +) + +env.CppUnitTest( + target='modifier_add_to_set_test', + source='modifier_add_to_set_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_bit_test', + source='modifier_bit_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_compare_test', + source='modifier_compare_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_current_date_test', + source='modifier_current_date_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_inc_test', + source='modifier_inc_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_object_replace_test', + source='modifier_object_replace_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_pop_test', + source='modifier_pop_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_push_sorter_test', + source='modifier_push_sorter_test.cpp', + LIBDEPS=[ + 'update', + ], +) + +env.CppUnitTest( + target='modifier_pull_all_test', + source='modifier_pull_all_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_pull_test', + source='modifier_pull_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_push_test', + source='modifier_push_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_rename_test', + source='modifier_rename_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_set_test', + source='modifier_set_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_unset_test', + source='modifier_unset_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + 'update', + ], +) + +env.Library( + target='update_driver', + source=[ + 'modifier_table.cpp', + 'update_driver.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/bson', + '$BUILD_DIR/mongo/db/common', + '$BUILD_DIR/mongo/db/query/query_planner', + '$BUILD_DIR/mongo/update_index_data', + 'update', + ], +) + +env.CppUnitTest( + target='modifier_table_test', + source='modifier_table_test.cpp', + LIBDEPS=[ + 'update_driver', + ], +) + +env.CppUnitTest( + target='update_driver_test', + source='update_driver_test.cpp', + LIBDEPS=[ + '$BUILD_DIR/mongo/mutable_bson_test_utils', + '$BUILD_DIR/mongo/db/query/query_planner', + 'update_driver', + ], +) diff --git a/src/mongo/db/ops/count.cpp b/src/mongo/db/ops/count.cpp index c63b4e5d8c6..d6c8b62efcc 100644 --- a/src/mongo/db/ops/count.cpp +++ b/src/mongo/db/ops/count.cpp @@ -1,7 +1,7 @@ // count.cpp /** - * Copyright (C) 2008 10gen Inc. + * Copyright (C) 2013 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -14,119 +14,185 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "count.h" +#include "mongo/db/ops/count.h" -#include "../client.h" -#include "../clientcursor.h" -#include "../namespace.h" -#include "../queryutil.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/util/elapsed_tracker.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/curop.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/type_explain.h" -namespace mongo { +namespace { + + using namespace mongo; + + /** + * Ask 'runner' for a summary of the plan it is using to run the count command, + * and store this information in 'currentOp'. + * + * Returns true if the planSummary was copied to 'currentOp' and false otherwise. + */ + bool setPlanSummary(Runner* runner, CurOp* currentOp) { + if (NULL != currentOp) { + PlanInfo* rawInfo; + Status s = runner->getInfo(NULL, &rawInfo); + if (s.isOK()) { + scoped_ptr planInfo(rawInfo); + currentOp->debug().planSummary = planInfo->planSummary.c_str(); + return true; + } + } - namespace { + return false; + } - /** - * Specialized Cursor creation rules that the count operator provides to the query - * processing system. These rules limit the performance overhead when counting index keys - * matching simple predicates. See SERVER-1752. - */ - class CountPlanPolicies : public QueryPlanSelectionPolicy { +} // namespace + +namespace mongo { - virtual string name() const { return "CountPlanPolicies"; } + static long long applySkipLimit(long long num, const BSONObj& cmd) { + BSONElement s = cmd["skip"]; + BSONElement l = cmd["limit"]; - virtual bool requestMatcher() const { - // Avoid using a Matcher when a Cursor will exactly match a query. - return false; + if (s.isNumber()) { + num = num - s.numberLong(); + if (num < 0) { + num = 0; } + } - virtual bool requestIntervalCursor() const { - // Request use of an IntervalBtreeCursor when the index bounds represent a single - // btree interval. This Cursor implementation is optimized for performing counts - // between two endpoints. - return true; + if (l.isNumber()) { + long long limit = l.numberLong(); + if (limit < 0) { + limit = -limit; } - } _countPlanPolicies; + // 0 means no limit. + if (limit < num && limit != 0) { + num = limit; + } + } + return num; } - - long long runCount( const char *ns, const BSONObj &cmd, string &err, int &errCode ) { + + long long runCount( const string& ns, const BSONObj &cmd, string &err, int &errCode ) { + // Lock 'ns'. Client::Context cx(ns); - NamespaceDetails *d = nsdetails( ns ); - if ( !d ) { + Collection* collection = cx.db()->getCollection(ns); + + if (NULL == collection) { err = "ns missing"; return -1; } + BSONObj query = cmd.getObjectField("query"); - + + BSONObj hintObj; + if (Object == cmd["hint"].type()) { + hintObj = cmd["hint"].Obj(); + } + else if (String == cmd["hint"].type()) { + const std::string hint = cmd.getStringField("hint"); + hintObj = BSON("$hint" << hint); + } + // count of all objects - if ( query.isEmpty() ) { - return applySkipLimit( d->stats.nrecords , cmd ); + if (query.isEmpty()) { + return applySkipLimit(collection->numRecords(), cmd); } - - long long count = 0; + + Runner* rawRunner; long long skip = cmd["skip"].numberLong(); long long limit = cmd["limit"].numberLong(); - if( limit < 0 ){ - limit = -limit; + if (limit < 0) { + limit = -limit; + } + + uassertStatusOK(getRunnerCount(collection, query, hintObj, &rawRunner)); + auto_ptr runner(rawRunner); + + // Get a pointer to the current operation. We will try to copy the planSummary + // there so that it appears in db.currentOp() and the slow query log. + Client& client = cc(); + CurOp* currentOp = client.curop(); + + // Have we copied the planSummary to 'currentOp' yet? + bool gotPlanSummary = false; + + // Try to copy the plan summary to the 'currentOp'. + if (!gotPlanSummary) { + gotPlanSummary = setPlanSummary(runner.get(), currentOp); } - shared_ptr cursor = - NamespaceDetailsTransient::getCursor( ns, - query, - BSONObj(), - _countPlanPolicies ); - ClientCursor::Holder ccPointer; - ElapsedTracker timeToStartYielding( 256, 20 ); try { - while( cursor->ok() ) { - if ( !ccPointer ) { - if ( timeToStartYielding.intervalHasElapsed() ) { - // Lazily construct a ClientCursor, avoiding a performance regression when scanning a very - // small number of documents. - ccPointer.reset( new ClientCursor( QueryOption_NoCursorTimeout, cursor, ns ) ); - } + const ScopedRunnerRegistration safety(runner.get()); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + long long count = 0; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, NULL))) { + // Try to copy the plan summary to the 'currentOp'. We need to try again + // here because we might not have chosen a plan until after the first + // call to getNext(...). + if (!gotPlanSummary) { + gotPlanSummary = setPlanSummary(runner.get(), currentOp); } - else if ( !ccPointer->yieldSometimes( ClientCursor::MaybeCovered ) || - !cursor->ok() ) { - break; + + if (skip > 0) { + --skip; } - - if ( cursor->currentMatches() && !cursor->getsetdup( cursor->currLoc() ) ) { - - if ( skip > 0 ) { - --skip; - } - else { - ++count; - if ( limit > 0 && count >= limit ) { - break; - } + else { + ++count; + // Fast-path. There's no point in iterating all over the runner if limit + // is set. + if (count >= limit && limit != 0) { + break; } } - cursor->advance(); } - ccPointer.reset(); + + // Try to copy the plan summary to the 'currentOp', if we haven't already. This + // could happen if, for example, the count is 0. + if (!gotPlanSummary) { + gotPlanSummary = setPlanSummary(runner.get(), currentOp); + } + + // Emulate old behavior and return the count even if the runner was killed. This + // happens when the underlying collection is dropped. return count; - } - catch ( const DBException &e ) { + catch (const DBException &e) { err = e.toString(); errCode = e.getCode(); } - catch ( const std::exception &e ) { + catch (const std::exception &e) { err = e.what(); errCode = 0; } + // Historically we have returned zero in many count assertion cases - see SERVER-2291. log() << "Count with ns: " << ns << " and query: " << query << " failed with exception: " << err << " code: " << errCode << endl; + return -2; } diff --git a/src/mongo/db/ops/count.h b/src/mongo/db/ops/count.h index 624dd9e7b8d..5cfe801f979 100644 --- a/src/mongo/db/ops/count.h +++ b/src/mongo/db/ops/count.h @@ -1,7 +1,7 @@ // count.h /** - * Copyright (C) 2008 10gen Inc. + * Copyright (C) 2013 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -14,17 +14,32 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "../jsobj.h" -#include "../diskloc.h" +#include "mongo/db/jsobj.h" namespace mongo { - + /** + * 'ns' is the namespace we're counting on. + * * { count: "collectionname"[, query: ] } - * @return -1 on ns does not exist error and other errors, 0 on other errors, otherwise the match count. + * + * @return -1 on ns does not exist error and other errors, 0 on other errors, otherwise the + * match count. */ - long long runCount(const char *ns, const BSONObj& cmd, string& err, int& errCode ); - + long long runCount(const std::string& ns, const BSONObj& cmd, string& err, int& errCode ); + } // namespace mongo diff --git a/src/mongo/db/ops/delete.cpp b/src/mongo/db/ops/delete.cpp index 8d7b1cf9d3b..cbc4c392140 100644 --- a/src/mongo/db/ops/delete.cpp +++ b/src/mongo/db/ops/delete.cpp @@ -1,158 +1,52 @@ -// delete.cpp - /** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" -#include "delete.h" -#include "../queryutil.h" -#include "../oplog.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/util/stacktrace.h" + * Copyright (C) 2008 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/delete.h" + +#include "mongo/db/ops/delete_executor.h" +#include "mongo/db/ops/delete_request.h" namespace mongo { - + /* ns: namespace, e.g. . pattern: the "where" clause / criteria justOne: stop after 1 match god: allow access to system namespaces, and don't yield */ - long long deleteObjects(const char *ns, BSONObj pattern, bool justOne, bool logop, bool god, RemoveSaver * rs ) { - if( !god ) { - if ( strstr(ns, ".system.") ) { - /* note a delete from system.indexes would corrupt the db - if done here, as there are pointers into those objects in - NamespaceDetails. - */ - uassert(12050, "cannot delete from system namespace", legalClientSystemNS( ns , true ) ); - } - if ( strchr( ns , '$' ) ) { - log() << "cannot delete from collection with reserved $ in name: " << ns << endl; - uassert( 10100 , "cannot delete from collection with reserved $ in name", strchr(ns, '$') == 0 ); - } - } - - { - NamespaceDetails *d = nsdetails( ns ); - if ( ! d ) - return 0; - uassert( 10101 , "can't remove from a capped collection" , ! d->isCapped() ); - } - - long long nDeleted = 0; - - shared_ptr< Cursor > creal = NamespaceDetailsTransient::getCursor( ns, pattern ); - - if( !creal->ok() ) - return nDeleted; - - shared_ptr< Cursor > cPtr = creal; - auto_ptr cc( new ClientCursor( QueryOption_NoCursorTimeout, cPtr, ns) ); - cc->setDoingDeletes( true ); - - CursorId id = cc->cursorid(); - - bool canYield = !god && !(creal->matcher() && creal->matcher()->docMatcher().atomic()); - - do { - // TODO: we can generalize this I believe - // - bool willNeedRecord = (creal->matcher() && creal->matcher()->needRecord()) || pattern.isEmpty() || isSimpleIdQuery( pattern ); - if ( ! willNeedRecord ) { - // TODO: this is a total hack right now - // check if the index full encompasses query - - if ( pattern.nFields() == 1 && - str::equals( pattern.firstElement().fieldName() , creal->indexKeyPattern().firstElement().fieldName() ) ) - willNeedRecord = true; - } - - if ( canYield && ! cc->yieldSometimes( willNeedRecord ? ClientCursor::WillNeed : ClientCursor::MaybeCovered ) ) { - cc.release(); // has already been deleted elsewhere - // TODO should we assert or something? - break; - } - if ( !cc->ok() ) { - break; // if we yielded, could have hit the end - } - - // this way we can avoid calling prepareToYield() every time (expensive) - // as well as some other nuances handled - cc->setDoingDeletes( true ); - - DiskLoc rloc = cc->currLoc(); - BSONObj key = cc->currKey(); - - bool match = creal->currentMatches(); - - cc->advance(); - - if ( ! match ) - continue; - - // SERVER-5198 Advance past the document to be modified, but see SERVER-5725. - while( cc->ok() && rloc == cc->currLoc() ) { - cc->advance(); - } - - bool foundAllResults = ( justOne || !cc->ok() ); - - if ( !foundAllResults ) { - // NOTE: Saving and restoring a btree cursor's position was historically described - // as slow here. - cc->c()->prepareToTouchEarlierIterate(); - } - - if ( logop ) { - BSONElement e; - if( BSONObj::make( rloc.rec() ).getObjectID( e ) ) { - BSONObjBuilder b; - b.append( e ); - bool replJustOne = true; - logOp( "d", ns, b.done(), 0, &replJustOne ); - } - else { - problem() << "deleted object without id, not logging" << endl; - } - } - - theDataFileMgr.deleteRecord(ns, rloc.rec(), rloc); - nDeleted++; - if ( foundAllResults ) { - break; - } - cc->c()->recoverFromTouchingEarlierIterate(); - - if( !god ) - getDur().commitIfNeeded(); - - if( debug && god && nDeleted == 100 ) - log() << "warning high number of deletes with god=true which could use significant memory" << endl; - } - while ( cc->ok() ); - - if ( cc.get() && ClientCursor::find( id , false ) == 0 ) { - // TODO: remove this and the id declaration above if this doesn't trigger - // if it does, then i'm very confused (ERH 06/2011) - error() << "this should be impossible" << endl; - printStackTrace(); - cc.release(); - } - - return nDeleted; + long long deleteObjects(const StringData& ns, BSONObj pattern, bool justOne, bool logop, bool god) { + NamespaceString nsString(ns); + DeleteRequest request(nsString); + request.setQuery(pattern); + request.setMulti(!justOne); + request.setUpdateOpLog(logop); + request.setGod(god); + DeleteExecutor executor(&request); + return executor.execute(); } -} +} // namespace mongo diff --git a/src/mongo/db/ops/delete.h b/src/mongo/db/ops/delete.h index 1cf0fbb78ad..c1ebe4f2211 100644 --- a/src/mongo/db/ops/delete.h +++ b/src/mongo/db/ops/delete.h @@ -1,33 +1,44 @@ // delete.h /** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ + * Copyright (C) 2008 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #pragma once -#include "mongo/pch.h" -#include "../jsobj.h" -#include "../clientcursor.h" +#include "mongo/db/jsobj.h" namespace mongo { - class RemoveSaver; - // If justOne is true, deletedId is set to the id of the deleted object. - long long deleteObjects(const char *ns, BSONObj pattern, bool justOne, bool logop = false, bool god=false, RemoveSaver * rs=0); - + long long deleteObjects(const StringData& ns, + BSONObj pattern, + bool justOne, + bool logop = false, + bool god = false); } diff --git a/src/mongo/db/ops/delete_executor.cpp b/src/mongo/db/ops/delete_executor.cpp new file mode 100644 index 00000000000..c5afe38374a --- /dev/null +++ b/src/mongo/db/ops/delete_executor.cpp @@ -0,0 +1,198 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ops/delete_executor.h" + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/curop.h" +#include "mongo/db/ops/delete_request.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/lite_parsed_query.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + DeleteExecutor::DeleteExecutor(const DeleteRequest* request) : + _request(request), + _canonicalQuery(), + _isQueryParsed(false) { + } + + DeleteExecutor::~DeleteExecutor() {} + + Status DeleteExecutor::prepare() { + if (_isQueryParsed) + return Status::OK(); + + dassert(!_canonicalQuery.get()); + + if (CanonicalQuery::isSimpleIdQuery(_request->getQuery())) { + _isQueryParsed = true; + return Status::OK(); + } + + CanonicalQuery* cqRaw; + Status status = CanonicalQuery::canonicalize(_request->getNamespaceString().ns(), + _request->getQuery(), + &cqRaw); + if (status.isOK()) { + _canonicalQuery.reset(cqRaw); + _isQueryParsed = true; + } + else if (status == ErrorCodes::NoClientContext) { + // _isQueryParsed is still false, but execute() will try again under the lock. + status = Status::OK(); + } + return status; + } + + long long DeleteExecutor::execute() { + uassertStatusOK(prepare()); + uassert(17417, + mongoutils::str::stream() << + "DeleteExecutor::prepare() failed to parse query " << _request->getQuery(), + _isQueryParsed); + const bool logop = _request->shouldCallLogOp(); + const NamespaceString& ns(_request->getNamespaceString()); + if (!_request->isGod()) { + if (ns.isSystem()) { + uassert(12050, + "cannot delete from system namespace", + legalClientSystemNS(ns.ns(), true)); + } + if (ns.ns().find('$') != string::npos) { + log() << "cannot delete from collection with reserved $ in name: " << ns << endl; + uasserted( 10100, "cannot delete from collection with reserved $ in name" ); + } + } + + massert(17418, + mongoutils::str::stream() << + "dbname = " << currentClient.get()->database()->name() << + "; ns = " << ns.ns(), + currentClient.get()->database()->name() == nsToDatabaseSubstring(ns.ns())); + Collection* collection = currentClient.get()->database()->getCollection(ns.ns()); + if (NULL == collection) { + return 0; + } + + uassert(10101, + str::stream() << "cannot remove from a capped collection: " << ns.ns(), + !collection->isCapped()); + + uassert(ErrorCodes::NotMaster, + str::stream() << "Not primary while removing from " << ns.ns(), + !logop || isMasterNs(ns.ns().c_str())); + + long long nDeleted = 0; + + const bool canYield = !_request->isGod() && ( + _canonicalQuery.get() ? + !QueryPlannerCommon::hasNode(_canonicalQuery->root(), MatchExpression::ATOMIC) : + LiteParsedQuery::isQueryIsolated(_request->getQuery())); + + Runner* rawRunner; + if (_canonicalQuery.get()) { + uassertStatusOK(getRunner(collection, _canonicalQuery.release(), &rawRunner)); + } + else { + CanonicalQuery* ignored; + uassertStatusOK(getRunner(collection, + ns.ns(), + _request->getQuery(), + &rawRunner, + &ignored)); + } + + auto_ptr runner(rawRunner); + auto_ptr safety; + + if (canYield) { + safety.reset(new ScopedRunnerRegistration(runner.get())); + runner->setYieldPolicy(Runner::YIELD_AUTO); + } + + DiskLoc rloc; + Runner::RunnerState state; + CurOp* curOp = cc().curop(); + int oldYieldCount = curOp->numYields(); + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &rloc))) { + if (oldYieldCount != curOp->numYields()) { + uassert(ErrorCodes::NotMaster, + str::stream() << "No longer primary while removing from " << ns.ns(), + !logop || isMasterNs(ns.ns().c_str())); + oldYieldCount = curOp->numYields(); + } + BSONObj toDelete; + + // TODO: do we want to buffer docs and delete them in a group rather than + // saving/restoring state repeatedly? + runner->saveState(); + collection->deleteDocument(rloc, false, false, logop ? &toDelete : NULL ); + runner->restoreState(); + + nDeleted++; + + if (logop) { + if ( toDelete.isEmpty() ) { + problem() << "deleted object without id, not logging" << endl; + } + else { + bool replJustOne = true; + logOp("d", ns.ns().c_str(), toDelete, 0, &replJustOne); + } + } + + if (!_request->isMulti()) { + break; + } + + if (!_request->isGod()) { + getDur().commitIfNeeded(); + } + + if (debug && _request->isGod() && nDeleted == 100) { + log() << "warning high number of deletes with god=true " + << " which could use significant memory b/c we don't commit journal"; + } + } + + return nDeleted; + } + +} // namespace mongo diff --git a/src/mongo/db/ops/delete_executor.h b/src/mongo/db/ops/delete_executor.h new file mode 100644 index 00000000000..9e32ba25044 --- /dev/null +++ b/src/mongo/db/ops/delete_executor.h @@ -0,0 +1,106 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" + +namespace mongo { + + class CanonicalQuery; + class DeleteRequest; + + /** + * Implementation of the processing of a delete operation in a mongod. + * + * The executor has two important methods, prepare() and execute(). The prepare() method can + * run without locks, and does whatever parsing and precomputation can be done without access to + * database data. The execute method performs the delete, but the caller must already hold the + * appropriate database lock. + * + * Expected usage is approximately: + * DeleteRequest request(...); + * // configure request + * DeleteExecutor executor(&request); + * uassertStatusOK(executor.prepare()); + * // Get locks, get ready to execute. + * try { + * long long nDeleted = executor.execute(); + * } + * catch (const DBException& ex) { + * // Error handling. + * } + */ + class DeleteExecutor { + MONGO_DISALLOW_COPYING(DeleteExecutor); + public: + /** + * Constructs a delete executor. + * + * The object pointed to by "request" must stay in scope for the life of the constructed + * executor. + */ + explicit DeleteExecutor(const DeleteRequest* request); + + ~DeleteExecutor(); + + /** + * Performs preparatory work that does not require database locks. + * + * Returns Status::OK() on success. Other results indicate that the executor will not run + * correctly, and should be abandoned. + * + * Calling prepare() is optional. It is available for situations in which the user + * wishes to do as much work as possible before acquiring database locks. + */ + Status prepare(); + + /** + * Execute a delete. Requires the caller to hold the database lock on the + * appropriate resources for the request. + * + * Returns the number of documents deleted. + */ + long long execute(); + + private: + /// Unowned pointer to the request object that this executor will process. + const DeleteRequest* const _request; + + /// Parsed query object, or NULL if the query proves to be an id hack query. + std::auto_ptr _canonicalQuery; + + /// Flag indicating if the query has been successfully parsed. + bool _isQueryParsed; + + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/delete_request.h b/src/mongo/db/ops/delete_request.h new file mode 100644 index 00000000000..4e10cb409c2 --- /dev/null +++ b/src/mongo/db/ops/delete_request.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { + + class DeleteRequest { + MONGO_DISALLOW_COPYING(DeleteRequest); + public: + explicit DeleteRequest(const NamespaceString& nsString) : + _nsString(nsString), + _multi(false), + _logop(false), + _god(false) {} + + void setQuery(const BSONObj& query) { _query = query; } + void setMulti(bool multi = true) { _multi = multi; } + void setUpdateOpLog(bool logop = true) { _logop = logop; } + void setGod(bool god = true) { _god = god; } + + const NamespaceString& getNamespaceString() const { return _nsString; } + const BSONObj& getQuery() const { return _query; } + bool isMulti() const { return _multi; } + bool shouldCallLogOp() const { return _logop; } + bool isGod() const { return _god; } + + std::string toString() const; + + private: + const NamespaceString& _nsString; + BSONObj _query; + bool _multi; + bool _logop; + bool _god; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/field_checker.cpp b/src/mongo/db/ops/field_checker.cpp new file mode 100644 index 00000000000..da6607ac229 --- /dev/null +++ b/src/mongo/db/ops/field_checker.cpp @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/field_checker.h" + +#include "mongo/base/error_codes.h" +#include "mongo/db/field_ref.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + using mongoutils::str::stream; + +namespace fieldchecker { + + Status isUpdatable(const FieldRef& field) { + const size_t numParts = field.numParts(); + + if (numParts == 0) { + return Status(ErrorCodes::EmptyFieldName, + "An empty update path is not valid."); + } + + for (size_t i = 0; i != numParts; ++i) { + const StringData part = field.getPart(i); + + if (part.empty()) { + return Status(ErrorCodes::EmptyFieldName, + mongoutils::str::stream() << "The update path '" + << field.dottedField() + << "' contains an empty field name, which is not allowed."); + } + } + + return Status::OK(); + } + + bool isPositional(const FieldRef& fieldRef, size_t* pos, size_t* count) { + + // 'count' is optional. + size_t dummy; + if (count == NULL) { + count = &dummy; + } + + *count = 0; + size_t size = fieldRef.numParts(); + for (size_t i=0; i 0; + } + +} // namespace fieldchecker +} // namespace mongo diff --git a/src/mongo/db/ops/field_checker.h b/src/mongo/db/ops/field_checker.h new file mode 100644 index 00000000000..544ea131074 --- /dev/null +++ b/src/mongo/db/ops/field_checker.h @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" + +namespace mongo { + + class FieldRef; + + namespace fieldchecker { + + /** + * Returns OK if all the below conditions on 'field' are valid: + * + Non-empty + * + Does not start or end with a '.' + * Otherwise returns a code indicating cause of failure. + */ + Status isUpdatable(const FieldRef& field); + + /** + * Returns true, the position 'pos' of the first $-sign if present in 'fieldRef', and + * how many other $-signs were found in 'count'. Otherwise return false. + * + * Note: + * isPositional assumes that the field is updatable. Call isUpdatable() above to + * verify. + */ + bool isPositional(const FieldRef& fieldRef, size_t* pos, size_t* count = NULL); + + } // namespace fieldchecker + +} // namespace mongo diff --git a/src/mongo/db/ops/field_checker_test.cpp b/src/mongo/db/ops/field_checker_test.cpp new file mode 100644 index 00000000000..5b49d721fa7 --- /dev/null +++ b/src/mongo/db/ops/field_checker_test.cpp @@ -0,0 +1,94 @@ +/** + * Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/ops/field_checker.h" + +#include "mongo/base/error_codes.h" +#include "mongo/base/status.h" +#include "mongo/db/field_ref.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::ErrorCodes; + using mongo::FieldRef; + using mongo::fieldchecker::isUpdatable; + using mongo::fieldchecker::isPositional; + using mongo::Status; + + TEST(IsUpdatable, Basics) { + FieldRef fieldRef("x"); + ASSERT_OK(isUpdatable(fieldRef)); + } + + TEST(IsUpdatable, DottedFields) { + FieldRef fieldRef("x.y.z"); + ASSERT_OK(isUpdatable(fieldRef)); + } + + TEST(IsUpdatable, EmptyFields) { + FieldRef fieldRef(""); + ASSERT_NOT_OK(isUpdatable(fieldRef)); + + FieldRef fieldRefDot("."); + ASSERT_NOT_OK(isUpdatable(fieldRefDot)); + + /* TODO: Re-enable after review + FieldRef fieldRefDollar; + fieldRefDollar.parse("$"); + ASSERT_NOT_OK(isUpdatable(fieldRefDollar)); + +*/ + + FieldRef fieldRefADot("a."); + ASSERT_NOT_OK(isUpdatable(fieldRefADot)); + + FieldRef fieldRefDotB(".b"); + ASSERT_NOT_OK(isUpdatable(fieldRefDotB)); + + FieldRef fieldRefEmptyMiddle; + fieldRefEmptyMiddle.parse("a..b"); + ASSERT_NOT_OK(isUpdatable(fieldRefEmptyMiddle)); + } + + // Positional checks + TEST(isPositional, EntireArrayItem) { + FieldRef fieldRefPositional("a.$"); + size_t pos; + size_t count; + ASSERT_TRUE(isPositional(fieldRefPositional, &pos, &count)); + ASSERT_EQUALS(pos, 1u); + ASSERT_EQUALS(count, 1u); + } + + TEST(isPositional, ArraySubObject) { + FieldRef fieldRefPositional("a.$.b"); + size_t pos; + size_t count; + ASSERT_TRUE(isPositional(fieldRefPositional, &pos, &count)); + ASSERT_EQUALS(pos, 1u); + ASSERT_EQUALS(count, 1u); + } + + TEST(isPositional, MultiplePositional) { + FieldRef fieldRefPositional("a.$.b.$.c"); + size_t pos; + size_t count; + ASSERT_TRUE(isPositional(fieldRefPositional, &pos, &count)); + ASSERT_EQUALS(pos, 1u); + ASSERT_EQUALS(count, 2u); + } +} // unnamed namespace diff --git a/src/mongo/db/ops/insert.cpp b/src/mongo/db/ops/insert.cpp new file mode 100644 index 00000000000..04f43d3a3a6 --- /dev/null +++ b/src/mongo/db/ops/insert.cpp @@ -0,0 +1,206 @@ +// insert.cpp + +/** + * Copyright (C) 2008 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/insert.h" +#include "mongo/db/structure/catalog/namespace.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + using namespace mongoutils; + + StatusWith fixDocumentForInsert( const BSONObj& doc ) { + if ( doc.objsize() > BSONObjMaxUserSize ) + return StatusWith( ErrorCodes::BadValue, + str::stream() + << "object to insert too large" + << ". size in bytes: " << doc.objsize() + << ", max size: " << BSONObjMaxUserSize ); + + bool firstElementIsId = doc.firstElement().fieldNameStringData() == "_id"; + bool hasTimestampToFix = false; + { + BSONObjIterator i( doc ); + while ( i.more() ) { + BSONElement e = i.next(); + + if ( e.type() == Timestamp && e.timestampValue() == 0 ) { + // we replace Timestamp(0,0) at the top level with a correct value + // in the fast pass, we just mark that we want to swap + hasTimestampToFix = true; + } + + const char* fieldName = e.fieldName(); + + if ( fieldName[0] == '$' ) { + return StatusWith( ErrorCodes::BadValue, + str::stream() + << "Document can't have $ prefixed field names: " + << e.fieldName() ); + } + + // check no regexp for _id (SERVER-9502) + // also, disallow undefined and arrays + if ( str::equals( fieldName, "_id") ) { + if ( e.type() == RegEx ) { + return StatusWith( ErrorCodes::BadValue, + "can't use a regex for _id" ); + } + if ( e.type() == Undefined ) { + return StatusWith( ErrorCodes::BadValue, + "can't use a undefined for _id" ); + } + if ( e.type() == Array ) { + return StatusWith( ErrorCodes::BadValue, + "can't use an array for _id" ); + } + if ( e.type() == Object ) { + BSONObj o = e.Obj(); + Status s = o.storageValidEmbedded(); + if ( !s.isOK() ) + return StatusWith( s ); + } + } + + } + } + + if ( firstElementIsId && !hasTimestampToFix ) + return StatusWith( BSONObj() ); + + bool hadId = firstElementIsId; + + BSONObjIterator i( doc ); + + BSONObjBuilder b( doc.objsize() + 16 ); + if ( firstElementIsId ) { + b.append( doc.firstElement() ); + i.next(); + } + else { + BSONElement e = doc["_id"]; + if ( e.type() ) { + b.append( e ); + hadId = true; + } + else { + b.appendOID( "_id", NULL, true ); + } + } + + while ( i.more() ) { + BSONElement e = i.next(); + if ( hadId && e.fieldNameStringData() == "_id" ) { + // no-op + } + else if ( e.type() == Timestamp && e.timestampValue() == 0 ) { + mutex::scoped_lock lk(OpTime::m); + b.append( e.fieldName(), OpTime::now(lk) ); + } + else { + b.append( e ); + } + } + return StatusWith( b.obj() ); + } + + Status userAllowedWriteNS( const StringData& ns ) { + return userAllowedWriteNS( nsToDatabaseSubstring( ns ), nsToCollectionSubstring( ns ) ); + } + + Status userAllowedWriteNS( const NamespaceString& ns ) { + return userAllowedWriteNS( ns.db(), ns.coll() ); + } + + Status userAllowedWriteNS( const StringData& db, const StringData& coll ) { + if ( coll == "system.profile" ) { + return Status( ErrorCodes::BadValue, + str::stream() << "cannot write to '" << db << ".system.profile'" ); + } + return userAllowedCreateNS( db, coll ); + } + + Status userAllowedCreateNS( const StringData& db, const StringData& coll ) { + // validity checking + + if ( db.size() == 0 ) + return Status( ErrorCodes::BadValue, "db cannot be blank" ); + + if ( !NamespaceString::validDBName( db ) ) + return Status( ErrorCodes::BadValue, "invalid db name" ); + + if ( coll.size() == 0 ) + return Status( ErrorCodes::BadValue, "collection cannot be blank" ); + + if ( !NamespaceString::validCollectionName( coll ) ) + return Status( ErrorCodes::BadValue, "invalid collection name" ); + + if ( db.size() + 1 /* dot */ + coll.size() > Namespace::MaxNsColletionLen ) + return Status( ErrorCodes::BadValue, + str::stream() + << "fully qualified namespace " << db << '.' << coll << " is too long " + << "(max is " << Namespace::MaxNsColletionLen << " bytes)" ); + + // check spceial areas + + if ( db == "system" ) + return Status( ErrorCodes::BadValue, "cannot use 'system' database" ); + + + if ( coll.startsWith( "system." ) ) { + if ( coll == "system.indexes" ) return Status::OK(); + if ( coll == "system.js" ) return Status::OK(); + if ( coll == "system.profile" ) return Status::OK(); + if ( coll == "system.users" ) return Status::OK(); + if ( db == "admin" ) { + if ( coll == "system.version" ) return Status::OK(); + if ( coll == "system.roles" ) return Status::OK(); + if ( coll == "system.new_users" ) return Status::OK(); + if ( coll == "system.backup_users" ) return Status::OK(); + } + if ( db == "local" ) { + if ( coll == "system.replset" ) return Status::OK(); + } + return Status( ErrorCodes::BadValue, + str::stream() << "cannot write to '" << db << "." << coll << "'" ); + } + + // some special rules + + if ( coll.find( ".system." ) != string::npos ) { + // this matches old (2.4 and older) behavior, but I'm not sure its a good idea + return Status( ErrorCodes::BadValue, + str::stream() << "cannot write to '" << db << "." << coll << "'" ); + } + + return Status::OK(); + } + +} diff --git a/src/mongo/db/ops/insert.h b/src/mongo/db/ops/insert.h new file mode 100644 index 00000000000..b449bfb6301 --- /dev/null +++ b/src/mongo/db/ops/insert.h @@ -0,0 +1,57 @@ +// insert.h + +/** + * Copyright (C) 2008 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" + +namespace mongo { + + /** + * if doc is ok, then return is BSONObj() + * otherwise, BSONObj is what should be inserted instead + */ + StatusWith fixDocumentForInsert( const BSONObj& doc ); + + + /** + * Returns Status::OK() if this namespace is valid for user write operations. If not, returns + * an error Status. + */ + Status userAllowedWriteNS( const StringData& db, const StringData& coll ); + Status userAllowedWriteNS( const StringData& ns ); + Status userAllowedWriteNS( const NamespaceString& ns ); + + /** + * Returns Status::OK() if the namespace described by (db, coll) is valid for user create + * operations. If not, returns an error Status. + */ + Status userAllowedCreateNS( const StringData& db, const StringData& coll ); + +} diff --git a/src/mongo/db/ops/log_builder.cpp b/src/mongo/db/ops/log_builder.cpp new file mode 100644 index 00000000000..8b58de50f1c --- /dev/null +++ b/src/mongo/db/ops/log_builder.cpp @@ -0,0 +1,173 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/log_builder.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + using mutablebson::Document; + using mutablebson::Element; + namespace str = mongoutils::str; + + namespace { + const char kSet[] = "$set"; + const char kUnset[] = "$unset"; + } // namespace + + inline Status LogBuilder::addToSection(Element newElt, + Element* section, + const char* sectionName) { + + // If we don't already have this section, try to create it now. + if (!section->ok()) { + + // If we already have object replacement data, we can't also have section entries. + if (hasObjectReplacement()) + return Status( + ErrorCodes::IllegalOperation, + "LogBuilder: Invalid attempt to add a $set/$unset entry" + "to a log with an existing object replacement"); + + Document& doc = _logRoot.getDocument(); + + // We should not already have an element with the section name under the root. + dassert(_logRoot[sectionName] == doc.end()); + + // Construct a new object element to represent this section in the log. + const Element newElement = doc.makeElementObject(sectionName); + if (!newElement.ok()) + return Status(ErrorCodes::InternalError, + "LogBuilder: failed to construct Object Element for $set/$unset"); + + // Enqueue the new section under the root, and record it as our out parameter. + Status result = _logRoot.pushBack(newElement); + if (!result.isOK()) + return result; + *section = newElement; + + // Invalidate attempts to add an object replacement, now that we have a named + // section under the root. + _objectReplacementAccumulator = doc.end(); + } + + // Whatever transpired, we should now have an ok accumulator for the section, and not + // have a replacement accumulator. + dassert(section->ok()); + dassert(!_objectReplacementAccumulator.ok()); + + // Enqueue the provided element to the section and propagate the result. + return section->pushBack(newElt); + } + + Status LogBuilder::addToSets(Element elt) { + return addToSection(elt, &_setAccumulator, kSet); + } + + Status LogBuilder::addToSetsWithNewFieldName(const StringData& name, + const mutablebson::Element val) { + mutablebson::Element elemToSet = + _logRoot.getDocument().makeElementWithNewFieldName(name, val); + if (!elemToSet.ok()) + return Status(ErrorCodes::InternalError, + str::stream() << "Could not create new '" + << name << "' element from existing element '" + << val.getFieldName() << "' of type " + << typeName(val.getType())); + + return addToSets(elemToSet); + } + + Status LogBuilder::addToSetsWithNewFieldName(const StringData& name, + const BSONElement& val){ + mutablebson::Element elemToSet = + _logRoot.getDocument().makeElementWithNewFieldName(name, val); + if (!elemToSet.ok()) + return Status(ErrorCodes::InternalError, + str::stream() << "Could not create new '" + << name << "' element from existing element '" + << val.fieldName() << "' of type " + << typeName(val.type())); + + return addToSets(elemToSet); + } + + Status LogBuilder::addToSets(const StringData& name, const SafeNum& val){ + mutablebson::Element elemToSet = _logRoot.getDocument().makeElementSafeNum(name, val); + if (!elemToSet.ok()) + return Status(ErrorCodes::InternalError, + str::stream() << "Could not create new '" + << name << "' SafeNum from " + << val.debugString()); + + return addToSets(elemToSet); + } + + Status LogBuilder::addToUnsets(StringData path) { + mutablebson::Element logElement = _logRoot.getDocument().makeElementBool(path, true); + if (!logElement.ok()) + return Status(ErrorCodes::InternalError, + str::stream() << "Cannot create $unset oplog entry for path" << path); + + return addToSection(logElement, &_unsetAccumulator, kUnset); + } + + Status LogBuilder::getReplacementObject(Element* outElt) { + + // If the replacement accumulator is not ok, we must have started a $set or $unset + // already, so an object replacement is not permitted. + if (!_objectReplacementAccumulator.ok()) { + dassert(_setAccumulator.ok() || _unsetAccumulator.ok()); + return Status( + ErrorCodes::IllegalOperation, + "LogBuilder: Invalid attempt to obtain the object replacement slot " + "for a log containing $set or $unset entries"); + } + + if (hasObjectReplacement()) + return Status( + ErrorCodes::IllegalOperation, + "LogBuilder: Invalid attempt to acquire the replacement object " + "in a log with existing object replacement data"); + + // OK to enqueue object replacement items. + *outElt = _objectReplacementAccumulator; + return Status::OK(); + } + + inline bool LogBuilder::hasObjectReplacement() const { + if (!_objectReplacementAccumulator.ok()) + return false; + + dassert(!_setAccumulator.ok()); + dassert(!_unsetAccumulator.ok()); + + return _objectReplacementAccumulator.hasChildren(); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/log_builder.h b/src/mongo/db/ops/log_builder.h new file mode 100644 index 00000000000..7f422e06224 --- /dev/null +++ b/src/mongo/db/ops/log_builder.h @@ -0,0 +1,122 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +namespace mongo { + + /** LogBuilder abstracts away some of the details of producing a properly constructed oplog + * update entry. It manages separate regions into which it accumulates $set and $unset + * operations, and distinguishes object replacement style oplog generation from + * $set/$unset style generation and prevents admixture. + */ + class LogBuilder { + public: + /** Construct a new LogBuilder. Log entries will be recorded as new children under the + * 'logRoot' Element, which must be of type mongo::Object and have no children. + */ + inline LogBuilder(mutablebson::Element logRoot) + : _logRoot(logRoot) + , _objectReplacementAccumulator(_logRoot) + , _setAccumulator(_logRoot.getDocument().end()) + , _unsetAccumulator(_setAccumulator) { + dassert(logRoot.isType(mongo::Object)); + dassert(!logRoot.hasChildren()); + } + + /** Return the Document to which the logging root belongs. */ + inline mutablebson::Document& getDocument() { + return _logRoot.getDocument(); + } + + /** Add the given Element as a new entry in the '$set' section of the log. If a $set + * section does not yet exist, it will be created. If this LogBuilder is currently + * configured to contain an object replacement, the request to add to the $set section + * will return an Error. + */ + Status addToSets(mutablebson::Element elt); + + /** + * Convenience method which calls addToSets after + * creating a new Element to wrap the SafeNum value. + * + * If any problem occurs then the operation will stop and return that error Status. + */ + Status addToSets(const StringData& name, const SafeNum& val); + + /** + * Convenience method which calls addToSets after + * creating a new Element to wrap the old one. + * + * If any problem occurs then the operation will stop and return that error Status. + */ + Status addToSetsWithNewFieldName(const StringData& name, const mutablebson::Element val); + + /** + * Convenience method which calls addToSets after + * creating a new Element to wrap the old one. + * + * If any problem occurs then the operation will stop and return that error Status. + */ + Status addToSetsWithNewFieldName(const StringData& name, const BSONElement& val); + + /** Add the given path as a new entry in the '$unset' section of the log. If an + * '$unset' section does not yet exist, it will be created. If this LogBuilder is + * currently configured to contain an object replacement, the request to add to the + * $unset section will return an Error. + */ + Status addToUnsets(StringData path); + + /** Obtain, via the out parameter 'outElt', a pointer to the mongo::Object type Element + * to which the components of an object replacement should be recorded. It is an error + * to call this if any Elements have been added by calling either addToSets or + * addToUnsets, and attempts to do so will return a non-OK Status. Similarly, if there + * is already object replacement data recorded for this log, the call will fail. + */ + Status getReplacementObject(mutablebson::Element* outElt); + + private: + // Returns true if the object replacement accumulator is valid and has children, false + // otherwise. + inline bool hasObjectReplacement() const; + + inline Status addToSection( + mutablebson::Element newElt, + mutablebson::Element* section, + const char* sectionName); + + mutablebson::Element _logRoot; + mutablebson::Element _objectReplacementAccumulator; + mutablebson::Element _setAccumulator; + mutablebson::Element _unsetAccumulator; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/log_builder_test.cpp b/src/mongo/db/ops/log_builder_test.cpp new file mode 100644 index 00000000000..8b5ee877ecf --- /dev/null +++ b/src/mongo/db/ops/log_builder_test.cpp @@ -0,0 +1,266 @@ +/** + * Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/ops/log_builder.h" + +#include "mongo/base/status.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/json.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/safe_num.h" + +namespace { + + namespace mmb = mongo::mutablebson; + using mongo::LogBuilder; + + TEST(LogBuilder, Initialization) { + mmb::Document doc; + LogBuilder lb(doc.root()); + ASSERT_EQUALS(&doc, &lb.getDocument()); + } + + TEST(LogBuilder, AddOneToSet) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + const mmb::Element elt_ab = doc.makeElementInt("a.b", 1); + ASSERT_TRUE(elt_ab.ok()); + ASSERT_OK(lb.addToSets(elt_ab)); + + ASSERT_EQUALS(mongo::fromjson("{ $set : { 'a.b' : 1 } }"), doc); + } + + TEST(LogBuilder, AddElementToSet) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + const mmb::Element elt_ab = doc.makeElementInt("", 1); + ASSERT_TRUE(elt_ab.ok()); + ASSERT_OK(lb.addToSetsWithNewFieldName("a.b", elt_ab)); + + ASSERT_EQUALS(mongo::fromjson("{ $set : { 'a.b' : 1 } }"), doc); + } + + TEST(LogBuilder, AddBSONElementToSet) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mongo::BSONObj obj = mongo::fromjson("{'':1}"); + + ASSERT_OK(lb.addToSetsWithNewFieldName("a.b", obj.firstElement())); + + ASSERT_EQUALS(mongo::fromjson("{ $set : { 'a.b' : 1 } }"), doc); + } + + TEST(LogBuilder, AddSafeNumToSet) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mongo::BSONObj obj = mongo::fromjson("{'':1}"); + + ASSERT_OK(lb.addToSets("a.b", mongo::SafeNum(1))); + + ASSERT_EQUALS(mongo::fromjson("{ $set : { 'a.b' : 1 } }"), doc); + } + + TEST(LogBuilder, AddOneToUnset) { + mmb::Document doc; + LogBuilder lb(doc.root()); + ASSERT_OK(lb.addToUnsets("x.y")); + ASSERT_EQUALS(mongo::fromjson("{ $unset : { 'x.y' : true } }"), doc); + } + + TEST(LogBuilder, AddOneToEach) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + const mmb::Element elt_ab = doc.makeElementInt("a.b", 1); + ASSERT_TRUE(elt_ab.ok()); + ASSERT_OK(lb.addToSets(elt_ab)); + + ASSERT_OK(lb.addToUnsets("x.y")); + + ASSERT_EQUALS( + mongo::fromjson( + "{ " + " $set : { 'a.b' : 1 }, " + " $unset : { 'x.y' : true } " + "}" + ), doc); + } + + TEST(LogBuilder, AddOneObjectReplacementEntry) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mmb::Element replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_OK(lb.getReplacementObject(&replacement)); + ASSERT_TRUE(replacement.ok()); + ASSERT_TRUE(replacement.isType(mongo::Object)); + + const mmb::Element elt_a = doc.makeElementInt("a", 1); + ASSERT_TRUE(elt_a.ok()); + ASSERT_OK(replacement.pushBack(elt_a)); + + ASSERT_EQUALS(mongo::fromjson("{ a : 1 }"), doc); + } + + TEST(LogBuilder, AddTwoObjectReplacementEntry) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mmb::Element replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_OK(lb.getReplacementObject(&replacement)); + ASSERT_TRUE(replacement.ok()); + ASSERT_TRUE(replacement.isType(mongo::Object)); + + const mmb::Element elt_a = doc.makeElementInt("a", 1); + ASSERT_TRUE(elt_a.ok()); + ASSERT_OK(replacement.pushBack(elt_a)); + + const mmb::Element elt_b = doc.makeElementInt("b", 2); + ASSERT_TRUE(elt_b.ok()); + ASSERT_OK(replacement.pushBack(elt_b)); + + ASSERT_EQUALS(mongo::fromjson("{ a : 1, b: 2 }"), doc); + } + + TEST(LogBuilder, VerifySetsAreGrouped) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + const mmb::Element elt_ab = doc.makeElementInt("a.b", 1); + ASSERT_TRUE(elt_ab.ok()); + ASSERT_OK(lb.addToSets(elt_ab)); + + const mmb::Element elt_xy = doc.makeElementInt("x.y", 1); + ASSERT_TRUE(elt_xy.ok()); + ASSERT_OK(lb.addToSets(elt_xy)); + + ASSERT_EQUALS( + mongo::fromjson( + "{ $set : {" + " 'a.b' : 1, " + " 'x.y' : 1 " + "} }" + ), doc); + } + + TEST(LogBuilder, VerifyUnsetsAreGrouped) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + ASSERT_OK(lb.addToUnsets("a.b")); + ASSERT_OK(lb.addToUnsets("x.y")); + + ASSERT_EQUALS( + mongo::fromjson( + "{ $unset : {" + " 'a.b' : true, " + " 'x.y' : true " + "} }" + ), doc); + } + + TEST(LogBuilder, PresenceOfSetPreventsObjectReplacement) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mmb::Element replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_OK(lb.getReplacementObject(&replacement)); + ASSERT_TRUE(replacement.ok()); + + const mmb::Element elt_ab = doc.makeElementInt("a.b", 1); + ASSERT_TRUE(elt_ab.ok()); + ASSERT_OK(lb.addToSets(elt_ab)); + + replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_NOT_OK(lb.getReplacementObject(&replacement)); + ASSERT_FALSE(replacement.ok()); + } + + TEST(LogBuilder, PresenceOfUnsetPreventsObjectReplacement) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mmb::Element replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_OK(lb.getReplacementObject(&replacement)); + ASSERT_TRUE(replacement.ok()); + + const mmb::Element elt_ab = doc.makeElementInt("a.b", 1); + ASSERT_TRUE(elt_ab.ok()); + ASSERT_OK(lb.addToSets(elt_ab)); + + replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_NOT_OK(lb.getReplacementObject(&replacement)); + ASSERT_FALSE(replacement.ok()); + } + + TEST(LogBuilder, CantAddSetWithObjectReplacementDataPresent) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mmb::Element replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_OK(lb.getReplacementObject(&replacement)); + ASSERT_TRUE(replacement.ok()); + ASSERT_OK(replacement.appendInt("a", 1)); + + mmb::Element setCandidate = doc.makeElementInt("x", 0); + ASSERT_NOT_OK(lb.addToSets(setCandidate)); + } + + TEST(LogBuilder, CantAddUnsetWithObjectReplacementDataPresent) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mmb::Element replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_OK(lb.getReplacementObject(&replacement)); + ASSERT_TRUE(replacement.ok()); + ASSERT_OK(replacement.appendInt("a", 1)); + + ASSERT_NOT_OK(lb.addToUnsets("x")); + } + + // Ensure that once you have obtained the object replacement slot and mutated it, that the + // object replacement slot becomes in accessible. This is a bit paranoid, since in practice + // the modifier conflict detection logic should prevent that outcome at a higher level, but + // preventing it here costs us nothing and add an extra safety check. + TEST(LogBuilder, CantReacquireObjectReplacementData) { + mmb::Document doc; + LogBuilder lb(doc.root()); + + mmb::Element replacement = doc.end(); + ASSERT_FALSE(replacement.ok()); + ASSERT_OK(lb.getReplacementObject(&replacement)); + ASSERT_TRUE(replacement.ok()); + ASSERT_OK(replacement.appendInt("a", 1)); + + mmb::Element again = doc.end(); + ASSERT_FALSE(again.ok()); + ASSERT_NOT_OK(lb.getReplacementObject(&again)); + ASSERT_FALSE(again.ok()); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_add_to_set.cpp b/src/mongo/db/ops/modifier_add_to_set.cpp new file mode 100644 index 00000000000..4eb01f40eeb --- /dev/null +++ b/src/mongo/db/ops/modifier_add_to_set.cpp @@ -0,0 +1,432 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_add_to_set.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace mb = mutablebson; + namespace str = mongoutils::str; + + namespace { + + template + void deduplicate(mb::Element parent, Ordering comp, Equality equal) { + + // First, build a vector of the children. + std::vector children; + mb::Element current = parent.leftChild(); + while (current.ok()) { + children.push_back(current); + current = current.rightSibling(); + } + + // Then, sort the child vector with our comparator. + std::sort(children.begin(), children.end(), comp); + + // Next, remove duplicates by walking the vector. + std::vector::iterator where = children.begin(); + const std::vector::iterator end = children.end(); + + while( where != end ) { + std::vector::iterator next = where; ++next; + while (next != end && equal(*where, *next)) { + next->remove(); + ++next; + } + where = next; + } + } + + } // namespace + + struct ModifierAddToSet::PreparedState { + + PreparedState(mb::Document& doc) + : doc(doc) + , idxFound(0) + , elemFound(doc.end()) + , addAll(false) + , elementsToAdd() + , noOp(false) { + } + + // Document that is going to be changed. + mb::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mb::Element elemFound; + + // Are we adding all of the $each elements, or just a subset? + bool addAll; + + // Values to be applied. + std::vector elementsToAdd; + + // True if this update is a no-op + bool noOp; + }; + + ModifierAddToSet::ModifierAddToSet() + : ModifierInterface () + , _fieldRef() + , _posDollar(0) + , _valDoc() + , _val(_valDoc.end()) { + } + + ModifierAddToSet::~ModifierAddToSet() { + } + + Status ModifierAddToSet::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + // Perform standard field name and updateable checks. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + // TODO: The driver could potentially do this re-writing. + + // If the type of the value is 'Object', we might be dealing with a $each. See if that + // is the case. + if (modExpr.type() == mongo::Object) { + BSONElement modExprObjPayload = modExpr.embeddedObject().firstElement(); + if (!modExprObjPayload.eoo() && StringData(modExprObjPayload.fieldName()) == "$each") { + // It is a $each. Verify that the payload is an array as is required for $each, + // set our flag, and store the array as our value. + if (modExprObjPayload.type() != mongo::Array) { + return Status(ErrorCodes::BadValue, + str::stream() << "The argument to $each in $addToSet must " + "be an array but it was of type " + << typeName(modExprObjPayload.type())); + } + + status = _valDoc.root().appendElement(modExprObjPayload); + if (!status.isOK()) + return status; + + _val = _valDoc.root().leftChild(); + + deduplicate(_val, mb::woLess(false), mb::woEqual(false)); + } + } + + // If this wasn't an 'each', turn it into one. No need to sort or de-dup since we only + // have one element. + if (_val == _valDoc.end()) { + mb::Element each = _valDoc.makeElementArray("$each"); + + status = each.appendElement(modExpr); + if (!status.isOK()) + return status; + + status = _valDoc.root().pushBack(each); + if (!status.isOK()) + return status; + + _val = each; + } + + // Check if no invalid data (such as fields with '$'s) are being used in the $each + // clause. + mb::ConstElement valCursor = _val.leftChild(); + while (valCursor.ok()) { + const BSONType type = valCursor.getType(); + dassert(valCursor.hasValue()); + switch(type) { + case mongo::Object: { + Status s = valCursor.getValueObject().storageValidEmbedded(); + if (!s.isOK()) + return s; + + break; + } + case mongo::Array: { + Status s = valCursor.getValueArray().storageValidEmbedded(); + if (!s.isOK()) + return s; + + break; + } + default: + break; + } + + valCursor = valCursor.rightSibling(); + } + + return Status::OK(); + } + + Status ModifierAddToSet::prepare(mb::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_posDollar) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_posDollar, matchedField); + } + + // Locate the field name in 'root'. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + } else if (!status.isOK()) { + return status; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_fieldRef; + + // + // in-place and no-op logic + // + + // If the field path is not fully present, then this mod cannot be in place, nor is a + // noOp. + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + // If no target element exists, we will simply be creating a new array. + _preparedState->addAll = true; + return Status::OK(); + } + + // This operation only applies to arrays + if (_preparedState->elemFound.getType() != mongo::Array) { + mb::Element idElem = mb::findElementNamed(root.leftChild(), "_id"); + return Status( + ErrorCodes::BadValue, + str::stream() << "Cannot apply $addToSet to a non-array field. Field named '" + << _preparedState->elemFound.getFieldName() + << "' has a non-array type " + << typeName(_preparedState->elemFound.getType()) + << " in the document " + << idElem.toString()); + } + + // If the array is empty, then we don't need to check anything: all of the values are + // going to be added. + if (!_preparedState->elemFound.hasChildren()) { + _preparedState->addAll = true; + return Status::OK(); + } + + // For each value in the $each clause, compare it against the values in the array. If + // the element is not present, record it as one to add. + mb::Element eachIter = _val.leftChild(); + while (eachIter.ok()) { + mb::Element where = mb::findElement( + _preparedState->elemFound.leftChild(), + mb::woEqualTo(eachIter, false)); + if (!where.ok()) { + // The element was not found. Record the element from $each as one to be added. + _preparedState->elementsToAdd.push_back(eachIter); + } + eachIter = eachIter.rightSibling(); + } + + // If we didn't find any elements to add, then this is a no-op. + if (_preparedState->elementsToAdd.empty()) { + _preparedState->noOp = execInfo->noOp = true; + } + + return Status::OK(); + } + + Status ModifierAddToSet::apply() const { + dassert(_preparedState->noOp == false); + + // TODO: The contents of this block are lifted directly from $push. + + // If the array field is not there, create it as an array and attach it to the + // document. + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + + // Creates the array element + mb::Document& doc = _preparedState->doc; + StringData lastPart = _fieldRef.getPart(_fieldRef.numParts() - 1); + mb::Element baseArray = doc.makeElementArray(lastPart); + if (!baseArray.ok()) { + return Status(ErrorCodes::InternalError, "can't create new base array"); + } + + // Now, we can be in two cases here, as far as attaching the element being set + // goes: (a) none of the parts in the element's path exist, or (b) some parts of + // the path exist but not all. + if (!_preparedState->elemFound.ok()) { + _preparedState->elemFound = doc.root(); + _preparedState->idxFound = 0; + } + else { + _preparedState->idxFound++; + } + + // createPathAt() will complete the path and attach 'elemToSet' at the end of it. + Status status = pathsupport::createPathAt(_fieldRef, + _preparedState->idxFound, + _preparedState->elemFound, + baseArray); + if (!status.isOK()) { + return status; + } + + // Point to the base array just created. The subsequent code expects it to exist + // already. + _preparedState->elemFound = baseArray; + } + + if (_preparedState->addAll) { + + // If we are adding all the values, we can just walk over _val; + + mb::Element where = _val.leftChild(); + while (where.ok()) { + + dassert(where.hasValue()); + + mb::Element toAdd = _preparedState->doc.makeElement(where.getValue()); + Status status = _preparedState->elemFound.pushBack(toAdd); + if (!status.isOK()) + return status; + + where = where.rightSibling(); + } + + } else { + + // Otherwise, we aren't adding all the values, and we need to add exactly those + // elements that were found to be missing during our scan in prepare. + std::vector::const_iterator where = + _preparedState->elementsToAdd.begin(); + + const std::vector::const_iterator end = + _preparedState->elementsToAdd.end(); + + for ( ; where != end; ++where) { + + dassert(where->hasValue()); + + mb::Element toAdd = _preparedState->doc.makeElement(where->getValue()); + Status status = _preparedState->elemFound.pushBack(toAdd); + if (!status.isOK()) + return status; + } + } + + return Status::OK(); + } + + Status ModifierAddToSet::log(LogBuilder* logBuilder) const { + + // TODO: This is copied more or less identically from $push. As a result, it copies the + // behavior in $push that relies on 'apply' having been called unless this is a no-op. + + // TODO We can log just a positional set in several cases. For now, let's just log the + // full resulting array. + + // We'd like to create an entry such as {$set: {: []}} under + // 'logRoot'. We start by creating the {$set: ...} Element. + mb::Document& doc = logBuilder->getDocument(); + + // Then we create the {:[]} Element, that is, an empty array. + mb::Element logElement = doc.makeElementArray(_fieldRef.dottedField()); + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, "cannot create details for $addToSet mod"); + } + + // Fill up the empty array. + mb::Element curr = _preparedState->elemFound.leftChild(); + while (curr.ok()) { + + dassert(curr.hasValue()); + + // We need to copy each array entry from the resulting document to the log + // document. + mb::Element currCopy = doc.makeElementWithNewFieldName( + StringData(), + curr.getValue()); + if (!currCopy.ok()) { + return Status(ErrorCodes::InternalError, "could create copy element"); + } + Status status = logElement.pushBack(currCopy); + if (!status.isOK()) { + return Status(ErrorCodes::BadValue, + str::stream() << "Could not append entry for $addToSet oplog entry." + << "Underlying cause: " << status.toString()); + } + curr = curr.rightSibling(); + } + + return logBuilder->addToSets(logElement); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_add_to_set.h b/src/mongo/db/ops/modifier_add_to_set.h new file mode 100644 index 00000000000..065f6c83657 --- /dev/null +++ b/src/mongo/db/ops/modifier_add_to_set.h @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierAddToSet : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierAddToSet); + + public: + + ModifierAddToSet(); + virtual ~ModifierAddToSet(); + + /** Goes over the array item(s) that are going to be set- unioned and converts them + * internally to a mutable bson. Both single and $each forms are supported. Returns OK + * if the item(s) are valid otherwise returns a status describing the error. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** Decides which portion of the array items that are going to be set-unioned to root's + * document and fills in 'execInfo' accordingly. Returns OK if the document has a + * valid array to set-union to, othwise returns a status describing the error. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** Updates the Element used in prepare with the effects of the $addToSet operation. */ + virtual Status apply() const; + + /** Converts the effects of this $addToSet into one or more equivalent $set operations. */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _posDollar; + + // Array of values to be set-union'ed onto target. + mutablebson::Document _valDoc; + mutablebson::Element _val; + + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_add_to_set_test.cpp b/src/mongo/db/ops/modifier_add_to_set_test.cpp new file mode 100644 index 00000000000..78e1880d417 --- /dev/null +++ b/src/mongo/db/ops/modifier_add_to_set_test.cpp @@ -0,0 +1,393 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_add_to_set.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::ModifierAddToSet; + using mongo::ModifierInterface; + using mongo::Status; + using mongo::StringData; + using mongo::fromjson; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate a $addToSet mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _modObj(modObj) + , _mod() { + ASSERT_OK(_mod.init(_modObj["$addToSet"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierAddToSet& mod() { + return _mod; + } + + private: + BSONObj _modObj; + ModifierAddToSet _mod; + }; + + TEST(Init, FailToInitWithInvalidValue) { + BSONObj modObj; + ModifierAddToSet mod; + + modObj = fromjson("{ $addToSet : { a : { 'x.$.y' : 'bad' } } }"); + ASSERT_NOT_OK(mod.init(modObj["$addToSet"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + modObj = fromjson("{ $addToSet : { a : { $each : [ { 'x.$.y' : 'bad' } ] } } }"); + ASSERT_NOT_OK(mod.init(modObj["$addToSet"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An int is not valid after $each + modObj = fromjson("{ $addToSet : { a : { $each : 0 } } }"); + ASSERT_NOT_OK(mod.init(modObj["$addToSet"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An object is not valid after $each + modObj = fromjson("{ $addToSet : { a : { $each : { a : 1 } } } }"); + ASSERT_NOT_OK(mod.init(modObj["$addToSet"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, ParsesSimple) { + Mod(fromjson("{ $addToSet : { a : 1 } }")); + Mod(fromjson("{ $addToSet : { a : 'foo' } }")); + Mod(fromjson("{ $addToSet : { a : {} } }")); + Mod(fromjson("{ $addToSet : { a : { x : 1 } } }")); + Mod(fromjson("{ $addToSet : { a : [] } }")); + Mod(fromjson("{ $addToSet : { a : [1, 2] } } }")); + Mod(fromjson("{ $addToSet : { 'a.b' : 1 } }")); + Mod(fromjson("{ $addToSet : { 'a.b' : 'foo' } }")); + Mod(fromjson("{ $addToSet : { 'a.b' : {} } }")); + Mod(fromjson("{ $addToSet : { 'a.b' : { x : 1} } }")); + Mod(fromjson("{ $addToSet : { 'a.b' : [] } }")); + Mod(fromjson("{ $addToSet : { 'a.b' : [1, 2] } } }")); + } + + TEST(Init, ParsesEach) { + Mod(fromjson("{ $addToSet : { a : { $each : [] } } }")); + Mod(fromjson("{ $addToSet : { a : { $each : [ 1 ] } } }")); + Mod(fromjson("{ $addToSet : { a : { $each : [ 1, 2 ] } } }")); + Mod(fromjson("{ $addToSet : { a : { $each : [ 1, 2, 1 ] } } }")); + Mod(fromjson("{ $addToSet : { a : { $each : [ {} ] } } }")); + Mod(fromjson("{ $addToSet : { a : { $each : [ { x : 1 } ] } } }")); + Mod(fromjson("{ $addToSet : { a : { $each : [ { x : 1 }, { y : 2 } ] } } }")); + Mod(fromjson("{ $addToSet : { a : { $each : [ { x : 1 }, { y : 2 }, { x : 1 } ] } } }")); + } + + TEST(SimpleMod, PrepareOKTargetNotFound) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(SimpleMod, PrepareOKTargetFound) { + Document doc(fromjson("{ a : [ 1 ] }")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1 ] } }"), logDoc); + } + + TEST(SimpleMod, PrepareInvalidTargetNumber) { + Document doc(fromjson("{ a : 1 }")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, PrepareInvalidTarget) { + Document doc(fromjson("{ a : {} }")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, ApplyAndLogEmptyDocument) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1 ] } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogEmptyArray) { + Document doc(fromjson("{ a : [] }")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1 ] } }"), logDoc); + } + + TEST(SimpleEachMod, ApplyAndLogEmptyDocument) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $addToSet : { a : { $each : [1, 2, 3] } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1, 2, 3 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1, 2, 3 ] } }"), logDoc); + } + + TEST(SimpleEachMod, ApplyAndLogEmptyArray) { + Document doc(fromjson("{ a : [] }")); + Mod mod(fromjson("{ $addToSet : { a : { $each : [1, 2, 3] } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1, 2, 3 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1, 2, 3 ] } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogPopulatedArray) { + Document doc(fromjson("{ a : [ 'x' ] }")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 'x', 1 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 'x', 1 ] } }"), logDoc); + } + + TEST(SimpleEachMod, ApplyAndLogPopulatedArray) { + Document doc(fromjson("{ a : [ 'x' ] }")); + Mod mod(fromjson("{ $addToSet : { a : { $each : [1, 2, 3] } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 'x', 1, 2, 3 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 'x', 1, 2, 3 ] } }"), logDoc); + } + + TEST(NoOp, AddOneExistingIsNoOp) { + Document doc(fromjson("{ a : [ 1, 2, 3 ] }")); + Mod mod(fromjson("{ $addToSet : { a : 1 } }")); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1, 2, 3 ] } }"), logDoc); + } + + TEST(NoOp, AddSeveralExistingIsNoOp) { + Document doc(fromjson("{ a : [ 1, 2, 3 ] }")); + Mod mod(fromjson("{ $addToSet : { a : { $each : [1, 2] } } }")); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1, 2, 3 ] } }"), logDoc); + } + + TEST(NoOp, AddAllExistingIsNoOp) { + Document doc(fromjson("{ a : [ 1, 2, 3 ] }")); + Mod mod(fromjson("{ $addToSet : { a : { $each : [1, 2, 3] } } }")); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1, 2, 3 ] } }"), logDoc); + } + + TEST(Deduplication, ExistingDuplicatesArePreserved) { + Document doc(fromjson("{ a : [ 1, 1, 2, 1, 2, 2 ] }")); + Mod mod(fromjson("{ $addToSet : { a : 3 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1, 1, 2, 1, 2, 2, 3] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1, 1, 2, 1, 2, 2, 3] } }"), logDoc); + } + + TEST(Deduplication, NewDuplicatesAreElided) { + Document doc(fromjson("{ a : [ 1, 1, 2, 1, 2, 2 ] }")); + Mod mod(fromjson("{ $addToSet : { a : { $each : [ 4, 1, 3, 2, 3, 1, 3, 3, 2, 4] } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1, 1, 2, 1, 2, 2, 4, 3] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [ 1, 1, 2, 1, 2, 2, 4, 3] } }"), logDoc); + } + + TEST(Regressions, SERVER_12848) { + // Proof that the mod works ok (the real issue was in validate). + + Document doc(fromjson("{ _id : 1, a : [ 1, [ ] ] }")); + Mod mod(fromjson("{ $addToSet : { 'a.1' : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ _id : 1, a : [ 1, [ 1 ] ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { 'a.1' : [ 1 ] } }"), logDoc); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_base.h b/src/mongo/db/ops/modifier_base.h deleted file mode 100644 index cc1e65d8ade..00000000000 --- a/src/mongo/db/ops/modifier_base.h +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "mongo/base/status.h" -#include "mongo/base/string_data.h" -#include "mongo/bson/mutable/mutable_bson.h" -#include "mongo/db/ops/field_ref.h" - -namespace mongo { - - /** - * Abstract base class for update "modifiers" (a.k.a "$ operators"). To create a new - * operator, implement a new derived class. - * - * A typical call sequence for the class is: - * - * + init() with the mod arguments - * - * + For each document that is being touched on that update, the following methods are - * going to be called once for that document and in the order the calls appear here. - * - * + prepareMod() to check if mod is viable over the document - * - * + applyMod(), effectively computing the update - * - * + logMod() registering the change in the log for replication purposes - * - * Again, a modifier implementation may rely on these last three calls being made and in - * that particular order and therefore can keep and reuse state between these calls, when - * appropriate. - * - * TODO: - * For a reference implementation, see modifier_identity.{h,cpp} used in tests. - */ - class ModifierBase { - public: - virtual ~ModifierBase() { } - - /** - * Returns OK and extracts the parameters for this given mod from 'modExpr'. For - * instance, for a $inc, extracts the increment value. The init() method would be - * called only once per operand, that is, if a { $inc: { a: 1, b: 1 } } is issued, - * there would be one instance of the operator working on 'a' and one on 'b'. In each - * case, init() would be called once with the respective bson element. - * - * If 'modExpr' is invalid, returns an error status with a reason description. - * - * Note: - * - * + An operator may assume the modExpr passed here will be unchanged throughout all - * the mod object lifetime and also that the modExrp's lifetime exceeds the life - * time of this mod. Therefore, taking references to elements inside modExpr is - * valid. - */ - virtual Status init( const BSONElement& modExpr ) = 0; - - /** - * Returns OK if it would be correct to apply this mod over the document 'root' (e.g, if - * we're $inc-ing a field, is that field numeric in the current doc?). - * - * If the field this mod is targeted to contains a $-positional parameter, that value - * can be bound with 'matchedField', passed by the caller. - * - * In addition, the call also identifies which fields(s) of 'root' the mod is interested - * in changing (note that the modifier may want to add a field that's not present in - * the document). The call also determines whether it could modify the document in - * place and whether it is a no-op for the given document. All this information is in - * the passed 'execInfo', which is filled inside the call. - * - * If the mod cannot be applied over 'root', returns an error status with a reason - * description. - */ - struct ExecInfo { - // The fields of concern to the driver: no other op may modify the fields listed here - FieldRef* fieldRef[2]; // not owned here - bool inPlace; - bool noOp; - }; - virtual Status prepareMod( const mutablebson::Element& root, - const StringData& matchedField, - /* IN-OUT */ ExecInfo* execInfo ) = 0; - - /** - * Returns OK and modifies (or adds) an element (or elements) from 'root'. This may - * act on multiple fields but should only be called once per operator. - * - * For this call to be issued, the call to 'prepareElem' must have necessarily turned - * off 'ExecMode.noOp', ie this mod over this document is not a no-op. - * - * If the mod could not be applied, returns an error status with a reason description. - */ - virtual Status applyMod( /* IN-OUT */ mutablebson::Element* root ) = 0; - - /** - * Returns OK and registers the result of this mod in 'logRoot', the document that - * would eventually become a log entry. The mod must have kept enough state to - * be able to produce the log record (see idempotency note below). - * - * If the mod could not be logged, returns an error status with a reason description. - * - * Idempotency Note: - * - * + The modifier must log a mod that is idempotent, ie, applying it more than once - * to a base collection would produce the same result as applying it only once. For - * example, a $inc can be switched to a $set for the resulting incremented value, - * for logging purposes. An array based operator may check the contents of the - * array before operating on it. - */ - virtual Status logMod( mutablebson::Element* logRoot ) = 0; - - }; - -} // namespace mongo diff --git a/src/mongo/db/ops/modifier_bit.cpp b/src/mongo/db/ops/modifier_bit.cpp new file mode 100644 index 00000000000..6c26dfeea8c --- /dev/null +++ b/src/mongo/db/ops/modifier_bit.cpp @@ -0,0 +1,300 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_bit.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace mb = mutablebson; + namespace str = mongoutils::str; + + struct ModifierBit::PreparedState { + + PreparedState(mutablebson::Document& doc) + : doc(doc) + , idxFound(0) + , elemFound(doc.end()) + , noOp(false) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element elemFound; + + // Value to be applied. + SafeNum newValue; + + // True if this update is a no-op + bool noOp; + }; + + ModifierBit::ModifierBit() + : ModifierInterface () + , _fieldRef() + , _posDollar(0) + , _ops() { + } + + ModifierBit::~ModifierBit() { + } + + Status ModifierBit::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + // Perform standard field name and updateable checks. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + if (modExpr.type() != mongo::Object) + return Status(ErrorCodes::BadValue, + str::stream() << "The $bit modifier is not compatible with a " + << typeName(modExpr.type()) + << ". You must pass in an embedded document: " + "{$bit: {field: {and/or/xor: #}}"); + + BSONObjIterator opsIterator(modExpr.embeddedObject()); + + while (opsIterator.more()) { + BSONElement curOp = opsIterator.next(); + + const StringData payloadFieldName = curOp.fieldName(); + + SafeNumOp op = NULL; + + if (payloadFieldName == "and") { + op = &SafeNum::bitAnd; + } + else if (payloadFieldName == "or") { + op = &SafeNum::bitOr; + } + else if (payloadFieldName == "xor") { + op = &SafeNum::bitXor; + } + else { + return Status( + ErrorCodes::BadValue, + str::stream() << "The $bit modifier only supports 'and', 'or', and 'xor', not '" + << payloadFieldName + << "' which is an unknown operator: {" << curOp << "}"); + } + + if ((curOp.type() != mongo::NumberInt) && + (curOp.type() != mongo::NumberLong)) + return Status( + ErrorCodes::BadValue, + str::stream() << "The $bit modifier field must be an Integer(32/64 bit); a '" + << typeName(curOp.type()) + << "' is not supported here: {" << curOp << "}"); + + const OpEntry entry = {SafeNum(curOp), op}; + _ops.push_back(entry); + } + + dassert(!_ops.empty()); + + return Status::OK(); + } + + Status ModifierBit::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_posDollar) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_posDollar, matchedField); + } + + // Locate the field name in 'root'. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + } + else if (!status.isOK()) { + return status; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_fieldRef; + + // + // in-place and no-op logic + // + + // If the field path is not fully present, then this mod cannot be in place, nor is a + // noOp. + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + // If no target element exists, the value we will write is the result of applying + // the operation to a zero-initialized integer element. + _preparedState->newValue = apply(SafeNum(static_cast(0))); + return Status::OK(); + } + + if (!_preparedState->elemFound.isIntegral()) { + mb::Element idElem = mb::findElementNamed(root.leftChild(), "_id"); + return Status( + ErrorCodes::BadValue, + str::stream() << "Cannot apply $bit to a value of non-integral type." + << idElem.toString() + << " has the field " << _preparedState->elemFound.getFieldName() + << " of non-integer type " + << typeName(_preparedState->elemFound.getType())); + } + + const SafeNum currentValue = _preparedState->elemFound.getValueSafeNum(); + + // Apply the op over the existing value and the mod value, and capture the result. + _preparedState->newValue = apply(currentValue); + + if (!_preparedState->newValue.isValid()) { + // TODO: Include list of ops, if that is easy, at some future point. + return Status(ErrorCodes::BadValue, + str::stream() << "Failed to apply $bit operations to current value: " + << currentValue.debugString()); + } + // If the values are identical (same type, same value), then this is a no-op. + if (_preparedState->newValue.isIdentical(currentValue)) { + _preparedState->noOp = execInfo->noOp = true; + return Status::OK(); + } + + return Status::OK(); + } + + Status ModifierBit::apply() const { + dassert(_preparedState->noOp == false); + + // If there's no need to create any further field part, the $bit is simply a value + // assignment. + if (_preparedState->elemFound.ok() && + _preparedState->idxFound == (_fieldRef.numParts() - 1)) { + return _preparedState->elemFound.setValueSafeNum(_preparedState->newValue); + } + + // + // Complete document path logic + // + + // Creates the final element that's going to be $set in 'doc'. + mutablebson::Document& doc = _preparedState->doc; + StringData lastPart = _fieldRef.getPart(_fieldRef.numParts() - 1); + mutablebson::Element elemToSet = doc.makeElementSafeNum(lastPart, _preparedState->newValue); + if (!elemToSet.ok()) { + return Status(ErrorCodes::InternalError, "can't create new element"); + } + + // Now, we can be in two cases here, as far as attaching the element being set goes: + // (a) none of the parts in the element's path exist, or (b) some parts of the path + // exist but not all. + if (!_preparedState->elemFound.ok()) { + _preparedState->elemFound = doc.root(); + _preparedState->idxFound = 0; + } + else { + _preparedState->idxFound++; + } + + // createPathAt() will complete the path and attach 'elemToSet' at the end of it. + return pathsupport::createPathAt(_fieldRef, + _preparedState->idxFound, + _preparedState->elemFound, + elemToSet); + } + + Status ModifierBit::log(LogBuilder* logBuilder) const { + + mutablebson::Element logElement = logBuilder->getDocument().makeElementSafeNum( + _fieldRef.dottedField(), + _preparedState->newValue); + + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, + str::stream() << "Could not append entry to $bit oplog entry: " + << "set '" << _fieldRef.dottedField() << "' -> " + << _preparedState->newValue.debugString() ); + } + return logBuilder->addToSets(logElement); + + } + + SafeNum ModifierBit::apply(SafeNum value) const { + OpEntries::const_iterator where = _ops.begin(); + const OpEntries::const_iterator end = _ops.end(); + for (; where != end; ++where) + value = (value.*(where->op))(where->val); + return value; + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_bit.h b/src/mongo/db/ops/modifier_bit.h new file mode 100644 index 00000000000..60fb4ab5e8c --- /dev/null +++ b/src/mongo/db/ops/modifier_bit.h @@ -0,0 +1,100 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/ops/modifier_interface.h" +#include "mongo/util/safe_num.h" + +namespace mongo { + + class LogBuilder; + + class ModifierBit : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierBit); + + public: + + ModifierBit(); + virtual ~ModifierBit(); + + /** + * A 'modExpr' is a BSONElement {: } coming from a $bit mod such as + * {$bit: {}}. init() extracts the field name, the + * operation subtype, and the value to be assigned to it from 'modExpr'. It returns OK + * if successful or a status describing the error. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** Validates the potential application of the init'ed mod to the given Element and + * configures the internal state of the mod as necessary. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** Updates the Element used in prepare with the effects of the $bit operation */ + virtual Status apply() const; + + /** Converts the effects of this $bit into an equivalent $set */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + SafeNum apply(SafeNum value) const; + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _posDollar; + + // The operator on SafeNum that we will invoke. + typedef SafeNum (SafeNum::* SafeNumOp)(const SafeNum&) const; + + struct OpEntry { + SafeNum val; + SafeNumOp op; + }; + + typedef std::vector OpEntries; + + OpEntries _ops; + + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_bit_test.cpp b/src/mongo/db/ops/modifier_bit_test.cpp new file mode 100644 index 00000000000..fb1d79b7837 --- /dev/null +++ b/src/mongo/db/ops/modifier_bit_test.cpp @@ -0,0 +1,736 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_bit.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::ModifierBit; + using mongo::ModifierInterface; + using mongo::Status; + using mongo::StringData; + using mongo::fromjson; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate a $bit mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _modObj(modObj) + , _mod() { + ASSERT_OK(_mod.init(_modObj["$bit"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierBit& mod() { return _mod; } + + private: + BSONObj _modObj; + ModifierBit _mod; + }; + + + TEST(Init, FailToInitWithInvalidValue) { + BSONObj modObj; + ModifierBit mod; + + // String is an invalid $bit argument + modObj = fromjson("{ $bit : { a : '' } }"); + ASSERT_NOT_OK(mod.init(modObj["$bit"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // Array is an invalid $bit argument + modObj = fromjson("{ $bit : { a : [] } }"); + ASSERT_NOT_OK(mod.init(modObj["$bit"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An object with value not in ('and', 'or') is an invalid $bit argument + modObj = fromjson("{ $bit : { a : { foo : 4 } } }"); + ASSERT_NOT_OK(mod.init(modObj["$bit"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // The argument to the sub-operator must be numeric + modObj = fromjson("{ $bit : { a : { or : [] } } }"); + ASSERT_NOT_OK(mod.init(modObj["$bit"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{ $bit : { a : { or : 'foo' } } }"); + ASSERT_NOT_OK(mod.init(modObj["$bit"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // The argument to the sub-operator must be integral + modObj = fromjson("{ $bit : { a : { or : 1.0 } } }"); + ASSERT_NOT_OK(mod.init(modObj["$bit"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, ParsesAndInt) { + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(1))))); + } + + TEST(Init, ParsesOrInt) { + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(1))))); + } + + TEST(Init, ParsesXorInt) { + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(1))))); + } + + TEST(Init, ParsesAndLong) { + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(1))))); + } + + TEST(Init, ParsesOrLong) { + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(1))))); + } + + TEST(Init, ParsesXorLong) { + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(1))))); + } + + TEST(SimpleMod, PrepareOKTargetNotFound) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $bit : { a : { and : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(SimpleMod, PrepareOKTargetFound) { + Document doc(fromjson("{ a : 1 }")); + Mod mod(fromjson("{ $bit : { a : { and : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1 } }"), logDoc); + } + + TEST(SimpleMod, PrepareSimpleNonNumericObject) { + Document doc(fromjson("{ a : {} }")); + Mod mod(fromjson("{ $bit : { a : { or : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, PrepareSimpleNonNumericArray) { + + Document doc(fromjson("{ a : [] }")); + Mod mod(fromjson("{ $bit : { a : { and : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, PrepareSimpleNonNumericString) { + Document doc(fromjson("{ a : '' }")); + Mod mod(fromjson("{ $bit : { a : { or : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, ApplyAndLogEmptyDocumentAnd) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $bit : { a : { and : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 0 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 0 } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogEmptyDocumentOr) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $bit : { a : { or : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1 } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogEmptyDocumentXor) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $bit : { a : { xor : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1 } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogSimpleDocumentAnd) { + Document doc(fromjson("{ a : 5 }")); + Mod mod(fromjson("{ $bit : { a : { and : 6 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 4 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 4 } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogSimpleDocumentOr) { + Document doc(fromjson("{ a : 5 }")); + Mod mod(fromjson("{ $bit : { a : { or : 6 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 7 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 7 } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogSimpleDocumentXor) { + Document doc(fromjson("{ a : 5 }")); + Mod mod(fromjson("{ $bit : { a : { xor : 6 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 3 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 3 } }"), logDoc); + } + + TEST(InPlace, IntToIntAndIsInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(1))), logDoc); + } + + TEST(InPlace, IntToIntOrIsInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(1))), logDoc); + } + + TEST(InPlace, IntToIntXorIsInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(0))), logDoc); + } + + TEST(InPlace, LongToLongAndIsInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(1))), logDoc); + } + + TEST(InPlace, LongToLongOrIsInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(1))), logDoc); + } + + TEST(InPlace, LongToLongXorIsInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(0))), logDoc); + } + + TEST(InPlace, IntToLongAndIsNotInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(InPlace, IntToLongOrIsNotInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(InPlace, IntToLongXorIsNotInPlace) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(NoOp, IntAnd) { + Document doc(BSON("a" << static_cast(0xABCD1234U))); + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(0xFFFFFFFFU))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(0xABCD1234U))), logDoc); + } + + TEST(NoOp, IntOr) { + Document doc(BSON("a" << static_cast(0xABCD1234U))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(0x0U))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(0xABCD1234U))), logDoc); + } + + TEST(NoOp, IntXor) { + Document doc(BSON("a" << static_cast(0xABCD1234U))); + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(0x0U))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(0xABCD1234U))), logDoc); + } + + TEST(NoOp, LongAnd) { + Document doc(BSON("a" << static_cast(0xABCD1234EF981234ULL))); + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << + static_cast(0xFFFFFFFFFFFFFFFFULL))))); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << + static_cast(0xABCD1234EF981234ULL))), logDoc); + } + + TEST(NoOp, LongOr) { + Document doc(BSON("a" << static_cast(0xABCD1234EF981234ULL))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(0x0ULL))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << + static_cast(0xABCD1234EF981234ULL))), logDoc); + } + + TEST(NoOp, LongXor) { + Document doc(BSON("a" << static_cast(0xABCD1234EF981234ULL))); + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(0x0ULL))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << + static_cast(0xABCD1234EF981234ULL))), logDoc); + } + + TEST(Upcasting, UpcastIntToLongAnd) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1 }"), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + } + + TEST(Upcasting, UpcastIntToLongOr) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1 }"), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + } + + TEST(Upcasting, UpcastIntToLongXor) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(0))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1 }"), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + } + + TEST(Upcasting, LongsStayLongsAnd) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("and" << static_cast(2))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 0 }"), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + } + + TEST(Upcasting, LongsStayLongsOr) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(2))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 3 }"), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + } + + TEST(Upcasting, LongsStayLongsXor) { + Document doc(BSON("a" << static_cast(1))); + Mod mod(BSON("$bit" << BSON("a" << BSON("xor" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 0 }"), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + } + + // The following tests are re-created from the previous $bit tests in updatetests.cpp. They + // are probably redundant with the tests above in various ways. + + TEST(DbUpdateTests, BitRewriteExistingField) { + Document doc(BSON("a" << static_cast(0))); + Mod mod(BSON("$bit" << BSON("a" << BSON("or" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(BSON("a" << static_cast(1)), doc); + ASSERT_EQUALS(mongo::NumberInt, doc.root()["a"].getType()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("a" << static_cast(1))), logDoc); + } + + TEST(DbUpdateTests, BitRewriteNonExistingField) { + Document doc(BSON("a" << static_cast(0))); + Mod mod(BSON("$bit" << BSON("b" << BSON("or" << static_cast(1))))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(BSON("a" << static_cast(0) << "b" << static_cast(1)), doc); + ASSERT_EQUALS(mongo::NumberInt, doc.root()["a"].getType()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("b" << static_cast(1))), logDoc); + } + + TEST(DbUpdateTests, Bit1_1) { + Document doc(BSON("_id" << 1 << "x" << 3)); + Mod mod(BSON("$bit" << BSON("x" << BSON("and" << 2)))); + const BSONObj result(BSON("_id" << 1 << "x" << (3 & 2))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + if (!execInfo.noOp) + ASSERT_OK(mod.apply()); + + ASSERT_EQUALS(result, doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("x" << (3 & 2))), logDoc); + } + + TEST(DbUpdateTests, Bit1_2) { + Document doc(BSON("_id" << 1 << "x" << 1)); + Mod mod(BSON("$bit" << BSON("x" << BSON("or" << 4)))); + const BSONObj result(BSON("_id" << 1 << "x" << (1 | 4))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + if (!execInfo.noOp) + ASSERT_OK(mod.apply()); + + ASSERT_EQUALS(result, doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("x" << (1 | 4))), logDoc); + } + + TEST(DbUpdateTests, Bit1_3) { + Document doc(BSON("_id" << 1 << "x" << 3)); + Mod mod1(BSON("$bit" << BSON("x" << BSON("and" << 2)))); + Mod mod2(BSON("$bit" << BSON("x" << BSON("or" << 8)))); + const BSONObj result(BSON("_id" << 1 << "x" << ((3 & 2) | 8))); + + ModifierInterface::ExecInfo execInfo1; + ASSERT_OK(mod1.prepare(doc.root(), "", &execInfo1)); + if (!execInfo1.noOp) + ASSERT_OK(mod1.apply()); + + ModifierInterface::ExecInfo execInfo2; + ASSERT_OK(mod2.prepare(doc.root(), "", &execInfo2)); + if (!execInfo2.noOp) + ASSERT_OK(mod2.apply()); + + ASSERT_EQUALS(result, doc); + } + + TEST(DbUpdateTests, Bit1_3_Combined) { + Document doc(BSON("_id" << 1 << "x" << 3)); + Mod mod(BSON("$bit" << BSON("x" << BSON("and" << 2 << "or" << 8)))); + const BSONObj result(BSON("_id" << 1 << "x" << ((3 & 2) | 8))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + if (!execInfo.noOp) + ASSERT_OK(mod.apply()); + + ASSERT_EQUALS(result, doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("x" << ((3 & 2) | 8))), logDoc); + } + + TEST(DbUpdateTests, Bit1_4) { + Document doc(BSON("_id" << 1 << "x" << 3)); + Mod mod1(BSON("$bit" << BSON("x" << BSON("or" << 2)))); + Mod mod2(BSON("$bit" << BSON("x" << BSON("and" << 8)))); + const BSONObj result(BSON("_id" << 1 << "x" << ((3 | 2) & 8))); + + ModifierInterface::ExecInfo execInfo1; + ASSERT_OK(mod1.prepare(doc.root(), "", &execInfo1)); + if (!execInfo1.noOp) + ASSERT_OK(mod1.apply()); + + ModifierInterface::ExecInfo execInfo2; + ASSERT_OK(mod2.prepare(doc.root(), "", &execInfo2)); + if (!execInfo2.noOp) + ASSERT_OK(mod2.apply()); + + ASSERT_EQUALS(result, doc); + } + + TEST(DbUpdateTests, Bit1_4_Combined) { + Document doc(BSON("_id" << 1 << "x" << 3)); + Mod mod(BSON("$bit" << BSON("x" << BSON("or" << 2 << "and" << 8)))); + const BSONObj result(BSON("_id" << 1 << "x" << ((3 | 2) & 8))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + if (!execInfo.noOp) + ASSERT_OK(mod.apply()); + + ASSERT_EQUALS(result, doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(BSON("$set" << BSON("x" << ((3 | 2) & 8))), logDoc); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_compare.cpp b/src/mongo/db/ops/modifier_compare.cpp new file mode 100644 index 00000000000..dee2b1ef2ef --- /dev/null +++ b/src/mongo/db/ops/modifier_compare.cpp @@ -0,0 +1,192 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/ops/modifier_compare.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + struct ModifierCompare::PreparedState { + + PreparedState(mutablebson::Document& targetDoc) + : doc(targetDoc) + , idxFound(0) + , elemFound(doc.end()) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element elemFound; + }; + + ModifierCompare::ModifierCompare(ModifierCompare::ModifierCompareMode mode) + : _mode(mode) + , _pathReplacementPosition(0) { + } + + ModifierCompare::~ModifierCompare() { + } + + Status ModifierCompare::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + _updatePath.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_updatePath); + if (!status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional( + _updatePath, &_pathReplacementPosition, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _updatePath.dottedField() << "'"); + } + + // Store value for later. + _val = modExpr; + return Status::OK(); + } + + Status ModifierCompare::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_pathReplacementPosition) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _updatePath.dottedField()); + } + _updatePath.setPart(_pathReplacementPosition, matchedField); + } + + // Locate the field name in 'root'. Note that we may not have all the parts in the path + // in the doc -- which is fine. Our goal now is merely to reason about whether this mod + // apply is a noOp or whether is can be in place. The remaining path, if missing, will + // be created during the apply. + Status status = pathsupport::findLongestPrefix(_updatePath, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + } + else if (!status.isOK()) { + return status; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_updatePath; + + const bool destExists = (_preparedState->elemFound.ok() && + _preparedState->idxFound == (_updatePath.numParts() - 1)); + if (!destExists) { + execInfo->noOp = false; + } + else { + const int compareVal = _preparedState->elemFound.compareWithBSONElement(_val, false); + execInfo->noOp = (compareVal == 0) || + ((_mode == ModifierCompare::MAX) ? + (compareVal > 0) : (compareVal < 0)); + } + + return Status::OK(); + } + + Status ModifierCompare::apply() const { + + const bool destExists = (_preparedState->elemFound.ok() && + _preparedState->idxFound == (_updatePath.numParts() - 1)); + // If there's no need to create any further field part, the $set is simply a value + // assignment. + if (destExists) { + return _preparedState->elemFound.setValueBSONElement(_val); + } + + mutablebson::Document& doc = _preparedState->doc; + StringData lastPart = _updatePath.getPart(_updatePath.numParts() - 1); + // If the element exists and is the same type, then that is what we want to work with + mutablebson::Element elemToSet = doc.makeElementWithNewFieldName(lastPart, _val); + if (!elemToSet.ok()) { + return Status(ErrorCodes::InternalError, "can't create new element"); + } + + // Now, we can be in two cases here, as far as attaching the element being set goes: + // (a) none of the parts in the element's path exist, or (b) some parts of the path + // exist but not all. + if (!_preparedState->elemFound.ok()) { + _preparedState->elemFound = doc.root(); + _preparedState->idxFound = 0; + } + else { + _preparedState->idxFound++; + } + + // createPathAt() will complete the path and attach 'elemToSet' at the end of it. + return pathsupport::createPathAt(_updatePath, + _preparedState->idxFound, + _preparedState->elemFound, + elemToSet); + } + + Status ModifierCompare::log(LogBuilder* logBuilder) const { + return logBuilder->addToSetsWithNewFieldName(_updatePath.dottedField(), _val); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_compare.h b/src/mongo/db/ops/modifier_compare.h new file mode 100644 index 00000000000..8aaa3a5e18e --- /dev/null +++ b/src/mongo/db/ops/modifier_compare.h @@ -0,0 +1,111 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierCompare : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierCompare); + + public: + + enum ModifierCompareMode { MAX, MIN }; + explicit ModifierCompare(ModifierCompareMode mode = MAX); + + virtual ~ModifierCompare(); + + // + // Modifier interface implementation + // + + /** + * A 'modExpr' is a BSONElement {: } coming from a $set mod such as + * {$set: {}}. init() extracts the field name and the value to be + * assigned to it from 'modExpr'. It returns OK if successful or a status describing + * the error. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** + * Looks up the field name in the sub-tree rooted at 'root', and binds, if necessary, + * the '$' field part using the 'matchedfield' number. prepare() returns OK and + * fills in 'execInfo' with information of whether this mod is a no-op on 'root' and + * whether it is an in-place candidate. Otherwise, returns a status describing the + * error. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** + * Applies the prepared mod over the element 'root' specified in the prepare() + * call. Returns OK if successful or a status describing the error. + */ + virtual Status apply() const; + + /** + * Adds a log entry to logRoot corresponding to the operation applied here. Returns OK + * if successful or a status describing the error. + */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // Compare mode: min/max + const ModifierCompareMode _mode; + + // Access to each component of fieldName that's the target of this mod. + FieldRef _updatePath; + + // 0 or index for $-positional in _updatePath. + size_t _pathReplacementPosition; + + // Element of the mod expression. + BSONElement _val; + + // The instance of the field in the provided doc. This state is valid after a + // prepare() was issued and until a log() is issued. The document this mod is + // being prepared against must be live throughout all the calls. + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_compare_test.cpp b/src/mongo/db/ops/modifier_compare_test.cpp new file mode 100644 index 00000000000..eba04776b1f --- /dev/null +++ b/src/mongo/db/ops/modifier_compare_test.cpp @@ -0,0 +1,296 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_compare.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::ModifierCompare; + using mongo::ModifierInterface; + using mongo::Status; + using mongo::StringData; + using mongo::fromjson; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + const char kModNameMin[] = "$min"; + const char kModNameMax[] = "$max"; + + /** Helper to build and manipulate a $min/max mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _modObj(modObj) + , _mod((modObj.firstElement().fieldNameStringData() == "$min") ? + ModifierCompare::MIN : + ModifierCompare::MAX) { + const StringData& modName = modObj.firstElement().fieldName(); + ASSERT_OK(_mod.init(modObj[modName].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierCompare& mod() { return _mod; } + + private: + BSONObj _modObj; + ModifierCompare _mod; + }; + + TEST(Init, ValidValues) { + BSONObj modObj; + ModifierCompare mod; + + modObj = fromjson("{ $min : { a : 2 } }"); + ASSERT_OK(mod.init(modObj[kModNameMin].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{ $max : { a : 1 } }"); + ASSERT_OK(mod.init(modObj[kModNameMax].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{ $min : { a : {$date : 0 } } }"); + ASSERT_OK(mod.init(modObj[kModNameMin].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(ExistingNumber, MaxSameNumber) { + Document doc(fromjson("{a: 1 }")); + Mod mod(fromjson("{$max: {a: 1} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(ExistingNumber, MinSameNumber) { + Document doc(fromjson("{a: 1 }")); + Mod mod(fromjson("{$min: {a: 1} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(ExistingNumber, MaxNumberIsLess) { + Document doc(fromjson("{a: 1 }")); + Mod mod(fromjson("{$max: {a: 0} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(ExistingNumber, MinNumberIsMore) { + Document doc(fromjson("{a: 1 }")); + Mod mod(fromjson("{$min: {a: 2} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(ExistingDouble, MaxSameValInt) { + Document doc(fromjson("{a: 1.0 }")); + Mod mod(BSON("$max" << BSON("a" << 1LL))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(ExistingDoubleZero, MaxSameValIntZero) { + Document doc(fromjson("{a: 0.0 }")); + Mod mod(BSON("$max" << BSON("a" << 0LL))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(ExistingDoubleZero, MinSameValIntZero) { + Document doc(fromjson("{a: 0.0 }")); + Mod mod(BSON("$min" << BSON("a" << 0LL))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(MissingField, MinNumber) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{$min: {a: 0} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(fromjson("{a : 0}"), doc); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 0 } }"), logDoc); + } + + TEST(ExistingNumber, MinNumber) { + Document doc(fromjson("{a: 1 }")); + Mod mod(fromjson("{$min: {a: 0} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(fromjson("{a : 0}"), doc); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 0 } }"), logDoc); + } + + TEST(MissingField, MaxNumber) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{$max: {a: 0} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(fromjson("{a : 0}"), doc); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 0 } }"), logDoc); + } + + TEST(ExistingNumber, MaxNumber) { + Document doc(fromjson("{a: 1 }")); + Mod mod(fromjson("{$max: {a: 2} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(fromjson("{a : 2}"), doc); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 2 } }"), logDoc); + } + + TEST(ExistingDate, MaxDate) { + Document doc(fromjson("{a: {$date: 0} }")); + Mod mod(fromjson("{$max: {a: {$date: 123123123}} }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(fromjson("{a: {$date: 123123123}}"), doc); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$set: {a: {$date: 123123123}} }"), logDoc); + } + + TEST(ExistingEmbeddedDoc, MaxDoc) { + Document doc(fromjson("{a: {b: 2}}")); + Mod mod(fromjson("{$max: {a: {b: 3}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(fromjson("{a: {b: 3}}}"), doc); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$set: {a: {b: 3}} }"), logDoc); + } + + TEST(ExistingEmbeddedDoc, MaxNumber) { + Document doc(fromjson("{a: {b: 2}}")); + Mod mod(fromjson("{$max: {a: 3}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_current_date.cpp b/src/mongo/db/ops/modifier_current_date.cpp new file mode 100644 index 00000000000..8e721def6db --- /dev/null +++ b/src/mongo/db/ops/modifier_current_date.cpp @@ -0,0 +1,277 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_current_date.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + namespace { + const char kType[] = "$type"; + const char kDate[] = "date"; + const char kTimestamp[] = "timestamp"; + } + + struct ModifierCurrentDate::PreparedState { + + PreparedState(mutablebson::Document& doc) + : doc(doc) + , elemFound(doc.end()) + , idxFound(0) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element elemFound; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + }; + + ModifierCurrentDate::ModifierCurrentDate() + : _pathReplacementPosition(0) + , _typeIsDate(true) { + } + + ModifierCurrentDate::~ModifierCurrentDate() { + } + + Status ModifierCurrentDate::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + _updatePath.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_updatePath); + if (!status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_updatePath, + &_pathReplacementPosition, + &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _updatePath.dottedField() << "'"); + } + + // Validate and store the type to produce + switch (modExpr.type()) { + case Bool: + _typeIsDate = true; + break; + case Object: { + const BSONObj argObj = modExpr.embeddedObject(); + const BSONElement typeElem = argObj.getField(kType); + bool badInput = typeElem.eoo() || !(typeElem.type() == String); + + if (!badInput) { + std::string typeVal = typeElem.String(); + badInput = !(typeElem.String() == kDate || typeElem.String() == kTimestamp); + if (!badInput) + _typeIsDate = (typeVal == kDate); + + if (!badInput) { + // Check to make sure only the $type field was given as an arg + BSONObjIterator i( argObj ); + const bool onlyHasTypeField = ((i.next().fieldNameStringData() == kType) + && i.next().eoo()); + if (!onlyHasTypeField) { + return Status(ErrorCodes::BadValue, + str::stream() << + "The only valid field of the option is '$type': " + "{$currentDate: {field : {$type: 'date/timestamp'}}}; " + << "arg: " << argObj); + } + + } + + } + + if (badInput) { + return Status(ErrorCodes::BadValue, + "The '$type' string field is required " + "to be 'date' or 'timestamp': " + "{$currentDate: {field : {$type: 'date'}}}"); + } + break; + } + default: + return Status(ErrorCodes::BadValue, + str::stream() << typeName(modExpr.type()) + << " is not valid type for $currentDate." + " Please use a boolean ('true')" + " or a $type expression ({$type: 'timestamp/date'})."); + } + + return Status::OK(); + } + + Status ModifierCurrentDate::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_pathReplacementPosition) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _updatePath.dottedField()); + } + _updatePath.setPart(_pathReplacementPosition, matchedField); + } + + // Locate the field name in 'root'. Note that we may not have all the parts in the path + // in the doc -- which is fine. Our goal now is merely to reason about whether this mod + // apply is a noOp or whether is can be in place. The remaining path, if missing, will + // be created during the apply. + Status status = pathsupport::findLongestPrefix(_updatePath, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + } + else if (!status.isOK()) { + return status; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_updatePath; + + return Status::OK(); + } + + Status ModifierCurrentDate::apply() const { + + const bool destExists = (_preparedState->elemFound.ok() && + _preparedState->idxFound == (_updatePath.numParts() - 1)); + + mutablebson::Document& doc = _preparedState->doc; + StringData lastPart = _updatePath.getPart(_updatePath.numParts() - 1); + // If the element exists and is the same type, then that is what we want to work with + mutablebson::Element elemToSet = destExists ? + _preparedState->elemFound: + doc.end() ; + + if (!destExists) { + // Creates the final element that's going to be $set in 'doc'. + // fills in the value with place-holder/empty + + elemToSet = _typeIsDate ? + doc.makeElementDate(lastPart, Date_t()) : + doc.makeElementTimestamp(lastPart, OpTime()); + + if (!elemToSet.ok()) { + return Status(ErrorCodes::InternalError, "can't create new element"); + } + + // Now, we can be in two cases here, as far as attaching the element being set goes: + // (a) none of the parts in the element's path exist, or (b) some parts of the path + // exist but not all. + if (!_preparedState->elemFound.ok()) { + _preparedState->elemFound = doc.root(); + _preparedState->idxFound = 0; + } + else { + _preparedState->idxFound++; + } + + // createPathAt() will complete the path and attach 'elemToSet' at the end of it. + Status s = pathsupport::createPathAt(_updatePath, + _preparedState->idxFound, + _preparedState->elemFound, + elemToSet); + if (!s.isOK()) + return s; + } + + dassert(elemToSet.ok()); + + // By the time we are here the element is in place and we just need to update the value + if (_typeIsDate) { + const mongo::Date_t now = mongo::jsTime(); + Status s = elemToSet.setValueDate(now); + if (!s.isOK()) + return s; + } + else { + mutex::scoped_lock lk(OpTime::m); + const OpTime timestamp = OpTime::now(lk); + Status s = elemToSet.setValueTimestamp(timestamp); + if (!s.isOK()) + return s; + } + + // Set the elemFound, idxFound to the changed element for oplog logging. + _preparedState->elemFound = elemToSet; + _preparedState->idxFound = (_updatePath.numParts() - 1); + + return Status::OK(); + } + + Status ModifierCurrentDate::log(LogBuilder* logBuilder) const { + // TODO: None of this checks should be needed unless someone calls if we are a noOp + // When we cleanup we should build in testing that no-one calls in the noOp case + const bool destExists = (_preparedState->elemFound.ok() && + _preparedState->idxFound == (_updatePath.numParts() - 1)); + + // If the destination doesn't exist then we have nothing to log. + // This would only happen if apply isn't called or fails when it had to create an element + if (!destExists) + return Status::OK(); + + return logBuilder->addToSetsWithNewFieldName(_updatePath.dottedField(), + _preparedState->elemFound); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_current_date.h b/src/mongo/db/ops/modifier_current_date.h new file mode 100644 index 00000000000..d85cd6e1d2b --- /dev/null +++ b/src/mongo/db/ops/modifier_current_date.h @@ -0,0 +1,90 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierCurrentDate : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierCurrentDate); + + public: + + ModifierCurrentDate(); + virtual ~ModifierCurrentDate(); + + /** + * A 'modExpr' is a BSONElement {: } coming + * from a $currentDate mod such as + * {$currentDate: { _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_current_date_test.cpp b/src/mongo/db/ops/modifier_current_date_test.cpp new file mode 100644 index 00000000000..0b1f64c5a00 --- /dev/null +++ b/src/mongo/db/ops/modifier_current_date_test.cpp @@ -0,0 +1,365 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_current_date.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::ModifierCurrentDate; + using mongo::ModifierInterface; + using mongo::OpTime; + using mongo::Status; + using mongo::StringData; + using mongo::fromjson; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** + * Helper to validate oplog entries in the tests below. + */ + void validateOplogEntry(BSONObj& oplogFormat, Document& doc){ + // Ensure that the field is the same + ASSERT_EQUALS(oplogFormat.firstElement().fieldName(), + doc.root().leftChild().getFieldName()); + + // Ensure the field names are the same + ASSERT_EQUALS(oplogFormat.firstElement().embeddedObject().firstElement().fieldName(), + doc.root().leftChild().leftChild().getFieldName()); + + // Ensure the type is the same in the document as the oplog + ASSERT_EQUALS(oplogFormat.firstElement().embeddedObject().firstElement().type(), + doc.root().leftChild().leftChild().getType()); + } + + /** Helper to build and manipulate a $currentDate mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _modObj(modObj) + , _mod() { + ASSERT_OK(_mod.init(_modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierCurrentDate& mod() { return _mod; } + + private: + BSONObj _modObj; + ModifierCurrentDate _mod; + }; + + TEST(Init, ValidValues) { + BSONObj modObj; + ModifierCurrentDate mod; + + modObj = fromjson("{ $currentDate : { a : true } }"); + ASSERT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{ $currentDate : { a : {$type : 'timestamp' } } }"); + ASSERT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{ $currentDate : { a : {$type : 'date' } } }"); + ASSERT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, FailToInitWithInvalidValue) { + BSONObj modObj; + ModifierCurrentDate mod; + + // String is an invalid $currentDate argument + modObj = fromjson("{ $currentDate : { a : 'Oct 11, 2001' } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // Array is an invalid $currentDate argument + modObj = fromjson("{ $currentDate : { a : [] } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // Number is an invalid $currentDate argument + modObj = fromjson("{ $currentDate : { a : 1 } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // Regex is an invalid $currentDate argument + modObj = fromjson("{ $currentDate : { a : /1/ } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An object with missing $type field is an invalid $currentDate argument + modObj = fromjson("{ $currentDate : { a : { foo : 4 } } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An object with extra fields, including the $type field is bad + modObj = fromjson("{ $currentDate : { a : { $type: 'date', foo : 4 } } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An object with extra fields, including the $type field is bad + modObj = fromjson("{ $currentDate : { a : { foo: 4, $type : 'date' } } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An object with non-date/timestamp $type field is an invalid $currentDate argument + modObj = fromjson("{ $currentDate : { a : { $type : 4 } } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // An object with non-date/timestamp $type field is an invalid $currentDate argument + modObj = fromjson("{ $currentDate : { a : { $type : 'foo' } } }"); + ASSERT_NOT_OK(mod.init(modObj["$currentDate"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(BoolInput, EmptyStartDoc) { + Document doc(fromjson("{ }")); + Mod mod(fromjson("{ $currentDate : { a : true } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + BSONObj olderDateObj = fromjson("{ a : { $date : 0 } }"); + ASSERT_OK(mod.apply()); + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { a : { $date : 0 } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(DateInput, EmptyStartDoc) { + Document doc(fromjson("{ }")); + Mod mod(fromjson("{ $currentDate : { a : {$type: 'date' } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + BSONObj olderDateObj = fromjson("{ a : { $date : 0 } }"); + ASSERT_OK(mod.apply()); + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { a : { $date : 0 } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(TimestampInput, EmptyStartDoc) { + Document doc(fromjson("{ }")); + Mod mod(fromjson("{ $currentDate : { a : {$type : 'timestamp' } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + mongo::OpTime ts; + BSONObj olderDateObj = BSON("a" << ts); + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { a : { $timestamp : {t:0, i:0} } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(BoolInput, ExistingStringDoc) { + Document doc(fromjson("{ a: 'a' }")); + Mod mod(fromjson("{ $currentDate : { a : true } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + BSONObj olderDateObj = fromjson("{ a : { $date : 0 } }"); + ASSERT_OK(mod.apply()); + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { a : { $date : 0 } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(BoolInput, ExistingDateDoc) { + Document doc(fromjson("{ a: {$date: 0 } }")); + Mod mod(fromjson("{ $currentDate : { a : true } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + BSONObj olderDateObj = fromjson("{ a : { $date : 0 } }"); + ASSERT_OK(mod.apply()); + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { a : { $date : 0 } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(DateInput, ExistingDateDoc) { + Document doc(fromjson("{ a: {$date: 0 } }")); + Mod mod(fromjson("{ $currentDate : { a : {$type: 'date' } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + BSONObj olderDateObj = fromjson("{ a : { $date : 0 } }"); + ASSERT_OK(mod.apply()); + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { a : { $date : 0 } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(TimestampInput, ExistingDateDoc) { + Document doc(fromjson("{ a: {$date: 0 } }")); + Mod mod(fromjson("{ $currentDate : { a : {$type : 'timestamp' } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a", execInfo.fieldRef[0]->dottedField()); + + mongo::OpTime ts; + BSONObj olderDateObj = BSON("a" << ts); + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); //Same Size as Date + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { a : { $timestamp : {t:0, i:0} } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(TimestampInput, ExistingEmbeddedDateDoc) { + Document doc(fromjson("{ a: {b: {$date: 0 } } }")); + Mod mod(fromjson("{ $currentDate : { 'a.b' : {$type : 'timestamp' } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a.b", execInfo.fieldRef[0]->dottedField()); + + mongo::OpTime ts; + BSONObj olderDateObj = BSON("a" << BSON( "b" << ts)); + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); //Same Size as Date + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { 'a.b' : { $timestamp : {t:0, i:0} } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + + TEST(DottedTimestampInput, EmptyStartDoc) { + Document doc(fromjson("{ }")); + Mod mod(fromjson("{ $currentDate : { 'a.b' : {$type : 'timestamp' } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS("a.b", execInfo.fieldRef[0]->dottedField()); + + mongo::OpTime ts; + BSONObj olderDateObj = BSON("a" << BSON( "b" << ts)); + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_LESS_THAN(olderDateObj, doc.getObject()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + BSONObj oplogFormat = fromjson("{ $set : { 'a.b' : { $timestamp : {t:0, i:0} } } }"); + validateOplogEntry(oplogFormat, logDoc); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_inc.cpp b/src/mongo/db/ops/modifier_inc.cpp new file mode 100644 index 00000000000..ad5364317aa --- /dev/null +++ b/src/mongo/db/ops/modifier_inc.cpp @@ -0,0 +1,297 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_inc.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace mb = mutablebson; + namespace str = mongoutils::str; + + struct ModifierInc::PreparedState { + + PreparedState(mutablebson::Document& doc) + : doc(doc) + , idxFound(0) + , elemFound(doc.end()) + , newValue() + , noOp(false) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element elemFound; + + // Value to be applied + SafeNum newValue; + + // This $inc is a no-op? + bool noOp; + }; + + ModifierInc::ModifierInc(ModifierIncMode mode) + : ModifierInterface () + , _mode(mode) + , _fieldRef() + , _posDollar(0) + , _val() { + } + + ModifierInc::~ModifierInc() { + } + + Status ModifierInc::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + // + // field name analysis + // + + // Perform standard field name and updateable checks. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + // + // value analysis + // + + if (!modExpr.isNumber()) { + // TODO: Context for mod error messages would be helpful + // include mod code, etc. + return Status(ErrorCodes::TypeMismatch, + str::stream() << "Cannot " + << (_mode == MODE_INC ? "increment" : "multiply") + << " with non-numeric argument: {" + << modExpr << "}"); + } + + _val = modExpr; + dassert(_val.isValid()); + + return Status::OK(); + } + + Status ModifierInc::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_posDollar) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_posDollar, matchedField); + } + + // Locate the field name in 'root'. Note that we may not have all the parts in the path + // in the doc -- which is fine. Our goal now is merely to reason about whether this mod + // apply is a noOp or whether is can be in place. The remaining path, if missing, will + // be created during the apply. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + } + else if (!status.isOK()) { + return status; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_fieldRef; + + // Capture the value we are going to write. At this point, there may not be a value + // against which to operate, so the result will be simply _val. + _preparedState->newValue = _val; + + // + // in-place and no-op logic + // + // If the field path is not fully present, then this mod cannot be in place, nor is a + // noOp. + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + + // For multiplication, we treat ops against missing as yielding zero. We take + // advantage here of the promotion rules for SafeNum; the expression below will + // always yield a zero of the same type of operand that the user provided + // (e.g. double). + if (_mode == MODE_MUL) + _preparedState->newValue *= SafeNum(static_cast(0)); + + return Status::OK(); + } + + // If the value being $inc'ed is the same as the one already in the doc, than this is a + // noOp. + if (!_preparedState->elemFound.isNumeric()) { + mb::Element idElem = mb::findFirstChildNamed(root, "_id"); + return Status( + ErrorCodes::TypeMismatch, + str::stream() << "Cannot apply " + << (_mode == MODE_INC ? "$inc" : "$mul") + << " to a value of non-numeric type. {" + << idElem.toString() + << "} has the field '" << _preparedState->elemFound.getFieldName() + << "' of non-numeric type " + << typeName(_preparedState->elemFound.getType())); + } + const SafeNum currentValue = _preparedState->elemFound.getValueSafeNum(); + + // Update newValue w.r.t to the current value of the found element. + if (_mode == MODE_INC) + _preparedState->newValue += currentValue; + else + _preparedState->newValue *= currentValue; + + // If the result of the addition is invalid, we must return an error. + if (!_preparedState->newValue.isValid()) { + mb::Element idElem = mb::findFirstChildNamed(root, "_id"); + return Status(ErrorCodes::BadValue, + str::stream() << "Failed to apply $inc operations to current value (" + << currentValue.debugString() << ") for document {" + << idElem.toString() << "}"); + } + + // If the values are identical (same type, same value), then this is a no-op. + if (_preparedState->newValue.isIdentical(currentValue)) { + _preparedState->noOp = execInfo->noOp = true; + return Status::OK(); + } + + return Status::OK(); + } + + Status ModifierInc::apply() const { + dassert(_preparedState->noOp == false); + + // If there's no need to create any further field part, the $inc is simply a value + // assignment. + if (_preparedState->elemFound.ok() && + _preparedState->idxFound == (_fieldRef.numParts() - 1)) { + return _preparedState->elemFound.setValueSafeNum(_preparedState->newValue); + } + + // + // Complete document path logic + // + + // Creates the final element that's going to be $set in 'doc'. + mutablebson::Document& doc = _preparedState->doc; + StringData lastPart = _fieldRef.getPart(_fieldRef.numParts() - 1); + mutablebson::Element elemToSet = doc.makeElementSafeNum(lastPart, _preparedState->newValue); + if (!elemToSet.ok()) { + return Status(ErrorCodes::InternalError, "can't create new element"); + } + + // Now, we can be in two cases here, as far as attaching the element being set goes: + // (a) none of the parts in the element's path exist, or (b) some parts of the path + // exist but not all. + if (!_preparedState->elemFound.ok()) { + _preparedState->elemFound = doc.root(); + _preparedState->idxFound = 0; + } + else { + _preparedState->idxFound++; + } + + // createPathAt() will complete the path and attach 'elemToSet' at the end of it. + return pathsupport::createPathAt(_fieldRef, + _preparedState->idxFound, + _preparedState->elemFound, + elemToSet); + } + + Status ModifierInc::log(LogBuilder* logBuilder) const { + + dassert(_preparedState->newValue.isValid()); + + // We'd like to create an entry such as {$set: {: }} under 'logRoot'. + // We start by creating the {$set: ...} Element. + mutablebson::Document& doc = logBuilder->getDocument(); + + // Then we create the {: } Element. + mutablebson::Element logElement = doc.makeElementSafeNum( + _fieldRef.dottedField(), + _preparedState->newValue); + + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, + str::stream() << "Could not append entry to " + << (_mode == MODE_INC ? "$inc" : "$mul") + << " oplog entry: " + << "set '" << _fieldRef.dottedField() << "' -> " + << _preparedState->newValue.debugString() ); + } + + // Now, we attach the {: } Element under the {$set: ...} segment. + return logBuilder->addToSets(logElement); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_inc.h b/src/mongo/db/ops/modifier_inc.h new file mode 100644 index 00000000000..3a93590fcbc --- /dev/null +++ b/src/mongo/db/ops/modifier_inc.h @@ -0,0 +1,101 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierInc : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierInc); + + public: + + // TODO: This is a shortcut to implementing $mul by hijacking $inc. In the near future, + // we should consider either pulling $mul into its own operator, or creating a general + // purpose "numeric binary op" operator. Potentially, that operator could also subsume + // $bit (thought there are some subtleties, like that $bit can have multiple + // operations, and doing so with arbirary math operations introduces potential + // associativity difficulties). At the very least, if this mechanism is retained, then + // this class should be renamed at some point away from ModifierInc. + enum ModifierIncMode { + MODE_INC, + MODE_MUL + }; + + ModifierInc(ModifierIncMode mode = MODE_INC); + virtual ~ModifierInc(); + + /** + * A 'modExpr' is a BSONElement {: } coming from a $inc mod such as + * {$inc: {}}. init() extracts the field name and the value to be + * assigned to it from 'modExpr'. It returns OK if successful or a status describing + * the error. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** Evaluates the validity of applying $inc to the identified node, and computes + * effects, handling upcasting and overflow as necessary. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** Updates the node passed in prepare with the results of the $inc */ + virtual Status apply() const; + + /** Converts the result of the $inc into an equivalent $set under logRoot */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + const ModifierIncMode _mode; + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _posDollar; + + // Element of the $set expression. + SafeNum _val; + + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_inc_test.cpp b/src/mongo/db/ops/modifier_inc_test.cpp new file mode 100644 index 00000000000..68d3db810e6 --- /dev/null +++ b/src/mongo/db/ops/modifier_inc_test.cpp @@ -0,0 +1,524 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_inc.h" + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::ModifierInc; + using mongo::ModifierInterface; + using mongo::NumberInt; + using mongo::Status; + using mongo::StringData; + using mongo::fromjson; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + using mongo::mutablebson::countChildren; + + /** Helper to build and manipulate a $inc mod. */ + class Mod { + public: + + explicit Mod(BSONObj modObj) + : _modObj(modObj) + , _mod(mongoutils::str::equals(modObj.firstElement().fieldName(), "$mul") ? + ModifierInc::MODE_MUL : ModifierInc::MODE_INC) { + const StringData& modName = modObj.firstElement().fieldName(); + ASSERT_OK(_mod.init(_modObj[modName].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierInc& mod() { return _mod; } + + private: + BSONObj _modObj; + ModifierInc _mod; + }; + + TEST(Init, FailToInitWithInvalidValue) { + BSONObj modObj; + ModifierInc mod; + + // String is an invalid increment argument + modObj = fromjson("{ $inc : { a : '' } }"); + ASSERT_NOT_OK(mod.init(modObj["$inc"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // Object is an invalid increment argument + modObj = fromjson("{ $inc : { a : {} } }"); + ASSERT_NOT_OK(mod.init(modObj["$inc"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + // Array is an invalid increment argument + modObj = fromjson("{ $inc : { a : [] } }"); + ASSERT_NOT_OK(mod.init(modObj["$inc"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, InitParsesNumberInt) { + Mod incMod(BSON("$inc" << BSON("a" << static_cast(1)))); + } + + TEST(Init, InitParsesNumberLong) { + Mod incMod(BSON("$inc" << BSON("a" << static_cast(1)))); + } + + TEST(Init, InitParsesNumberDouble) { + Mod incMod(BSON("$inc" << BSON("a" << 1.0))); + } + + TEST(SimpleMod, PrepareSimpleOK) { + Document doc(fromjson("{ a : 1 }")); + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(SimpleMod, PrepareSimpleNonNumericObject) { + Document doc(fromjson("{ a : {} }")); + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(incMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, PrepareSimpleNonNumericArray) { + + Document doc(fromjson("{ a : [] }")); + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(incMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, PrepareSimpleNonNumericString) { + Document doc(fromjson("{ a : '' }")); + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(incMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, ApplyAndLogEmptyDocument) { + Document doc(fromjson("{}")); + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1 } }"), logDoc); + } + + TEST(SimpleMod, LogWithoutApplyEmptyDocument) { + Document doc(fromjson("{}")); + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1 } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogSimpleDocument) { + Document doc(fromjson("{ a : 2 }")); + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 3 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 3 } }"), logDoc); + } + + TEST(DottedMod, ApplyAndLogSimpleDocument) { + Document doc(fromjson("{ a : { b : 2 } }")); + Mod incMod(fromjson("{ $inc: { 'a.b' : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : { b : 3 } }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { 'a.b' : 3 } }"), logDoc); + } + + TEST(InPlace, IntToInt) { + Document doc(BSON("a" << static_cast(1))); + Mod incMod(BSON("$inc" << BSON("a" << static_cast(1)))); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(InPlace, LongToLong) { + Document doc(BSON("a" << static_cast(1))); + Mod incMod(BSON("$inc" << BSON("a" << static_cast(1)))); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(InPlace, DoubleToDouble) { + Document doc(BSON("a" << 1.0)); + Mod incMod(BSON("$inc" << BSON("a" << 1.0 ))); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(NoOp, Int) { + Document doc(BSON("a" << static_cast(1))); + Mod incMod(BSON("$inc" << BSON("a" << static_cast(0)))); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(NoOp, Long) { + Document doc(BSON("a" << static_cast(1))); + Mod incMod(BSON("$inc" << BSON("a" << static_cast(0)))); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(NoOp, Double) { + Document doc(BSON("a" << 1.0)); + Mod incMod(BSON("$inc" << BSON("a" << 0.0))); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(Upcasting, UpcastIntToLong) { + // Checks that $inc : NumberLong(0) turns a NumberInt into a NumberLong and logs it + // correctly. + Document doc(BSON("a" << static_cast(1))); + ASSERT_EQUALS(mongo::NumberInt, doc.root()["a"].getType()); + + Mod incMod(BSON("$inc" << BSON("a" << static_cast(0)))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1 }"), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1 } }"), logDoc); + ASSERT_EQUALS(mongo::NumberLong, logDoc.root()["$set"]["a"].getType()); + } + + TEST(Upcasting, UpcastIntToDouble) { + // Checks that $inc : 0.0 turns a NumberInt into a NumberDouble and logs it + // correctly. + Document doc(BSON("a" << static_cast(1))); + ASSERT_EQUALS(mongo::NumberInt, doc.root()["a"].getType()); + + Mod incMod(fromjson("{ $inc : { a : 0.0 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1.0 }"), doc); + ASSERT_EQUALS(mongo::NumberDouble, doc.root()["a"].getType()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1.0 } }"), logDoc); + ASSERT_EQUALS(mongo::NumberDouble, logDoc.root()["$set"]["a"].getType()); + } + + TEST(Upcasting, UpcastLongToDouble) { + // Checks that $inc : 0.0 turns a NumberLong into a NumberDouble and logs it + // correctly. + Document doc(BSON("a" << static_cast(1))); + ASSERT_EQUALS(mongo::NumberLong, doc.root()["a"].getType()); + + Mod incMod(fromjson("{ $inc : { a : 0.0 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 1.0 }"), doc); + ASSERT_EQUALS(mongo::NumberDouble, doc.root()["a"].getType()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 1.0 } }"), logDoc); + ASSERT_EQUALS(mongo::NumberDouble, logDoc.root()["$set"]["a"].getType()); + } + + TEST(Upcasting, DoublesStayDoubles) { + // Checks that $inc : 0 doesn't change a NumberDouble away from double + Document doc(fromjson("{ a : 1.0 }")); + ASSERT_EQUALS(mongo::NumberDouble, doc.root()["a"].getType()); + + Mod incMod(fromjson("{ $inc : { a : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 2.0 }"), doc); + ASSERT_EQUALS(mongo::NumberDouble, doc.root()["a"].getType()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : 2.0 } }"), logDoc); + ASSERT_EQUALS(mongo::NumberDouble, logDoc.root()["$set"]["a"].getType()); + } + + // The only interesting overflow cases are int->long via increment: we never overflow to + // double, and we never decrease precision on decrement. + + TEST(Spilling, OverflowIntToLong) { + const int initial_value = std::numeric_limits::max(); + + Document doc(BSON("a" << static_cast(initial_value))); + ASSERT_EQUALS(mongo::NumberInt, doc.root()["a"].getType()); + + Mod incMod(fromjson("{ $inc : { a : 1 } }")); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + const long long target_value = static_cast(initial_value) + 1; + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(BSON("a" << target_value), doc); + } + + TEST(Spilling, UnderflowIntToLong) { + const int initial_value = std::numeric_limits::min(); + + Document doc(BSON("a" << static_cast(initial_value))); + ASSERT_EQUALS(mongo::NumberInt, doc.root()["a"].getType()); + + Mod incMod(fromjson("{ $inc : { a : -1 } }")); + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + const long long target_value = static_cast(initial_value) - 1; + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(BSON("a" << target_value), doc); + } + + TEST(Lifecycle, IncModCanBeReused) { + Document doc1(fromjson("{ a : 1 }")); + Document doc2(fromjson("{ a : 1 }")); + + Mod incMod(fromjson("{ $inc: { a : 1 }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc1.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_TRUE(doc1.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 2 }"), doc1); + + ASSERT_OK(incMod.prepare(doc2.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_TRUE(doc2.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 2 }"), doc2); + } + + // Given the current implementation of $mul, we really only need one test for + // $mul. However, in the future, we should probably write additional ones, or, perhaps find + // a way to run all the obove tests in both modes. + TEST(Multiplication, ApplyAndLogSimpleDocument) { + Document doc(fromjson("{ a : { b : 2 } }")); + Mod incMod(fromjson("{ $mul: { 'a.b' : 3 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : { b : 6 } }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { 'a.b' : 6 } }"), logDoc); + } + + TEST(Multiplication, ApplyAndLogMissingElement) { + Document doc(fromjson("{ a : 0 }")); + Mod incMod(fromjson("{ $mul : { b : 3 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : 0, b : 0 }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(incMod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { b : 0 } }"), logDoc); + } + + TEST(Multiplication, ApplyMissingElementInt) { + const int int_zero = 0; + const int int_three = 3; + + Document doc(BSON("a" << int_zero)); + Mod incMod(BSON("$mul" << BSON("b" << int_three))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(BSON("a" << int_zero << "b" << int_zero), doc); + ASSERT_EQUALS(mongo::NumberInt, doc.root().rightChild().getType()); + } + + TEST(Multiplication, ApplyMissingElementLongLong) { + const long long ll_zero = 0; + const long long ll_three = 3; + + Document doc(BSON("a" << ll_zero)); + Mod incMod(BSON("$mul" << BSON("b" << ll_three))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(BSON("a" << ll_zero << "b" << ll_zero), doc); + ASSERT_EQUALS(mongo::NumberLong, doc.root().rightChild().getType()); + } + + TEST(Multiplication, ApplyMissingElementDouble) { + const double double_zero = 0; + const double double_three = 3; + + Document doc(BSON("a" << double_zero)); + Mod incMod(BSON("$mul" << BSON("b" << double_three))); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(incMod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(incMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(BSON("a" << double_zero << "b" << 0), doc); + ASSERT_EQUALS(mongo::NumberDouble, doc.root().rightChild().getType()); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_interface.h b/src/mongo/db/ops/modifier_interface.h new file mode 100644 index 00000000000..873d18e78c3 --- /dev/null +++ b/src/mongo/db/ops/modifier_interface.h @@ -0,0 +1,195 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + class LogBuilder; + + /** + * Abstract base class for update "modifiers" (a.k.a "$ operators"). To create a new + * operator, implement a new derived class. + * + * A typical call sequence for the class is: + * + * + init() with the mod arguments + * + * + For each document that is being touched on that update, the following methods are + * going to be called once for that document and in the order the calls appear here. + * + * + prepare() to check if mod is viable over the document + * + * + apply(), effectively computing the update + * + * + log() registering the change in the log for replication purposes + * + * Again, a modifier implementation may rely on these last three calls being made and in + * that particular order and therefore can keep and reuse state between these calls, when + * appropriate. + * + * TODO: + * For a reference implementation, see modifier_identity.{h,cpp} used in tests. + */ + class ModifierInterface { + public: + virtual ~ModifierInterface() { } + + struct Options; + /** + * Returns OK and extracts the parameters for this given mod from 'modExpr'. For + * instance, for a $inc, extracts the increment value. The init() method would be + * called only once per operand, that is, if a { $inc: { a: 1, b: 1 } } is issued, + * there would be one instance of the operator working on 'a' and one on 'b'. In each + * case, init() would be called once with the respective bson element. + * + * If 'modExpr' is invalid, returns an error status with a reason description. + * + * The optional bool out parameter 'positional', if provided, will be set to 'true' if + * the mod requires matched field details to be provided when calling 'prepare'. The + * field is optional since this is a hint to the caller about what work is needed to + * correctly invoke 'prepare'. It is always legal to provide any match details + * unconditionally. The value set in 'positional' if any, is only meaningful if 'init' + * returns an OK status. + * + * Note: + * + * + An operator may assume the modExpr passed here will be unchanged throughout all + * the mod object lifetime and also that the modExrp's lifetime exceeds the life + * time of this mod. Therefore, taking references to elements inside modExpr is + * valid. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL) = 0; + + /** + * Returns OK if it would be correct to apply this mod over the document 'root' (e.g, if + * we're $inc-ing a field, is that field numeric in the current doc?). + * + * If the field this mod is targeted to contains a $-positional parameter, that value + * can be bound with 'matchedField', passed by the caller. + * + * In addition, the call also identifies which fields(s) of 'root' the mod is interested + * in changing (note that the modifier may want to add a field that's not present in + * the document). The call also determines whether it could modify the document in + * place and whether it is a no-op for the given document. All this information is in + * the passed 'execInfo', which is filled inside the call. + * + * If the mod cannot be applied over 'root', returns an error status with a reason + * description. + * + * Note that you must provide a meaningful 'matchedField' here, unless 'init' set + * 'positional' to 'false', in which case you may pass an empty StringData object. + */ + struct ExecInfo; + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + /* IN-OUT */ ExecInfo* execInfo) = 0; + + /** + * Returns OK and modifies (or adds) an element (or elements) from the 'root' passed on + * the prepareMod call. This may act on multiple fields but should only be called once + * per operator. + * + * For this call to be issued, the call to 'prepareElem' must have necessarily turned + * off 'ExecInfo.noOp', ie this mod over this document is not a no-op. + * + * If the mod could not be applied, returns an error status with a reason description. + */ + virtual Status apply() const = 0 ; + + /** + * Returns OK and records the result of this mod in the provided LogBuilder. The mod + * must have kept enough state to be able to produce the log record (see idempotency + * note below). This call may be issued even if apply() was not. + * + * If the mod could not be logged, returns an error status with a reason description. + * + * Idempotency Note: + * + * + The modifier must log a mod that is idempotent, ie, applying it more than once + * to a base collection would produce the same result as applying it only once. For + * example, a $inc can be switched to a $set for the resulting incremented value, + * for logging purposes. An array based operator may check the contents of the + * array before operating on it. + */ + virtual Status log(LogBuilder* logBuilder) const = 0; + }; + + /** + * Options used to control Modifier behavior + */ + struct ModifierInterface::Options { + Options() : fromReplication(false), enforceOkForStorage(true) {} + Options(bool repl, bool ofs) : fromReplication(repl), enforceOkForStorage(ofs) {} + + static Options normal() { return Options(false, true); } + static Options fromRepl() { return Options(true, false); } + static Options unchecked() { return Options(false, false); } + + bool fromReplication; + bool enforceOkForStorage; + }; + + struct ModifierInterface::ExecInfo { + static const int MAX_NUM_FIELDS = 2; + + /** + * An update mod may specify that it wishes to the applied only if the context + * of the update turns out a certain way. + */ + enum UpdateContext { + // This mod wants to be applied only if the update turns out to be an insert. + INSERT_CONTEXT, + + // This mod wants to be applied only if the update is not an insert. + UPDATE_CONTEXT, + + // This mod doesn't care if the update will be an update or an upsert. + ANY_CONTEXT + }; + + ExecInfo() : noOp(false), context(ANY_CONTEXT) { + for (int i = 0; i < MAX_NUM_FIELDS; i++) { + fieldRef[i] = NULL; + } + } + + // The fields of concern to the driver: no other op may modify the fields listed here. + FieldRef* fieldRef[MAX_NUM_FIELDS]; // not owned here + bool noOp; + UpdateContext context; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_object_replace.cpp b/src/mongo/db/ops/modifier_object_replace.cpp new file mode 100644 index 00000000000..4748286b9d8 --- /dev/null +++ b/src/mongo/db/ops/modifier_object_replace.cpp @@ -0,0 +1,194 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_object_replace.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + namespace { + const char idFieldName[] = "_id"; + + void fixupTimestamps( const BSONObj& obj ) { + BSONObjIterator i(obj); + while (i.more()) { + BSONElement e = i.next(); + + // Skip _id field -- we do not replace it + if (e.type() == Timestamp && e.fieldNameStringData() != idFieldName) { + // performance note, this locks a mutex: + unsigned long long ×tamp = + *(reinterpret_cast( + const_cast(e.value()))); + if (timestamp == 0) { + mutex::scoped_lock lk(OpTime::m); + timestamp = OpTime::now(lk).asDate(); + } + } + } + } + } + + struct ModifierObjectReplace::PreparedState { + + PreparedState(mutablebson::Document* targetDoc) + : doc(*targetDoc) + , noOp(false) { + } + + // Document that is going to be changed + mutablebson::Document& doc; + + // This is a no op + bool noOp; + + }; + + ModifierObjectReplace::ModifierObjectReplace() : _val() { + } + + ModifierObjectReplace::~ModifierObjectReplace() { + } + + Status ModifierObjectReplace::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + if (modExpr.type() != Object) { + // Impossible, really since the caller check this already... + return Status(ErrorCodes::BadValue, + str::stream() << "Document replacement expects a complete document" + " but the type supplied was " + << modExpr.type()); + } + + // Object replacements never have positional operator. + if (positional) + *positional = false; + + // We make a copy of the object here because the update driver does not guarantees, in + // the case of object replacement, that the modExpr is going to outlive this mod. + _val = modExpr.embeddedObject().getOwned(); + fixupTimestamps(_val); + + return Status::OK(); + } + + Status ModifierObjectReplace::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + _preparedState.reset(new PreparedState(&root.getDocument())); + + // objectSize checked by binaryEqual (optimization) + BSONObj objOld = root.getDocument().getObject(); + if (objOld.binaryEqual(_val)) { + _preparedState->noOp = true; + execInfo->noOp = true; + } + + return Status::OK(); + } + + Status ModifierObjectReplace::apply() const { + dassert(!_preparedState->noOp); + + // Remove the contents of the provided doc. + mutablebson::Document& doc = _preparedState->doc; + mutablebson::Element current = doc.root().leftChild(); + mutablebson::Element srcIdElement = doc.end(); + while (current.ok()) { + mutablebson::Element toRemove = current; + current = current.rightSibling(); + + // Skip _id field element -- it should not change + if (toRemove.getFieldName() == idFieldName) { + srcIdElement = toRemove; + continue; + } + + Status status = toRemove.remove(); + if (!status.isOK()) { + return status; + } + } + + // Insert the provided contents instead. + BSONElement dstIdElement; + BSONObjIterator it(_val); + while (it.more()) { + BSONElement elem = it.next(); + if (elem.fieldNameStringData() == idFieldName) { + dstIdElement = elem; + + // Do not duplicate _id field + if (srcIdElement.ok()) { + if (srcIdElement.compareWithBSONElement(dstIdElement, true) != 0) { + return Status(ErrorCodes::ImmutableField, + str::stream() << "The _id field cannot be changed from {" + << srcIdElement.toString() << "} to {" + << dstIdElement.toString() << "}."); + } + continue; + } + } + + Status status = doc.root().appendElement(elem); + if (!status.isOK()) { + return status; + } + } + + return Status::OK(); + } + + Status ModifierObjectReplace::log(LogBuilder* logBuilder) const { + + mutablebson::Document& doc = logBuilder->getDocument(); + + mutablebson::Element replacementObject = doc.end(); + Status status = logBuilder->getReplacementObject(&replacementObject); + + if (status.isOK()) { + mutablebson::Element current = _preparedState->doc.root().leftChild(); + while (current.ok()) { + status = replacementObject.appendElement(current.getValue()); + if (!status.isOK()) + return status; + current = current.rightSibling(); + } + } + + return status; + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_object_replace.h b/src/mongo/db/ops/modifier_object_replace.h new file mode 100644 index 00000000000..c8bc2ee4b5f --- /dev/null +++ b/src/mongo/db/ops/modifier_object_replace.h @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierObjectReplace : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierObjectReplace); + + public: + + ModifierObjectReplace(); + + // + // Modifier interface implementation + // + + virtual ~ModifierObjectReplace(); + + + /** + * Returns true and takes the embedded object contained in 'modExpr' to be the object + * we're replacing for. The field name of 'modExpr' is ignored. If 'modExpr' is in an + * unexpected format or if it can't be parsed for some reason, returns an error status + * describing the error. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** + * Registers the that 'root' is in the document that we want to fully replace. + * prepare() returns OK and always fills 'execInfo' with true for + * noOp. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** + * Replaces the document passed in prepare() for the object passed in init(). Returns + * OK if successful or a status describing the error. + */ + virtual Status apply() const; + + /** + * Adds a log entry to logRoot corresponding to full object replacement. Returns OK if + * successful or a status describing the error. + */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // Object to replace with. + BSONObj _val; + + // The document whose value needs to be replaced. This state is valid after a prepare() + // was issued and until a log() is issued. The document this mod is being prepared + // against must e live throughout all the calls. + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_object_replace_test.cpp b/src/mongo/db/ops/modifier_object_replace_test.cpp new file mode 100644 index 00000000000..f6c450c68fb --- /dev/null +++ b/src/mongo/db/ops/modifier_object_replace_test.cpp @@ -0,0 +1,309 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_object_replace.h" + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/bson/optime.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::fromjson; + using mongo::LogBuilder; + using mongo::ModifierInterface; + using mongo::ModifierObjectReplace; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::countChildren; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + using mongo::mutablebson::findFirstChildNamed; + using mongo::NumberInt; + using mongo::Status; + using mongo::StringData; + + /** Helper to build and manipulate a $set mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _mod() { + _modObj = modObj; + ASSERT_OK(_mod.init(BSON("" << modObj).firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierObjectReplace& mod() { return _mod; } + BSONObj& obj() { return _modObj; } + + private: + ModifierObjectReplace _mod; + BSONObj _modObj; + }; + + // Normal replacements below + TEST(Normal, SingleFieldDoc){ + Document doc(fromjson("{_id:1, a:1}")); + Mod mod(fromjson("{_id:1, b:12}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, fromjson("{_id:1, b:12}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + TEST(Normal, ComplexDoc){ + Document doc(fromjson("{_id:1, a:1}")); + Mod mod(fromjson("{_id:1, b:[123], c: {r:true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, fromjson("{_id:1, b:[123], c: {r:true}}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + TEST(Normal, OnlyIdField){ + Document doc(fromjson("{}")); + Mod mod(fromjson("{_id:1}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, mod.obj()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + // These updates have to do with updates without an _id field + // (the existing _id isn't removed) + TEST(IdLeft, EmptyDocReplacement){ + Document doc(fromjson("{_id:1}")); + Mod mod(fromjson("{}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, fromjson("{_id:1}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + TEST(IdLeft, EmptyDoc){ + Document doc(fromjson("{_id:1}")); + Mod mod(fromjson("{}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, fromjson("{_id:1}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + TEST(IdLeft, SingleFieldAddition){ + Document doc(fromjson("{_id:1}")); + Mod mod(fromjson("{a:1}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, fromjson("{_id:1, a:1}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + TEST(IdLeft, SingleFieldReplaced){ + Document doc(fromjson("{a: []}")); + Mod mod(fromjson("{a:10}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, mod.obj()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + TEST(IdLeft, SwapFields){ + Document doc(fromjson("{_id:1, a:1}")); + Mod mod(fromjson("{b:1}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, fromjson("{_id:1, b:1}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(doc, logDoc); + } + + TEST(IdImmutable, ReplaceIdNumber){ + Document doc(fromjson("{_id:1, a:1}")); + Mod mod(fromjson("{_id:2}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_NOT_OK(mod.apply()); + } + + TEST(IdImmutable, ReplaceIdNumberSameVal){ + Document doc(fromjson("{_id:1, a:1}")); + Mod mod(fromjson("{_id:2}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_NOT_OK(mod.apply()); + } + + TEST(IdImmutable, ReplaceEmbeddedId){ + Document doc(fromjson("{_id:{a:1, b:2}, a:1}")); + Mod mod(fromjson("{_id:{b:2, a:1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + ASSERT_NOT_OK(mod.apply()); + } + + TEST(Timestamp, IdNotReplaced){ + Document doc(fromjson("{}")); + Mod mod(fromjson("{_id:Timestamp(0,0), a:1}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_EQUALS(doc, mod.obj()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(mod.obj(), logDoc); + + Element idElem = findFirstChildNamed(logDoc.root(), "_id"); + ASSERT(idElem.ok()); + ASSERT(idElem.getValueTimestamp().isNull()); + + } + + TEST(Timestamp, ReplaceAll){ + Document doc(fromjson("{}")); + Mod mod(fromjson("{a:Timestamp(0,0), r:1, x:1, b:Timestamp(0,0)}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + + Element elem = findFirstChildNamed(doc.root(), "a"); + ASSERT(elem.ok()); + ASSERT_NOT_EQUALS(0U, elem.getValueTimestamp().getSecs()); + ASSERT_NOT_EQUALS(0U, elem.getValueTimestamp().getInc()); + + elem = findFirstChildNamed(doc.root(), "b"); + ASSERT(elem.ok()); + ASSERT_NOT_EQUALS(0U, elem.getValueTimestamp().getSecs()); + ASSERT_NOT_EQUALS(0U, elem.getValueTimestamp().getInc()); + } + +} // unnamed namespace diff --git a/src/mongo/db/ops/modifier_pop.cpp b/src/mongo/db/ops/modifier_pop.cpp new file mode 100644 index 00000000000..dada6d66be7 --- /dev/null +++ b/src/mongo/db/ops/modifier_pop.cpp @@ -0,0 +1,213 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_pop.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace mb = mutablebson; + namespace str = mongoutils::str; + + struct ModifierPop::PreparedState { + + PreparedState(mutablebson::Document* targetDoc) + : doc(*targetDoc) + , elementToRemove(doc.end()) + , pathFoundIndex(0) + , pathFoundElement(doc.end()) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Element to be removed + mutablebson::Element elementToRemove; + + // Index in _fieldRef for which an Element exist in the document. + size_t pathFoundIndex; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element pathFoundElement; + }; + + ModifierPop::ModifierPop() + : _fieldRef() + , _positionalPathIndex(0) + , _fromTop(false) { + } + + ModifierPop::~ModifierPop() { + } + + Status ModifierPop::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + // + // field name analysis + // + + // Break down the field name into its 'dotted' components (aka parts) and check that + // there are no empty parts. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, + &_positionalPathIndex, + &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + // + // value analysis + // + + // TODO: tighten validation to numbers and just 1/-1 explicitly + //if (!modExpr.isNumber()) { + // return Status(ErrorCodes::BadValue, "Must be a number"); + //} + + _fromTop = (modExpr.isNumber() && modExpr.number() < 0) ? true : false; + + return Status::OK(); + } + + Status ModifierPop::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(&root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_positionalPathIndex) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_positionalPathIndex, matchedField); + } + + // Locate the field name in 'root'. Note that if we don't have the full path in the + // doc, there isn't anything to unset, really. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->pathFoundIndex, + &_preparedState->pathFoundElement); + // Check if we didn't find the full path + if (status.isOK()) { + const bool destExists = (_preparedState->pathFoundIndex == (_fieldRef.numParts()-1)); + if (!destExists) { + execInfo->noOp = true; + } else { + // If the path exists, we require the target field to be already an + // array. + if (_preparedState->pathFoundElement.getType() != Array) { + mb::Element idElem = mb::findFirstChildNamed(root, "_id"); + return Status( + ErrorCodes::BadValue, + str::stream() << "Can only $pop from arrays. {" + << idElem.toString() + << "} has the field '" + << _preparedState->pathFoundElement.getFieldName() + << "' of non-array type " + << typeName(_preparedState->pathFoundElement.getType())); + } + + // No children, nothing to do -- not an error state + if (!_preparedState->pathFoundElement.hasChildren()) { + execInfo->noOp = true; + } else { + _preparedState->elementToRemove = _fromTop ? + _preparedState->pathFoundElement.leftChild() : + _preparedState->pathFoundElement.rightChild(); + } + } + } else { + // Let the caller know we can't do anything given the mod, _fieldRef, and doc. + execInfo->noOp = true; + _preparedState->pathFoundElement = root.getDocument().end(); + + //okay if path not found + if (status.code() == ErrorCodes::NonExistentPath) + status = Status::OK(); + } + + // Let the caller know what field we care about + execInfo->fieldRef[0] = &_fieldRef; + + return status; + } + + Status ModifierPop::apply() const { + return _preparedState->elementToRemove.remove(); + } + + Status ModifierPop::log(LogBuilder* logBuilder) const { + // log document + mutablebson::Document& doc = logBuilder->getDocument(); + const bool pathExists = _preparedState->pathFoundElement.ok() && + (_preparedState->pathFoundIndex == (_fieldRef.numParts() - 1)); + + if (!pathExists) + return logBuilder->addToUnsets(_fieldRef.dottedField()); + + // value for the logElement ("field.path.name": ) + mutablebson::Element logElement = doc.makeElementWithNewFieldName( + _fieldRef.dottedField(), + _preparedState->pathFoundElement); + + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, + str::stream() << "Could not append entry to $pop oplog entry: " + << "set '" << _fieldRef.dottedField() << "' -> " + << _preparedState->pathFoundElement.toString() ); + } + return logBuilder->addToSets(logElement); + } +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_pop.h b/src/mongo/db/ops/modifier_pop.h new file mode 100644 index 00000000000..00b8991dbbf --- /dev/null +++ b/src/mongo/db/ops/modifier_pop.h @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierPop : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierPop); + + public: + + ModifierPop(); + virtual ~ModifierPop(); + + /** + * The format of this modifier ($pop) is {: }. + * If the value is number and greater than -1 then an element is removed from the bottom, + * otherwise the top. Currently the value can be any anything but we document + * the use of the numbers "1, -1" only. + * + * Ex. $pop: {'a':1} will remove the last item from this array: [1,2,3] -> [1,2] + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + + virtual Status apply() const; + + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _positionalPathIndex; + + // element position to remove from + bool _fromTop; + + // The instance of the field in the provided doc. + // This data is valid after prepare, for use by log and apply + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_pop_test.cpp b/src/mongo/db/ops/modifier_pop_test.cpp new file mode 100644 index 00000000000..e33ace66db9 --- /dev/null +++ b/src/mongo/db/ops/modifier_pop_test.cpp @@ -0,0 +1,316 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_pop.h" + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/json.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::Array; + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::fromjson; + using mongo::ModifierInterface; + using mongo::ModifierPop; + using mongo::Status; + using mongo::StringData; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate a $pop mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) { + _modObj = modObj; + ASSERT_OK(_mod.init(_modObj["$pop"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierPop& mod() { return _mod; } + + BSONObj modObj() { return _modObj; } + + private: + ModifierPop _mod; + BSONObj _modObj; + }; + + // + // Test init values which aren't numbers. + // These are going to cause a pop from the bottom. + // + TEST(Init, StringArg) { + BSONObj modObj = fromjson("{$pop: {a: 'hi'}}"); + ModifierPop mod; + ASSERT_OK(mod.init(modObj["$pop"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, BoolTrueArg) { + BSONObj modObj = fromjson("{$pop: {a: true}}"); + ModifierPop mod; + ASSERT_OK(mod.init(modObj["$pop"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, BoolFalseArg) { + BSONObj modObj = fromjson("{$pop: {a: false}}"); + ModifierPop mod; + ASSERT_OK(mod.init(modObj["$pop"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(MissingField, AllButApply) { + Document doc(fromjson("{a: [1,2]}")); + Mod mod(fromjson("{$pop: {s: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "s"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$unset: {'s': true}}"), logDoc); + } + + TEST(SimpleMod, PrepareBottom) { + Document doc(fromjson("{a: [1,2]}")); + Mod mod(fromjson("{$pop: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(SimpleMod, PrepareApplyBottomO) { + Document doc(fromjson("{a: [1,2]}")); + Mod mod(fromjson("{$pop: {a: 0}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(("{a: [1]}")), doc); + } + + TEST(SimpleMod, PrepareTop) { + Document doc(fromjson("{a: [1,2]}")); + Mod mod(fromjson("{$pop: {a: -1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(SimpleMod, ApplyTopPop) { + Document doc(fromjson("{a: [1,2]}")); + Mod mod(fromjson("{$pop: {a: -1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(("{a: [2]}")), doc); + } + + TEST(SimpleMod, ApplyBottomPop) { + Document doc(fromjson("{a: [1,2]}")); + Mod mod(fromjson("{$pop: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(("{a: [1]}")), doc); + } + + TEST(SimpleMod, ApplyLogBottomPop) { + Document doc(fromjson("{a: [1,2]}")); + Mod mod(fromjson("{$pop: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(("{a:[1]}")), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$set: {a: [1]}}"), logDoc); + } + + TEST(EmptyArray, PrepareNoOp) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{$pop: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(SingleElemArray, ApplyLog) { + Document doc(fromjson("{a: [1]}")); + Mod mod(fromjson("{$pop: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(("{a:[]}")), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$set: {a: []}}"), logDoc); + } + + TEST(ArrayOfArray, ApplyLogPop) { + Document doc(fromjson("{a: [[1,2], 1]}")); + Mod mod(fromjson("{$pop: {'a.0': 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(("{a:[[1], 1]}")), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$set: { 'a.0': [1]}}"), logDoc); + } + + TEST(ArrayOfArray, ApplyLogPopOnlyElement) { + Document doc(fromjson("{a: [[1], 1]}")); + Mod mod(fromjson("{$pop: {'a.0': 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(("{a:[[], 1]}")), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$set: { 'a.0': []}}"), logDoc); + } + + TEST(Prepare, MissingPath) { + Document doc(fromjson("{ a : [1, 2] }")); + Mod mod(fromjson("{ $pop : { b : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + // from SERVER-12846 + TEST(Prepare, MissingArrayElementPath) { + Document doc(fromjson("{_id : 1, a : [1, 2]}")); + Mod mod(fromjson("{ $pop : { 'a.3' : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(Prepare, FromArrayElementPath) { + Document doc(fromjson("{ a : [1, 2] }")); + Mod mod(fromjson("{ $pop : { 'a.0' : 1 } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + +} // unnamed namespace diff --git a/src/mongo/db/ops/modifier_pull.cpp b/src/mongo/db/ops/modifier_pull.cpp new file mode 100644 index 00000000000..54044dae423 --- /dev/null +++ b/src/mongo/db/ops/modifier_pull.cpp @@ -0,0 +1,299 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_pull.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace mb = mutablebson; + namespace str = mongoutils::str; + + struct ModifierPull::PreparedState { + + PreparedState(mb::Document& doc) + : doc(doc) + , idxFound(0) + , elemFound(doc.end()) + , elementsToRemove() + , noOp(false) { + } + + // Document that is going to be changed. + mb::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mb::Element elemFound; + + // Values to be removed. + std::vector elementsToRemove; + + // True if this update is a no-op + bool noOp; + }; + + ModifierPull::ModifierPull() + : ModifierInterface() + , _fieldRef() + , _posDollar(0) + , _exprElt() + , _exprObj() + , _matchExpr() + , _matcherOnPrimitive(false) + , _preparedState() { + } + + ModifierPull::~ModifierPull() { + } + + Status ModifierPull::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + // Perform standard field name and updateable checks. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + _exprElt = modExpr; + + // If the element in the mod is actually an object or a regular expression, we need to + // build a matcher, instead of just doing an equality comparision. + if ((_exprElt.type() == mongo::Object) || (_exprElt.type() == mongo::RegEx)) { + if (_exprElt.type() == Object) { + _exprObj = _exprElt.embeddedObject(); + + // If not is not a query operator, then it is a primitive. + _matcherOnPrimitive = (_exprObj.firstElement().getGtLtOp() != 0); + + // If the object is primitive then wrap it up into an object. + if (_matcherOnPrimitive) + _exprObj = BSON( "" << _exprObj ); + } + else { + // For a regex, we also need to wrap and treat like a primitive. + _matcherOnPrimitive = true; + _exprObj = _exprElt.wrap(""); + } + + // Build the matcher around the object we built above. + StatusWithMatchExpression parseResult = MatchExpressionParser::parse(_exprObj); + if (!parseResult.isOK()) + return parseResult.getStatus(); + + _matchExpr.reset(parseResult.getValue()); + } + + return Status::OK(); + } + + Status ModifierPull::prepare(mb::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_posDollar) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_posDollar, matchedField); + } + + // Locate the field name in 'root'. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + } else if (!status.isOK()) { + return status; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_fieldRef; + + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + // If no target element exists, then there is nothing to do here. + _preparedState->noOp = execInfo->noOp = true; + return Status::OK(); + } + + // This operation only applies to arrays + if (_preparedState->elemFound.getType() != mongo::Array) + return Status( + ErrorCodes::BadValue, + "Cannot apply $pull to a non-array value"); + + // If the array is empty, there is nothing to pull, so this is a noop. + if (!_preparedState->elemFound.hasChildren()) { + _preparedState->noOp = execInfo->noOp = true; + return Status::OK(); + } + + // Walk the values in the array + mb::Element cursor = _preparedState->elemFound.leftChild(); + while (cursor.ok()) { + if (isMatch(cursor)) + _preparedState->elementsToRemove.push_back(cursor); + cursor = cursor.rightSibling(); + } + + // If we didn't find any elements to add, then this is a no-op, and therefore in place. + if (_preparedState->elementsToRemove.empty()) { + _preparedState->noOp = execInfo->noOp = true; + } + + return Status::OK(); + } + + Status ModifierPull::apply() const { + dassert(_preparedState->noOp == false); + + dassert(_preparedState->elemFound.ok() && + _preparedState->idxFound == (_fieldRef.numParts() - 1)); + + std::vector::const_iterator where = _preparedState->elementsToRemove.begin(); + const std::vector::const_iterator end = _preparedState->elementsToRemove.end(); + for ( ; where != end; ++where) + const_cast(*where).remove(); + + return Status::OK(); + } + + Status ModifierPull::log(LogBuilder* logBuilder) const { + + mb::Document& doc = logBuilder->getDocument(); + + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + + // If we didn't find the element that we wanted to pull from, we log an unset for + // that element. + return logBuilder->addToUnsets(_fieldRef.dottedField()); + + } else { + + // TODO: This is copied more or less identically from $push. As a result, it copies the + // behavior in $push that relies on 'apply' having been called unless this is a no-op. + + // TODO We can log just a positional unset in several cases. For now, let's just log + // the full resulting array. + + // We'd like to create an entry such as {$set: {: []}} under + // 'logRoot'. We start by creating the {$set: ...} Element. + + // Then we create the {:[]} Element, that is, an empty array. + mb::Element logElement = doc.makeElementArray(_fieldRef.dottedField()); + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, "cannot create details for $pull mod"); + } + + mb::Element curr = _preparedState->elemFound.leftChild(); + while (curr.ok()) { + + dassert(curr.hasValue()); + + // We need to copy each array entry from the resulting document to the log + // document. + mb::Element currCopy = doc.makeElementWithNewFieldName( + StringData(), + curr.getValue()); + if (!currCopy.ok()) { + return Status(ErrorCodes::InternalError, "could create copy element"); + } + Status status = logElement.pushBack(currCopy); + if (!status.isOK()) { + return Status(ErrorCodes::BadValue, "could not append entry for $pull log"); + } + curr = curr.rightSibling(); + } + + return logBuilder->addToSets(logElement); + } + } + + bool ModifierPull::isMatch(mutablebson::ConstElement element) { + + // TODO: We are assuming that 'element' hasValue is true. That might be OK if the + // conflict detection logic will prevent us from ever seeing a deserialized element, + // but are we sure about that? + + dassert(element.hasValue()); + + if (!_matchExpr) + return (element.compareWithBSONElement(_exprElt, false) == 0); + + if (_matcherOnPrimitive) { + // TODO: This is kinda slow. + BSONObj candidate = element.getValue().wrap(""); + return _matchExpr->matchesBSON(candidate); + } + + if (element.getType() != Object) + return false; + + return _matchExpr->matchesBSON(element.getValueObject()); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_pull.h b/src/mongo/db/ops/modifier_pull.h new file mode 100644 index 00000000000..b415ea0682d --- /dev/null +++ b/src/mongo/db/ops/modifier_pull.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class MatchExpression; + + class ModifierPull : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierPull); + + public: + ModifierPull(); + virtual ~ModifierPull(); + + /** Evaluates the array items to be removed and the match expression. */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** Decides which portion of the array items will be removed from the provided element */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** Updates the Element used in prepare with the effects of the $pull operation. */ + virtual Status apply() const; + + /** Converts the effects of this $pull into one or more equivalent $unset operations. */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + bool isMatch(mutablebson::ConstElement element); + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _posDollar; + + // If we aren't using a matcher, we just keep modExpr as _exprElt and use that to match + // with woCompare. + BSONElement _exprElt; + + // If we are using a matcher, we need to keep around a BSONObj for it. + BSONObj _exprObj; + + // If we are using the matcher, this is the match expression we built around _exprObj. + scoped_ptr _matchExpr; + bool _matcherOnPrimitive; + + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_pull_all.cpp b/src/mongo/db/ops/modifier_pull_all.cpp new file mode 100644 index 00000000000..2a3cbb855eb --- /dev/null +++ b/src/mongo/db/ops/modifier_pull_all.cpp @@ -0,0 +1,252 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_pull_all.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace mb = mutablebson; + namespace str = mongoutils::str; + + struct ModifierPullAll::PreparedState { + + PreparedState(mutablebson::Document* targetDoc) + : doc(*targetDoc) + , pathFoundIndex(0) + , pathFoundElement(doc.end()) + , applyCalled(false) + , elementsToRemove() { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t pathFoundIndex; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element pathFoundElement; + + bool applyCalled; + + // Elements to be removed + vector elementsToRemove; + }; + + namespace { + + struct mutableElementEqualsBSONElement : std::unary_function + { + mutableElementEqualsBSONElement(const mutablebson::Element& elem) : _what(elem) {} + bool operator()(const BSONElement& elem) const { + return _what.compareWithBSONElement(elem, false) == 0; + } + const mutablebson::Element& _what; + }; + } // namespace + + ModifierPullAll::ModifierPullAll() + : _fieldRef() + , _positionalPathIndex(0) + , _elementsToFind() { + } + + ModifierPullAll::~ModifierPullAll() { + } + + Status ModifierPullAll::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + // + // field name analysis + // + + // Break down the field name into its 'dotted' components (aka parts) and check that + // there are no empty parts. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, + &_positionalPathIndex, + &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + // + // value analysis + // + + if (modExpr.type() != Array) { + return Status(ErrorCodes::BadValue, + str::stream() << "$pullAll requires an array argument but was given a " + << typeName(modExpr.type())); + } + + // store the stuff to remove later + _elementsToFind = modExpr.Array(); + + return Status::OK(); + } + + Status ModifierPullAll::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(&root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_positionalPathIndex) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_positionalPathIndex, matchedField); + } + + // Locate the field name in 'root'. Note that if we don't have the full path in the + // doc, there isn't anything to unset, really. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->pathFoundIndex, + &_preparedState->pathFoundElement); + // Check if we didn't find the full path + if (status.isOK()) { + const bool destExists = (_preparedState->pathFoundIndex == (_fieldRef.numParts()-1)); + + if (!destExists) { + execInfo->noOp = true; + } else { + // If the path exists, we require the target field to be already an + // array. + if (_preparedState->pathFoundElement.getType() != Array) { + mb::Element idElem = mb::findElementNamed(root.leftChild(), "_id"); + return Status( + ErrorCodes::BadValue, + str::stream() << "Can only apply $pullAll to an array. " + << idElem.toString() + << " has the field " + << _preparedState->pathFoundElement.getFieldName() + << " of non-array type " + << typeName(_preparedState->pathFoundElement.getType())); + } + + // No children, nothing to do -- not an error state + if (!_preparedState->pathFoundElement.hasChildren()) { + execInfo->noOp = true; + } else { + mutablebson::Element elem = _preparedState->pathFoundElement.leftChild(); + while (elem.ok()) { + if (std::find_if(_elementsToFind.begin(), + _elementsToFind.end(), + mutableElementEqualsBSONElement(elem)) + != _elementsToFind.end()) { + _preparedState->elementsToRemove.push_back(elem); + } + elem = elem.rightSibling(); + } + + // Nothing to remove so it is a noOp. + if (_preparedState->elementsToRemove.empty()) + execInfo->noOp = true; + } + } + } else { + // Let the caller know we can't do anything given the mod, _fieldRef, and doc. + execInfo->noOp = true; + + + //okay if path not found + if (status.code() == ErrorCodes::NonExistentPath) + status = Status::OK(); + } + + // Let the caller know what field we care about + execInfo->fieldRef[0] = &_fieldRef; + + return status; + } + + Status ModifierPullAll::apply() const { + _preparedState->applyCalled = true; + + vector::const_iterator curr = + _preparedState->elementsToRemove.begin(); + const vector::const_iterator end = + _preparedState->elementsToRemove.end(); + for ( ; curr != end; ++curr) { + const_cast(*curr).remove(); + } + return Status::OK(); + } + + Status ModifierPullAll::log(LogBuilder* logBuilder) const { + // log document + mutablebson::Document& doc = logBuilder->getDocument(); + const bool pathExists = _preparedState->pathFoundElement.ok() && + (_preparedState->pathFoundIndex == (_fieldRef.numParts() - 1)); + + if (!pathExists) + return logBuilder->addToUnsets(_fieldRef.dottedField()); + + // value for the logElement ("field.path.name": ) + mutablebson::Element logElement = doc.makeElementWithNewFieldName( + _fieldRef.dottedField(), + _preparedState->pathFoundElement); + + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, + str::stream() << "Could not append entry to $pullAll oplog entry: " + << "set '" << _fieldRef.dottedField() << "' -> " + << _preparedState->pathFoundElement.toString() ); + } + return logBuilder->addToSets(logElement); + } +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_pull_all.h b/src/mongo/db/ops/modifier_pull_all.h new file mode 100644 index 00000000000..54e3ae571e2 --- /dev/null +++ b/src/mongo/db/ops/modifier_pull_all.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierPullAll : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierPullAll); + + public: + + ModifierPullAll(); + virtual ~ModifierPullAll(); + + /** + * The modifier $pullAll takes an array of values to match literally, and remove + * + * Ex. {$pullAll : { : []}} + * {$pullAll :{ array : [1,2] } } will transform {array: [1,2,3]} -> {array: [3]} + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + + virtual Status apply() const; + + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _positionalPathIndex; + + // The instance of the field in the provided doc. + // This data is valid after prepare, for use by log and apply + struct PreparedState; + scoped_ptr _preparedState; + + // User specified elements to remove + std::vector _elementsToFind; + + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_pull_all_test.cpp b/src/mongo/db/ops/modifier_pull_all_test.cpp new file mode 100644 index 00000000000..5c266e0a8ed --- /dev/null +++ b/src/mongo/db/ops/modifier_pull_all_test.cpp @@ -0,0 +1,250 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_pull_all.h" + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::ModifierPullAll; + using mongo::ModifierInterface; + using mongo::NumberInt; + using mongo::Status; + using mongo::StringData; + using mongo::fromjson; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate the mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _modObj(modObj) + , _mod() { + ASSERT_OK(_mod.init(_modObj["$pullAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierPullAll& mod() { return _mod; } + + private: + BSONObj _modObj; + ModifierPullAll _mod; + }; + + TEST(Init, BadThings) { + BSONObj modObj; + ModifierPullAll mod; + + modObj = fromjson("{$pullAll: {a:1}}"); + ASSERT_NOT_OK(mod.init(modObj["$pullAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{$pullAll: {a:'test'}}"); + ASSERT_NOT_OK(mod.init(modObj["$pullAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{$pullAll: {a:{}}}"); + ASSERT_NOT_OK(mod.init(modObj["$pullAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + modObj = fromjson("{$pullAll: {a:true}}"); + ASSERT_NOT_OK(mod.init(modObj["$pullAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + } + + TEST(PrepareApply, SimpleNumber) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : { a : [1] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : ['a', {r:1, b:2}] }"), doc); + } + + TEST(PrepareApply, MissingElement) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : { a : ['r'] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [1, 'a', {r:1, b:2}] }"), doc); + } + + TEST(PrepareApply, TwoElements) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : { a : [1, 'a'] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [{r:1, b:2}] }"), doc); + } + + TEST(EmptyResult, RemoveEverythingOutOfOrder) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : {a : [ {r:1, b:2}, 1, 'a' ] }}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [] } }"), logDoc); + } + + TEST(EmptyResult, RemoveEverythingInOrder) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : { a : [1, 'a', {r:1, b:2} ] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [] } }"), logDoc); + } + + TEST(EmptyResult, RemoveEverythingAndThenSome) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : { a : [2,3,1,'r', {r:1, b:2}, 'a' ] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [] } }"), logDoc); + } + + TEST(PrepareLog, MissingPullValue) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : { a : [2] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [1, 'a', {r:1, b:2}] } }"), logDoc); + } + + TEST(PrepareLog, MissingPath) { + Document doc(fromjson("{ a : [1, 'a', {r:1, b:2}] }")); + Mod mod(fromjson("{ $pullAll : { b : [1] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $unset : { b : true } }"), logDoc); + } + + TEST(Prepare, MissingArrayElementPath) { + Document doc(fromjson("{ a : [1, 2] }")); + Mod mod(fromjson("{ $pullAll : { 'a.2' : [1] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(Prepare, FromArrayElementPath) { + Document doc(fromjson("{ a : [1, 2] }")); + Mod mod(fromjson("{ $pullAll : { 'a.0' : [1] } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_pull_test.cpp b/src/mongo/db/ops/modifier_pull_test.cpp new file mode 100644 index 00000000000..af457dd8cbc --- /dev/null +++ b/src/mongo/db/ops/modifier_pull_test.cpp @@ -0,0 +1,576 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_pull.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::LogBuilder; + using mongo::ModifierPull; + using mongo::ModifierInterface; + using mongo::Status; + using mongo::StringData; + using mongo::fromjson; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate a $pull mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _modObj(modObj) + , _mod() { + ASSERT_OK(_mod.init(_modObj["$pull"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierPull& mod() { + return _mod; + } + + private: + BSONObj _modObj; + ModifierPull _mod; + }; + + TEST(SimpleMod, PrepareOKTargetNotFound) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $unset : { a : true } }"), logDoc); + } + + TEST(SimpleMod, PrepareOKTargetFound) { + Document doc(fromjson("{ a : [ 0, 1, 2, 3 ] }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + } + + TEST(SimpleMod, PrepareInvalidTargetString) { + Document doc(fromjson("{ a : 'foo' }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, PrepareInvalidTargetObject) { + Document doc(fromjson("{ a : { 'foo' : 'bar' } }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(mod.prepare(doc.root(), "", &execInfo)); + } + + TEST(SimpleMod, PrepareAndLogEmptyDocument) { + Document doc(fromjson("{}")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $unset : { a : true } }"), logDoc); + } + + TEST(SimpleMod, PrepareAndLogMissingElementAfterFoundPath) { + Document doc(fromjson("{ a : { b : { c : {} } } }")); + Mod mod(fromjson("{ $pull : { 'a.b.c.d' : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b.c.d"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $unset : { 'a.b.c.d' : true } }"), logDoc); + } + + TEST(SimpleMod, PrepareAndLogEmptyArray) { + Document doc(fromjson("{ a : [] }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [] } }"), logDoc); + } + + TEST(SimpleMod, PullMatchesNone) { + Document doc(fromjson("{ a : [2, 3, 4, 5] }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [2, 3, 4, 5] } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogPullMatchesOne) { + Document doc(fromjson("{ a : [0, 1, 2, 3, 4, 5] }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1, 2, 3, 4, 5 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [1, 2, 3, 4, 5] } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogPullMatchesSeveral) { + Document doc(fromjson("{ a : [0, 1, 0, 2, 0, 3, 0, 4, 0, 5] }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [ 1, 2, 3, 4, 5 ] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [1, 2, 3, 4, 5] } }"), logDoc); + } + + TEST(SimpleMod, ApplyAndLogPullMatchesAll) { + Document doc(fromjson("{ a : [0, -1, -2, -3, -4, -5] }")); + Mod mod(fromjson("{ $pull : { a : { $lt : 1 } } }")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ a : [] }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{ $set : { a : [] } }"), logDoc); + } + + TEST(ComplexMod, ApplyAndLogComplexDocAndMatching1) { + + const char* const strings[] = { + // Document: + "{ a : { b : [ { x : 1 }, { y : 'y' }, { x : 2 }, { z : 'z' } ] } }", + + // Modifier: + "{ $pull : { 'a.b' : { $or : [ " + " { 'y' : { $exists : true } }, " + " { 'z' : { $exists : true } } " + "] } } }", + + // Document result: + "{ a : { b : [ { x : 1 }, { x : 2 } ] } }", + + // Log result: + "{ $set : { 'a.b' : [ { x : 1 }, { x : 2 } ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(ComplexMod, ApplyAndLogComplexDocAndMatching2) { + + const char* const strings[] = { + // Document: + "{ a : { b : [ { x : 1 }, { y : 'y' }, { x : 2 }, { z : 'z' } ] } }", + + // Modifier: + "{ $pull : { 'a.b' : { 'y' : { $exists : true } } } }", + + // Document result: + "{ a : { b : [ { x : 1 }, { x : 2 }, { z : 'z' } ] } }", + + // Log result: + "{ $set : { 'a.b' : [ { x : 1 }, { x : 2 }, { z : 'z' } ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(ComplexMod, ApplyAndLogComplexDocAndMatching3) { + + const char* const strings[] = { + // Document: + "{ a : { b : [ { x : 1 }, { y : 'y' }, { x : 2 }, { z : 'z' } ] } }", + + // Modifier: + "{ $pull : { 'a.b' : { $in : [ { x : 1 }, { y : 'y' } ] } } }", + + // Document result: + "{ a : { b : [ { x : 2 }, { z : 'z' } ] } }", + + // Log result: + "{ $set : { 'a.b' : [ { x : 2 }, { z : 'z' } ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(ValueMod, ApplyAndLogScalarValueMod) { + + const char* const strings[] = { + // Document: + "{ a : [1, 2, 1, 2, 1, 2] }", + + // Modifier: + "{ $pull : { a : 1 } }", + + // Document result: + "{ a : [ 2, 2, 2] }", + + // Log result: + "{ $set : { a : [ 2, 2, 2 ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(ValueMod, ApplyAndLogObjectValueMod) { + + const char* const strings[] = { + // Document: + "{ a : [ { x : 1 }, { y : 2 }, { x : 1 }, { y : 2 } ] }", + + // Modifier: + "{ $pull : { a : { y : 2 } } }", + + // Document result: + "{ a : [ { x : 1 }, { x : 1 }] }", + + // Log result: + "{ $set : { a : [ { x : 1 }, { x : 1 } ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(DocumentationTests, Example1) { + const char* const strings[] = { + // Document: + "{ flags: ['vme', 'de', 'pse', 'tsc', 'msr', 'pae', 'mce' ] }", + + // Modifier: + "{ $pull: { flags: 'msr' } }", + + // Document result: + "{ flags: ['vme', 'de', 'pse', 'tsc', 'pae', 'mce' ] }", + + // Log result: + "{ $set : { flags: ['vme', 'de', 'pse', 'tsc', 'pae', 'mce' ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "flags"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(DocumentationTests, Example2a) { + const char* const strings[] = { + // Document: + "{ votes: [ 3, 5, 6, 7, 7, 8 ] }", + + // Modifier: + "{ $pull: { votes: 7 } }", + + // Document result: + "{ votes: [ 3, 5, 6, 8 ] }", + + // Log result: + "{ $set : { votes: [ 3, 5, 6, 8 ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "votes"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(DocumentationTests, Example2b) { + const char* const strings[] = { + // Document: + "{ votes: [ 3, 5, 6, 7, 7, 8 ] }", + + // Modifier: + "{ $pull: { votes: { $gt: 6 } } }", + + // Document result: + "{ votes: [ 3, 5, 6 ] }", + + // Log result: + "{ $set : { votes: [ 3, 5, 6 ] } }" + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "votes"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(MatchingEdgeCases, NonObjectShortCircuit) { + const char* const strings[] = { + "{ a: [ { x: 1 }, 2 ] }", + + "{ $pull: { a: { x: 1 } } }", + + "{ a: [ 2 ] }", + + "{ $set : { a: [ 2 ] } }", + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + + TEST(MatchingRegressions, SERVER_3988) { + const char* const strings[] = { + "{ x: 1, y: [ 2, 3, 4, 'abc', 'xyz' ] }", + + "{ $pull: { y: /yz/ } }", + + "{ x: 1, y: [ 2, 3, 4, 'abc' ] }", + + "{ $set : { y: [ 2, 3, 4, 'abc' ] } }", + }; + + Document doc(fromjson(strings[0])); + Mod mod(fromjson(strings[1])); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "y"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson(strings[2]), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod.log(&logBuilder)); + ASSERT_EQUALS(fromjson(strings[3]), logDoc); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_push.cpp b/src/mongo/db/ops/modifier_push.cpp new file mode 100644 index 00000000000..4002ed71f50 --- /dev/null +++ b/src/mongo/db/ops/modifier_push.cpp @@ -0,0 +1,721 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_push.h" + +#include +#include + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace mb = mutablebson; + namespace str = mongoutils::str; + + namespace { + + const char kEach[] = "$each"; + const char kSlice[] = "$slice"; + const char kSort[] = "$sort"; + const char kPosition[] = "$position"; + + bool isPatternElement(const BSONElement& pattern) { + if (!pattern.isNumber()) { + return false; + } + + // Patterns can be only 1 or -1. + double val = pattern.Number(); + if (val != 1 && val != -1) { + return false; + } + + return true; + } + + bool inEachMode(const BSONElement& modExpr) { + if (modExpr.type() != Object) { + return false; + } + BSONObj obj = modExpr.embeddedObject(); + if (obj[kEach].type() == EOO) { + return false; + } + return true; + } + + Status parseEachMode(ModifierPush::ModifierPushMode pushMode, + const BSONElement& modExpr, + BSONElement* eachElem, + BSONElement* sliceElem, + BSONElement* sortElem, + BSONElement* positionElem) { + + Status status = Status::OK(); + + // If in $pushAll mode, all we need is the array. + if (pushMode == ModifierPush::PUSH_ALL) { + if (modExpr.type() != Array) { + return Status(ErrorCodes::BadValue, "$pushAll requires an array"); + } + *eachElem = modExpr; + return Status::OK(); + } + + // The $each clause must be an array. + *eachElem = modExpr.embeddedObject()[kEach]; + if (eachElem->type() != Array) { + return Status(ErrorCodes::BadValue, + str::stream() << "The argument to $each in $push must be" + " an array but it was of type " + << typeName(eachElem->type())); + } + + // There must be only one $each clause. + bool seenEach = false; + BSONObjIterator itMod(modExpr.embeddedObject()); + while (itMod.more()) { + BSONElement modElem = itMod.next(); + if (mongoutils::str::equals(modElem.fieldName(), kEach)) { + if (seenEach) { + return Status(ErrorCodes::BadValue, + "Only one $each clause is supported."); + } + seenEach = true; + } + } + + // Slice, sort, position are optional and may be present in any order. + bool seenSlice = false; + bool seenSort = false; + bool seenPosition = false; + BSONObjIterator itPush(modExpr.embeddedObject()); + while (itPush.more()) { + BSONElement elem = itPush.next(); + if (mongoutils::str::equals(elem.fieldName(), kSlice)) { + if (seenSlice) { + return Status(ErrorCodes::BadValue, + "Only one $slice clause is supported."); + } + *sliceElem = elem; + seenSlice = true; + } + else if (mongoutils::str::equals(elem.fieldName(), kSort)) { + if (seenSort) { + return Status(ErrorCodes::BadValue, + "Only one $sort clause is supported."); + } + *sortElem = elem; + seenSort = true; + } + else if (mongoutils::str::equals(elem.fieldName(), kPosition)) { + if (seenPosition) { + return Status(ErrorCodes::BadValue, + "Only one $position clause is supported."); + } + *positionElem = elem; + seenPosition = true; + } + else if (!mongoutils::str::equals(elem.fieldName(), kEach)) { + return Status(ErrorCodes::BadValue, + str::stream() << "Unrecognized clause in $push: " + << elem.fieldNameStringData()); + } + } + + return Status::OK(); + } + + } // unnamed namespace + + struct ModifierPush::PreparedState { + + PreparedState(mutablebson::Document* targetDoc) + : doc(*targetDoc) + , idxFound(0) + , elemFound(doc.end()) + , arrayPreModSize(0) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element elemFound; + + size_t arrayPreModSize; + + }; + + ModifierPush::ModifierPush(ModifierPush::ModifierPushMode pushMode) + : _fieldRef() + , _posDollar(0) + , _eachMode(false) + , _eachElem() + , _slicePresent(false) + , _slice(0) + , _sortPresent(false) + , _startPosition(std::numeric_limits::max()) + , _sort() + , _pushMode(pushMode) + , _val() { + } + + ModifierPush::~ModifierPush() { + } + + Status ModifierPush::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + // + // field name analysis + // + + // Break down the field name into its 'dotted' components (aka parts) and check that + // the field is fit for updates. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + // + // value analysis + // + + // Are the target push values safe to store? + BSONElement sliceElem; + BSONElement sortElem; + BSONElement positionElem; + switch (modExpr.type()) { + + case Array: + if (_pushMode == PUSH_ALL) { + _eachMode = true; + Status status = parseEachMode(PUSH_ALL, + modExpr, + &_eachElem, + &sliceElem, + &sortElem, + &positionElem); + if (!status.isOK()) { + return status; + } + } + else { + _val = modExpr; + } + break; + + case Object: + if (_pushMode == PUSH_ALL) { + return Status(ErrorCodes::BadValue, + str::stream() << "$pushAll requires an array of values " + "but was given an embedded document."); + } + + // If any known clause ($each, $slice, or $sort) is present, we'd assume + // we're using the $each variation of push and would parse accodingly. + _eachMode = inEachMode(modExpr); + if (_eachMode) { + Status status = parseEachMode(PUSH_NORMAL, + modExpr, + &_eachElem, + &sliceElem, + &sortElem, + &positionElem); + if (!status.isOK()) { + return status; + } + } + else { + _val = modExpr; + } + break; + + default: + if (_pushMode == PUSH_ALL) { + return Status(ErrorCodes::BadValue, + str::stream() << "$pushAll requires an array of values " + "but was given an " + << typeName(modExpr.type())); + } + + _val = modExpr; + break; + } + + // Is slice present and correct? + if (sliceElem.type() != EOO) { + if (_pushMode == PUSH_ALL) { + return Status(ErrorCodes::BadValue, "cannot use $slice in $pushAll"); + } + + if (!sliceElem.isNumber()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The value for $slice must " + "be a numeric value not a " + << typeName(sliceElem.type())); + } + + // TODO: Cleanup and unify numbers wrt getting int32/64 bson values (from doubles) + + // If the value of slice is not fraction, even if it's a double, we allow it. The + // reason here is that the shell will use doubles by default unless told otherwise. + const double doubleVal = sliceElem.numberDouble(); + if (doubleVal - static_cast(doubleVal) != 0) { + return Status(ErrorCodes::BadValue, + "The $slice value in $push cannot be fractional"); + } + + _slice = sliceElem.numberLong(); + _slicePresent = true; + } + + // Is position present and correct? + if (positionElem.type() != EOO) { + if (_pushMode == PUSH_ALL) { + return Status(ErrorCodes::BadValue, "cannot use $position in $pushAll"); + } + + if (!positionElem.isNumber()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The value for $position must " + "be a positive numeric value not a " + << typeName(positionElem.type())); + } + + // TODO: Cleanup and unify numbers wrt getting int32/64 bson values (from doubles) + + // If the value of position is not fraction, even if it's a double, we allow it. The + // reason here is that the shell will use doubles by default unless told otherwise. + const double doubleVal = positionElem.numberDouble(); + if (doubleVal - static_cast(doubleVal) != 0) { + return Status(ErrorCodes::BadValue, + "The $position value in $push cannot be fractional"); + } + + if (static_cast(numeric_limits::max()) < doubleVal) { + return Status(ErrorCodes::BadValue, + "The $position value in $push is too large a number."); + } + + if (static_cast(numeric_limits::min()) > doubleVal) { + return Status(ErrorCodes::BadValue, + "The $position value in $push is too small a number."); + } + + const int64_t tempVal = positionElem.numberLong(); + if (tempVal < 0) + return Status(ErrorCodes::BadValue, + "The $position value in $push must be positive."); + + _startPosition = size_t(tempVal); + } + + // Is sort present and correct? + if (sortElem.type() != EOO) { + if (_pushMode == PUSH_ALL) { + return Status(ErrorCodes::BadValue, + "cannot use $sort in $pushAll"); + } + + if (sortElem.type() != Object && !sortElem.isNumber()) { + return Status(ErrorCodes::BadValue, + "The $sort is invalid: use 1/-1 to sort the whole element, " + "or {field:1/-1} to sort embedded fields"); + } + + if (sortElem.isABSONObj()) { + BSONObj sortObj = sortElem.embeddedObject(); + if (sortObj.isEmpty()) { + return Status(ErrorCodes::BadValue, + "The $sort pattern is empty when it should be a set of fields."); + } + + // Check if the sort pattern is sound. + BSONObjIterator sortIter(sortObj); + while (sortIter.more()) { + + BSONElement sortPatternElem = sortIter.next(); + + // We require either : 1 or -1 for asc and desc. + if (!isPatternElement(sortPatternElem)) { + return Status(ErrorCodes::BadValue, + "The $sort element value must be either 1 or -1"); + } + + // All fields parts must be valid. + FieldRef sortField(sortPatternElem.fieldName()); + if (sortField.numParts() == 0) { + return Status(ErrorCodes::BadValue, + "The $sort field cannot be empty"); + } + + for (size_t i = 0; i < sortField.numParts(); i++) { + if (sortField.getPart(i).size() == 0) { + return Status(ErrorCodes::BadValue, + str::stream() << "The $sort field is a dotted field " + "but has an empty part: " + << sortField.dottedField()); + } + } + } + + _sort = PatternElementCmp(sortElem.embeddedObject()); + } + else { + // Ensure the sortElem number is valid. + if (!isPatternElement(sortElem)) { + return Status(ErrorCodes::BadValue, + "The $sort element value must be either 1 or -1"); + } + + _sort = PatternElementCmp(BSON("" << sortElem.number())); + } + + _sortPresent = true; + } + + return Status::OK(); + } + + Status ModifierPush::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(&root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_posDollar) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_posDollar, matchedField); + } + + // Locate the field name in 'root'. Note that we may not have all the parts in the path + // in the doc -- which is fine. Our goal now is merely to reason about whether this mod + // apply is a noOp or whether is can be in place. The remainin path, if missing, will + // be created during the apply. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + + } + else if (status.isOK()) { + + const bool destExists = (_preparedState->idxFound == (_fieldRef.numParts()-1)); + // If the path exists, we require the target field to be already an + // array. + if (destExists && _preparedState->elemFound.getType() != Array) { + mb::Element idElem = mb::findFirstChildNamed(root, "_id"); + return Status(ErrorCodes::BadValue, + str::stream() << "The field '" << _fieldRef.dottedField() << "'" + << " must be an array but is of type " + << typeName(_preparedState->elemFound.getType()) + << " in document {" << idElem.toString() << "}"); + } + } + else { + return status; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_fieldRef; + + return Status::OK(); + } + + namespace { + Status pushFirstElement(mb::Element& arrayElem, + const size_t arraySize, + const size_t pos, + mb::Element& elem) { + + + // Empty array or pushing to the front + if (arraySize == 0 || pos == 0) { + return arrayElem.pushFront(elem); + } + else { + + // Push position is at the end, or beyond + if (pos >= arraySize) { + return arrayElem.pushBack(elem); + } + + const size_t appendPos = pos - 1; + mutablebson::Element fromElem = getNthChild(arrayElem, appendPos); + + // This should not be possible since the checks above should + // cover us but error just in case + if (!fromElem.ok()){ + return Status(ErrorCodes::InvalidLength, + str::stream() << "The specified position (" << appendPos << "/" + << pos + << ") is invalid based on the length ( " + << arraySize + << ") of the array"); + } + + return fromElem.addSiblingRight(elem); + } + } + } //unamed namespace + + Status ModifierPush::apply() const { + + Status status = Status::OK(); + + // + // Applying a $push with an $clause has the following steps + // 1. Create the doc array we'll push into, if it is not there + // 2. Add the items in the $each array (or the simple $push) to the doc array + // 3. Sort the resulting array according to $sort clause, if present + // 4. Trim the resulting array according the $slice clasue, if present + // + // TODO There are _lots_ of optimization opportunities that we'll consider once the + // test coverage is adequate. + // + + // 1. If the array field is not there, create it as an array and attach it to the + // document. + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts()-1)) { + + // Creates the array element + mutablebson::Document& doc = _preparedState->doc; + StringData lastPart = _fieldRef.getPart(_fieldRef.numParts()-1); + mutablebson::Element baseArray = doc.makeElementArray(lastPart); + if (!baseArray.ok()) { + return Status(ErrorCodes::InternalError, "can't create new base array"); + } + + // Now, we can be in two cases here, as far as attaching the element being set + // goes: (a) none of the parts in the element's path exist, or (b) some parts of + // the path exist but not all. + if (!_preparedState->elemFound.ok()) { + _preparedState->elemFound = doc.root(); + _preparedState->idxFound = 0; + } + else { + _preparedState->idxFound++; + } + + // createPathAt() will complete the path and attach 'elemToSet' at the end of it. + status = pathsupport::createPathAt(_fieldRef, + _preparedState->idxFound, + _preparedState->elemFound, + baseArray); + if (!status.isOK()) { + return status; + } + + // Point to the base array just created. The subsequent code expects it to exist + // already. + _preparedState->elemFound = baseArray; + + } + + // This is the count of the array before we change it, or 0 if missing from the doc. + _preparedState->arrayPreModSize = countChildren(_preparedState->elemFound); + + // 2. Add new elements to the array either by going over the $each array or by + // appending the (old style $push) element. + if (_eachMode || _pushMode == PUSH_ALL) { + BSONObjIterator itEach(_eachElem.embeddedObject()); + + // When adding more than one element we keep track of the previous one + // so we can add right siblings to it. + mutablebson::Element prevElem = _preparedState->doc.end(); + + // The first element is special below + bool first = true; + + while (itEach.more()) { + BSONElement eachItem = itEach.next(); + mutablebson::Element elem = + _preparedState->doc.makeElementWithNewFieldName(StringData(), eachItem); + + if (first) { + status = pushFirstElement(_preparedState->elemFound, + _preparedState->arrayPreModSize, + _startPosition, + elem); } + else { + status = prevElem.addSiblingRight(elem); + } + + if (!status.isOK()) { + return status; + } + + // For the next iteration the previous element will be the left sibling + prevElem = elem; + first = false; + } + } + else { + mutablebson::Element elem = + _preparedState->doc.makeElementWithNewFieldName(StringData(), _val); + if (!elem.ok()) { + return Status(ErrorCodes::InternalError, "can't wrap element being $push-ed"); + } + return pushFirstElement(_preparedState->elemFound, + _preparedState->arrayPreModSize, + _startPosition, + elem); + } + + // 3. Sort the resulting array, if $sort was requested. + if (_sortPresent) { + sortChildren(_preparedState->elemFound, _sort); + } + + // 4. Trim the resulting array according to $slice, if present. + if (_slicePresent) { + + // Slice 0 means to remove all + if (_slice == 0) { + while(_preparedState->elemFound.ok() && + _preparedState->elemFound.rightChild().ok()) { + _preparedState->elemFound.rightChild().remove(); + } + } + + const int64_t numChildren = mutablebson::countChildren(_preparedState->elemFound); + int64_t countRemoved = std::max(static_cast(0), numChildren - abs(_slice)); + + // If _slice is negative, remove from the bottom, otherwise from the top + const bool removeFromEnd = (_slice > 0); + + // Either start at right or left depending if we are taking from top or bottom + mutablebson::Element curr = removeFromEnd ? + _preparedState->elemFound.rightChild() : + _preparedState->elemFound.leftChild(); + while (curr.ok() && countRemoved > 0) { + mutablebson::Element toRemove = curr; + // Either go right or left depending if we are taking from top or bottom + curr = removeFromEnd ? curr.leftSibling() : curr.rightSibling(); + + status = toRemove.remove(); + if (!status.isOK()) { + return status; + } + countRemoved--; + } + } + + return status; + } + + Status ModifierPush::log(LogBuilder* logBuilder) const { + + // The start position to use for positional (ordinal) updates to the array + // (We will increment as we append elements to the oplog entry so can't be const) + size_t position = _preparedState->arrayPreModSize; + + // NOTE: Idempotence Requirement + // In the case that the document does't have an array or it is empty we need to make sure + // that the first time the field gets filled with items that it is a full set of the array. + + // If we sorted, sliced, or added the first items to the array, make a full array copy. + const bool doFullCopy = _slicePresent || _sortPresent + || (position == 0) // first element in new/empty array + || (_startPosition < _preparedState->arrayPreModSize); // add in middle + + if (doFullCopy) { + return logBuilder->addToSetsWithNewFieldName(_fieldRef.dottedField(), + _preparedState->elemFound); + } + else { + // Set only the positional elements appended + if (_eachMode || _pushMode == PUSH_ALL) { + // For each input element log it as a posisional $set + BSONObjIterator itEach(_eachElem.embeddedObject()); + while (itEach.more()) { + BSONElement eachItem = itEach.next(); + // value for the logElement ("field.path.name.N": ) + const std::string positionalName = + mongoutils::str::stream() << _fieldRef.dottedField() << "." << position++; + + Status s = logBuilder->addToSetsWithNewFieldName(positionalName, eachItem); + if (!s.isOK()) + return s; + } + + return Status::OK(); + } + else { + // single value for the logElement ("field.path.name.N": ) + const std::string positionalName = + mongoutils::str::stream() << _fieldRef.dottedField() << "." << position++; + + return logBuilder->addToSetsWithNewFieldName(positionalName, _val); + } + } + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_push.h b/src/mongo/db/ops/modifier_push.h new file mode 100644 index 00000000000..9abe6d7d0e0 --- /dev/null +++ b/src/mongo/db/ops/modifier_push.h @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" +#include "mongo/db/ops/modifier_push_sorter.h" + +namespace mongo { + + class LogBuilder; + + class ModifierPush : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierPush); + + public: + + enum ModifierPushMode { PUSH_NORMAL, PUSH_ALL }; + explicit ModifierPush(ModifierPushMode mode= PUSH_NORMAL); + + // + // Modifier interface implementation + // + + virtual ~ModifierPush(); + + /** + * A 'modExpr' here is a BSONElement {: , , } coming from a $push mod such as {$set: {x: $each[{a:1}], $slice:3, + * $sort{b:1}}}. init() extracts and validates the field name and the clauses. It + * returns OK if successful or a status describing the error. + * + * There are currently a few restrictions concerning the clauses (but all can be + * lifted): + * + $slice can be negative only (ie, slicing from the recent end) + * + $sort requires $slice to be present + * + $sort can only sort objects (as opposed to basic types), so it only takes + * object as patterns + * + Because of the previous, $sort requires that the array being pushed to be made + * of objects + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** + * Locates the array to be pushed into in the 'root', if it exists, and fills in + * execInfo accordingly. Returns true if $push would succeed in 'root', otherwise + * return a status describing the error. + * + * Note that a $push is never in-place. The cost of checking if it is a no-op makes it + * so that we don't do such check either. As such, execInfo is always filled with + * 'false' for those two options. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** + * Pushes the array into the prepared position and "sort/slice"s the resulting array + * according to that call's instructions. + */ + virtual Status apply() const; + + /** + * $push currently logs the entire resulting array as a $set. + * + * TODO Log a positional $set in the array, whenever possible. + */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _posDollar; + + // Clauses for the $push that are filled when the $each variation of the command is used. + bool _eachMode; + BSONElement _eachElem; + bool _slicePresent; + int64_t _slice; + bool _sortPresent; + size_t _startPosition; + + PatternElementCmp _sort; + + // Whether this mod is supposed to be parsed as a $pushAll. + const ModifierPushMode _pushMode; + + // Simple (old style) push value when the $each variation of the command is not + // used. The _eachMode flag would be off if we're this mode. + BSONElement _val; + + // The instance of the field in the provided doc. This state is valid after a + // prepare() was issued and until a log() is issued. The document this mod is + // being prepared against must be live throughout all the calls. + struct PreparedState; + scoped_ptr _preparedState; + + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_push_sorter.h b/src/mongo/db/ops/modifier_push_sorter.h new file mode 100644 index 00000000000..930ab539d57 --- /dev/null +++ b/src/mongo/db/ops/modifier_push_sorter.h @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" + +namespace mongo { + + // Extracts the value for 'pattern' for both 'lhs' and 'rhs' and return true if 'lhs' < + // 'rhs'. We expect that both 'lhs' and 'rhs' be key patterns. + struct PatternElementCmp { + BSONObj sortPattern; + bool useWholeValue; + + PatternElementCmp() + : sortPattern(BSONObj()) + , useWholeValue(true) {} + + PatternElementCmp(const BSONObj& pattern) + : sortPattern(pattern) + , useWholeValue(pattern.hasField("")){ + } + + bool operator()(const mutablebson::Element& lhs, const mutablebson::Element& rhs) const { + if (useWholeValue) { + const int comparedValue = lhs.compareWithElement( rhs, false ); + + const bool reversed = (sortPattern.firstElement().number() < 0 ); + + return (reversed ? comparedValue > 0 : comparedValue < 0); + } + else { + //TODO: Push on to mutable in the future, and to support non-contiguous Elements. + BSONObj lhsObj = lhs.getType() == Object ? + lhs.getValueObject() : + lhs.getValue().wrap(""); + BSONObj rhsObj = rhs.getType() == Object ? + rhs.getValueObject() : + rhs.getValue().wrap(""); + + BSONObj lhsKey = lhsObj.extractFields(sortPattern, true); + BSONObj rhsKey = rhsObj.extractFields(sortPattern, true); + + return lhsKey.woCompare(rhsKey, sortPattern) < 0; + } + } + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_push_sorter_test.cpp b/src/mongo/db/ops/modifier_push_sorter_test.cpp new file mode 100644 index 00000000000..4df1c4dfadc --- /dev/null +++ b/src/mongo/db/ops/modifier_push_sorter_test.cpp @@ -0,0 +1,179 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_push_sorter.h" + +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::fromjson; + using mongo::PatternElementCmp; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + using mongo::mutablebson::sortChildren; + + class ObjectArray : public mongo::unittest::Test { + public: + ObjectArray() : _doc(), _size(0) {} + + virtual void setUp() { + Element arr = _doc.makeElementArray("x"); + ASSERT_TRUE(arr.ok()); + ASSERT_OK(_doc.root().pushBack(arr)); + } + + void addObj(BSONObj obj) { + ASSERT_LESS_THAN_OR_EQUALS(_size, 3u); + _objs[_size] = obj; + _size++; + + ASSERT_OK(_doc.root()["x"].appendObject(mongo::StringData(), obj)); + } + + BSONObj getOrigObj(size_t i) { + return _objs[i]; + } + + BSONObj getSortedObj(size_t i) { + return getArray()[i].getValueObject(); + } + + Element getArray() { + return _doc.root()["x"]; + } + + private: + Document _doc; + BSONObj _objs[3]; + size_t _size; + }; + + TEST_F(ObjectArray, NormalOrder) { + addObj(fromjson("{b:1, a:1}")); + addObj(fromjson("{a:3, b:2}")); + addObj(fromjson("{b:3, a:2}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{'a':1,'b':1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(2)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(1)); + } + + TEST_F(ObjectArray, MixedOrder) { + addObj(fromjson("{b:1, a:1}")); + addObj(fromjson("{a:3, b:2}")); + addObj(fromjson("{b:3, a:2}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{b:1,a:-1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(1)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(2)); + } + + TEST_F(ObjectArray, ExtraFields) { + addObj(fromjson("{b:1, c:2, a:1}")); + addObj(fromjson("{c:1, a:3, b:2}")); + addObj(fromjson("{b:3, a:2}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{a:1,b:1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(2)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(1)); + } + + TEST_F(ObjectArray, MissingFields) { + addObj(fromjson("{a:2, b:2}")); + addObj(fromjson("{a:1}")); + addObj(fromjson("{a:3, b:3, c:3}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{b:1,c:1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(1)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(2)); + } + + TEST_F(ObjectArray, NestedFields) { + addObj(fromjson("{a:{b:{c:2, d:0}}}")); + addObj(fromjson("{a:{b:{c:1, d:2}}}")); + addObj(fromjson("{a:{b:{c:3, d:1}}}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{'a.b':1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(1)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(2)); + } + + TEST_F(ObjectArray, SimpleNestedFields) { + addObj(fromjson("{a:{b: -1}}")); + addObj(fromjson("{a:{b: -100}}")); + addObj(fromjson("{a:{b: 34}}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{'a.b':1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(1)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(2)); + } + + TEST_F(ObjectArray, NestedInnerObjectDescending) { + addObj(fromjson("{a:{b:{c:2, d:0}}}")); + addObj(fromjson("{a:{b:{c:1, d:2}}}")); + addObj(fromjson("{a:{b:{c:3, d:1}}}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{'a.b.d':-1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(2)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(1)); + } + + TEST_F(ObjectArray, NestedInnerObjectAscending) { + addObj(fromjson("{a:{b:{c:2, d:0}}}")); + addObj(fromjson("{a:{b:{c:1, d:2}}}")); + addObj(fromjson("{a:{b:{c:3, d:1}}}")); + + sortChildren(getArray(), PatternElementCmp(fromjson("{'a.b.d':1}"))); + + ASSERT_EQUALS(getOrigObj(0), getSortedObj(0)); + ASSERT_EQUALS(getOrigObj(2), getSortedObj(1)); + ASSERT_EQUALS(getOrigObj(1), getSortedObj(2)); + } + +} // unnamed namespace diff --git a/src/mongo/db/ops/modifier_push_test.cpp b/src/mongo/db/ops/modifier_push_test.cpp new file mode 100644 index 00000000000..88811026205 --- /dev/null +++ b/src/mongo/db/ops/modifier_push_test.cpp @@ -0,0 +1,1455 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_push.h" + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/ordering.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/mongoutils/str.h" + +namespace { + + using mongo::BSONObj; + using mongo::BSONObjBuilder; + using mongo::BSONArrayBuilder; + using mongo::fromjson; + using mongo::LogBuilder; + using mongo::ModifierInterface; + using mongo::ModifierPush; + using mongo::NumberInt; + using mongo::Ordering; + using mongo::Status; + using mongo::StringData; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::countChildren; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + using std::sort; + using std::vector; + + void combineVec(const vector& origVec, + const vector& modVec, + int32_t slice, + vector* combined) { + + using namespace std; + combined->clear(); + + // Slice 0 means the result is empty + if (slice == 0) + return; + + // Combine both vectors + *combined = origVec; + combined->insert(combined->end(), modVec.begin(), modVec.end()); + + // Remove sliced items + bool removeFromFront = (slice < 0); + + // if abs(slice) is larger than the size, nothing to do. + if (abs(slice) >= int32_t(combined->size())) + return; + + if (removeFromFront) { + // Slice is negative. + int32_t removeCount = combined->size() + slice; + combined->erase(combined->begin(), combined->begin() + removeCount); + } + else { + combined->resize(std::min(combined->size(), size_t(slice))); + } + } + + /** + * Comparator between two BSONObjects that takes in consideration only the keys and + * direction described in the sort pattern. + */ + struct ProjectKeyCmp { + BSONObj sortPattern; + bool useWholeValue; + + ProjectKeyCmp(BSONObj pattern) : sortPattern(pattern) { + useWholeValue = pattern.hasField(""); + } + + int operator()(const BSONObj& left, const BSONObj& right) const { + int ret = 0; + if (useWholeValue) { + ret = left.woCompare( right, Ordering::make(sortPattern), false ); + } else { + BSONObj lhsKey = left.extractFields(sortPattern, true); + BSONObj rhsKey = right.extractFields(sortPattern, true); + ret = lhsKey.woCompare(rhsKey, sortPattern); + } + return ret < 0; + } + }; + + void combineAndSortVec(const vector& origVec, + const vector& modVec, + int32_t slice, + BSONObj sortOrder, + vector* combined) { + + combined->clear(); + + // Slice 0 means the result is empty + if (slice == 0) + return; + + *combined = origVec; + combined->insert(combined->end(), modVec.begin(), modVec.end()); + + sort(combined->begin(), combined->end(), ProjectKeyCmp(sortOrder)); + + // Remove sliced items + bool removeFromFront = (slice < 0); + + // if abs(slice) is larger than the size, nothing to do. + if (abs(slice) >= int32_t(combined->size())) + return; + + if (removeFromFront) { + // Slice is negative. + int32_t removeCount = combined->size() + slice; + combined->erase(combined->begin(), combined->begin() + removeCount); + } + else { + combined->resize(std::min(combined->size(), size_t(slice))); + } + } + + // + // Init testing (module field checking, which is done in 'fieldchecker' + // + + TEST(Init, SimplePush) { + BSONObj modObj = fromjson("{$push: {x: 0}}"); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + // + // If present, is the $each clause valid? + // + + TEST(Init, PushEachNormal) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2]}}}"); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachMixed) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, {a: 2}]}}}"); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachObject) { + // $each must be an array + BSONObj modObj = fromjson("{$push: {x: {$each: {'0': 1}}}}"); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachSimpleType) { + // $each must be an array. + BSONObj modObj = fromjson("{$push: {x: {$each: 1}}}"); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachEmpty) { + BSONObj modObj = fromjson("{$push: {x: {$each: []}}}"); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachInvalidType) { + // $each must be an array. + BSONObj modObj = fromjson("{$push: {x: {$each: {b: 1}}}}"); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + // + // If present, is the $slice clause valid? + // + + TEST(Init, PushEachWithSliceBottom) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2], $slice: -3}}}"); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithSliceTop) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2], $slice: 3}}}"); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithInvalidSliceObject) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2], $slice: {a: 1}}}}"); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithInvalidSliceDouble) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2], $slice: -2.1}}}"); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithValidSliceDouble) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2], $slice: -2.0}}}"); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithUnsupportedFullSlice) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2], $slice: [1,2]}}}"); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithWrongTypeSlice) { + BSONObj modObj = fromjson("{$push: {x: {$each: [1, 2], $slice: '-1'}}}"); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + // + // If present, is the sort $sort clause valid? + // + + TEST(Init, PushEachWithObjectSort) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {a:1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithNumbericSort) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort:1 }}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithInvalidSortType) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: [{a:1}]}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachDuplicateSortPattern) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: [{a:1,a:1}]}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithInvalidSortValue) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {a:100}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithEmptySortField) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {'':1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithEmptyDottedSortField) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {'.':1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithMissingSortFieldSuffix) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {'a.':1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithMissingSortFieldPreffix) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {'.b':1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithMissingSortFieldMiddle) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {'a..b':1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithEmptySort) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort:{} }}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + // + // If in $pushAll semantics, do we check the array and that nothing else is there? + // + + TEST(Init, PushAllSimple) { + BSONObj modObj = fromjson("{$pushAll: {x: [0]}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushAllMultiple) { + BSONObj modObj = fromjson("{$pushAll: {x: [1,2,3]}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushAllObject) { + BSONObj modObj = fromjson("{$pushAll: {x: [{a:1},{a:2}]}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushAllMixed) { + BSONObj modObj = fromjson("{$pushAll: {x: [1,{a:2}]}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushAllWrongType) { + BSONObj modObj = fromjson("{$pushAll: {x: 1}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_NOT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushAllNotArray) { + BSONObj modObj = fromjson("{$pushAll: {x: {a:1}}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_NOT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + // + // Are all clauses present? Is anything extroneous? Is anything duplicated? + // + + TEST(Init, PushEachWithSortMissingSlice) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $sort:{a:1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachInvalidClause) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $xxx: -1, $sort:{a:1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachExtraField) { + const char* c = "{$push: {x: {$each: [{a:1},{a:2}], $slice: -2.0, $sort: {a:1}, b: 1}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachDuplicateSortClause) { + const char* c = "{$push: {x:{$each:[{a:1},{a:2}], $slice:-2.0, $sort:{a:1}, $sort:{a:1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachDuplicateSliceClause) { + const char* c = "{$push: {x: {$each:[{a:1},{a:2}], $slice:-2.0, $slice:-2, $sort:{a:1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachDuplicateEachClause) { + const char* c = "{$push: {x: {$each:[{a:1}], $each:[{a:2}], $slice:-3, $sort:{a:1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_NOT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithSliceFirst) { + const char* c = "{$push: {x: {$slice: -2.0, $each: [{a:1},{a:2}], $sort: {a:1}}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(Init, PushEachWithSortFirst) { + const char* c = "{$push: {x: {$sort: {a:1}, $slice: -2.0, $each: [{a:1},{a:2}]}}}"; + BSONObj modObj = fromjson(c); + ModifierPush mod; + ASSERT_OK(mod.init(modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + // + // Simple mod + // + + /** Helper to build and manipulate a $push or a $pushAll mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) + : _mod(mongoutils::str::equals(modObj.firstElement().fieldName(), "$pushAll") ? + ModifierPush::PUSH_ALL : ModifierPush::PUSH_NORMAL) { + _modObj = modObj; + const StringData& modName = modObj.firstElement().fieldName(); + ASSERT_OK(_mod.init(_modObj[modName].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierPush& mod() { return _mod; } + + private: + ModifierPush _mod; + BSONObj _modObj; + }; + + TEST(SimpleMod, PrepareNonArray) { + Document doc(fromjson("{a: 1}")); + Mod pushMod(fromjson("{$push: {a: 1}}")); + + ModifierInterface::ExecInfo dummy; + ASSERT_NOT_OK(pushMod.prepare(doc.root(), "", &dummy)); + } + + TEST(SimpleMod, PrepareApplyEmpty) { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [1]}}"), logDoc); + } + + TEST(SimpleMod, PrepareApplyInexistent) { + Document doc(fromjson("{}")); + Mod pushMod(fromjson("{$push: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [1]}}"), logDoc); + } + + TEST(SimpleMod, PrepareApplyNormal) { + Document doc(fromjson("{a: [0]}")); + Mod pushMod(fromjson("{$push: {a: 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [0,1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.1':1}}"), logDoc); + } + + // + // Simple object mod + // + + TEST(SimpleObjMod, PrepareNonArray) { + Document doc(fromjson("{a: 1}")); + Mod pushMod(fromjson("{$push: {a: {b: 1}}}")); + + ModifierInterface::ExecInfo dummy; + ASSERT_NOT_OK(pushMod.prepare(doc.root(), "", &dummy)); + } + + TEST(SimpleObjMod, PrepareApplyEmpty) { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: {b: 1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [{b:1}]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [{b:1}]}}"), logDoc); + } + + TEST(SimpleObjMod, PrepareApplyInexistent) { + Document doc(fromjson("{}")); + Mod pushMod(fromjson("{$push: {a: {b: 1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [{b:1}]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [{b:1}]}}"), logDoc); + } + + TEST(SimpleObjMod, PrepareApplyNormal) { + Document doc(fromjson("{a: [{b:0}]}")); + Mod pushMod(fromjson("{$push: {a: {b: 1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [{b:0},{b:1}]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.1':{b:1}}}"), logDoc); + } + + TEST(SimpleObjMod, PrepareApplyDotted) { + Document doc(fromjson("{ _id : 1 , " + " question : 'a', " + " choices : { " + " first : { choice : 'b' }, " + " second : { choice : 'c' } }" + "}")); + Mod pushMod(fromjson("{$push: {'choices.first.votes': 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "choices.first.votes"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson( "{ _id : 1 , " + " question : 'a', " + " choices : { " + " first : { choice : 'b', votes: [1]}, " + " second : { choice : 'c' } }" + "}"), + doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'choices.first.votes':[1]}}"), logDoc); + } + + + // + // $pushAll Variation + // + + TEST(PushAll, PrepareApplyEmpty) { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$pushAll: {a: [1]}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{a: [1]}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(logDoc, fromjson("{$set: {a: [1]}}")); + } + + TEST(PushAll, PrepareApplyInexistent) { + Document doc(fromjson("{}")); + Mod pushMod(fromjson("{$pushAll: {a: [1]}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{a: [1]}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(logDoc, fromjson("{$set: {a: [1]}}")); + } + + TEST(PushAll, PrepareApplyNormal) { + Document doc(fromjson("{a: [0]}")); + Mod pushMod(fromjson("{$pushAll: {a: [1,2]}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{a: [0,1,2]}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(logDoc, fromjson("{$set: {'a.1': 1, 'a.2':2}}")); + } + + // + // Simple $each mod + // + + TEST(SimpleEachMod, PrepareNonArray) { + Document doc(fromjson("{a: 1}")); + Mod pushMod(fromjson("{$push: {a: {$each: [1]}}}")); + + ModifierInterface::ExecInfo dummy; + ASSERT_NOT_OK(pushMod.prepare(doc.root(), "", &dummy)); + } + + TEST(SimpleEachMod, PrepareApplyEmpty) { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: {$each: [1]}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [1]}}"), logDoc); + } + + TEST(SimpleEachMod, PrepareApplyInexistent) { + Document doc(fromjson("{}")); + Mod pushMod(fromjson("{$push: {a: {$each: [1]}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [1]}}"), logDoc); + } + + TEST(SimpleEachMod, PrepareApplyInexistentMultiple) { + Document doc(fromjson("{}")); + Mod pushMod(fromjson("{$push: {a: {$each: [1, 2]}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1, 2]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [1, 2]}}"), logDoc); + } + + TEST(SimpleEachMod, PrepareApplyNormal) { + Document doc(fromjson("{a: [0]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [1]}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [0,1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.1': 1}}"), logDoc); + } + + TEST(SimpleEachMod, PrepareApplyNormalMultiple) { + Document doc(fromjson("{a: [0]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [1,2]}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [0,1,2]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.1': 1, 'a.2':2}}"), logDoc); + } + + /** + * Slice variants + */ + TEST(SlicePushEach, TopOne) { + Document doc(fromjson("{a: [3]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [2, -1], $slice:1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [3]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [3]}}"), logDoc); + } + + /** + * Sort for scalar (whole) array elements + */ + TEST(SortPushEach, NumberSort) { + Document doc(fromjson("{a: [3]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [2, -1], $sort:1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [-1,2,3]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [-1, 2, 3]}}"), logDoc); + } + + TEST(SortPushEach, NumberSortReverse) { + Document doc(fromjson("{a: [3]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [4, -1], $sort:-1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [4,3,-1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: [4,3,-1]}}"), logDoc); + } + + TEST(SortPushEach, MixedSortWhole) { + Document doc(fromjson("{a: [3, 't', {b:1}, {a:1}]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [4, -1], $sort:1}}}")); + const BSONObj expectedObj = fromjson("{a: [-1,3,4,'t', {a:1}, {b:1}]}"); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(expectedObj, doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(BSON("$set" << expectedObj), logDoc); + } + + TEST(SortPushEach, MixedSortWholeReverse) { + Document doc(fromjson("{a: [3, 't', {b:1}, {a:1}]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [4, -1], $sort:-1}}}")); + const BSONObj expectedObj = fromjson("{a: [{b:1}, {a:1}, 't', 4, 3, -1]}"); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(expectedObj, doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(BSON("$set" << expectedObj), logDoc); + } + + TEST(SortPushEach, MixedSortEmbeddedField) { + Document doc(fromjson("{a: [3, 't', {b:1}, {a:1}]}")); + Mod pushMod(fromjson("{$push: {a: {$each: [4, -1], $sort:{a:1}}}}")); + const BSONObj expectedObj = fromjson("{a: [3, 't', {b: 1}, 4, -1, {a: 1}]}"); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(expectedObj, doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(BSON("$set" << expectedObj), logDoc); + } + + /** + * This fixture supports building $push mods with parameterized $each arrays and $slices. + * It always assume that the array being operated on is called 'a'. To build a mod, one + * issues a set*Mod() call. + * + * The setSimpleMod() call will build a $each array of numbers. The setObjectMod() call + * will build a $each array with object. Both these calls take the slice as a parameter as + * well. + * + * Here's a typical test case flow: + * + Determine what the original document's 'a' array would contain + * + Ditto for the $push's $each arrray + * + Loop over slice value + * + Apply the $push with current slice value to the doc + * + Use the fixture/helpers to combine and slice the mod's and original's 'a' + * array + * + Build a document with the above and check against the one generated by the mod apply + */ + class SlicedMod : public mongo::unittest::Test { + public: + SlicedMod() : _mod() {} + + virtual void setUp() { + // no op; set all state using the setMod() call + } + + /** Sets up the mod to be {$push: {a: {$each: [], $slice: }}} */ + void setSimpleMod(int32_t slice, const vector& eachArray) { + + BSONArrayBuilder arrBuilder; + for (vector::const_iterator it = eachArray.begin(); it != eachArray.end(); ++it) { + arrBuilder.append(*it); + } + + _modObj = BSON("$push" << + BSON("a" + << BSON("$each" << arrBuilder.arr() << + "$slice" << slice))); + + ASSERT_OK(_mod.init(_modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + /** Sets up the mod to be {$push: {a: {$each:[,...], $slice:, $sort:}}} */ + void setSortMod(int32_t slice, const vector& eachArray, BSONObj sort) { + + BSONArrayBuilder arrBuilder; + for (vector::const_iterator it = eachArray.begin(); + it != eachArray.end(); + ++it) { + arrBuilder.append(*it); + } + + _modObj = BSON("$push" << + BSON("a" + << BSON("$each" << arrBuilder.arr() << + "$slice" << slice << + "$sort" << sort))); + + ASSERT_OK(_mod.init(_modObj["$push"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + /** Returns an object {a: [<'vec's content>]} */ + BSONObj getObjectUsing(const vector& vec) { + + BSONArrayBuilder arrBuilder; + for (vector::const_iterator it = vec.begin(); it != vec.end(); ++it) { + arrBuilder.append(*it); + } + + BSONObjBuilder builder; + builder.appendArray("a", arrBuilder.obj()); + + return builder.obj(); + } + + /** Returns an object {a: [<'vec's content>]} */ + BSONObj getObjectUsing(const vector& vec) { + + BSONArrayBuilder arrBuilder; + for (vector::const_iterator it = vec.begin(); it != vec.end(); ++it) { + arrBuilder.append(*it); + } + + BSONObjBuilder builder; + builder.appendArray("a", arrBuilder.obj()); + + return builder.obj(); + } + + ModifierPush& mod() { return _mod; } + + BSONObj modObj() { return _modObj; } + + private: + ModifierPush _mod; + BSONObj _modObj; + vector _eachArray; + }; + + TEST_F(SlicedMod, SimpleArrayFromEmpty) { + // We'll simulate the original document having {a: []} and the mod being + // {$push: {a: {$each: [1], $slice: <-2..0>}}} + vector docArray; + vector eachArray; + eachArray.push_back(1); + + for (int32_t slice = -2; slice <= 0; slice++) { + setSimpleMod(slice, eachArray); + Document doc(getObjectUsing(docArray /* {a: []} */)); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod().prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod().apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + vector combinedVec; + combineVec(docArray, /* a: [] */ + eachArray, /* a: [1] */ + slice, + &combinedVec); + ASSERT_EQUALS(getObjectUsing(combinedVec), doc); + } + } + + TEST_F(SlicedMod, SimpleArrayFromExisting) { + // We'll simulate the original document having {a: [2,3]} and the mod being + // {$push: {a: {$each: [1], $slice: <-4..0>}}} + vector docArray; + docArray.push_back(2); + docArray.push_back(3); + vector eachArray; + eachArray.push_back(1); + + for (int32_t slice = -4; slice <= 0; slice++) { + setSimpleMod(slice, eachArray); + Document doc(getObjectUsing(docArray /* {a: [2, 3]} */)); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod().prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod().apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + vector combinedVec; + combineVec(docArray, /* a: [2, 3] */ + eachArray, /* a: [1] */ + slice, + &combinedVec); + ASSERT_EQUALS(getObjectUsing(combinedVec), doc); + } + } + + TEST_F(SlicedMod, ObjectArrayFromEmpty) { + // We'll simulate the original document having {a: []} and the mod being + // {$push: {a: {$each: [{a:2,b:1}], $slice: <-4..0>}, $sort: {a:-1/1,b:-1/1}} + vector docArray; + vector eachArray; + eachArray.push_back(fromjson("{a:2,b:1}")); + eachArray.push_back(fromjson("{a:1,b:2}")); + + for (int32_t aOrB = 0; aOrB < 2 ; aOrB++) { + for (int32_t sortA = 0; sortA < 2; sortA++) { + for (int32_t sortB = 0; sortB < 2; sortB++) { + for (int32_t slice = -3; slice <= 3; slice++) { + + BSONObj sortOrder; + if (aOrB == 0) { + sortOrder = BSON("a" << (sortA ? 1 : -1) << "b" << (sortB ? 1 : -1)); + } + else { + sortOrder = BSON("b" << (sortB ? 1 : -1) << "a" << (sortA ? 1 : -1)); + } + + setSortMod(slice, eachArray, sortOrder); + Document doc(getObjectUsing(docArray /* {a: []} */)); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod().prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod().apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + vector combinedVec; + combineAndSortVec(docArray, /* a: [] */ + eachArray, /* a: [{a:2,b:1},{a:1,b:2}] */ + slice, + sortOrder, + &combinedVec); + ASSERT_EQUALS(getObjectUsing(combinedVec), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod().log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(BSON("$set" << getObjectUsing(combinedVec)), logDoc); + + } + } + } + } + } + + TEST_F(SlicedMod, ObjectArrayFromExisting) { + // We'll simulate the original document having {a: [{a:2,b:3},{a:3,:b1}]} and the mod being + // {$push: {a: {$each: [{a:2,b:1}], $slice: <-4..0>}, $sort: {a:-1/1,b:-1/1}} + vector docArray; + docArray.push_back(fromjson("{a:2,b:3}")); + docArray.push_back(fromjson("{a:3,b:1}")); + vector eachArray; + eachArray.push_back(fromjson("{a:2,b:1}")); + + for (int32_t aOrB = 0; aOrB < 2 ; aOrB++) { + for (int32_t sortA = 0; sortA < 2; sortA++) { + for (int32_t sortB = 0; sortB < 2; sortB++) { + for (int32_t slice = -4; slice <= 4; slice++) { + + BSONObj sortOrder; + if (aOrB == 0) { + sortOrder = BSON("a" << (sortA ? 1 : -1) << "b" << (sortB ? 1 : -1)); + } + else { + sortOrder = BSON("b" << (sortB ? 1 : -1) << "a" << (sortA ? 1 : -1)); + } + + setSortMod(slice, eachArray, sortOrder); + Document doc(getObjectUsing(docArray /* {a: [{a:2,b:b},{a:3,:b1}]} */)); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod().prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(mod().apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + + vector combinedVec; + combineAndSortVec(docArray, /* a: [{a:2,b:3},{a:3,:b1}] */ + eachArray, /* a: [{a:2,b:1}] */ + slice, + sortOrder, + &combinedVec); + ASSERT_EQUALS(getObjectUsing(combinedVec), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(mod().log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(BSON("$set" << getObjectUsing(combinedVec)), logDoc); + + } + } + } + } + } + + // Push to position tests + + TEST(ToPosition, BadInputs) { + const char* const bad[] = { + "{$push: {a: { $each: [1], $position:-1}}}", + "{$push: {a: { $each: [1], $position:'s'}}}", + "{$push: {a: { $each: [1], $position:{}}}}", + "{$push: {a: { $each: [1], $position:[0]}}}", + "{$push: {a: { $each: [1], $position:1.1211212}}}", + "{$push: {a: { $each: [1], $position:3.000000000001}}}", + "{$push: {a: { $each: [1], $position:1.2}}}", + "{$push: {a: { $each: [1], $position:-1.2}}}", + "{$push: {a: { $each: [1], $position:9223372036854775810}}}", + "{$push: {a: { $each: [1], $position:-9223372036854775810}}}", + NULL, + }; + + int i = 0; + while(bad[i] != NULL) + { + ModifierPush pushMod(ModifierPush::PUSH_NORMAL); + BSONObj modObj = fromjson(bad[i]); + ASSERT_NOT_OK(pushMod.init(modObj.firstElement().embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + i++; + } + } + + TEST(ToPosition, GoodInputs) { + { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position: NumberLong(1)}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + } + { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:NumberInt(100)}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + } + { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:1.0}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + } + { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:1000000}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + } + { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:0}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + } + } + + TEST(ToPosition, EmptyArrayFront) { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:0}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a':[1]}}"), logDoc); + } + + TEST(ToPosition, EmptyArrayBackBigPosition) { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:1000}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a':[1]}}"), logDoc); + } + + TEST(ToPosition, EmptyArrayBack) { + Document doc(fromjson("{a: []}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a':[1]}}"), logDoc); + } + + TEST(ToPosition, Front) { + Document doc(fromjson("{a: [0]}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:0}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [1, 0]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a':[1, 0]}}"), logDoc); + } + + TEST(ToPosition, Back) { + Document doc(fromjson("{a: [0]}")); + Mod pushMod(fromjson("{$push: {a: { $each: [1], $position:100}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(pushMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(pushMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [0,1]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(pushMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.1':1}}"), logDoc); + } + +} // unnamed namespace diff --git a/src/mongo/db/ops/modifier_rename.cpp b/src/mongo/db/ops/modifier_rename.cpp new file mode 100644 index 00000000000..c2a86e5cb0d --- /dev/null +++ b/src/mongo/db/ops/modifier_rename.cpp @@ -0,0 +1,316 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_rename.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + struct ModifierRename::PreparedState { + + PreparedState(mutablebson::Element root) + : doc(root.getDocument()) + , fromElemFound(doc.end()) + , toIdxFound(0) + , toElemFound(doc.end()) + , applyCalled(false){ + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // The element to rename + mutablebson::Element fromElemFound; + + // Index in _fieldRef for which an Element exist in the document. + size_t toIdxFound; + + // Element to remove (in the destination position) + mutablebson::Element toElemFound; + + // Was apply called? + bool applyCalled; + + }; + + ModifierRename::ModifierRename() + : _fromFieldRef() + , _toFieldRef() { + } + + ModifierRename::~ModifierRename() { + } + + Status ModifierRename::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + if (modExpr.type() != String) { + return Status(ErrorCodes::BadValue, + str::stream() << "The 'to' field for $rename must be a string: " + << modExpr); + } + + // Extract the field names from the mod expression + + _fromFieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fromFieldRef); + if (!status.isOK()) + return status; + + _toFieldRef.parse(modExpr.String()); + status = fieldchecker::isUpdatable(_toFieldRef); + if (!status.isOK()) + return status; + + // TODO: Remove this restriction and make a noOp to lift restriction + // Old restriction is that if the fields are the same then it is not allowed. + if (_fromFieldRef == _toFieldRef) + return Status(ErrorCodes::BadValue, + str::stream() << "The source and target field for $rename must differ: " + << modExpr); + + // TODO: Remove this restriction by allowing moving deeping from the 'from' path + // Old restriction is that if the to/from is on the same path it fails + if (_fromFieldRef.isPrefixOf(_toFieldRef) || _toFieldRef.isPrefixOf(_fromFieldRef)){ + return Status(ErrorCodes::BadValue, + str::stream() << "The source and target field for $rename must " + "not be on the same path: " + << modExpr); + } + // TODO: We can remove this restriction as long as there is only one, + // or it is the same array -- should think on this a bit. + // + // If a $-positional operator was used it is an error + size_t dummyPos; + if (fieldchecker::isPositional(_fromFieldRef, &dummyPos)) + return Status(ErrorCodes::BadValue, + str::stream() << "The source field for $rename may not be dynamic: " + << _fromFieldRef.dottedField()); + else if (fieldchecker::isPositional(_toFieldRef, &dummyPos)) + return Status(ErrorCodes::BadValue, + str::stream() << "The destination field for $rename may not be dynamic: " + << _toFieldRef.dottedField()); + + if (positional) + *positional = false; + + return Status::OK(); + } + + Status ModifierRename::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + // Rename doesn't work with positional fields ($) + dassert(matchedField.empty()); + + _preparedState.reset(new PreparedState(root)); + + // Locate the to field name in 'root', which must exist. + size_t fromIdxFound; + Status status = pathsupport::findLongestPrefix(_fromFieldRef, + root, + &fromIdxFound, + &_preparedState->fromElemFound); + + const bool sourceExists = (_preparedState->fromElemFound.ok() && + fromIdxFound == (_fromFieldRef.numParts() - 1)); + + // If we can't find the full element in the from field then we can't do anything. + if (!status.isOK() || !sourceExists) { + execInfo->noOp = true; + _preparedState->fromElemFound = root.getDocument().end(); + + // TODO: remove this special case from existing behavior + if (status.code() == ErrorCodes::PathNotViable) { + return status; + } + + return Status::OK(); + } + + // Ensure no array in ancestry if what we found is not at the root + mutablebson::Element curr = _preparedState->fromElemFound.parent(); + if (curr != curr.getDocument().root()) + while (curr.ok() && (curr != curr.getDocument().root())) { + if (curr.getType() == Array) + return Status(ErrorCodes::BadValue, + str::stream() << "The source field cannot be an array element, '" + << _fromFieldRef.dottedField() << "' in doc with " + << findElementNamed(root.leftChild(), "_id").toString() + << " has an array field called '" << curr.getFieldName() << "'"); + curr = curr.parent(); + } + + // "To" side validation below + + status = pathsupport::findLongestPrefix(_toFieldRef, + root, + &_preparedState->toIdxFound, + &_preparedState->toElemFound); + + // FindLongestPrefix may return not viable or any other error and then we cannot proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + // Not an error condition as we will create the "to" path as needed. + } else if (!status.isOK()) { + return status; + } + + const bool destExists = _preparedState->toElemFound.ok() && + (_preparedState->toIdxFound == (_toFieldRef.numParts()-1)); + + // Ensure no array in ancestry of "to" Element + // Set to either parent, or node depending on if the full path element was found + curr = (destExists ? _preparedState->toElemFound.parent() : _preparedState->toElemFound); + if (curr != curr.getDocument().root()) { + while (curr.ok()) { + if (curr.getType() == Array) + return Status(ErrorCodes::BadValue, + str::stream() + << "The destination field cannot be an array element, '" + << _fromFieldRef.dottedField() << "' in doc with " + << findElementNamed(root.leftChild(), "_id").toString() + << " has an array field called '" << curr.getFieldName() << "'"); + curr = curr.parent(); + } + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_fromFieldRef; + execInfo->fieldRef[1] = &_toFieldRef; + + execInfo->noOp = false; + + return Status::OK(); + } + + Status ModifierRename::apply() const { + dassert(_preparedState->fromElemFound.ok()); + + _preparedState->applyCalled = true; + + // Remove from source + Status removeStatus = _preparedState->fromElemFound.remove(); + if (!removeStatus.isOK()) { + return removeStatus; + } + + // If there's no need to create any further field part, the op is simply a value + // assignment. + const bool destExists = _preparedState->toElemFound.ok() && + (_preparedState->toIdxFound == (_toFieldRef.numParts()-1)); + + if (destExists) { + removeStatus = _preparedState->toElemFound.remove(); + if (!removeStatus.isOK()) { + return removeStatus; + } + } + + // Creates the final element that's going to be the in 'doc'. + mutablebson::Document& doc = _preparedState->doc; + StringData lastPart = _toFieldRef.getPart(_toFieldRef.numParts()-1); + mutablebson::Element elemToSet = doc.makeElementWithNewFieldName( + lastPart, + _preparedState->fromElemFound); + if (!elemToSet.ok()) { + return Status(ErrorCodes::InternalError, "can't create new element"); + } + + // Find the new place to put the "to" element: + // createPathAt does not use existing prefix elements so we + // need to get the prefix match position for createPathAt below + size_t tempIdx = 0; + mutablebson::Element tempElem = doc.end(); + Status status = pathsupport::findLongestPrefix(_toFieldRef, + doc.root(), + &tempIdx, + &tempElem); + + // createPathAt will complete the path and attach 'elemToSet' at the end of it. + return pathsupport::createPathAt(_toFieldRef, + tempElem == doc.end() ? 0 : tempIdx + 1, + tempElem == doc.end() ? doc.root() : tempElem, + elemToSet); + } + + Status ModifierRename::log(LogBuilder* logBuilder) const { + + // If there was no element found then it was a noop, so return immediately + if (!_preparedState->fromElemFound.ok()) + return Status::OK(); + + // debug assert if apply not called, since we found an element to move. + dassert(_preparedState->applyCalled); + + const bool isPrefix = _fromFieldRef.isPrefixOf(_toFieldRef); + const StringData setPath = + (isPrefix ? _fromFieldRef : _toFieldRef).dottedField(); + const StringData unsetPath = + isPrefix ? StringData() : _fromFieldRef.dottedField(); + const bool doUnset = !isPrefix; + + // We'd like to create an entry such as {$set: {: }} under 'logRoot'. + // We start by creating the {$set: ...} Element. + mutablebson::Document& doc = logBuilder->getDocument(); + + // Create the {: } Element. Note that we log the mod with a + // dotted field, if it was applied over a dotted field. The rationale is that the + // secondary may be in a different state than the primary and thus make different + // decisions about creating the intermediate path in _fieldRef or not. + mutablebson::Element logElement = doc.makeElementWithNewFieldName( + setPath, _preparedState->fromElemFound.getValue()); + + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, "cannot create details for $rename mod"); + } + + // Now, we attach the {: } Element under the {$set: ...} section. + Status status = logBuilder->addToSets(logElement); + + if (status.isOK() && doUnset) { + // Create the {: } Element. Note that we log the mod with a + // dotted field, if it was applied over a dotted field. The rationale is that the + // secondary may be in a different state than the primary and thus make different + // decisions about creating the intermediate path in _fieldRef or not. + status = logBuilder->addToUnsets(unsetPath); + } + + return status; + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_rename.h b/src/mongo/db/ops/modifier_rename.h new file mode 100644 index 00000000000..21ac594024d --- /dev/null +++ b/src/mongo/db/ops/modifier_rename.h @@ -0,0 +1,99 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + /** + * The $rename modifier moves the field from source to the destination to perform + * the rename. + * + * Example: {$rename: {:}} where both are field names + * Start with {a:1} and applying a {$rename: {"a":"b"} } produces {b:1} + **/ + class ModifierRename : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierRename); + + public: + + ModifierRename(); + virtual ~ModifierRename(); + + /** + * We will check that the to/from are valid paths; in prepare more validation is done + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** + * In prepare we will ensure that all restrictions are met: + * -- The 'from' field exists, and is valid, else it is a no-op + * -- The 'to' field is valid as a destination + * -- The 'to' field is not on the path (or the same path) as the 'from' field + * -- Neither 'to' nor 'from' have an array ancestor + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** + * We will transform the document by first making sure that the 'to' element + * is empty before moving the 'from' element there. + */ + virtual Status apply() const; + + /** + * For the oplog entry we will generate an $unset on the 'from' field, and $set for + * the 'to' field. If no 'from' element is found then function will return immediately. + */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // The source and destination fields + FieldRef _fromFieldRef; + FieldRef _toFieldRef; + + // The state carried over from prepare for apply/log + struct PreparedState; + scoped_ptr _preparedState; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_rename_test.cpp b/src/mongo/db/ops/modifier_rename_test.cpp new file mode 100644 index 00000000000..606337bdd7c --- /dev/null +++ b/src/mongo/db/ops/modifier_rename_test.cpp @@ -0,0 +1,395 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_rename.h" + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::fromjson; + using mongo::LogBuilder; + using mongo::ModifierInterface; + using mongo::NumberInt; + using mongo::ModifierRename; + using mongo::Status; + using mongo::StringData; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate the mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) { + _modObj = modObj; + ASSERT_OK(_mod.init(_modObj["$rename"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierRename& mod() { return _mod; } + + private: + ModifierRename _mod; + BSONObj _modObj; + }; + + /** + * These test negative cases: + * -- No '$' support for positional operator + * -- No empty field names (ex. .a, b. ) + * -- Can't rename to an invalid fieldname (empty fieldname part) + */ + TEST(InvalidInit, FromDbTests) { + ModifierRename mod; + ASSERT_NOT_OK(mod.init(fromjson("{'a.$':'b'}").firstElement(), + ModifierInterface::Options::normal())); + ASSERT_NOT_OK(mod.init(fromjson("{'a':'b.$'}").firstElement(), + ModifierInterface::Options::normal())); + ASSERT_NOT_OK(mod.init(fromjson("{'.b':'a'}").firstElement(), + ModifierInterface::Options::normal())); + ASSERT_NOT_OK(mod.init(fromjson("{'b.':'a'}").firstElement(), + ModifierInterface::Options::normal())); + ASSERT_NOT_OK(mod.init(fromjson("{'b':'.a'}").firstElement(), + ModifierInterface::Options::normal())); + ASSERT_NOT_OK(mod.init(fromjson("{'b':'a.'}").firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(MissingFrom, InitPrepLog) { + Document doc(fromjson("{a: 2}")); + Mod setMod(fromjson("{$rename: {'b':'a'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(MissingFromDotted, InitPrepLog) { + Document doc(fromjson("{a: {r:2}}")); + Mod setMod(fromjson("{$rename: {'a.b':'a.c'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + ASSERT_TRUE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(BasicInit, DifferentRoots) { + Document doc(fromjson("{a: 2}")); + Mod setMod(fromjson("{$rename: {'a':'f.g'}}")); + } + + TEST(MoveOnSamePath, MoveUp) { + ModifierRename mod; + ASSERT_NOT_OK(mod.init(fromjson("{'b.a':'b'}").firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(MoveOnSamePath, MoveDown) { + ModifierRename mod; + ASSERT_NOT_OK(mod.init(fromjson("{'b':'b.a'}").firstElement(), + ModifierInterface::Options::normal())); + } + + TEST(MissingTo, SimpleNumberAtRoot) { + Document doc(fromjson("{a: 2}")); + Mod setMod(fromjson("{$rename: {'a':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b:2}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'b': 2}, $unset: {'a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(SimpleReplace, SameLevel) { + Document doc(fromjson("{a: 2, b: 1}")); + Mod setMod(fromjson("{$rename: {'a':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b:2}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'b': 2}, $unset: {'a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(SimpleReplace, FromDottedElement) { + Document doc(fromjson("{a: {c: {d: 6}}, b: 1}")); + Mod setMod(fromjson("{$rename: {'a.c':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.c"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{a: {}, b:{ d: 6}}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'b': {d: 6}}, $unset: {'a.c': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(DottedTo, MissingCompleteTo) { + Document doc(fromjson("{a: 2, b: 1, c: {}}")); + Mod setMod(fromjson("{$rename: {'a':'c.r.d'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "c.r.d"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b:1, c: { r: { d: 2}}}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'c.r.d': 2}, $unset: {'a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(DottedTo, ToIsCompletelyMissing) { + Document doc(fromjson("{a: 2}")); + Mod setMod(fromjson("{$rename: {'a':'b.c.d'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b.c.d"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b: {c: {d: 2}}}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'b.c.d': 2}, $unset: {'a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(FromArrayOfEmbeddedDocs, ToMissingDottedField) { + Document doc(fromjson("{a: [ {a:2, b:1} ] }")); + Mod setMod(fromjson("{$rename: {'a':'b.c.d'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b.c.d"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b: {c: {d: [ {a:2, b:1} ]}}}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'b.c.d': [ {a:2, b:1} ]}, $unset: {'a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(FromArrayOfEmbeddedDocs, ToArray) { + Document doc(fromjson("{a: [ {a:2, b:1} ] }")); + Mod setMod(fromjson("{$rename: {'a.a':'a.b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(Arrays, MoveInto) { + Document doc(fromjson("{a: [1, 2], b:2}")); + Mod setMod(fromjson("{$rename: {'b':'a.2'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(Arrays, MoveOut) { + Document doc(fromjson("{a: [1, 2]}")); + Mod setMod(fromjson("{$rename: {'a.0':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(Arrays, MoveNonexistantEmbeddedFieldOut) { + Document doc(fromjson("{a: [{a:1}, {b:2}]}")); + Mod setMod(fromjson("{$rename: {'a.a':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(Arrays, MoveEmbeddedFieldOutWithElementNumber) { + Document doc(fromjson("{a: [{a:1}, {b:2}]}")); + Mod setMod(fromjson("{$rename: {'a.0.a':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(Arrays, ReplaceArrayField) { + Document doc(fromjson("{a: 2, b: []}")); + Mod setMod(fromjson("{$rename: {'a':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b:2}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'b': 2}, $unset: {'a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + + TEST(Arrays, ReplaceWithArrayField) { + Document doc(fromjson("{a: [], b: 2}")); + Mod setMod(fromjson("{$rename: {'a':'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b:[]}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'b': []}, $unset: {'a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(LegacyData, CanRenameFromInvalidFieldName) { + Document doc(fromjson("{$a: 2}")); + Mod setMod(fromjson("{$rename: {'$a':'a'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "$a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{a:2}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set:{ 'a': 2}, $unset: {'$a': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + +} // namespace diff --git a/src/mongo/db/ops/modifier_set.cpp b/src/mongo/db/ops/modifier_set.cpp new file mode 100644 index 00000000000..4f015359437 --- /dev/null +++ b/src/mongo/db/ops/modifier_set.cpp @@ -0,0 +1,289 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_set.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + struct ModifierSet::PreparedState { + + PreparedState(mutablebson::Document* targetDoc) + : doc(*targetDoc) + , idxFound(0) + , elemFound(doc.end()) + , noOp(false) + , elemIsBlocking(false) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element elemFound; + + // This $set is a no-op? + bool noOp; + + // The element we find during a replication operation that blocks our update path + bool elemIsBlocking; + + }; + + ModifierSet::ModifierSet(ModifierSet::ModifierSetMode mode) + : _fieldRef() + , _posDollar(0) + , _setMode(mode) + , _val() + , _modOptions() { + } + + ModifierSet::~ModifierSet() { + } + + Status ModifierSet::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + // + // field name analysis + // + + // Break down the field name into its 'dotted' components (aka parts) and check that + // the field is fit for updates + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + // + // value analysis + // + + if (!modExpr.ok()) + return Status(ErrorCodes::BadValue, "cannot $set an empty value"); + + _val = modExpr; + _modOptions = opts; + + return Status::OK(); + } + + Status ModifierSet::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(&root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_posDollar) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_posDollar, matchedField); + } + + // Locate the field name in 'root'. Note that we may not have all the parts in the path + // in the doc -- which is fine. Our goal now is merely to reason about whether this mod + // apply is a noOp or whether is can be in place. The remainin path, if missing, will + // be created during the apply. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + + // FindLongestPrefix may say the path does not exist at all, which is fine here, or + // that the path was not viable or otherwise wrong, in which case, the mod cannot + // proceed. + if (status.code() == ErrorCodes::NonExistentPath) { + _preparedState->elemFound = root.getDocument().end(); + } + else if (_modOptions.fromReplication && status.code() == ErrorCodes::PathNotViable) { + // If we are coming from replication and it is an invalid path, + // then push on indicating that we had a blocking element, which we stopped at + _preparedState->elemIsBlocking = true; + } + else if (!status.isOK()) { + return status; + } + + if (_setMode == SET_ON_INSERT) { + execInfo->context = ModifierInterface::ExecInfo::INSERT_CONTEXT; + } + + // We register interest in the field name. The driver needs this info to sort out if + // there is any conflict among mods. + execInfo->fieldRef[0] = &_fieldRef; + + // + // in-place and no-op logic + // + + // If the field path is not fully present, then this mod cannot be in place, nor is a + // noOp. + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_fieldRef.numParts()-1)) { + return Status::OK(); + } + + // If the value being $set is the same as the one already in the doc, than this is a + // noOp. + if (_preparedState->elemFound.ok() && + _preparedState->idxFound == (_fieldRef.numParts()-1) && + _preparedState->elemFound.compareWithBSONElement(_val, false /*ignore field*/) == 0) { + execInfo->noOp = _preparedState->noOp = true; + } + + return Status::OK(); + } + + Status ModifierSet::apply() const { + dassert(!_preparedState->noOp); + + const bool destExists = _preparedState->elemFound.ok() && + _preparedState->idxFound == (_fieldRef.numParts()-1); + // If there's no need to create any further field part, the $set is simply a value + // assignment. + if (destExists) { + return _preparedState->elemFound.setValueBSONElement(_val); + } + + // + // Complete document path logic + // + + // Creates the final element that's going to be $set in 'doc'. + mutablebson::Document& doc = _preparedState->doc; + StringData lastPart = _fieldRef.getPart(_fieldRef.numParts()-1); + mutablebson::Element elemToSet = doc.makeElementWithNewFieldName(lastPart, _val); + if (!elemToSet.ok()) { + return Status(ErrorCodes::InternalError, "can't create new element"); + } + + // Now, we can be in two cases here, as far as attaching the element being set goes: + // (a) none of the parts in the element's path exist, or (b) some parts of the path + // exist but not all. + if (!_preparedState->elemFound.ok()) { + _preparedState->elemFound = doc.root(); + _preparedState->idxFound = 0; + } + else { + _preparedState->idxFound++; + } + + // Remove the blocking element, if we are from replication applier. See comment below. + if (_modOptions.fromReplication && !destExists && _preparedState->elemFound.ok() && + _preparedState->elemIsBlocking && + (!(_preparedState->elemFound.isType(Array)) || + !(_preparedState->elemFound.isType(Object))) + ) { + + /** + * With replication we want to be able to remove blocking elements for $set (only). + * The reason they are blocking elements is that they are not embedded documents + * (objects) nor an array (a special type of an embedded doc) and we cannot + * add children to them (because the $set path requires adding children past + * the blocking element). + * + * Imagine that we started with this: + * {_id:1, a:1} + {$set : {"a.b.c" : 1}} -> {_id:1, a: {b: {c:1}}} + * Above we found that element (a:1) is blocking at position 1. We now will replace + * it with an empty object so the normal logic below can be + * applied from the root (in this example case). + * + * Here is an array example: + * {_id:1, a:[1, 2]} + {$set : {"a.0.c" : 1}} -> {_id:1, a: [ {c:1}, 2]} + * The blocking element is "a.0" since it is a number, non-object, and we must + * then replace it with an empty object so we can add c:1 to that empty object + */ + + mutablebson::Element blockingElem = _preparedState->elemFound; + BSONObj newObj; + // Replace blocking non-object with an empty object + Status status = blockingElem.setValueObject(newObj); + if (!status.isOK()) { + return status; + } + } + + // createPathAt() will complete the path and attach 'elemToSet' at the end of it. + return pathsupport::createPathAt(_fieldRef, + _preparedState->idxFound, + _preparedState->elemFound, + elemToSet); + } + + Status ModifierSet::log(LogBuilder* logBuilder) const { + + // We'd like to create an entry such as {$set: {: }} under 'logRoot'. + // We start by creating the {$set: ...} Element. + mutablebson::Document& doc = logBuilder->getDocument(); + + // Create the {: } Element. Note that we log the mod with a + // dotted field, if it was applied over a dotted field. The rationale is that the + // secondary may be in a different state than the primary and thus make different + // decisions about creating the intermediate path in _fieldRef or not. + mutablebson::Element logElement = doc.makeElementWithNewFieldName( + _fieldRef.dottedField(), _val); + + if (!logElement.ok()) { + return Status(ErrorCodes::InternalError, "cannot create details for $set mod"); + } + + return logBuilder->addToSets(logElement); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_set.h b/src/mongo/db/ops/modifier_set.h new file mode 100644 index 00000000000..f4ff52d6e4c --- /dev/null +++ b/src/mongo/db/ops/modifier_set.h @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierSet : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierSet); + + public: + + enum ModifierSetMode { SET_NORMAL, SET_ON_INSERT }; + explicit ModifierSet(ModifierSetMode mode = SET_NORMAL); + + // + // Modifier interface implementation + // + + virtual ~ModifierSet(); + + /** + * A 'modExpr' is a BSONElement {: } coming from a $set mod such as + * {$set: {}}. init() extracts the field name and the value to be + * assigned to it from 'modExpr'. It returns OK if successful or a status describing + * the error. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** + * Looks up the field name in the sub-tree rooted at 'root', and binds, if necessary, + * the '$' field part using the 'matchedfield' number. prepare() returns OK and + * fills in 'execInfo' with information of whether this mod is a no-op on 'root' and + * whether it is an in-place candidate. Otherwise, returns a status describing the + * error. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** + * Applies the prepared mod over the element 'root' specified in the prepare() + * call. Returns OK if successful or a status describing the error. + */ + virtual Status apply() const; + + /** + * Adds a log entry to logRoot corresponding to the operation applied here. Returns OK + * if successful or a status describing the error. + */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _posDollar; + + // If on 'on insert' mode, We'd like to apply this mod only if we're in a upsert. + const ModifierSetMode _setMode; + + // Element of the $set expression. + BSONElement _val; + + // See the class comments in modifier_interface.h + ModifierInterface::Options _modOptions; + + // The instance of the field in the provided doc. This state is valid after a + // prepare() was issued and until a log() is issued. The document this mod is + // being prepared against must be live throughout all the calls. + struct PreparedState; + scoped_ptr _preparedState; + + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_set_test.cpp b/src/mongo/db/ops/modifier_set_test.cpp new file mode 100644 index 00000000000..48dbf556ba3 --- /dev/null +++ b/src/mongo/db/ops/modifier_set_test.cpp @@ -0,0 +1,756 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_set.h" + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::fromjson; + using mongo::LogBuilder; + using mongo::ModifierInterface; + using mongo::NumberInt; + using mongo::ModifierSet; + using mongo::Status; + using mongo::StringData; + using mongo::mutablebson::ConstElement; + using mongo::mutablebson::countChildren; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate a $set mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj, bool fromRepl = false) : + _mod(mongoutils::str::equals(modObj.firstElement().fieldName(), "$setOnInsert") ? + ModifierSet::SET_ON_INSERT : ModifierSet::SET_NORMAL) { + + _modObj = modObj; + const StringData& modName = modObj.firstElement().fieldName(); + ASSERT_OK(_mod.init(_modObj[modName].embeddedObject().firstElement(), + !fromRepl ? ModifierInterface::Options::normal(): + ModifierInterface::Options::fromRepl())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierSet& mod() { return _mod; } + + private: + ModifierSet _mod; + BSONObj _modObj; + }; + + // + // Init tests + // + + TEST(Init, EmptyOperation) { + BSONObj modObj = fromjson("{$set: {}}"); + ModifierSet mod; + ASSERT_NOT_OK(mod.init(modObj["$set"].embeddedObject().firstElement(), + ModifierInterface::Options::normal() )); + } + + // + // Simple Mods + // + + TEST(SimpleMod, PrepareNoOp) { + Document doc(fromjson("{a: 2}")); + Mod setMod(fromjson("{$set: {a: 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(SimpleMod, PrepareSetOnInsert) { + Document doc(fromjson("{a: 1}")); + Mod setMod(fromjson("{$setOnInsert: {a: 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + ASSERT_EQUALS(execInfo.context, ModifierInterface::ExecInfo::INSERT_CONTEXT); + } + + TEST(SimpleMod, PrepareApplyEmptyDocument) { + Document doc(fromjson("{}")); + Mod setMod(fromjson("{$set: {a: 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: 2}"), doc); + } + + TEST(SimpleMod, PrepareApplyInPlace) { + Document doc(fromjson("{a: 1}")); + Mod setMod(fromjson("{$set: {a: 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: 2}"), doc); + } + + TEST(SimpleMod, PrepareApplyOverridePath) { + Document doc(fromjson("{a: {b: 1}}")); + Mod setMod(fromjson("{$set: {a: 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: 2}"), doc); + } + + TEST(SimpleMod, PrepareApplyChangeType) { + Document doc(fromjson("{a: 'str'}")); + Mod setMod(fromjson("{$set: {a: 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: 2}"), doc); + } + + TEST(SimpleMod, PrepareApplyNewPath) { + Document doc(fromjson("{b: 1}")); + Mod setMod(fromjson("{$set: {a: 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{b: 1, a: 2}"), doc); + } + + TEST(SimpleMod, LogNormal) { + BSONObj obj = fromjson("{a: 1}"); + Mod setMod(fromjson("{$set: {a: 2}}")); + + Document doc(obj); + ModifierInterface::ExecInfo dummy; + ASSERT_OK(setMod.prepare(doc.root(), "", &dummy)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {a: 2}}"), logDoc); + } + + // + // Simple dotted mod + // + + TEST(DottedMod, PrepareNoOp) { + Document doc(fromjson("{a: {b: 2}}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(DottedMod, PreparePathNotViable) { + Document doc(fromjson("{a:1}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(DottedMod, PreparePathNotViableArrray) { + Document doc(fromjson("{a:[{b:1}]}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(DottedMod, PrepareApplyInPlace) { + Document doc(fromjson("{a: {b: 1}}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {b: 2}}"), doc); + } + + TEST(DottedMod, PrepareApplyChangeType) { + Document doc(fromjson("{a: {b: 'str'}}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {b: 2}}"), doc); + } + + TEST(DottedMod, PrepareApplyChangePath) { + Document doc(fromjson("{a: {b: {c: 1}}}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {b: 2}}"), doc); + } + + TEST(DottedMod, PrepareApplyExtendPath) { + Document doc(fromjson("{a: {c: 1}}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {c: 1, b: 2}}"), doc); + } + + TEST(DottedMod, PrepareApplyNewPath) { + Document doc(fromjson("{c: 1}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{c: 1, a: {b: 2}}"), doc); + } + + TEST(DottedMod, PrepareApplyEmptyDoc) { + Document doc(fromjson("{}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {b: 2}}"), doc); + } + + TEST(DottedMod, PrepareApplyFieldWithDot) { + Document doc(fromjson("{'a.b':4}")); + Mod setMod(fromjson("{$set: {'a.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{'a.b':4, a: {b: 2}}"), doc); + } + + // + // Indexed mod + // + + TEST(IndexedMod, PrepareNoOp) { + Document doc(fromjson("{a: [{b: 0},{b: 1},{b: 2}]}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(IndexedMod, PrepareNonViablePath) { + Document doc(fromjson("{a: 0}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(IndexedMod, PrepareApplyInPlace) { + Document doc(fromjson("{a: [{b: 0},{b: 1},{b: 1}]}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [{b: 0},{b: 1},{b: 2}]}"), doc); + } + + TEST(IndexedMod, PrepareApplyNormalArray) { + Document doc(fromjson("{a: [{b: 0},{b: 1}]}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [{b: 0},{b: 1},{b: 2}]}"), doc); + } + + TEST(IndexedMod, PrepareApplyPaddingArray) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [{b: 0},null,{b: 2}]}"), doc); + } + + TEST(IndexedMod, PrepareApplyNumericObject) { + Document doc(fromjson("{a: {b: 0}}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {b: 0, '2': {b: 2}}}"), doc); + } + + TEST(IndexedMod, PrepareApplyNumericField) { + Document doc(fromjson("{a: {'2': {b: 1}}}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {'2': {b: 2}}}"), doc); + } + + TEST(IndexedMod, PrepareApplyExtendNumericField) { + Document doc(fromjson("{a: {'2': {c: 1}}}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {'2': {c: 1, b: 2}}}"), doc); + } + + TEST(IndexedMod, PrepareApplyEmptyObject) { + Document doc(fromjson("{a: {}}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {'2': {b: 2}}}"), doc); + } + + TEST(IndexedMod, PrepareApplyEmptyArray) { + Document doc(fromjson("{a: []}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [null, null, {b: 2}]}"), doc); + } + + TEST(IndexedMod, PrepareApplyInexistent) { + Document doc(fromjson("{}")); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.2.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {'2': {b: 2}}}"), doc); + } + + TEST(IndexedMod, LogNormal) { + BSONObj obj = fromjson("{a: [{b:0}, {b:1}]}"); + Document doc(obj); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.2.b': 2}}"), logDoc); + } + + TEST(IndexedMod, LogEmptyArray) { + BSONObj obj = fromjson("{a: []}"); + Document doc(obj); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.2.b': 2}}"), logDoc); + } + + TEST(IndexedMod, LogEmptyObject) { + BSONObj obj = fromjson("{a: []}"); + Document doc(obj); + Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'a.2.b': 2}}"), logDoc); + } + + // + // Indexed complex mod + // + + TEST(IndexedComplexMod, PrepareNoOp) { + Document doc(fromjson("{a: [{b: {c: 0, d: 0}}, {b: {c: 1, d: 1}}]}}")); + Mod setMod(fromjson("{$set: {'a.1.b': {c: 1, d: 1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.b"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(IndexedComplexMod, PrepareSameStructure) { + Document doc(fromjson("{a: [{b: {c: 0, d: 0}}, {b: {c: 1, xxx: 1}}]}}")); + Mod setMod(fromjson("{$set: {'a.1.b': {c: 1, d: 1}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.b"); + ASSERT_FALSE(execInfo.noOp); + } + + // + // Replication version where viable paths don't block modification + // + TEST(NonViablePathWithoutRepl, ControlRun) { + Document doc(fromjson("{a: 1}")); + Mod setMod(fromjson("{$set: {'a.1.b': 1}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); + } + + TEST(NonViablePathWithRepl, SingleField) { + Document doc(fromjson("{_id:1, a: 1}")); + Mod setMod(fromjson("{$set: {'a.1.b': 1}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{_id:1, a: {'1': {b: 1}}}"), doc); + } + + TEST(NonViablePathWithRepl, SingleFieldNoId) { + Document doc(fromjson("{a: 1}")); + Mod setMod(fromjson("{$set: {'a.1.b': 1}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {'1': {b: 1}}}"), doc); + } + + TEST(NonViablePathWithRepl, NestedField) { + Document doc(fromjson("{_id:1, a: {a: 1}}")); + Mod setMod(fromjson("{$set: {'a.a.1.b': 1}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.a.1.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{_id:1, a: {a: {'1': {b: 1}}}}"), doc); + } + + TEST(NonViablePathWithRepl, DoubleNestedField) { + Document doc(fromjson("{_id:1, a: {b: {c: 1}}}")); + Mod setMod(fromjson("{$set: {'a.b.c.d': 2}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b.c.d"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{_id:1, a: {b: {c: {d: 2}}}}"), doc); + } + + TEST(NonViablePathWithRepl, NestedFieldNoId) { + Document doc(fromjson("{a: {a: 1}}")); + Mod setMod(fromjson("{$set: {'a.a.1.b': 1}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.a.1.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: {a: {'1': {b: 1}}}}"), doc); + } + + TEST(NonViablePathWithRepl, ReplayArrayFieldNotAppendedItermediate) { + Document doc(fromjson("{_id: 0, a: [1, {b: [1]}]}")); + Mod setMod(fromjson("{$set: {'a.0.b': [0,2]}}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{_id: 0, a: [{b: [0,2]}, {b: [1]}]}"), doc); + } + + // Cases from users/issues/jstests + TEST(JsTestIssues, Set6) { + Document doc(fromjson("{_id: 1, r: {a:1, b:2}}")); + Mod setMod(fromjson("{$set: {'r.a': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "r.a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{_id: 1, r: {a:2, b:2}}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'r.a': 2}}"), logDoc); + } + + // Test which failed before and is here to verify correct execution. + TEST(JsTestIssues, Set6FromRepl) { + Document doc(fromjson("{_id: 1, r: {a:1, b:2}}")); + Mod setMod(fromjson("{$set: { 'r.a': 2}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "r.a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_TRUE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{_id: 1, r: {a:2, b:2} }"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(countChildren(logDoc.root()), 1u); + ASSERT_EQUALS(fromjson("{$set: {'r.a': 2}}"), logDoc); + } + + TEST(Ephemeral, ApplySetModToEphemeralDocument) { + // The following mod when applied to a document constructed node by node exposed a + // latent debug only defect in mutable BSON, so this is more a test of mutable than + // $set. + Document doc; + Element x = doc.makeElementObject("x"); + doc.root().pushBack(x); + Element a = doc.makeElementInt("a", 100); + x.pushBack(a); + + Mod setMod(fromjson("{ $set: { x: { a: 100, b: 2 }}}"), true); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "x"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{ x : { a : 100, b : 2 } }"), doc); + } + +} // unnamed namespace diff --git a/src/mongo/db/ops/modifier_table.cpp b/src/mongo/db/ops/modifier_table.cpp new file mode 100644 index 00000000000..31d4095160a --- /dev/null +++ b/src/mongo/db/ops/modifier_table.cpp @@ -0,0 +1,176 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_table.h" + +#include +#include + +#include "mongo/base/init.h" +#include "mongo/base/status.h" +#include "mongo/db/ops/modifier_add_to_set.h" +#include "mongo/db/ops/modifier_bit.h" +#include "mongo/db/ops/modifier_compare.h" +#include "mongo/db/ops/modifier_current_date.h" +#include "mongo/db/ops/modifier_inc.h" +#include "mongo/db/ops/modifier_pop.h" +#include "mongo/db/ops/modifier_pull.h" +#include "mongo/db/ops/modifier_pull_all.h" +#include "mongo/db/ops/modifier_push.h" +#include "mongo/db/ops/modifier_rename.h" +#include "mongo/db/ops/modifier_set.h" +#include "mongo/db/ops/modifier_unset.h" +#include "mongo/platform/unordered_map.h" + +namespace mongo { +namespace modifiertable { + + namespace { + + struct ModifierEntry { + string name; + ModifierType type; + + ModifierEntry(const StringData& name, ModifierType type) + : name(name.toString()) + , type(type) { + } + }; + + typedef unordered_map NameMap; + + NameMap* MODIFIER_NAME_MAP; + + void init(NameMap* nameMap) { + ModifierEntry* entryAddToSet = new ModifierEntry("$addToSet", MOD_ADD_TO_SET); + nameMap->insert(make_pair(StringData(entryAddToSet->name), entryAddToSet)); + + ModifierEntry* entryBit = new ModifierEntry("$bit", MOD_BIT); + nameMap->insert(make_pair(StringData(entryBit->name), entryBit)); + + ModifierEntry* entryCurrentDate = new ModifierEntry("$currentDate", MOD_CURRENTDATE); + nameMap->insert(make_pair(StringData(entryCurrentDate->name), entryCurrentDate)); + + ModifierEntry* entryInc = new ModifierEntry("$inc", MOD_INC); + nameMap->insert(make_pair(StringData(entryInc->name), entryInc)); + + ModifierEntry* entryMax = new ModifierEntry("$max", MOD_MAX); + nameMap->insert(make_pair(StringData(entryMax->name), entryMax)); + + ModifierEntry* entryMin = new ModifierEntry("$min", MOD_MIN); + nameMap->insert(make_pair(StringData(entryMin->name), entryMin)); + + ModifierEntry* entryMul = new ModifierEntry("$mul", MOD_MUL); + nameMap->insert(make_pair(StringData(entryMul->name), entryMul)); + + ModifierEntry* entryPop = new ModifierEntry("$pop", MOD_POP); + nameMap->insert(make_pair(StringData(entryPop->name), entryPop)); + + ModifierEntry* entryPull = new ModifierEntry("$pull", MOD_PULL); + nameMap->insert(make_pair(StringData(entryPull->name), entryPull)); + + ModifierEntry* entryPullAll = new ModifierEntry("$pullAll", MOD_PULL_ALL); + nameMap->insert(make_pair(StringData(entryPullAll->name), entryPullAll)); + + ModifierEntry* entryPush = new ModifierEntry("$push", MOD_PUSH); + nameMap->insert(make_pair(StringData(entryPush->name), entryPush)); + + ModifierEntry* entryPushAll = new ModifierEntry("$pushAll", MOD_PUSH_ALL); + nameMap->insert(make_pair(StringData(entryPushAll->name), entryPushAll)); + + ModifierEntry* entrySet = new ModifierEntry("$set", MOD_SET); + nameMap->insert(make_pair(StringData(entrySet->name), entrySet)); + + ModifierEntry* entrySetOnInsert = new ModifierEntry("$setOnInsert", MOD_SET_ON_INSERT); + nameMap->insert(make_pair(StringData(entrySetOnInsert->name), entrySetOnInsert)); + + ModifierEntry* entryRename = new ModifierEntry("$rename", MOD_RENAME); + nameMap->insert(make_pair(StringData(entryRename->name), entryRename)); + + ModifierEntry* entryUnset = new ModifierEntry("$unset", MOD_UNSET); + nameMap->insert(make_pair(StringData(entryUnset->name), entryUnset)); + } + + } // unnamed namespace + + MONGO_INITIALIZER(ModifierTable)(InitializerContext* context) { + MODIFIER_NAME_MAP = new NameMap; + init(MODIFIER_NAME_MAP); + + return Status::OK(); + } + + ModifierType getType(const StringData& typeStr) { + NameMap::const_iterator it = MODIFIER_NAME_MAP->find(typeStr); + if (it == MODIFIER_NAME_MAP->end()) { + return MOD_UNKNOWN; + } + return it->second->type; + } + + ModifierInterface* makeUpdateMod(ModifierType modType) { + switch (modType) { + case MOD_ADD_TO_SET: + return new ModifierAddToSet; + case MOD_BIT: + return new ModifierBit; + case MOD_CURRENTDATE: + return new ModifierCurrentDate; + case MOD_INC: + return new ModifierInc(ModifierInc::MODE_INC); + case MOD_MAX: + return new ModifierCompare(ModifierCompare::MAX); + case MOD_MIN: + return new ModifierCompare(ModifierCompare::MIN); + case MOD_MUL: + return new ModifierInc(ModifierInc::MODE_MUL); + case MOD_POP: + return new ModifierPop; + case MOD_PULL: + return new ModifierPull; + case MOD_PULL_ALL: + return new ModifierPullAll; + case MOD_PUSH: + return new ModifierPush(ModifierPush::PUSH_NORMAL); + case MOD_PUSH_ALL: + return new ModifierPush(ModifierPush::PUSH_ALL); + case MOD_SET: + return new ModifierSet(ModifierSet::SET_NORMAL); + case MOD_SET_ON_INSERT: + return new ModifierSet(ModifierSet::SET_ON_INSERT); + case MOD_RENAME: + return new ModifierRename; + case MOD_UNSET: + return new ModifierUnset; + default: + return NULL; + } + } + +} // namespace modifiertable +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_table.h b/src/mongo/db/ops/modifier_table.h new file mode 100644 index 00000000000..69aadab0839 --- /dev/null +++ b/src/mongo/db/ops/modifier_table.h @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { +namespace modifiertable { + + // NOTE: Please update jstests/verify_update_mods.js or include a jstest for any new mods + enum ModifierType { + MOD_ADD_TO_SET, + MOD_BIT, + MOD_CURRENTDATE, + MOD_INC, + MOD_MAX, + MOD_MIN, + MOD_MUL, + MOD_POP, + MOD_PULL, + MOD_PULL_ALL, + MOD_PUSH, + MOD_PUSH_ALL, + MOD_SET, + MOD_SET_ON_INSERT, + MOD_RENAME, + MOD_UNSET, + MOD_UNKNOWN + }; + + /** + * Returns the modifier type for 'typeStr', if it was recognized as an existing update + * mod, or MOD_UNKNOWN otherwise. + */ + ModifierType getType(const StringData& typeStr); + + /** + * Instantiate an update mod that corresponds to 'modType' or NULL if 'modType' is not + * valid. The ownership of the new object is the caller's. + */ + ModifierInterface* makeUpdateMod(ModifierType modType); + +} // namespace modifiertable +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_table_test.cpp b/src/mongo/db/ops/modifier_table_test.cpp new file mode 100644 index 00000000000..23f568b1981 --- /dev/null +++ b/src/mongo/db/ops/modifier_table_test.cpp @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_table.h" + +#include + +#include "mongo/db/ops/modifier_interface.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using namespace mongo::modifiertable; + + using mongo::ModifierInterface; + using std::auto_ptr; + + TEST(getType, Normal) { + ASSERT_EQUALS(getType("$set"), MOD_SET); + ASSERT_EQUALS(getType("$AModThatDoesn'tExist"), MOD_UNKNOWN); + ASSERT_EQUALS(getType("NotAModExpression"), MOD_UNKNOWN); + } + + TEST(makeUpdateMod, Normal) { + auto_ptr mod; + + mod.reset(makeUpdateMod(MOD_SET)); + ASSERT_NOT_EQUALS(mod.get(), static_cast(0)); + + mod.reset(makeUpdateMod(MOD_UNKNOWN)); + ASSERT_EQUALS(mod.get(), static_cast(0)); + } + +} // unnamed namespace diff --git a/src/mongo/db/ops/modifier_unset.cpp b/src/mongo/db/ops/modifier_unset.cpp new file mode 100644 index 00000000000..b5f55f86708 --- /dev/null +++ b/src/mongo/db/ops/modifier_unset.cpp @@ -0,0 +1,185 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/modifier_unset.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/ops/field_checker.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + struct ModifierUnset::PreparedState { + + PreparedState(mutablebson::Document* targetDoc) + : doc(*targetDoc) + , idxFound(0) + , elemFound(doc.end()) + , noOp(false) { + } + + // Document that is going to be changed. + mutablebson::Document& doc; + + // Index in _fieldRef for which an Element exist in the document. + size_t idxFound; + + // Element corresponding to _fieldRef[0.._idxFound]. + mutablebson::Element elemFound; + + // This $set is a no-op? + bool noOp; + + }; + + ModifierUnset::ModifierUnset() + : _fieldRef() + , _posDollar(0) + , _val() { + } + + ModifierUnset::~ModifierUnset() { + } + + Status ModifierUnset::init(const BSONElement& modExpr, const Options& opts, + bool* positional) { + + // + // field name analysis + // + + // Perform standard field name and updateable checks. + _fieldRef.parse(modExpr.fieldName()); + Status status = fieldchecker::isUpdatable(_fieldRef); + if (! status.isOK()) { + return status; + } + + // If a $-positional operator was used, get the index in which it occurred + // and ensure only one occurrence. + size_t foundCount; + bool foundDollar = fieldchecker::isPositional(_fieldRef, &_posDollar, &foundCount); + + if (positional) + *positional = foundDollar; + + if (foundDollar && foundCount > 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Too many positional (i.e. '$') elements found in path '" + << _fieldRef.dottedField() << "'"); + } + + + // + // value analysis + // + + // Unset takes any value, since there is no semantics attached to such value. + _val = modExpr; + + return Status::OK(); + } + + Status ModifierUnset::prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo) { + + _preparedState.reset(new PreparedState(&root.getDocument())); + + // If we have a $-positional field, it is time to bind it to an actual field part. + if (_posDollar) { + if (matchedField.empty()) { + return Status(ErrorCodes::BadValue, + str::stream() << "The positional operator did not find the match " + "needed from the query. Unexpanded update: " + << _fieldRef.dottedField()); + } + _fieldRef.setPart(_posDollar, matchedField); + } + + + // Locate the field name in 'root'. Note that if we don't have the full path in the + // doc, there isn't anything to unset, really. + Status status = pathsupport::findLongestPrefix(_fieldRef, + root, + &_preparedState->idxFound, + &_preparedState->elemFound); + if (!status.isOK() || + _preparedState->idxFound != (_fieldRef.numParts() -1)) { + execInfo->noOp = _preparedState->noOp = true; + execInfo->fieldRef[0] = &_fieldRef; + + return Status::OK(); + } + + // If there is indeed something to unset, we register so, along with the interest in + // the field name. The driver needs this info to sort out if there is any conflict + // among mods. + execInfo->fieldRef[0] = &_fieldRef; + + // The only way for an $unset to be inplace is for its target field to be the last one + // of the object. That is, it is always the right child on its paths. The current + // rationale is that there should be no holes in a BSONObj and, to be in place, no + // field boundaries must change. + // + // TODO: + // mutablebson::Element curr = _preparedState->elemFound; + // while (curr.ok()) { + // if (curr.rightSibling().ok()) { + // } + // curr = curr.parent(); + // } + + return Status::OK(); + } + + Status ModifierUnset::apply() const { + dassert(!_preparedState->noOp); + + // Our semantics says that, if we're unseting an element of an array, we swap that + // value to null. The rationale is that we don't want other array elements to change + // indices. (That could be achieved with $pull-ing element from it.) + if (_preparedState->elemFound.parent().ok() && + _preparedState->elemFound.parent().getType() == Array) { + return _preparedState->elemFound.setValueNull(); + } + else { + return _preparedState->elemFound.remove(); + } + } + + Status ModifierUnset::log(LogBuilder* logBuilder) const { + return logBuilder->addToUnsets(_fieldRef.dottedField()); + } + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_unset.h b/src/mongo/db/ops/modifier_unset.h new file mode 100644 index 00000000000..3e4a5fc2b22 --- /dev/null +++ b/src/mongo/db/ops/modifier_unset.h @@ -0,0 +1,104 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" + +namespace mongo { + + class LogBuilder; + + class ModifierUnset : public ModifierInterface { + MONGO_DISALLOW_COPYING(ModifierUnset); + + public: + + ModifierUnset(); + + // + // Modifier interface implementation + // + + virtual ~ModifierUnset(); + + /** + * A 'modExpr' is a BSONElement {: } coming from a $set mod such as + * {$unset: {}}. init() extracts the field name and the value to be + * assigned to it from 'modExpr'. It returns OK if successful or a status describing + * the error. + */ + virtual Status init(const BSONElement& modExpr, const Options& opts, + bool* positional = NULL); + + /** + * Locates the field to be removed under the 'root' element, if it exist, and fills in + * 'execInfo' accordingly. Return OK if successful or a status describing the error. + */ + virtual Status prepare(mutablebson::Element root, + const StringData& matchedField, + ExecInfo* execInfo); + + /** + * Removes the found element from the document. If such element was inside an array, + * removal means setting that array position to 'null'. + */ + virtual Status apply() const; + + /** + * Adds the exact $unset mod to the log. + */ + virtual Status log(LogBuilder* logBuilder) const; + + private: + + // Access to each component of fieldName that's the target of this mod. + FieldRef _fieldRef; + + // 0 or index for $-positional in _fieldRef. + size_t _posDollar; + + // Element of the $set expression. + BSONElement _val; + + // The instance of the field in the provided doc. This state is valid after a + // prepare() was issued and until a log() is issued. The document this mod is + // being prepared against must be live throughout all the calls. + struct PreparedState; + scoped_ptr _preparedState; + + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/modifier_unset_test.cpp b/src/mongo/db/ops/modifier_unset_test.cpp new file mode 100644 index 00000000000..d181bd57d99 --- /dev/null +++ b/src/mongo/db/ops/modifier_unset_test.cpp @@ -0,0 +1,451 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/ops/modifier_unset.h" + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::Array; + using mongo::BSONObj; + using mongo::fromjson; + using mongo::LogBuilder; + using mongo::ModifierInterface; + using mongo::ModifierUnset; + using mongo::Status; + using mongo::StringData; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + + /** Helper to build and manipulate a $set mod. */ + class Mod { + public: + Mod() : _mod() {} + + explicit Mod(BSONObj modObj) { + _modObj = modObj; + ASSERT_OK(_mod.init(_modObj["$unset"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + } + + Status prepare(Element root, + const StringData& matchedField, + ModifierInterface::ExecInfo* execInfo) { + return _mod.prepare(root, matchedField, execInfo); + } + + Status apply() const { + return _mod.apply(); + } + + Status log(LogBuilder* logBuilder) const { + return _mod.log(logBuilder); + } + + ModifierUnset& mod() { return _mod; } + + BSONObj modObj() { return _modObj; } + + private: + ModifierUnset _mod; + BSONObj _modObj; + }; + + // + // Simple mod + // + + TEST(SimpleMod, PrepareNoOp) { + Document doc(fromjson("{}")); + Mod modUnset(fromjson("{$unset: {a: true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(SimpleMod, PrepareApplyNormal) { + Document doc(fromjson("{a: 1, b: 2}")); + Mod modUnset(fromjson("{$unset: {a: true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{b: 2}"), doc); + } + + TEST(SimpleMod, PrepareApplyInPlace) { + Document doc(fromjson("{x: 0, a: 1}")); + Mod modUnset(fromjson("{$unset: {a: true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); // TODO turn in-place on for this. + ASSERT_EQUALS(fromjson("{x: 0}"), doc); + } + + TEST(SimpleMod, PrepareApplyGeneratesEmptyDocument) { + Document doc(fromjson("{a: 1}")); + Mod modUnset(fromjson("{$unset: {a: true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); // TODO turn in-place on for this. + ASSERT_EQUALS(fromjson("{}"), doc); + } + + TEST(SimpleMod, PrepareApplyUnsetSubtree) { + Document doc(fromjson("{a: {b: 1}, c: 2}")); + Mod modUnset(fromjson("{$unset: {a: true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{c: 2}"), doc); + } + + TEST(SimpleMod, LogNormal) { + BSONObj obj = fromjson("{a: 1}"); + Document doc(obj); + Mod modUnset(fromjson("{$unset: {a: true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_FALSE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(modUnset.log(&logBuilder)); + ASSERT_EQUALS(modUnset.modObj(), logDoc); + } + + // + // Dotted mod + // + + TEST(DottedMod, PrepareNoOp) { + Document doc(fromjson("{c:2}")); + Mod modUnset(fromjson("{$unset: {'a.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(DottedMod, PrepareApplyNormal) { + Document doc(fromjson("{a: {b: 1}, c: 2}")); + Mod modUnset(fromjson("{$unset: {'a.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a:{}, c:2}"), doc); + } + + TEST(DottedMod, PrepareApplyInPlace) { + Document doc(fromjson("{x: 0, a: {b: 1}}")); + Mod modUnset(fromjson("{$unset: {'a.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); // TODO turn in-place on for this. + ASSERT_EQUALS(fromjson("{x: 0, a:{}}"), doc); + } + + TEST(DottedMod, PrepareApplyUnsetNestedSubobject) { + Document doc(fromjson("{a: {b: {c: 1}}}")); + Mod modUnset(fromjson("{$unset: {'a.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); // TODO turn in-place on for this. + ASSERT_EQUALS(fromjson("{a: {}}"), doc); + } + + // + // Indexed mod + // + + TEST(IndexedMod, PrepareNoOp) { + Document doc(fromjson("{a:[]}")); + Mod modUnset(fromjson("{$unset: {'a.0': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(IndexedMod, PrepareApplyNormal) { + Document doc(fromjson("{a:[0,1,2]}")); + Mod modUnset(fromjson("{$unset: {'a.0': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a:[null,1,2]}"), doc); + } + + TEST(IndexedMod, PrepareApplyInPlace) { + Document doc(fromjson("{b:1, a:[1]}")); + Mod modUnset(fromjson("{$unset: {'a.0': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); // TODO turn in-place on for this. + ASSERT_EQUALS(fromjson("{b:1, a:[null]}"), doc); + } + + TEST(IndexedMod, PrepareApplyInPlaceNuance) { + // Can't change the encoding in the middle of a bson stream. + Document doc(fromjson("{a:[1], b:1}")); + Mod modUnset(fromjson("{$unset: {'a.0': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a:[null], b:1}"), doc); + } + + TEST(IndexedMod, PrepareApplyInnerObject) { + Document doc(fromjson("{a:[{b:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.0.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a:[{}]}"), doc); + } + + TEST(IndexedMod, PrepareApplyObject) { + Document doc(fromjson("{a:[{b:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.0': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a:[null]}"), doc); + } + + TEST(IndexedMod, LogNormal) { + Document doc(fromjson("{a:[0,1,2]}")); + Mod modUnset(fromjson("{$unset: {'a.0': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "", &execInfo)); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(modUnset.log(&logBuilder)); + ASSERT_EQUALS(modUnset.modObj(), logDoc); + } + + // + // Positional mod + // + + TEST(PositionalMod, PrepareNoOp) { + Document doc(fromjson("{a:[{b:0}]}")); + Mod modUnset(fromjson("{$unset: {'a.$.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "1", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.b"); + ASSERT_TRUE(execInfo.noOp); + } + + TEST(PositionalMod, PrepareMissingPositional) { + Document doc(fromjson("{a:[{b:0},{c:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.$.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_NOT_OK(modUnset.prepare(doc.root(), "" /* no position */, &execInfo)); + } + + TEST(PositionalMod, PrepareApplyNormal) { + Document doc(fromjson("{a:[{b:0},{c:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.$.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "0", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [{}, {c:1}]}"), doc); + } + + TEST(PositionalMod, PrepareApplyObject) { + Document doc(fromjson("{a:[{b:0},{c:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.$': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "0", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{a: [null, {c:1}]}"), doc); + } + + TEST(PositionalMod, PrepareApplyInPlace) { + Document doc(fromjson("{b:1, a:[{b:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.$.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "0", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); // TODO turn in-place on for this. + ASSERT_EQUALS(fromjson("{b:1, a:[{}]}"), doc); + } + + TEST(PositionalMod, LogNormal) { + Document doc(fromjson("{b:1, a:[{b:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.$.b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "0", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.b"); + ASSERT_FALSE(execInfo.noOp); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(modUnset.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$unset: {'a.0.b': true}}"), logDoc); + } + + TEST(LegacyData, CanUnsetInvalidField) { + Document doc(fromjson("{b:1, a:[{$b:1}]}")); + Mod modUnset(fromjson("{$unset: {'a.$.$b': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(modUnset.prepare(doc.root(), "0", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.$b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(modUnset.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(fromjson("{b:1, a:[{}]}"), doc); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + ASSERT_OK(modUnset.log(&logBuilder)); + ASSERT_EQUALS(fromjson("{$unset: {'a.0.$b': true}}"), logDoc); + } + + +} // unnamed namespace diff --git a/src/mongo/db/ops/path_support.cpp b/src/mongo/db/ops/path_support.cpp new file mode 100644 index 00000000000..d46ad1aa977 --- /dev/null +++ b/src/mongo/db/ops/path_support.cpp @@ -0,0 +1,264 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/path_support.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { +namespace pathsupport { + + namespace { + + bool isNumeric(const StringData& str, size_t* num) { + size_t res = 0; + for (size_t i = 0; i < str.size(); ++i) { + if (str[i] < '0' || str[i] > '9') { + return false; + } + else { + res = res * 10 + (str[i] - '0'); + } + } + *num = res; + return true; + } + + Status maybePadTo(mutablebson::Element* elemArray, + size_t sizeRequired) { + dassert(elemArray->getType() == Array); + + if (sizeRequired > kMaxPaddingAllowed) { + return Status(ErrorCodes::CannotBackfillArray, + mongoutils::str::stream() << "can't backfill array to larger than " + << kMaxPaddingAllowed << " elements"); + } + + size_t currSize = mutablebson::countChildren(*elemArray); + if (sizeRequired > currSize) { + size_t toPad = sizeRequired - currSize; + for (size_t i = 0; i < toPad; i++) { + Status status = elemArray->appendNull(""); + if (!status.isOK()) { + return status; + } + } + } + return Status::OK(); + } + + } // unnamed namespace + + Status findLongestPrefix(const FieldRef& prefix, + mutablebson::Element root, + size_t* idxFound, + mutablebson::Element* elemFound) { + + // If root is empty or the prefix is so, there's no point in looking for a prefix. + const size_t prefixSize = prefix.numParts(); + if (!root.hasChildren() || prefixSize == 0) { + return Status(ErrorCodes::NonExistentPath, + "either the document or the path are empty"); + } + + // Loop through prefix's parts. At each iteration, check that the part ('curr') exists + // in 'root' and that the type of the previous part ('prev') allows for children. + mutablebson::Element curr = root; + mutablebson::Element prev = root; + size_t i = 0; + size_t numericPart = 0; + bool viable = true; + for (;i < prefixSize; i++) { + + // If prefix wants to reach 'curr' by applying a non-numeric index to an array + // 'prev', or if 'curr' wants to traverse a leaf 'prev', then we'd be in a + // non-viable path (see definition on the header file). + StringData prefixPart = prefix.getPart(i); + prev = curr; + switch (curr.getType()) { + + case Object: + curr = prev[prefixPart]; + break; + + case Array: + if (!isNumeric(prefixPart, &numericPart)) { + viable = false; + } else { + curr = prev[numericPart]; + } + break; + + default: + viable = false; + } + + // If we couldn't find the next field part of the prefix in the document or if the + // field part we're in constitutes a non-viable path, we can stop looking. + if (!curr.ok() || !viable) { + break; + } + } + + // We broke out of the loop because one of four things happened. (a) 'prefix' and + // 'root' have nothing in common, (b) 'prefix' is not viable in 'root', (c) not all the + // parts in 'prefix' exist in 'root', or (d) all parts do. In each case, we need to + // figure out what index and Element pointer to return. + if (i == 0) { + return Status(ErrorCodes::NonExistentPath, + "cannot find path in the document"); + } + else if (!viable) { + *idxFound = i - 1; + *elemFound = prev; + return Status(ErrorCodes::PathNotViable, + mongoutils::str::stream() << "cannot use the part (" << + prefix.getPart(i-1) << " of " << prefix.dottedField() << + ") to traverse the element ({" << + curr.toString() << "})"); + } + else if (curr.ok()) { + *idxFound = i - 1; + *elemFound = curr; + return Status::OK(); + } + else { + *idxFound = i - 1; + *elemFound = prev; + return Status::OK(); + } + } + + Status createPathAt(const FieldRef& prefix, + size_t idxFound, + mutablebson::Element elemFound, + mutablebson::Element newElem) { + Status status = Status::OK(); + + // Sanity check that 'idxField' is an actual part. + const size_t size = prefix.numParts(); + if (idxFound >= size) { + return Status(ErrorCodes::BadValue, "index larger than path size"); + } + + mutablebson::Document& doc = elemFound.getDocument(); + + // If we are creating children under an array and a numeric index is next, then perhaps + // we need padding. + size_t i = idxFound; + bool inArray = false; + if (elemFound.getType() == mongo::Array) { + size_t newIdx = 0; + if (!isNumeric(prefix.getPart(idxFound), &newIdx)) { + return Status(ErrorCodes::InvalidPath, "Array require numeric fields"); + } + + status = maybePadTo(&elemFound, newIdx); + if (!status.isOK()) { + return status; + } + + // If there is a next field, that would be an array element. We'd like to mark that + // field because we create array elements differently than we do regular objects. + if (++i < size) { + inArray = true; + } + } + + // Create all the remaining parts but the last one. + for (; i < size - 1 ; i++) { + mutablebson::Element elem = doc.makeElementObject(prefix.getPart(i)); + if (!elem.ok()) { + return Status(ErrorCodes::InternalError, "cannot create path"); + } + + // If this field is an array element, we wrap it in an object (because array + // elements are wraped in { "N": } objects. + if (inArray) { + // TODO pass empty StringData to makeElementObject, when that's supported. + mutablebson::Element arrayObj = doc.makeElementObject("" /* it's an array */); + if (!arrayObj.ok()) { + return Status(ErrorCodes::InternalError, "cannot create item on array"); + } + status = arrayObj.pushBack(elem); + if (!status.isOK()) { + return status; + } + status = elemFound.pushBack(arrayObj); + if (!status.isOK()) { + return status; + } + inArray = false; + } + else { + status = elemFound.pushBack(elem); + if (!status.isOK()) { + return status; + } + } + + elemFound = elem; + } + + // Attach the last element. Here again, if we're in a field that is an array element, + // we wrap it in an object first. + if (inArray) { + // TODO pass empty StringData to makeElementObject, when that's supported. + mutablebson::Element arrayObj = doc.makeElementObject("" /* it's an array */); + if (!arrayObj.ok()) { + return Status(ErrorCodes::InternalError, "cannot create item on array"); + } + + status = arrayObj.pushBack(newElem); + if (!status.isOK()) { + return status; + } + + status = elemFound.pushBack(arrayObj); + if (!status.isOK()) { + return status; + } + + } + else { + status = elemFound.pushBack(newElem); + if (!status.isOK()) { + return status; + } + } + + return Status::OK(); + } + +} // namespace pathsupport +} // namespace mongo diff --git a/src/mongo/db/ops/path_support.h b/src/mongo/db/ops/path_support.h new file mode 100644 index 00000000000..2641b39f658 --- /dev/null +++ b/src/mongo/db/ops/path_support.h @@ -0,0 +1,98 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/field_ref.h" +#include "mongo/platform/cstdint.h" + +namespace mongo { + + namespace pathsupport { + + // Cap on the number of nulls we'll add to an array if we're inserting to an index that + // doesn't exist. + static const size_t kMaxPaddingAllowed = 1500000; + + /** + * Finds the longest portion of 'prefix' that exists in document rooted at 'root' and is + * "viable." A viable path is one that, if fully created on a given doc, would not + * change the existing types of any fields in that doc. (See examples below.) + * + * If a prefix indeed exists, 'idxFound' is set to indicate how many parts in common + * 'prefix' and 'doc' have. 'elemFound' would point to the Element corresponding to + * prefix[idxFound] in 'doc'. The call would return an OK status in this case. + * + * If a prefix is not viable, returns a status "PathNotViable". 'idxFound' is set to + * indicate the part in the document that caused the path to be not viable. 'elemFound' + * would point to the Element corresponding to prefix[idxFound] in 'doc'. + * + * If a prefix does not exist, the call returns "NonExistentPath". 'elemFound' and + * 'idxFound' are indeterminate in this case. + * + * Definition of a "Viable Path": + * + * A field reference 'p_1.p_2.[...].p_n', where 'p_i' is a field part, is said to be + * a viable path in a given document D if the creation of each part 'p_i', 0 <= i < n + * in D does not force 'p_i' to change types. In other words, no existing 'p_i' in D + * may have a different type, other than the 'p_n'. + * + * 'a.b.c' is a viable path in {a: {b: {c: 1}}} + * 'a.b.c' is a viable path in {a: {b: {c: {d: 1}}}} + * 'a.b.c' is NOT a viable path in {a: {b: 1}}, because b would have changed types + * 'a.0.b' is a viable path in {a: [{b: 1}, {c: 1}]} + * 'a.0.b' is a viable path in {a: {"0": {b: 1}}} + * 'a.0.b' is NOT a viable path in {a: 1}, because a would have changed types + * 'a.5.b' is a viable path in in {a: []} (padding would occur) + */ + Status findLongestPrefix(const FieldRef& prefix, + mutablebson::Element root, + size_t* idxFound, + mutablebson::Element* elemFound); + + /** + * Creates the parts 'prefix[idxRoot]', 'prefix[idxRoot+1]', ..., + * 'prefix[-1]' under 'elemFound' and adds 'newElem' as a child of that + * path. Returns OK, if successful, or an error code describing why not, otherwise. + * + * createPathAt is designed to work with 'findLongestPrefix' in that it can create the + * field parts in 'prefix' that are missing from a given document. 'elemFound' points + * to the element in the doc that is the parent of prefix[idxRoot]. + */ + Status createPathAt(const FieldRef& prefix, + size_t idxRoot, + mutablebson::Element elemFound, + mutablebson::Element newElem); + + } // namespace pathsupport + +} // namespace mongo diff --git a/src/mongo/db/ops/path_support_test.cpp b/src/mongo/db/ops/path_support_test.cpp new file mode 100644 index 00000000000..b933b84d797 --- /dev/null +++ b/src/mongo/db/ops/path_support_test.cpp @@ -0,0 +1,450 @@ +/** + * Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/ops/path_support.h" + +#include + +#include "mongo/base/error_codes.h" +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/platform/cstdint.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/mongoutils/str.h" + +namespace { + + using mongo::BSONObj; + using mongo::ErrorCodes; + using mongo::FieldRef; + using mongo::fromjson; + using mongo::jstNULL; + using mongo::NumberInt; + using mongo::Object; + using mongo::pathsupport::findLongestPrefix; + using mongo::pathsupport::createPathAt; + using mongo::Status; + using mongo::StringData; + using mongo::mutablebson::countChildren; + using mongo::mutablebson::getNthChild; + using mongo::mutablebson::Document; + using mongo::mutablebson::Element; + using mongoutils::str::stream; + using std::string; + + class EmptyDoc : public mongo::unittest::Test { + public: + EmptyDoc() : _doc() {} + + Document& doc() { return _doc; } + + Element root() { return _doc.root(); } + + FieldRef& field() { return _field; } + + void setField(StringData str) { _field.parse(str); } + + private: + Document _doc; + FieldRef _field; + }; + + TEST_F(EmptyDoc, EmptyPath) { + setField(""); + + size_t idxFound; + Element elemFound = root(); + Status status = findLongestPrefix(field(), root(), &idxFound, &elemFound); + ASSERT_EQUALS(status, ErrorCodes::NonExistentPath); + } + + TEST_F(EmptyDoc, NewField) { + setField("a"); + + size_t idxFound; + Element elemFound = root(); + Status status = findLongestPrefix(field(), root(), &idxFound, &elemFound); + ASSERT_EQUALS(status, ErrorCodes::NonExistentPath); + + Element newElem = doc().makeElementInt("a", 1); + ASSERT_TRUE(newElem.ok()); + ASSERT_OK(createPathAt(field(), 0, root(), newElem)); + ASSERT_EQUALS(fromjson("{a: 1}"), doc()); + } + + class SimpleDoc : public mongo::unittest::Test { + public: + SimpleDoc() : _doc() {} + + virtual void setUp() { + // {a: 1} + ASSERT_OK(root().appendInt("a", 1)); + } + + Document& doc() { return _doc; } + + Element root() { return _doc.root(); } + + FieldRef& field() { return _field; } + void setField(StringData str) { _field.parse(str); } + + private: + Document _doc; + FieldRef _field; + }; + + TEST_F(SimpleDoc, EmptyPath) { + setField(""); + + size_t idxFound; + Element elemFound = root(); + Status status = findLongestPrefix(field(), root(), &idxFound, &elemFound); + ASSERT_EQUALS(status, ErrorCodes::NonExistentPath); + } + + TEST_F(SimpleDoc, SimplePath) { + setField("a"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]), 0); + } + + TEST_F(SimpleDoc, LongerPath) { + setField("a.b"); + + size_t idxFound; + Element elemFound = root(); + Status status = findLongestPrefix(field(), root(), &idxFound, &elemFound); + ASSERT_EQUALS(status, ErrorCodes::PathNotViable); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]), 0); + } + + TEST_F(SimpleDoc, NotCommonPrefix) { + setField("b"); + + size_t idxFound; + Element elemFound = root(); + Status status = findLongestPrefix(field(), root(), &idxFound, &elemFound); + ASSERT_EQUALS(status, ErrorCodes::NonExistentPath); + + // From this point on, handles the creation of the '.b' part that wasn't found. + Element newElem = doc().makeElementInt("b", 1); + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(countChildren(root()), 1u); + + ASSERT_OK(createPathAt(field(), 0, root(), newElem)); + ASSERT_EQUALS(newElem.getFieldName(), "b"); + ASSERT_EQUALS(newElem.getType(), NumberInt); + ASSERT_TRUE(newElem.hasValue()); + ASSERT_EQUALS(newElem.getValueInt(), 1); + + ASSERT_TRUE(newElem.parent().ok() /* root an ok parent */); + ASSERT_EQUALS(countChildren(root()), 2u); + ASSERT_EQUALS(root().leftChild().getFieldName(), "a"); + ASSERT_EQUALS(root().leftChild().rightSibling().getFieldName(), "b"); + ASSERT_EQUALS(root().rightChild().getFieldName(), "b"); + ASSERT_EQUALS(root().rightChild().leftSibling().getFieldName(), "a"); + } + + class NestedDoc : public mongo::unittest::Test { + public: + NestedDoc() : _doc() {} + + virtual void setUp() { + // {a: {b: {c: 1}}} + Element elemA = _doc.makeElementObject("a"); + ASSERT_TRUE(elemA.ok()); + Element elemB = _doc.makeElementObject("b"); + ASSERT_TRUE(elemB.ok()); + Element elemC = _doc.makeElementInt("c", 1); + ASSERT_TRUE(elemC.ok()); + + ASSERT_OK(elemB.pushBack(elemC)); + ASSERT_OK(elemA.pushBack(elemB)); + ASSERT_OK(root().pushBack(elemA)); + } + + Document& doc() { return _doc; } + + Element root() { return _doc.root(); } + + FieldRef& field() { return _field; } + void setField(StringData str) { _field.parse(str); } + + private: + Document _doc; + FieldRef _field; + }; + + TEST_F(NestedDoc, SimplePath) { + setField("a"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]), 0); + } + + TEST_F(NestedDoc, ShorterPath) { + setField("a.b"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_EQUALS(idxFound, 1U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]["b"]), 0); + } + + TEST_F(NestedDoc, ExactPath) { + setField("a.b.c"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 2U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]["b"]["c"]), 0); + } + + TEST_F(NestedDoc, LongerPath) { + // This would for 'c' to change from NumberInt to Object, which is invalid. + setField("a.b.c.d"); + + size_t idxFound; + Element elemFound = root(); + Status status = findLongestPrefix(field(), root(), &idxFound, &elemFound); + ASSERT_EQUALS(status.code(), ErrorCodes::PathNotViable); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 2U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]["b"]["c"]), 0); + + } + + TEST_F(NestedDoc, NewFieldNested) { + setField("a.b.d"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_EQUALS(idxFound, 1U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]["b"]), 0); + + // From this point on, handles the creation of the '.d' part that wasn't found. + Element newElem = doc().makeElementInt("d", 1); + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(countChildren(elemFound), 1u); // 'c' is a child of 'b' + + ASSERT_OK(createPathAt(field(), idxFound+1, elemFound, newElem)); + ASSERT_EQUALS(fromjson("{a: {b: {c: 1, d: 1}}}"), doc()); + } + + TEST_F(NestedDoc, NotStartingFromRoot) { + setField("b.c"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root()["a"], &idxFound, &elemFound)); + ASSERT_EQUALS(idxFound, 1U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]["b"]["c"]), 0); + } + + class ArrayDoc : public mongo::unittest::Test { + public: + ArrayDoc() : _doc() {} + + virtual void setUp() { + // {a: []} + Element elemA = _doc.makeElementArray("a"); + ASSERT_TRUE(elemA.ok()); + ASSERT_OK(root().pushBack(elemA)); + + // {a: [], b: [{c: 1}]} + Element elemB = _doc.makeElementArray("b"); + ASSERT_TRUE(elemB.ok()); + Element elemObj = _doc.makeElementObject("dummy" /* field name not used in array */); + ASSERT_TRUE(elemObj.ok()); + ASSERT_OK(elemObj.appendInt("c",1)); + ASSERT_OK(elemB.pushBack(elemObj)); + ASSERT_OK(root().pushBack(elemB)); + } + + Document& doc() { return _doc; } + + Element root() { return _doc.root(); } + + FieldRef& field() { return _field; } + + void setField(StringData str) { _field.parse(str); } + + private: + Document _doc; + FieldRef _field; + }; + + TEST_F(ArrayDoc, PathOnEmptyArray) { + setField("a.0"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["a"]), 0); + } + + TEST_F(ArrayDoc, PathOnPopulatedArray) { + setField("b.0"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 1U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["b"][0]), 0); + } + + TEST_F(ArrayDoc, MixedArrayAndObjectPath) { + setField("b.0.c"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 2U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["b"][0]["c"]), 0); + } + + TEST_F(ArrayDoc, ExtendingExistingObject) { + setField("b.0.d"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 1U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["b"][0]), 0); + + // From this point on, handles the creation of the '.0.d' part that wasn't found. + Element newElem = doc().makeElementInt("d", 1); + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(countChildren(elemFound), 1u); // '{c:1}' is a child of b.0 + + ASSERT_OK(createPathAt(field(), idxFound+1, elemFound, newElem)); + ASSERT_EQUALS(fromjson("{a: [], b: [{c:1, d:1}]}"), doc()); + } + + TEST_F(ArrayDoc, NewObjectInsideArray) { + setField("b.1.c"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["b"]), 0); + + // From this point on, handles the creation of the '.1.c' part that wasn't found. + Element newElem = doc().makeElementInt("c", 2); + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(countChildren(elemFound), 1u); // '{c:1}' is a child of 'b' + + ASSERT_OK(createPathAt(field(), idxFound+1, elemFound, newElem)); + ASSERT_EQUALS(fromjson("{a: [], b: [{c:1},{c:2}]}"), doc()); + } + + TEST_F(ArrayDoc, NewNestedObjectInsideArray) { + setField("b.1.c.d"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["b"]), 0); + + // From this point on, handles the creation of the '.1.c.d' part that wasn't found. + Element newElem = doc().makeElementInt("d", 2); + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(countChildren(elemFound), 1u); // '{c:1}' is a child of 'b' + + ASSERT_OK(createPathAt(field(), idxFound+1, elemFound, newElem)); + ASSERT_EQUALS(fromjson("{a: [], b: [{c:1},{c:{d:2}}]}"), doc()); + } + + TEST_F(ArrayDoc, ArrayPaddingNecessary) { + setField("b.5"); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["b"]), 0); + + // From this point on, handles the creation of the '.5' part that wasn't found. + Element newElem = doc().makeElementInt("", 1); + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(countChildren(elemFound), 1u); // '{c:1}' is a child of 'b' + + ASSERT_OK(createPathAt(field(), idxFound+1, elemFound, newElem)); + ASSERT_EQUALS(fromjson("{a: [], b: [{c:1},null,null,null,null,1]}"), doc()); + } + + TEST_F(ArrayDoc, ExcessivePaddingRequested) { + // Try to create an array item beyond what we're allowed to pad. + string paddedField = stream() << "b." << mongo::pathsupport::kMaxPaddingAllowed + 1;; + setField(paddedField); + + size_t idxFound; + Element elemFound = root(); + ASSERT_OK(findLongestPrefix(field(), root(), &idxFound, &elemFound)); + + // From this point on, try to create the padded part that wasn't found. + Element newElem = doc().makeElementInt("", 1); + ASSERT_TRUE(newElem.ok()); + ASSERT_EQUALS(countChildren(elemFound), 1u); // '{c:1}' is a child of 'b' + + Status status = createPathAt(field(), idxFound+1, elemFound, newElem); + ASSERT_EQUALS(status.code(), ErrorCodes::CannotBackfillArray); + } + + TEST_F(ArrayDoc, NonNumericPathInArray) { + setField("b.z"); + + size_t idxFound; + Element elemFound = root(); + Status status = findLongestPrefix(field(), root(), &idxFound, &elemFound); + ASSERT_EQUALS(status.code(), ErrorCodes::PathNotViable); + ASSERT_TRUE(elemFound.ok()); + ASSERT_EQUALS(idxFound, 0U); + ASSERT_EQUALS(elemFound.compareWithElement(root()["b"]), 0); + } + +} // unnamed namespace diff --git a/src/mongo/db/ops/query.cpp b/src/mongo/db/ops/query.cpp deleted file mode 100644 index 48a191b1bbb..00000000000 --- a/src/mongo/db/ops/query.cpp +++ /dev/null @@ -1,1060 +0,0 @@ -// query.cpp - -/** - * Copyright (C) 2008 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "pch.h" - -#include "mongo/db/ops/query.h" - -#include "mongo/bson/util/builder.h" -#include "mongo/db/clientcursor.h" -#include "mongo/db/commands.h" -#include "mongo/db/oplog.h" -#include "mongo/db/pagefault.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/queryoptimizer.h" -#include "mongo/db/queryoptimizercursor.h" -#include "mongo/db/replutil.h" -#include "mongo/db/scanandorder.h" -#include "mongo/s/d_logic.h" -#include "mongo/s/stale_exception.h" // for SendStaleConfigException -#include "mongo/server.h" - -namespace mongo { - - /* We cut off further objects once we cross this threshold; thus, you might get - a little bit more than this, it is a threshold rather than a limit. - */ - const int MaxBytesToReturnToClientAtOnce = 4 * 1024 * 1024; - - bool runCommands(const char *ns, BSONObj& jsobj, CurOp& curop, BufBuilder &b, BSONObjBuilder& anObjBuilder, bool fromRepl, int queryOptions) { - try { - return _runCommands(ns, jsobj, b, anObjBuilder, fromRepl, queryOptions); - } - catch( SendStaleConfigException& ){ - throw; - } - catch ( AssertionException& e ) { - verify( e.getCode() != SendStaleConfigCode && e.getCode() != RecvStaleConfigCode ); - - e.getInfo().append( anObjBuilder , "assertion" , "assertionCode" ); - curop.debug().exceptionInfo = e.getInfo(); - } - anObjBuilder.append("errmsg", "db assertion failure"); - anObjBuilder.append("ok", 0.0); - BSONObj x = anObjBuilder.done(); - b.appendBuf((void*) x.objdata(), x.objsize()); - return true; - } - - - BSONObj id_obj = fromjson("{\"_id\":1}"); - BSONObj empty_obj = fromjson("{}"); - - - //int dump = 0; - - /* empty result for error conditions */ - QueryResult* emptyMoreResult(long long cursorid) { - BufBuilder b(32768); - b.skip(sizeof(QueryResult)); - QueryResult *qr = (QueryResult *) b.buf(); - qr->cursorId = 0; // 0 indicates no more data to retrieve. - qr->startingFrom = 0; - qr->len = b.len(); - qr->setOperation(opReply); - qr->initializeResultFlags(); - qr->nReturned = 0; - b.decouple(); - return qr; - } - - QueryResult* processGetMore(const char* ns, - int ntoreturn, - long long cursorid, - CurOp& curop, - int pass, - bool& exhaust, - bool* isCursorAuthorized ) { - exhaust = false; - - int bufSize = 512 + sizeof( QueryResult ) + MaxBytesToReturnToClientAtOnce; - - BufBuilder b( bufSize ); - b.skip(sizeof(QueryResult)); - int resultFlags = ResultFlag_AwaitCapable; - int start = 0; - int n = 0; - - Client::ReadContext ctx(ns); - // call this readlocked so state can't change - replVerifyReadsOk(); - - ClientCursor::Pin p(cursorid); - ClientCursor *cc = p.c(); - - - if ( unlikely(!cc) ) { - LOGSOME << "getMore: cursorid not found " << ns << " " << cursorid << endl; - cursorid = 0; - resultFlags = ResultFlag_CursorNotFound; - } - else { - // check for spoofing of the ns such that it does not match the one originally there for the cursor - uassert(14833, "auth error", str::equals(ns, cc->ns().c_str())); - - *isCursorAuthorized = true; - - if ( pass == 0 ) - cc->updateSlaveLocation( curop ); - - int queryOptions = cc->queryOptions(); - - curop.debug().query = cc->query(); - curop.setQuery( cc->query() ); - - start = cc->pos(); - Cursor *c = cc->c(); - c->recoverFromYield(); - DiskLoc last; - - // This manager may be stale, but it's the state of chunking when the cursor was created. - ShardChunkManagerPtr manager = cc->getChunkManager(); - - while ( 1 ) { - if ( !c->ok() ) { - if ( c->tailable() ) { - /* when a tailable cursor hits "EOF", ok() goes false, and current() is null. however - advance() can still be retries as a reactivation attempt. when there is new data, it will - return true. that's what we are doing here. - */ - if ( c->advance() ) - continue; - - if( n == 0 && (queryOptions & QueryOption_AwaitData) && pass < 1000 ) { - return 0; - } - - break; - } - p.release(); - bool ok = ClientCursor::erase(cursorid); - verify(ok); - cursorid = 0; - cc = 0; - break; - } - - MatchDetails details; - if ( cc->fields && cc->fields->getArrayOpType() == Projection::ARRAY_OP_POSITIONAL ) { - // field projection specified, and contains an array operator - details.requestElemMatchKey(); - } - - // in some cases (clone collection) there won't be a matcher - if ( !c->currentMatches( &details ) ) { - } - else if ( manager && ! manager->belongsToMe( cc ) ){ - LOG(2) << "cursor skipping document in un-owned chunk: " << c->current() << endl; - } - else { - if( c->getsetdup(c->currLoc()) ) { - //out() << " but it's a dup \n"; - } - else { - last = c->currLoc(); - n++; - - cc->fillQueryResultFromObj( b, &details ); - - if ( ( ntoreturn && n >= ntoreturn ) || b.len() > MaxBytesToReturnToClientAtOnce ) { - c->advance(); - cc->incPos( n ); - break; - } - } - } - c->advance(); - - if ( ! cc->yieldSometimes( ( c->ok() && c->keyFieldsOnly() ) ? - ClientCursor::DontNeed : ClientCursor::WillNeed ) ) { - ClientCursor::erase(cursorid); - cursorid = 0; - cc = 0; - break; - } - } - - if ( cc ) { - if ( c->supportYields() ) { - ClientCursor::YieldData data; - verify( cc->prepareToYield( data ) ); - } - else { - cc->c()->noteLocation(); - } - cc->mayUpgradeStorage(); - cc->storeOpForSlave( last ); - exhaust = cc->queryOptions() & QueryOption_Exhaust; - } - } - - QueryResult *qr = (QueryResult *) b.buf(); - qr->len = b.len(); - qr->setOperation(opReply); - qr->_resultFlags() = resultFlags; - qr->cursorId = cursorid; - qr->startingFrom = start; - qr->nReturned = n; - b.decouple(); - - return qr; - } - - ResultDetails::ResultDetails() : - match(), - orderedMatch(), - loadedRecord(), - chunkSkip() { - } - - ExplainRecordingStrategy::ExplainRecordingStrategy - ( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo ) : - _ancillaryInfo( ancillaryInfo ) { - } - - shared_ptr ExplainRecordingStrategy::doneQueryInfo() { - shared_ptr ret = _doneQueryInfo(); - ret->setAncillaryInfo( _ancillaryInfo ); - return ret; - } - - NoExplainStrategy::NoExplainStrategy() : - ExplainRecordingStrategy( ExplainQueryInfo::AncillaryInfo() ) { - } - - shared_ptr NoExplainStrategy::_doneQueryInfo() { - verify( false ); - return shared_ptr(); - } - - MatchCountingExplainStrategy::MatchCountingExplainStrategy - ( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo ) : - ExplainRecordingStrategy( ancillaryInfo ), - _orderedMatches() { - } - - void MatchCountingExplainStrategy::noteIterate( const ResultDetails& resultDetails ) { - _noteIterate( resultDetails ); - if ( resultDetails.orderedMatch ) { - ++_orderedMatches; - } - } - - SimpleCursorExplainStrategy::SimpleCursorExplainStrategy - ( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo, - const shared_ptr &cursor ) : - MatchCountingExplainStrategy( ancillaryInfo ), - _cursor( cursor ), - _explainInfo( new ExplainSinglePlanQueryInfo() ) { - } - - void SimpleCursorExplainStrategy::notePlan( bool scanAndOrder, bool indexOnly ) { - _explainInfo->notePlan( *_cursor, scanAndOrder, indexOnly ); - } - - void SimpleCursorExplainStrategy::_noteIterate( const ResultDetails& resultDetails ) { - _explainInfo->noteIterate( resultDetails.match, - resultDetails.loadedRecord, - resultDetails.chunkSkip, - *_cursor ); - } - - void SimpleCursorExplainStrategy::noteYield() { - _explainInfo->noteYield(); - } - - shared_ptr SimpleCursorExplainStrategy::_doneQueryInfo() { - _explainInfo->noteDone( *_cursor ); - return _explainInfo->queryInfo(); - } - - QueryOptimizerCursorExplainStrategy::QueryOptimizerCursorExplainStrategy - ( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo, - const shared_ptr &cursor ) : - MatchCountingExplainStrategy( ancillaryInfo ), - _cursor( cursor ) { - } - - void QueryOptimizerCursorExplainStrategy::_noteIterate( const ResultDetails& resultDetails ) { - // Note ordered matches only; if an unordered plan is selected, the explain result will - // be updated with reviseN(). - _cursor->noteIterate( resultDetails.orderedMatch, - resultDetails.loadedRecord, - resultDetails.chunkSkip ); - } - - void QueryOptimizerCursorExplainStrategy::noteYield() { - _cursor->noteYield(); - } - - shared_ptr QueryOptimizerCursorExplainStrategy::_doneQueryInfo() { - return _cursor->explainQueryInfo(); - } - - ResponseBuildStrategy::ResponseBuildStrategy( const ParsedQuery &parsedQuery, - const shared_ptr &cursor, - BufBuilder &buf ) : - _parsedQuery( parsedQuery ), - _cursor( cursor ), - _queryOptimizerCursor( dynamic_pointer_cast( _cursor ) ), - _buf( buf ) { - } - - void ResponseBuildStrategy::resetBuf() { - _buf.reset(); - _buf.skip( sizeof( QueryResult ) ); - } - - BSONObj ResponseBuildStrategy::current( bool allowCovered, - ResultDetails* resultDetails ) const { - if ( _parsedQuery.returnKey() ) { - BSONObjBuilder bob; - bob.appendKeys( _cursor->indexKeyPattern(), _cursor->currKey() ); - return bob.obj(); - } - if ( allowCovered ) { - const Projection::KeyOnly *keyFieldsOnly = _cursor->keyFieldsOnly(); - if ( keyFieldsOnly ) { - return keyFieldsOnly->hydrate( _cursor->currKey() ); - } - } - resultDetails->loadedRecord = true; - BSONObj ret = _cursor->current(); - verify( ret.isValid() ); - return ret; - } - - OrderedBuildStrategy::OrderedBuildStrategy( const ParsedQuery &parsedQuery, - const shared_ptr &cursor, - BufBuilder &buf ) : - ResponseBuildStrategy( parsedQuery, cursor, buf ), - _skip( _parsedQuery.getSkip() ), - _bufferedMatches() { - } - - bool OrderedBuildStrategy::handleMatch( ResultDetails* resultDetails ) { - DiskLoc loc = _cursor->currLoc(); - if ( _cursor->getsetdup( loc ) ) { - return false; - } - if ( _skip > 0 ) { - --_skip; - return false; - } - BSONObj currentDocument = current( true, resultDetails ); - // Explain does not obey soft limits, so matches should not be buffered. - if ( !_parsedQuery.isExplain() ) { - fillQueryResultFromObj( _buf, _parsedQuery.getFields(), - currentDocument, &resultDetails->matchDetails, - ( _parsedQuery.showDiskLoc() ? &loc : 0 ) ); - ++_bufferedMatches; - } - resultDetails->match = true; - resultDetails->orderedMatch = true; - return true; - } - - ReorderBuildStrategy* ReorderBuildStrategy::make( const ParsedQuery& parsedQuery, - const shared_ptr& cursor, - BufBuilder& buf, - const QueryPlanSummary& queryPlan ) { - auto_ptr ret( new ReorderBuildStrategy( parsedQuery, cursor, buf ) ); - ret->init( queryPlan ); - return ret.release(); - } - - ReorderBuildStrategy::ReorderBuildStrategy( const ParsedQuery &parsedQuery, - const shared_ptr &cursor, - BufBuilder &buf ) : - ResponseBuildStrategy( parsedQuery, cursor, buf ), - _bufferedMatches() { - } - - void ReorderBuildStrategy::init( const QueryPlanSummary &queryPlan ) { - _scanAndOrder.reset( newScanAndOrder( queryPlan ) ); - } - - bool ReorderBuildStrategy::handleMatch( ResultDetails* resultDetails ) { - if ( _cursor->getsetdup( _cursor->currLoc() ) ) { - return false; - } - _handleMatchNoDedup( resultDetails ); - resultDetails->match = true; - return true; - } - - void ReorderBuildStrategy::_handleMatchNoDedup( ResultDetails* resultDetails ) { - DiskLoc loc = _cursor->currLoc(); - _scanAndOrder->add( current( false, resultDetails ), - _parsedQuery.showDiskLoc() ? &loc : 0 ); - } - - int ReorderBuildStrategy::rewriteMatches() { - cc().curop()->debug().scanAndOrder = true; - int ret = 0; - _scanAndOrder->fill( _buf, &_parsedQuery, ret ); - _bufferedMatches = ret; - return ret; - } - - ScanAndOrder * - ReorderBuildStrategy::newScanAndOrder( const QueryPlanSummary &queryPlan ) const { - verify( !_parsedQuery.getOrder().isEmpty() ); - verify( _cursor->ok() ); - const FieldRangeSet *fieldRangeSet = 0; - if ( queryPlan.valid() ) { - fieldRangeSet = queryPlan._fieldRangeSetMulti.get(); - } - else { - verify( _queryOptimizerCursor ); - fieldRangeSet = _queryOptimizerCursor->initialFieldRangeSet(); - } - verify( fieldRangeSet ); - return new ScanAndOrder( _parsedQuery.getSkip(), - _parsedQuery.getNumToReturn(), - _parsedQuery.getOrder(), - *fieldRangeSet ); - } - - HybridBuildStrategy* HybridBuildStrategy::make( const ParsedQuery& parsedQuery, - const shared_ptr& cursor, - BufBuilder& buf ) { - auto_ptr ret( new HybridBuildStrategy( parsedQuery, cursor, buf ) ); - ret->init(); - return ret.release(); - } - - HybridBuildStrategy::HybridBuildStrategy( const ParsedQuery &parsedQuery, - const shared_ptr &cursor, - BufBuilder &buf ) : - ResponseBuildStrategy( parsedQuery, cursor, buf ), - _orderedBuild( _parsedQuery, _cursor, _buf ), - _reorderedMatches() { - } - - void HybridBuildStrategy::init() { - _reorderBuild.reset( ReorderBuildStrategy::make( _parsedQuery, _cursor, _buf, - QueryPlanSummary() ) ); - } - - bool HybridBuildStrategy::handleMatch( ResultDetails* resultDetails ) { - if ( !_queryOptimizerCursor->currentPlanScanAndOrderRequired() ) { - return _orderedBuild.handleMatch( resultDetails ); - } - return handleReorderMatch( resultDetails ); - } - - bool HybridBuildStrategy::handleReorderMatch( ResultDetails* resultDetails ) { - DiskLoc loc = _cursor->currLoc(); - if ( _scanAndOrderDups.getsetdup( loc ) ) { - return false; - } - resultDetails->match = true; - try { - _reorderBuild->_handleMatchNoDedup( resultDetails ); - } catch ( const UserException &e ) { - if ( e.getCode() == ScanAndOrderMemoryLimitExceededAssertionCode ) { - if ( _queryOptimizerCursor->hasPossiblyExcludedPlans() ) { - _queryOptimizerCursor->clearIndexesForPatterns(); - throw QueryRetryException(); - } - else if ( _queryOptimizerCursor->runningInitialInOrderPlan() ) { - _queryOptimizerCursor->abortOutOfOrderPlans(); - return true; - } - } - throw; - } - return true; - } - - int HybridBuildStrategy::rewriteMatches() { - if ( !_queryOptimizerCursor->completePlanOfHybridSetScanAndOrderRequired() ) { - return _orderedBuild.rewriteMatches(); - } - _reorderedMatches = true; - resetBuf(); - return _reorderBuild->rewriteMatches(); - } - - int HybridBuildStrategy::bufferedMatches() const { - return _reorderedMatches ? - _reorderBuild->bufferedMatches() : - _orderedBuild.bufferedMatches(); - } - - void HybridBuildStrategy::finishedFirstBatch() { - _queryOptimizerCursor->abortOutOfOrderPlans(); - } - - QueryResponseBuilder *QueryResponseBuilder::make( const ParsedQuery &parsedQuery, - const shared_ptr &cursor, - const QueryPlanSummary &queryPlan, - const BSONObj &oldPlan ) { - auto_ptr ret( new QueryResponseBuilder( parsedQuery, cursor ) ); - ret->init( queryPlan, oldPlan ); - return ret.release(); - } - - QueryResponseBuilder::QueryResponseBuilder( const ParsedQuery &parsedQuery, - const shared_ptr &cursor ) : - _parsedQuery( parsedQuery ), - _cursor( cursor ), - _queryOptimizerCursor( dynamic_pointer_cast( _cursor ) ), - _buf( 32768 ) { // TODO be smarter here - } - - void QueryResponseBuilder::init( const QueryPlanSummary &queryPlan, const BSONObj &oldPlan ) { - _chunkManager = newChunkManager(); - _explain = newExplainRecordingStrategy( queryPlan, oldPlan ); - _builder = newResponseBuildStrategy( queryPlan ); - _builder->resetBuf(); - } - - bool QueryResponseBuilder::addMatch() { - ResultDetails resultDetails; - - if ( _parsedQuery.getFields() && _parsedQuery.getFields()->getArrayOpType() == Projection::ARRAY_OP_POSITIONAL ) { - // field projection specified, and contains an array operator - resultDetails.matchDetails.requestElemMatchKey(); - } - - bool match = - currentMatches( &resultDetails ) && - chunkMatches( &resultDetails ) && - _builder->handleMatch( &resultDetails ); - - _explain->noteIterate( resultDetails ); - return match; - } - - void QueryResponseBuilder::noteYield() { - _explain->noteYield(); - } - - bool QueryResponseBuilder::enoughForFirstBatch() const { - return _parsedQuery.enoughForFirstBatch( _builder->bufferedMatches(), _buf.len() ); - } - - bool QueryResponseBuilder::enoughTotalResults() const { - if ( _parsedQuery.isExplain() ) { - return _parsedQuery.enoughForExplain( _explain->orderedMatches() ); - } - return ( _parsedQuery.enough( _builder->bufferedMatches() ) || - _buf.len() >= MaxBytesToReturnToClientAtOnce ); - } - - void QueryResponseBuilder::finishedFirstBatch() { - _builder->finishedFirstBatch(); - } - - int QueryResponseBuilder::handoff( Message &result ) { - int rewriteCount = _builder->rewriteMatches(); - if ( _parsedQuery.isExplain() ) { - shared_ptr explainInfo = _explain->doneQueryInfo(); - if ( rewriteCount != -1 ) { - explainInfo->reviseN( rewriteCount ); - } - _builder->resetBuf(); - fillQueryResultFromObj( _buf, 0, explainInfo->bson() ); - result.appendData( _buf.buf(), _buf.len() ); - _buf.decouple(); - return 1; - } - if ( _buf.len() > 0 ) { - result.appendData( _buf.buf(), _buf.len() ); - _buf.decouple(); - } - return _builder->bufferedMatches(); - } - - ShardChunkManagerPtr QueryResponseBuilder::newChunkManager() const { - if ( !shardingState.needShardChunkManager( _parsedQuery.ns() ) ) { - return ShardChunkManagerPtr(); - } - return shardingState.getShardChunkManager( _parsedQuery.ns() ); - } - - shared_ptr QueryResponseBuilder::newExplainRecordingStrategy - ( const QueryPlanSummary &queryPlan, const BSONObj &oldPlan ) const { - if ( !_parsedQuery.isExplain() ) { - return shared_ptr( new NoExplainStrategy() ); - } - ExplainQueryInfo::AncillaryInfo ancillaryInfo; - ancillaryInfo._oldPlan = oldPlan; - if ( _queryOptimizerCursor ) { - return shared_ptr - ( new QueryOptimizerCursorExplainStrategy( ancillaryInfo, _queryOptimizerCursor ) ); - } - shared_ptr ret - ( new SimpleCursorExplainStrategy( ancillaryInfo, _cursor ) ); - ret->notePlan( queryPlan.valid() && queryPlan._scanAndOrderRequired, - queryPlan._keyFieldsOnly ); - return ret; - } - - shared_ptr QueryResponseBuilder::newResponseBuildStrategy - ( const QueryPlanSummary &queryPlan ) { - bool unordered = _parsedQuery.getOrder().isEmpty(); - bool empty = !_cursor->ok(); - bool singlePlan = !_queryOptimizerCursor; - bool singleOrderedPlan = - singlePlan && ( !queryPlan.valid() || !queryPlan._scanAndOrderRequired ); - CandidatePlanCharacter queryOptimizerPlans; - if ( _queryOptimizerCursor ) { - queryOptimizerPlans = _queryOptimizerCursor->initialCandidatePlans(); - } - if ( unordered || - empty || - singleOrderedPlan || - ( !singlePlan && !queryOptimizerPlans.mayRunOutOfOrderPlan() ) ) { - return shared_ptr - ( new OrderedBuildStrategy( _parsedQuery, _cursor, _buf ) ); - } - if ( singlePlan || - !queryOptimizerPlans.mayRunInOrderPlan() ) { - return shared_ptr - ( ReorderBuildStrategy::make( _parsedQuery, _cursor, _buf, queryPlan ) ); - } - return shared_ptr - ( HybridBuildStrategy::make( _parsedQuery, _queryOptimizerCursor, _buf ) ); - } - - bool QueryResponseBuilder::currentMatches( ResultDetails* resultDetails ) { - bool matches = _cursor->currentMatches( &resultDetails->matchDetails ); - if ( resultDetails->matchDetails.hasLoadedRecord() ) { - resultDetails->loadedRecord = true; - } - return matches; - } - - bool QueryResponseBuilder::chunkMatches( ResultDetails* resultDetails ) { - if ( !_chunkManager ) { - return true; - } - // TODO: should make this covered at some point - resultDetails->loadedRecord = true; - if ( _chunkManager->belongsToMe( _cursor->current() ) ) { - return true; - } - resultDetails->chunkSkip = true; - return false; - } - - /** - * Run a query with a cursor provided by the query optimizer, or FindingStartCursor. - * @yields the db lock. - */ - string queryWithQueryOptimizer( int queryOptions, const string& ns, - const BSONObj &jsobj, CurOp& curop, - const BSONObj &query, const BSONObj &order, - const shared_ptr &pq_shared, - const BSONObj &oldPlan, - const ConfigVersion &shardingVersionAtStart, - scoped_ptr& parentPageFaultSection, - scoped_ptr& noPageFault, - Message &result ) { - - const ParsedQuery &pq( *pq_shared ); - shared_ptr cursor; - QueryPlanSummary queryPlan; - - if ( pq.hasOption( QueryOption_OplogReplay ) ) { - cursor = FindingStartCursor::getCursor( ns.c_str(), query, order ); - } - else { - cursor = - NamespaceDetailsTransient::getCursor( ns.c_str(), - query, - order, - QueryPlanSelectionPolicy::any(), - pq_shared, - false, - &queryPlan ); - } - verify( cursor ); - - scoped_ptr queryResponseBuilder - ( QueryResponseBuilder::make( pq, cursor, queryPlan, oldPlan ) ); - bool saveClientCursor = false; - OpTime slaveReadTill; - ClientCursor::Holder ccPointer( new ClientCursor( QueryOption_NoCursorTimeout, cursor, - ns ) ); - - for( ; cursor->ok(); cursor->advance() ) { - - bool yielded = false; - if ( !ccPointer->yieldSometimes( ClientCursor::MaybeCovered, &yielded ) || - !cursor->ok() ) { - cursor.reset(); - queryResponseBuilder->noteYield(); - // !!! TODO The queryResponseBuilder still holds cursor. Currently it will not do - // anything unsafe with the cursor in handoff(), but this is very fragile. - // - // We don't fail the query since we're fine with returning partial data if the - // collection was dropped. - // NOTE see SERVER-2454. - // TODO This is wrong. The cursor could be gone if the closeAllDatabases command - // just ran. - break; - } - - if ( yielded ) { - queryResponseBuilder->noteYield(); - } - - if ( pq.getMaxScan() && cursor->nscanned() > pq.getMaxScan() ) { - break; - } - - if ( !queryResponseBuilder->addMatch() ) { - continue; - } - - // Note slave's position in the oplog. - if ( pq.hasOption( QueryOption_OplogReplay ) ) { - BSONObj current = cursor->current(); - BSONElement e = current["ts"]; - if ( e.type() == Date || e.type() == Timestamp ) { - slaveReadTill = e._opTime(); - } - } - - if ( !cursor->supportGetMore() || pq.isExplain() ) { - if ( queryResponseBuilder->enoughTotalResults() ) { - break; - } - } - else if ( queryResponseBuilder->enoughForFirstBatch() ) { - // if only 1 requested, no cursor saved for efficiency...we assume it is findOne() - if ( pq.wantMore() && pq.getNumToReturn() != 1 ) { - queryResponseBuilder->finishedFirstBatch(); - if ( cursor->advance() ) { - saveClientCursor = true; - } - } - break; - } - } - - if ( cursor ) { - if ( pq.hasOption( QueryOption_CursorTailable ) && pq.getNumToReturn() != 1 ) { - cursor->setTailable(); - } - - // If the tailing request succeeded. - if ( cursor->tailable() ) { - saveClientCursor = true; - } - } - - if ( ! shardingState.getVersion( ns ).isWriteCompatibleWith( shardingVersionAtStart ) ) { - // if the version changed during the query - // we might be missing some data - // and its safe to send this as mongos can resend - // at this point - throw SendStaleConfigException( ns , "version changed during initial query", shardingVersionAtStart, shardingState.getVersion( ns ) ); - } - - parentPageFaultSection.reset(0); - noPageFault.reset( new NoPageFaultsAllowed() ); - - int nReturned = queryResponseBuilder->handoff( result ); - - ccPointer.reset(); - long long cursorid = 0; - if ( saveClientCursor ) { - // Create a new ClientCursor, with a default timeout. - ccPointer.reset( new ClientCursor( queryOptions, cursor, ns, - jsobj.getOwned() ) ); - cursorid = ccPointer->cursorid(); - DEV tlog(2) << "query has more, cursorid: " << cursorid << endl; - if ( cursor->supportYields() ) { - ClientCursor::YieldData data; - ccPointer->prepareToYield( data ); - } - else { - ccPointer->c()->noteLocation(); - } - - // Save slave's position in the oplog. - if ( pq.hasOption( QueryOption_OplogReplay ) && !slaveReadTill.isNull() ) { - ccPointer->slaveReadTill( slaveReadTill ); - } - - if ( !ccPointer->ok() && ccPointer->c()->tailable() ) { - DEV tlog() << "query has no more but tailable, cursorid: " << cursorid << endl; - } - - if( queryOptions & QueryOption_Exhaust ) { - curop.debug().exhaust = true; - } - - // Set attributes for getMore. - ccPointer->setChunkManager( queryResponseBuilder->chunkManager() ); - ccPointer->setPos( nReturned ); - ccPointer->pq = pq_shared; - ccPointer->fields = pq.getFieldPtr(); - ccPointer.release(); - } - - QueryResult *qr = (QueryResult *) result.header(); - qr->cursorId = cursorid; - curop.debug().cursorid = ( cursorid == 0 ? -1 : qr->cursorId ); - qr->setResultFlagsToOk(); - // qr->len is updated automatically by appendData() - curop.debug().responseLength = qr->len; - qr->setOperation(opReply); - qr->startingFrom = 0; - qr->nReturned = nReturned; - - curop.debug().nscanned = ( cursor ? cursor->nscanned() : 0LL ); - curop.debug().ntoskip = pq.getSkip(); - curop.debug().nreturned = nReturned; - - return curop.debug().exhaust ? ns : ""; - } - - bool queryIdHack( const char* ns, const BSONObj& query, const ParsedQuery& pq, CurOp& curop, Message& result ) { - // notes: - // do not touch result inside of PageFaultRetryableSection area - - Client& currentClient = cc(); // only here since its safe and takes time - auto_ptr< QueryResult > qr; - - { - // this extra bracing is not strictly needed - // but makes it clear what the rules are in different spots - - scoped_ptr pgfs; - if ( ! currentClient.getPageFaultRetryableSection() ) - pgfs.reset( new PageFaultRetryableSection() ); - while ( 1 ) { - try { - - int n = 0; - bool nsFound = false; - bool indexFound = false; - - BSONObj resObject; // put inside since we don't own the memory - - Client::ReadContext ctx( ns , dbpath ); // read locks - replVerifyReadsOk(&pq); - - bool found = Helpers::findById( currentClient, ns, query, resObject, &nsFound, &indexFound ); - if ( nsFound && ! indexFound ) { - // we have to resort to a table scan - return false; - } - - if ( shardingState.needShardChunkManager( ns ) ) { - ShardChunkManagerPtr m = shardingState.getShardChunkManager( ns ); - if ( m && ! m->belongsToMe( resObject ) ) { - // I have something this _id - // but it doesn't belong to me - // so return nothing - resObject = BSONObj(); - found = false; - } - } - - BufBuilder bb(sizeof(QueryResult)+resObject.objsize()+32); - bb.skip(sizeof(QueryResult)); - - curop.debug().idhack = true; - if ( found ) { - n = 1; - fillQueryResultFromObj( bb , pq.getFields() , resObject ); - } - - qr.reset( (QueryResult *) bb.buf() ); - bb.decouple(); - qr->setResultFlagsToOk(); - qr->len = bb.len(); - - curop.debug().responseLength = bb.len(); - qr->setOperation(opReply); - qr->cursorId = 0; - qr->startingFrom = 0; - qr->nReturned = n; - - break; - } - catch ( PageFaultException& e ) { - e.touch(); - } - } - } - - result.setData( qr.release(), true ); - return true; - } - - /** - * Run a query -- includes checking for and running a Command. - * @return points to ns if exhaust mode. 0=normal mode - * @locks the db mutex for reading (and potentially for writing temporarily to create a new db). - * @yields the db mutex periodically after acquiring it. - * @asserts on scan and order memory exhaustion and other cases. - */ - string runQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) { - shared_ptr pq_shared( new ParsedQuery(q) ); - ParsedQuery& pq( *pq_shared ); - BSONObj jsobj = q.query; - int queryOptions = q.queryOptions; - const char *ns = q.ns; - - uassert( 16332 , "can't have an empty ns" , ns[0] ); - - if( logLevel >= 2 ) - log() << "runQuery called " << ns << " " << jsobj << endl; - - curop.debug().ns = ns; - curop.debug().ntoreturn = pq.getNumToReturn(); - curop.debug().query = jsobj; - curop.setQuery(jsobj); - - const NamespaceString nsString( ns ); - uassert( 16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid() ); - - // Run a command. - - if ( pq.couldBeCommand() ) { - curop.markCommand(); - BufBuilder bb; - bb.skip(sizeof(QueryResult)); - BSONObjBuilder cmdResBuf; - if ( runCommands(ns, jsobj, curop, bb, cmdResBuf, false, queryOptions) ) { - curop.debug().iscommand = true; - curop.debug().query = jsobj; - - auto_ptr< QueryResult > qr; - qr.reset( (QueryResult *) bb.buf() ); - bb.decouple(); - qr->setResultFlagsToOk(); - qr->len = bb.len(); - curop.debug().responseLength = bb.len(); - qr->setOperation(opReply); - qr->cursorId = 0; - qr->startingFrom = 0; - qr->nReturned = 1; - result.setData( qr.release(), true ); - } - else { - uasserted(13530, "bad or malformed command request?"); - } - return ""; - } - - bool explain = pq.isExplain(); - BSONObj order = pq.getOrder(); - BSONObj query = pq.getFilter(); - - /* The ElemIter will not be happy if this isn't really an object. So throw exception - here when that is true. - (Which may indicate bad data from client.) - */ - if ( query.objsize() == 0 ) { - out() << "Bad query object?\n jsobj:"; - out() << jsobj.toString() << "\n query:"; - out() << query.toString() << endl; - uassert( 10110 , "bad query object", false); - } - - - // Run a simple id query. - if ( ! (explain || pq.showDiskLoc()) && isSimpleIdQuery( query ) && !pq.hasOption( QueryOption_CursorTailable ) ) { - if ( queryIdHack( ns, query, pq, curop, result ) ) { - return ""; - } - } - - // sanity check the query and projection - if ( pq.getFields() != NULL ) - pq.getFields()->validateQuery( query ); - - // these now may stored in a ClientCursor or somewhere else, - // so make sure we use a real copy - jsobj = jsobj.getOwned(); - query = query.getOwned(); - order = order.getOwned(); - - bool hasRetried = false; - scoped_ptr pgfs; - scoped_ptr npfe; - while ( 1 ) { - - if ( ! cc().getPageFaultRetryableSection() ) { - verify( ! pgfs ); - pgfs.reset( new PageFaultRetryableSection() ); - } - - try { - Client::ReadContext ctx( ns , dbpath ); // read locks - const ConfigVersion shardingVersionAtStart = shardingState.getVersion( ns ); - - replVerifyReadsOk(&pq); - - if ( pq.hasOption( QueryOption_CursorTailable ) ) { - NamespaceDetails *d = nsdetails( ns ); - uassert( 13051, "tailable cursor requested on non capped collection", d && d->isCapped() ); - const BSONObj nat1 = BSON( "$natural" << 1 ); - if ( order.isEmpty() ) { - order = nat1; - } - else { - uassert( 13052, "only {$natural:1} order allowed for tailable cursor", order == nat1 ); - } - } - - - // Run a regular query. - - BSONObj oldPlan; - if ( ! hasRetried && explain && ! pq.hasIndexSpecifier() ) { - scoped_ptr mps( MultiPlanScanner::make( ns, query, order ) ); - oldPlan = mps->cachedPlanExplainSummary(); - } - - - return queryWithQueryOptimizer( queryOptions, ns, jsobj, curop, query, order, - pq_shared, oldPlan, shardingVersionAtStart, - pgfs, npfe, result ); - } - catch ( PageFaultException& e ) { - e.touch(); - } - catch ( const QueryRetryException & ) { - // In some cases the query may be retried if there is an in memory sort size assertion. - verify( ! hasRetried ); - hasRetried = true; - } - } - } - -} // namespace mongo diff --git a/src/mongo/db/ops/query.h b/src/mongo/db/ops/query.h deleted file mode 100644 index 1b67de7e247..00000000000 --- a/src/mongo/db/ops/query.h +++ /dev/null @@ -1,322 +0,0 @@ -// query.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/pch.h" -#include "../../util/net/message.h" -#include "../dbmessage.h" -#include "../jsobj.h" -#include "../diskloc.h" -#include "../explain.h" -#include "../../s/d_chunk_manager.h" - -// struct QueryOptions, QueryResult, QueryResultFlags in: - -namespace mongo { - - class ParsedQuery; - class QueryOptimizerCursor; - class QueryPlanSummary; - - /** - * Return a batch of results from a client OP_GET_MORE request. - * 'cursorid' - The id of the cursor producing results. - * 'isCursorAuthorized' - Set to true after a cursor with id 'cursorid' is authorized for use. - */ - QueryResult* processGetMore(const char* ns, - int ntoreturn, - long long cursorid, - CurOp& op, - int pass, - bool& exhaust, - bool* isCursorAuthorized); - - string runQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result); - - /** Exception indicating that a query should be retried from the beginning. */ - class QueryRetryException : public DBException { - public: - QueryRetryException() : DBException( "query retry exception" , 16083 ) { - return; - massert( 16083, "reserve 16083", true ); // Reserve 16083. - } - }; - - /** Metadata about matching and loading a single candidate result document from a Cursor. */ - struct ResultDetails { - ResultDetails(); - MatchDetails matchDetails; // Details on how the Matcher matched the query. - bool match; // Matched the query, was not a dup, was not skipped etc. - bool orderedMatch; // _match and belonged to an ordered query plan. - bool loadedRecord; // Record was loaded (to match or return the document). - bool chunkSkip; // Did not belong to an owned chunk range. - }; - - /** Interface for recording events that contribute to explain results. */ - class ExplainRecordingStrategy { - public: - ExplainRecordingStrategy( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo ); - virtual ~ExplainRecordingStrategy() {} - /** Note information about a single query plan. */ - virtual void notePlan( bool scanAndOrder, bool indexOnly ) {} - /** Note an iteration of the query. */ - virtual void noteIterate( const ResultDetails& resultDetails ) {} - /** Note that the query yielded. */ - virtual void noteYield() {} - /** @return number of ordered matches noted. */ - virtual long long orderedMatches() const { return 0; } - /** @return ExplainQueryInfo for a complete query. */ - shared_ptr doneQueryInfo(); - protected: - /** @return ExplainQueryInfo for a complete query, to be implemented by subclass. */ - virtual shared_ptr _doneQueryInfo() = 0; - private: - ExplainQueryInfo::AncillaryInfo _ancillaryInfo; - }; - - /** No explain events are recorded. */ - class NoExplainStrategy : public ExplainRecordingStrategy { - public: - NoExplainStrategy(); - private: - /** @asserts always. */ - virtual shared_ptr _doneQueryInfo(); - }; - - class MatchCountingExplainStrategy : public ExplainRecordingStrategy { - public: - MatchCountingExplainStrategy( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo ); - protected: - virtual void _noteIterate( const ResultDetails& resultDetails ) = 0; - private: - virtual void noteIterate( const ResultDetails& resultDetails ); - virtual long long orderedMatches() const { return _orderedMatches; } - long long _orderedMatches; - }; - - /** Record explain events for a simple cursor representing a single clause and plan. */ - class SimpleCursorExplainStrategy : public MatchCountingExplainStrategy { - public: - SimpleCursorExplainStrategy( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo, - const shared_ptr &cursor ); - private: - virtual void notePlan( bool scanAndOrder, bool indexOnly ); - virtual void _noteIterate( const ResultDetails& resultDetails ); - virtual void noteYield(); - virtual shared_ptr _doneQueryInfo(); - shared_ptr _cursor; - shared_ptr _explainInfo; - }; - - /** - * Record explain events for a QueryOptimizerCursor, which may record some explain information - * for multiple clauses and plans through an internal implementation. - */ - class QueryOptimizerCursorExplainStrategy : public MatchCountingExplainStrategy { - public: - QueryOptimizerCursorExplainStrategy( const ExplainQueryInfo::AncillaryInfo &ancillaryInfo, - const shared_ptr &cursor ); - private: - virtual void _noteIterate( const ResultDetails& resultDetails ); - virtual void noteYield(); - virtual shared_ptr _doneQueryInfo(); - shared_ptr _cursor; - }; - - /** Interface for building a query response in a supplied BufBuilder. */ - class ResponseBuildStrategy { - public: - /** - * @param queryPlan must be supplied if @param cursor is not a QueryOptimizerCursor and - * results must be sorted or read with a covered index. - */ - ResponseBuildStrategy( const ParsedQuery &parsedQuery, const shared_ptr &cursor, - BufBuilder &buf ); - virtual ~ResponseBuildStrategy() {} - /** - * Handle the current iterate of the supplied cursor as a (possibly duplicate) match. - * @return true if a match is found. - * @param resultDetails details of how the result is matched and loaded. - */ - virtual bool handleMatch( ResultDetails* resultDetails ) = 0; - - /** - * Write all matches into the buffer, overwriting existing data. - * @return number of matches written, or -1 if no op. - */ - virtual int rewriteMatches() { return -1; } - /** @return the number of matches that have been written to the buffer. */ - virtual int bufferedMatches() const = 0; - /** - * Callback when enough results have been read for the first batch, with potential handoff - * to getMore. - */ - virtual void finishedFirstBatch() {} - /** Reset the buffer. */ - void resetBuf(); - protected: - /** - * Return the document for the current iterate. Implements the $returnKey option. - * @param allowCovered enable covered index support. - * @param resultDetails details of how the result is loaded. - */ - BSONObj current( bool allowCovered, ResultDetails* resultDetails ) const; - const ParsedQuery &_parsedQuery; - shared_ptr _cursor; - shared_ptr _queryOptimizerCursor; - BufBuilder &_buf; - }; - - /** Build strategy for a cursor returning in order results. */ - class OrderedBuildStrategy : public ResponseBuildStrategy { - public: - OrderedBuildStrategy( const ParsedQuery &parsedQuery, const shared_ptr &cursor, - BufBuilder &buf ); - virtual bool handleMatch( ResultDetails* resultDetails ); - virtual int bufferedMatches() const { return _bufferedMatches; } - private: - int _skip; - int _bufferedMatches; - }; - - class ScanAndOrder; - - /** Build strategy for a cursor returning out of order results. */ - class ReorderBuildStrategy : public ResponseBuildStrategy { - public: - static ReorderBuildStrategy* make( const ParsedQuery& parsedQuery, - const shared_ptr& cursor, - BufBuilder& buf, - const QueryPlanSummary& queryPlan ); - virtual bool handleMatch( ResultDetails* resultDetails ); - /** Handle a match without performing deduping. */ - void _handleMatchNoDedup( ResultDetails* resultDetails ); - virtual int rewriteMatches(); - virtual int bufferedMatches() const { return _bufferedMatches; } - private: - ReorderBuildStrategy( const ParsedQuery& parsedQuery, - const shared_ptr& cursor, - BufBuilder& buf ); - void init( const QueryPlanSummary& queryPlan ); - ScanAndOrder *newScanAndOrder( const QueryPlanSummary &queryPlan ) const; - shared_ptr _scanAndOrder; - int _bufferedMatches; - }; - - /** Helper class for deduping DiskLocs */ - class DiskLocDupSet { - public: - /** @return true if dup, otherwise return false and insert. */ - bool getsetdup( const DiskLoc &loc ) { - pair::iterator, bool> p = _dups.insert(loc); - return !p.second; - } - private: - set _dups; - }; - - /** - * Build strategy for a QueryOptimizerCursor containing some in order and some out of order - * candidate plans. - */ - class HybridBuildStrategy : public ResponseBuildStrategy { - public: - static HybridBuildStrategy* make( const ParsedQuery& parsedQuery, - const shared_ptr& cursor, - BufBuilder& buf ); - private: - HybridBuildStrategy( const ParsedQuery &parsedQuery, - const shared_ptr &cursor, - BufBuilder &buf ); - void init(); - virtual bool handleMatch( ResultDetails* resultDetails ); - virtual int rewriteMatches(); - virtual int bufferedMatches() const; - virtual void finishedFirstBatch(); - bool handleReorderMatch( ResultDetails* resultDetails ); - DiskLocDupSet _scanAndOrderDups; - OrderedBuildStrategy _orderedBuild; - scoped_ptr _reorderBuild; - bool _reorderedMatches; - }; - - /** - * Builds a query response with the help of an ExplainRecordingStrategy and a - * ResponseBuildStrategy. - */ - class QueryResponseBuilder { - public: - /** - * @param queryPlan must be supplied if @param cursor is not a QueryOptimizerCursor and - * results must be sorted or read with a covered index. - */ - static QueryResponseBuilder *make( const ParsedQuery &parsedQuery, - const shared_ptr &cursor, - const QueryPlanSummary &queryPlan, - const BSONObj &oldPlan ); - /** @return true if the current iterate matches and is added. */ - bool addMatch(); - /** Note that a yield occurred. */ - void noteYield(); - /** @return true if there are enough results to return the first batch. */ - bool enoughForFirstBatch() const; - /** @return true if there are enough results to return the full result set. */ - bool enoughTotalResults() const; - /** - * Callback when enough results have been read for the first batch, with potential handoff - * to getMore. - */ - void finishedFirstBatch(); - /** - * Set the data portion of the supplied Message to a buffer containing the query results. - * @return the number of results in the buffer. - */ - int handoff( Message &result ); - /** A chunk manager found at the beginning of the query. */ - ShardChunkManagerPtr chunkManager() const { return _chunkManager; } - - private: - QueryResponseBuilder( const ParsedQuery &parsedQuery, const shared_ptr &cursor ); - void init( const QueryPlanSummary &queryPlan, const BSONObj &oldPlan ); - - ShardChunkManagerPtr newChunkManager() const; - shared_ptr newExplainRecordingStrategy - ( const QueryPlanSummary &queryPlan, const BSONObj &oldPlan ) const; - shared_ptr newResponseBuildStrategy - ( const QueryPlanSummary &queryPlan ); - /** - * @return true if the cursor's document matches the query. - * @param resultDetails describes how the document was matched and loaded. - */ - bool currentMatches( ResultDetails* resultDetails ); - /** - * @return true if the cursor's document is in a valid chunk range. - * @param resultDetails describes how the document was matched and loaded. - */ - bool chunkMatches( ResultDetails* resultDetails ); - const ParsedQuery &_parsedQuery; - shared_ptr _cursor; - shared_ptr _queryOptimizerCursor; - BufBuilder _buf; - ShardChunkManagerPtr _chunkManager; - shared_ptr _explain; - shared_ptr _builder; - }; - -} // namespace mongo diff --git a/src/mongo/db/ops/update.cpp b/src/mongo/db/ops/update.cpp index 835ecc4723c..0cea2598963 100644 --- a/src/mongo/db/ops/update.cpp +++ b/src/mongo/db/ops/update.cpp @@ -14,496 +14,930 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" + +#include "mongo/db/ops/update.h" + +#include // for memcpy -#include "mongo/db/oplog.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/damage_vector.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/ops/update_driver.h" +#include "mongo/db/ops/update_executor.h" +#include "mongo/db/ops/update_lifecycle.h" #include "mongo/db/pagefault.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/lite_parsed_query.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/runner_yield_policy.h" +#include "mongo/db/query/type_explain.h" #include "mongo/db/queryutil.h" -#include "mongo/client/dbclientinterface.h" +#include "mongo/db/repl/is_master.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/storage/record.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/update_index_data.h" +#include "mongo/platform/unordered_set.h" -#include "update.h" -#include "update_internal.h" +namespace mongo { -//#define DEBUGUPDATE(x) cout << x << endl; -#define DEBUGUPDATE(x) + namespace mb = mutablebson; + namespace { + + const char idFieldName[] = "_id"; + const FieldRef idFieldRef(idFieldName); + + // TODO: Make this a function on NamespaceString, or make it cleaner. + inline void validateUpdate(const char* ns , + const BSONObj& updateobj, + const BSONObj& patternOrig) { + uassert(10155 , "cannot update reserved $ collection", strchr(ns, '$') == 0); + if (strstr(ns, ".system.")) { + /* dm: it's very important that system.indexes is never updated as IndexDetails + has pointers into it */ + uassert(10156, + str::stream() << "cannot update system collection: " + << ns << " q: " << patternOrig << " u: " << updateobj, + legalClientSystemNS(ns , true)); + } + } -namespace mongo { + Status storageValid(const mb::Document&, const bool); + Status storageValid(const mb::ConstElement&, const bool); + Status storageValidChildren(const mb::ConstElement&, const bool); + + /** + * mutable::document storageValid check -- like BSONObj::_okForStorage + */ + Status storageValid(const mb::Document& doc, const bool deep = true) { + mb::ConstElement currElem = doc.root().leftChild(); + while (currElem.ok()) { + if (currElem.getFieldName() == idFieldName) { + switch (currElem.getType()) { + case RegEx: + case Array: + case Undefined: + return Status(ErrorCodes::InvalidIdField, + str::stream() << "The '_id' value cannot be of type " + << typeName(currElem.getType())); + default: + break; + } + } + Status s = storageValid(currElem, deep); + if (!s.isOK()) + return s; + currElem = currElem.rightSibling(); + } - void checkNoMods( BSONObj o ) { - BSONObjIterator i( o ); - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - uassert( 10154 , "Modifiers and non-modifiers cannot be mixed", e.fieldName()[ 0 ] != '$' ); + return Status::OK(); } - } - static void checkTooLarge(const BSONObj& newObj) { - uassert( 12522 , "$ operator made object too large" , newObj.objsize() <= BSONObjMaxUserSize ); - } + /** + * Validates an element that has a field name which starts with a dollar sign ($). + * In the case of a DBRef field ($id, $ref, [$db]) these fields may be valid in + * the correct order/context only. + */ + Status validateDollarPrefixElement(const mb::ConstElement elem, const bool deep) { + mb::ConstElement curr = elem; + StringData currName = elem.getFieldName(); + + // Found a $db field + if (currName == "$db") { + if (curr.getType() != String) { + return Status(ErrorCodes::InvalidDBRef, + str::stream() << "The DBRef $db field must be a String, not a " + << typeName(curr.getType())); + } + curr = curr.leftSibling(); - /* note: this is only (as-is) called for - - - not multi - - not mods is indexed - - not upsert - */ - static UpdateResult _updateById(bool isOperatorUpdate, - int idIdxNo, - ModSet* mods, - NamespaceDetails* d, - NamespaceDetailsTransient *nsdt, - bool su, - const char* ns, - const BSONObj& updateobj, - BSONObj patternOrig, - bool logop, - OpDebug& debug, - bool fromMigrate = false) { - - DiskLoc loc; - { - IndexDetails& i = d->idx(idIdxNo); - BSONObj key = i.getKeyFromQuery( patternOrig ); - loc = i.idxInterface().findSingle(i, i.head, key); - if( loc.isNull() ) { - // no upsert support in _updateById yet, so we are done. - return UpdateResult( 0 , 0 , 0 , BSONObj() ); + if (!curr.ok() || (curr.getFieldName() != "$id")) + return Status(ErrorCodes::InvalidDBRef, + "Found $db field without a $id before it, which is invalid."); + + currName = curr.getFieldName(); } - } - Record* r = loc.rec(); - if ( cc().allowedToThrowPageFaultException() && ! r->likelyInPhysicalMemory() ) { - throw PageFaultException( r ); - } + // Found a $id field + if (currName == "$id") { + Status s = storageValidChildren(curr, deep); + if (!s.isOK()) + return s; - /* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some - regular ones at the moment. */ - if ( isOperatorUpdate ) { - const BSONObj& onDisk = loc.obj(); - auto_ptr mss = mods->prepare( onDisk, false /* not an insertion */ ); + curr = curr.leftSibling(); + if (!curr.ok() || (curr.getFieldName() != "$ref")) { + return Status(ErrorCodes::InvalidDBRef, + "Found $id field without a $ref before it, which is invalid."); + } - if( mss->canApplyInPlace() ) { - mss->applyModsInPlace(true); - debug.fastmod = true; - DEBUGUPDATE( "\t\t\t updateById doing in place update" ); + currName = curr.getFieldName(); + } + + if (currName == "$ref") { + if (curr.getType() != String) { + return Status(ErrorCodes::InvalidDBRef, + str::stream() << "The DBRef $ref field must be a String, not a " + << typeName(curr.getType())); + } + + if (!curr.rightSibling().ok() || curr.rightSibling().getFieldName() != "$id") + return Status(ErrorCodes::InvalidDBRef, + str::stream() << "The DBRef $ref field must be " + "following by a $id field"); } else { - BSONObj newObj = mss->createNewFromMods(); - checkTooLarge(newObj); - verify(nsdt); - theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug); + // not an okay, $ prefixed field name. + return Status(ErrorCodes::DollarPrefixedFieldName, + str::stream() << "The dollar ($) prefixed field '" + << elem.getFieldName() << "' in '" + << mb::getFullName(elem) + << "' is not valid for storage."); + } - if ( logop ) { - DEV verify( mods->size() ); - BSONObj pattern = patternOrig; - BSONObj logObj = mss->getOpLogRewrite(); - DEBUGUPDATE( "\t rewrite update: " << logObj ); - - // It is possible that the entire mod set was a no-op over this document. We - // would have an empty log record in that case. If we call logOp, with an empty - // record, that would be replicated as "clear this record", which is not what - // we want. Therefore, to get a no-op in the replica, we simply don't log. - if ( logObj.nFields() ) { - logOp("u", ns, logObj, &pattern, 0, fromMigrate ); + return Status::OK(); + } + + Status storageValid(const mb::ConstElement& elem, const bool deep = true) { + if (!elem.ok()) + return Status(ErrorCodes::BadValue, "Invalid elements cannot be stored."); + + // Field names of elements inside arrays are not meaningful in mutable bson, + // so we do not want to validate them. + // + // TODO: Revisit how mutable handles array field names. We going to need to make + // this better if we ever want to support ordered updates that can alter the same + // element repeatedly; see SERVER-12848. + const bool childOfArray = elem.parent().ok() ? + (elem.parent().getType() == mongo::Array) : false; + + if (!childOfArray) { + StringData fieldName = elem.getFieldName(); + // Cannot start with "$", unless dbref + if (fieldName[0] == '$') { + Status status = validateDollarPrefixElement(elem, deep); + if (!status.isOK()) + return status; + } + else if (fieldName.find(".") != string::npos) { + // Field name cannot have a "." in it. + return Status(ErrorCodes::DottedFieldName, + str::stream() << "The dotted field '" + << elem.getFieldName() << "' in '" + << mb::getFullName(elem) + << "' is not valid for storage."); } } - return UpdateResult( 1 , 1 , 1 , BSONObj() ); - } // end $operator update + // Check children if there are any. + Status s = storageValidChildren(elem, deep); + if (!s.isOK()) + return s; - // regular update - BSONElementManipulator::lookForTimestamps( updateobj ); - checkNoMods( updateobj ); - verify(nsdt); - theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug ); - if ( logop ) { - logOp("u", ns, updateobj, &patternOrig, 0, fromMigrate ); + return Status::OK(); } - return UpdateResult( 1 , 0 , 1 , BSONObj() ); - } - UpdateResult _updateObjects( bool su, - const char* ns, - const BSONObj& updateobj, - const BSONObj& patternOrig, - bool upsert, - bool multi, - bool logop , - OpDebug& debug, - RemoveSaver* rs, - bool fromMigrate, - const QueryPlanSelectionPolicy& planPolicy, - bool forReplication ) { - - DEBUGUPDATE( "update: " << ns - << " update: " << updateobj - << " query: " << patternOrig - << " upsert: " << upsert << " multi: " << multi ); - - Client& client = cc(); - - debug.updateobj = updateobj; - - // The idea with these here it to make them loop invariant for - // multi updates, and thus be a bit faster for that case. The - // pointers may be left invalid on a failed or terminal yield - // recovery. - NamespaceDetails* d = nsdetails(ns); // can be null if an upsert... - NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get(ns); - - auto_ptr mods; - bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$'; - int modsIsIndexed = false; // really the # of indexes - if ( isOperatorUpdate ) { - mods.reset( new ModSet(updateobj, nsdt->indexKeys(), forReplication) ); - modsIsIndexed = mods->maxNumIndexUpdated(); - } + Status storageValidChildren(const mb::ConstElement& elem, const bool deep) { + if (!elem.hasChildren()) + return Status::OK(); - if( planPolicy.permitOptimalIdPlan() && !multi && isSimpleIdQuery(patternOrig) && d && - !modsIsIndexed ) { - int idxNo = d->findIdIndex(); - if( idxNo >= 0 ) { - debug.idhack = true; - - UpdateResult result = _updateById( isOperatorUpdate, - idxNo, - mods.get(), - d, - nsdt, - su, - ns, - updateobj, - patternOrig, - logop, - debug, - fromMigrate); - if ( result.existing || ! upsert ) { - return result; - } - else if ( upsert && ! isOperatorUpdate ) { - // this handles repl inserts - checkNoMods( updateobj ); - debug.upsert = true; - BSONObj no = updateobj; - theDataFileMgr.insertWithObjMod(ns, no, false, su); - if ( logop ) - logOp( "i", ns, no, 0, 0, fromMigrate ); - - return UpdateResult( 0 , 0 , 1 , no ); - } + mb::ConstElement curr = elem.leftChild(); + while (curr.ok()) { + Status s = storageValid(curr, deep); + if (!s.isOK()) + return s; + curr = curr.rightSibling(); } + + return Status::OK(); } - int numModded = 0; - debug.nscanned = 0; - shared_ptr c = - NamespaceDetailsTransient::getCursor( ns, patternOrig, BSONObj(), planPolicy ); - d = nsdetails(ns); - nsdt = &NamespaceDetailsTransient::get(ns); - bool autoDedup = c->autoDedup(); - - if( c->ok() ) { - set seenObjects; - MatchDetails details; - auto_ptr cc; - do { - - if ( cc.get() == 0 && - client.allowedToThrowPageFaultException() && - ! c->currLoc().isNull() && - ! c->currLoc().rec()->likelyInPhysicalMemory() ) { - throw PageFaultException( c->currLoc().rec() ); + /** + * This will verify that all updated fields are + * 1.) Valid for storage (checking parent to make sure things like DBRefs are valid) + * 2.) Compare updated immutable fields do not change values + * + * If updateFields is empty then it was replacement and/or we need to check all fields + */ + inline Status validate(const BSONObj& original, + const FieldRefSet& updatedFields, + const mb::Document& updated, + const std::vector* immutableAndSingleValueFields, + const ModifierInterface::Options& opts) { + + LOG(3) << "update validate options -- " + << " updatedFields: " << updatedFields + << " immutableAndSingleValueFields.size:" + << (immutableAndSingleValueFields ? immutableAndSingleValueFields->size() : 0) + << " fromRepl: " << opts.fromReplication + << " validate:" << opts.enforceOkForStorage; + + // 1.) Loop through each updated field and validate for storage + // and detect immutable field updates + + // The set of possibly changed immutable fields -- we will need to check their vals + FieldRefSet changedImmutableFields; + + // Check to see if there were no fields specified or if we are not validating + // The case if a range query, or query that didn't result in saved fields + if (updatedFields.empty() || !opts.enforceOkForStorage) { + if (opts.enforceOkForStorage) { + // No specific fields were updated so the whole doc must be checked + Status s = storageValid(updated, true); + if (!s.isOK()) + return s; } - bool atomic = c->matcher() && c->matcher()->docMatcher().atomic(); + // Check all immutable fields + if (immutableAndSingleValueFields) + changedImmutableFields.fillFrom(*immutableAndSingleValueFields); + } + else { - if ( ! atomic && debug.nscanned > 0 ) { - // we need to use a ClientCursor to yield - if ( cc.get() == 0 ) { - shared_ptr< Cursor > cPtr = c; - cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) ); + // TODO: Change impl so we don't need to create a new FieldRefSet + // -- move all conflict logic into static function on FieldRefSet? + FieldRefSet immutableFieldRef; + if (immutableAndSingleValueFields) + immutableFieldRef.fillFrom(*immutableAndSingleValueFields); + + FieldRefSet::const_iterator where = updatedFields.begin(); + const FieldRefSet::const_iterator end = updatedFields.end(); + for( ; where != end; ++where) { + const FieldRef& current = **where; + + // Find the updated field in the updated document. + mutablebson::ConstElement newElem = updated.root(); + size_t currentPart = 0; + while (newElem.ok() && currentPart < current.numParts()) + newElem = newElem[current.getPart(currentPart++)]; + + // newElem might be missing if $unset/$renamed-away + if (newElem.ok()) { + Status s = storageValid(newElem, true); + if (!s.isOK()) + return s; } + // Check if the updated field conflicts with immutable fields + immutableFieldRef.findConflicts(¤t, &changedImmutableFields); + } + } + + const bool checkIdField = (updatedFields.empty() && !original.isEmpty()) || + updatedFields.findConflicts(&idFieldRef, NULL); + + // Add _id to fields to check since it too is immutable + if (checkIdField) + changedImmutableFields.keepShortest(&idFieldRef); + else if (changedImmutableFields.empty()) { + // Return early if nothing changed which is immutable + return Status::OK(); + } + + LOG(4) << "Changed immutable fields: " << changedImmutableFields; + // 2.) Now compare values of the changed immutable fields (to make sure they haven't) + + const mutablebson::ConstElement newIdElem = updated.root()[idFieldName]; + + FieldRefSet::const_iterator where = changedImmutableFields.begin(); + const FieldRefSet::const_iterator end = changedImmutableFields.end(); + for( ; where != end; ++where ) { + const FieldRef& current = **where; + + // Find the updated field in the updated document. + mutablebson::ConstElement newElem = updated.root(); + size_t currentPart = 0; + while (newElem.ok() && currentPart < current.numParts()) + newElem = newElem[current.getPart(currentPart++)]; + + if (!newElem.ok()) { + if (original.isEmpty()) { + // If the _id is missing and not required, then skip this check + if (!(current.dottedField() == idFieldName)) + return Status(ErrorCodes::NoSuchKey, + mongoutils::str::stream() + << "After applying the update, the new" + << " document was missing the '" + << current.dottedField() + << "' (required and immutable) field."); - bool didYield; - if ( ! cc->yieldSometimes( ClientCursor::WillNeed, &didYield ) ) { - cc.release(); - break; } - if ( !c->ok() ) { - break; + else { + if (current.dottedField() != idFieldName) + return Status(ErrorCodes::ImmutableField, + mongoutils::str::stream() + << "After applying the update to the document with " + << newIdElem.toString() + << ", the '" << current.dottedField() + << "' (required and immutable) field was " + "found to have been removed --" + << original); } - - if ( didYield ) { - d = nsdetails(ns); - if ( ! d ) - break; - nsdt = &NamespaceDetailsTransient::get(ns); - if ( mods.get() ) { - mods->setIndexedStatus( nsdt->indexKeys() ); - modsIsIndexed = mods->maxNumIndexUpdated(); + } + else { + + // Find the potentially affected field in the original document. + const BSONElement oldElem = original.getFieldDotted(current.dottedField()); + const BSONElement oldIdElem = original.getField(idFieldName); + + // Ensure no arrays since neither _id nor shard keys can be in an array, or one. + mb::ConstElement currElem = newElem; + while (currElem.ok()) { + if (currElem.getType() == Array) { + return Status(ErrorCodes::NotSingleValueField, + mongoutils::str::stream() + << "After applying the update to the document {" + << (oldIdElem.ok() ? oldIdElem.toString() : + newIdElem.toString()) + << " , ...}, the (immutable) field '" + << current.dottedField() + << "' was found to be an array or array descendant."); } + currElem = currElem.parent(); + } + // If we have both (old and new), compare them. If we just have new we are good + if (oldElem.ok() && newElem.compareWithBSONElement(oldElem, false) != 0) { + return Status(ErrorCodes::ImmutableField, + mongoutils::str::stream() + << "After applying the update to the document {" + << (oldIdElem.ok() ? oldIdElem.toString() : + newIdElem.toString()) + << " , ...}, the (immutable) field '" << current.dottedField() + << "' was found to have been altered to " + << newElem.toString()); } + } + } - } // end yielding block + return Status::OK(); + } - debug.nscanned++; + Status recoverFromYield(const UpdateRequest& request, + UpdateDriver* driver, + Collection* collection) { - if ( mods.get() && mods->hasDynamicArray() ) { - details.requestElemMatchKey(); - } + const NamespaceString& nsString(request.getNamespaceString()); + // We yielded and recovered OK, and our cursor is still good. Details about + // our namespace may have changed while we were yielded, so we re-acquire + // them here. If we can't do so, escape the update loop. Otherwise, refresh + // the driver so that it knows about what is currently indexed. - if ( !c->currentMatches( &details ) ) { - c->advance(); - continue; + if (request.shouldCallLogOp() && !isMasterNs(nsString.ns().c_str())) { + return Status(ErrorCodes::NotMaster, mongoutils::str::stream() << + "Demoted from primary while performing update on " << nsString.ns()); + } + + Collection* oldCollection = collection; + collection = cc().database()->getCollection(nsString.ns()); + + // We should not get a new pointer to the same collection... + if (oldCollection && (oldCollection != collection)) + return Status(ErrorCodes::IllegalOperation, + str::stream() << "Collection changed during the Update: ok?" + << " old: " << oldCollection->ok() + << " new:" << collection->ok()); + + if (!collection) + return Status(ErrorCodes::IllegalOperation, + "Update aborted due to invalid state transitions after yield -- " + "collection pointer NULL."); + + if (!collection->ok()) + return Status(ErrorCodes::IllegalOperation, + "Update aborted due to invalid state transitions after yield -- " + "collection not ok()."); + + IndexCatalog* idxCatalog = collection->getIndexCatalog(); + if (!idxCatalog) + return Status(ErrorCodes::IllegalOperation, + "Update aborted due to invalid state transitions after yield -- " + "IndexCatalog pointer NULL."); + + if (!idxCatalog->ok()) + return Status(ErrorCodes::IllegalOperation, + "Update aborted due to invalid state transitions after yield -- " + "IndexCatalog not ok()."); + + if (request.getLifecycle()) { + UpdateLifecycle* lifecycle = request.getLifecycle(); + lifecycle->setCollection(collection); + + if (!lifecycle->canContinue()) { + return Status(ErrorCodes::IllegalOperation, + "Update aborted due to invalid state transitions after yield.", + 17270); } - Record* r = c->_current(); - DiskLoc loc = c->currLoc(); + driver->refreshIndexKeys(lifecycle->getIndexKeys()); + } + + return Status::OK(); + } - if ( c->getsetdup( loc ) && autoDedup ) { - c->advance(); - continue; + Status ensureIdAndFirst(mb::Document& doc) { + mb::Element idElem = mb::findFirstChildNamed(doc.root(), idFieldName); + + // Move _id as first element if it exists + if (idElem.ok()) { + if (idElem.leftSibling().ok()) { + Status s = idElem.remove(); + if (!s.isOK()) + return s; + s = doc.root().pushFront(idElem); + if (!s.isOK()) + return s; } + } + else { + // Create _id if the document does not currently have one. + idElem = doc.makeElementNewOID(idFieldName); + if (!idElem.ok()) + return Status(ErrorCodes::BadValue, + "Could not create new _id ObjectId element.", + 17268); + Status s = doc.root().pushFront(idElem); + if (!s.isOK()) + return s; + } - BSONObj js = BSONObj::make(r); + return Status::OK(); - BSONObj pattern = patternOrig; + } + } // namespace - if ( logop ) { - BSONObjBuilder idPattern; - BSONElement id; - // NOTE: If the matching object lacks an id, we'll log - // with the original pattern. This isn't replay-safe. - // It might make sense to suppress the log instead - // if there's no id. - if ( js.getObjectID( id ) ) { - idPattern.append( id ); - pattern = idPattern.obj(); - } - else { - uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi ); - } + UpdateResult update(const UpdateRequest& request, OpDebug* opDebug) { + + UpdateExecutor executor(&request, opDebug); + return executor.execute(); + } + + UpdateResult update( + const UpdateRequest& request, + OpDebug* opDebug, + UpdateDriver* driver, + CanonicalQuery* cq) { + + LOG(3) << "processing update : " << request; + + std::auto_ptr cqHolder(cq); + const NamespaceString& nsString = request.getNamespaceString(); + UpdateLifecycle* lifecycle = request.getLifecycle(); + const CurOp* curOp = cc().curop(); + Collection* collection = cc().database()->getCollection(nsString.ns()); + + validateUpdate(nsString.ns().c_str(), request.getUpdates(), request.getQuery()); + + + // TODO: This seems a bit circuitious. + opDebug->updateobj = request.getUpdates(); + + if (lifecycle) { + lifecycle->setCollection(collection); + driver->refreshIndexKeys(lifecycle->getIndexKeys()); + } + + Runner* rawRunner; + Status status = cq ? + getRunner(collection, cqHolder.release(), &rawRunner) : + getRunner(collection, nsString.ns(), request.getQuery(), &rawRunner, &cq); + uassert(17243, + "could not get runner " + request.getQuery().toString() + "; " + causedBy(status), + status.isOK()); + + // Create the runner and setup all deps. + auto_ptr runner(rawRunner); + + // Register Runner with ClientCursor + const ScopedRunnerRegistration safety(runner.get()); + + // Use automatic yield policy + runner->setYieldPolicy(Runner::YIELD_AUTO); + + // If the update was marked with '$isolated' (a.k.a '$atomic'), we are not allowed to + // yield while evaluating the update loop below. + const bool isolated = + (cq && QueryPlannerCommon::hasNode(cq->root(), MatchExpression::ATOMIC)) || + LiteParsedQuery::isQueryIsolated(request.getQuery()); + + // + // We'll start assuming we have one or more documents for this update. (Otherwise, + // we'll fall-back to insert case (if upsert is true).) + // + + // We are an update until we fall into the insert case below. + driver->setContext(ModifierInterface::ExecInfo::UPDATE_CONTEXT); + + int numMatched = 0; + + // If the update was in-place, we may see it again. This only matters if we're doing + // a multi-update; if we're not doing a multi-update we stop after one update and we + // won't see any more docs. + // + // For example: If we're scanning an index {x:1} and performing {$inc:{x:5}}, we'll keep + // moving the document forward and it will continue to reappear in our index scan. + // Unless the index is multikey, the underlying query machinery won't de-dup. + // + // If the update wasn't in-place we may see it again. Our query may return the new + // document and we wouldn't want to update that. + // + // So, no matter what, we keep track of where the doc wound up. + typedef unordered_set DiskLocSet; + const scoped_ptr updatedLocs(request.isMulti() ? new DiskLocSet : NULL); + + // Reset these counters on each call. We might re-enter this function to retry this + // update if we throw a page fault exception below, and we rely on these counters + // reflecting only the actions taken locally. In particlar, we must have the no-op + // counter reset so that we can meaningfully comapre it with numMatched above. + opDebug->nModified = 0; + + // -1 for these fields means we don't have a value. Once the update completes, we + // request these values from the plan executor. + opDebug->nscanned = -1; + opDebug->nscannedObjects = -1; + + // Get the cached document from the update driver. + mutablebson::Document& doc = driver->getDocument(); + mutablebson::DamageVector damages; + + // Used during iteration of docs + BSONObj oldObj; + + // Keep track if we have done a write in isolation mode, which will indicate we can't yield + bool isolationModeWriteOccured = false; + + // Get first doc, and location + Runner::RunnerState state = Runner::RUNNER_ADVANCED; + + // Keep track of yield count so we can see if one happens on the getNext() calls below + int oldYieldCount = curOp->numYields(); + + uassert(ErrorCodes::NotMaster, + mongoutils::str::stream() << "Not primary while updating " << nsString.ns(), + !request.shouldCallLogOp() || isMasterNs(nsString.ns().c_str())); + + while (true) { + // See if we have a write in isolation mode + isolationModeWriteOccured = isolated && (opDebug->nModified > 0); + + // Change to manual yielding (no yielding) if we have written in isolation mode + if (isolationModeWriteOccured) { + runner->setYieldPolicy(Runner::YIELD_MANUAL); + } + + // keep track of the yield count before calling getNext (which might yield). + oldYieldCount = curOp->numYields(); + + // Get next doc, and location + DiskLoc loc; + state = runner->getNext(&oldObj, &loc); + const bool didYield = (oldYieldCount != curOp->numYields()); + + if (state != Runner::RUNNER_ADVANCED) { + if (state == Runner::RUNNER_EOF) { + if (didYield) + uassertStatusOK(recoverFromYield(request, driver, collection)); + + // We have reached the logical end of the loop, so do yielding recovery + break; + } + else { + uassertStatusOK(Status(ErrorCodes::InternalError, + str::stream() << " Update query failed -- " + << Runner::statestr(state))); } + } - /* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some - regular ones at the moment. */ - if ( isOperatorUpdate ) { + // Refresh things after a yield. + if (didYield) + uassertStatusOK(recoverFromYield(request, driver, collection)); - if ( multi ) { - // go to next record in case this one moves - c->advance(); + // We fill this with the new locs of moved doc so we don't double-update. + if (updatedLocs && updatedLocs->count(loc) > 0) { + continue; + } - // Update operations are deduped for cursors that implement their own - // deduplication. In particular, some geo cursors are excluded. - if ( autoDedup ) { + // Found a matching document + numMatched++; - if ( seenObjects.count( loc ) ) { - continue; - } + // Ask the driver to apply the mods. It may be that the driver can apply those "in + // place", that is, some values of the old document just get adjusted without any + // change to the binary layout on the bson layer. It may be that a whole new + // document is needed to accomodate the new bson layout of the resulting document. + doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled); + BSONObj logObj; - // SERVER-5198 Advance past the document to be modified, provided - // deduplication is enabled, but see SERVER-5725. - while( c->ok() && loc == c->currLoc() ) { - c->advance(); - } - } - } - const BSONObj& onDisk = loc.obj(); + FieldRefSet updatedFields; - ModSet* useMods = mods.get(); + Status status = Status::OK(); + if (!driver->needMatchDetails()) { + // If we don't need match details, avoid doing the rematch + status = driver->update(StringData(), &doc, &logObj, &updatedFields); + } + else { + // If there was a matched field, obtain it. + MatchDetails matchDetails; + matchDetails.requestElemMatchKey(); - auto_ptr mymodset; - if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) { - useMods = mods->fixDynamicArray( details.elemMatchKey() ); - mymodset.reset( useMods ); - } + dassert(cq); + verify(cq->root()->matchesBSON(oldObj, &matchDetails)); - auto_ptr mss = useMods->prepare( onDisk, - false /* not an insertion */ ); + string matchedField; + if (matchDetails.hasElemMatchKey()) + matchedField = matchDetails.elemMatchKey(); - bool willAdvanceCursor = multi && c->ok() && ( modsIsIndexed || ! mss->canApplyInPlace() ); + // TODO: Right now, each mod checks in 'prepare' that if it needs positional + // data, that a non-empty StringData() was provided. In principle, we could do + // that check here in an else clause to the above conditional and remove the + // checks from the mods. - if ( willAdvanceCursor ) { - if ( cc.get() ) { - cc->setDoingDeletes( true ); - } - c->prepareToTouchEarlierIterate(); - } + status = driver->update(matchedField, &doc, &logObj, &updatedFields); + } - // If we've made it this far, "ns" must contain a valid collection name, and so - // is of the form "db.collection". Therefore, the following expression must - // always be valid. "system.users" updates must never be done in place, in - // order to ensure that they are validated inside DataFileMgr::updateRecord(.). - bool isSystemUsersMod = (NamespaceString(ns).coll == "system.users"); + if (!status.isOK()) { + uasserted(16837, status.reason()); + } - if ( !mss->isUpdateIndexed() && mss->canApplyInPlace() && !isSystemUsersMod ) { - mss->applyModsInPlace( true );// const_cast(onDisk) ); + // Ensure _id exists and is first + uassertStatusOK(ensureIdAndFirst(doc)); + + // If the driver applied the mods in place, we can ask the mutable for what + // changed. We call those changes "damages". :) We use the damages to inform the + // journal what was changed, and then apply them to the original document + // ourselves. If, however, the driver applied the mods out of place, we ask it to + // generate a new, modified document for us. In that case, the file manager will + // take care of the journaling details for us. + // + // This code flow is admittedly odd. But, right now, journaling is baked in the file + // manager. And if we aren't using the file manager, we have to do jounaling + // ourselves. + bool docWasModified = false; + BSONObj newObj; + const char* source = NULL; + bool inPlace = doc.getInPlaceUpdates(&damages, &source); + + // If something changed in the document, verify that no immutable fields were changed + // and data is valid for storage. + if ((!inPlace || !damages.empty()) ) { + if (!(request.isFromReplication() || request.isFromMigration())) { + const std::vector* immutableFields = NULL; + if (lifecycle) + immutableFields = lifecycle->getImmutableFields(); + + uassertStatusOK(validate(oldObj, + updatedFields, + doc, + immutableFields, + driver->modOptions()) ); + } + } - DEBUGUPDATE( "\t\t\t doing in place update" ); - if ( !multi ) - debug.fastmod = true; + // Save state before making changes + runner->saveState(); - if ( modsIsIndexed ) { - seenObjects.insert( loc ); - } + if (inPlace && !driver->modsAffectIndices()) { - d->paddingFits(); - } - else { - BSONObj newObj = mss->createNewFromMods(); - checkTooLarge(newObj); - DiskLoc newLoc = theDataFileMgr.updateRecord(ns, - d, - nsdt, - r, - loc, - newObj.objdata(), - newObj.objsize(), - debug); - - if ( newLoc != loc || modsIsIndexed ){ - // log() << "Moved obj " << newLoc.obj()["_id"] << " from " << loc << " to " << newLoc << endl; - // object moved, need to make sure we don' get again - seenObjects.insert( newLoc ); - } + // If a set of modifiers were all no-ops, we are still 'in place', but there is + // no work to do, in which case we want to consider the object unchanged. + if (!damages.empty() ) { - } + // Broadcast the mutation so that query results stay correct. + collection->cursorCache()->invalidateDocument(loc, INVALIDATION_MUTATION); - if ( logop ) { - DEV verify( mods->size() ); - BSONObj logObj = mss->getOpLogRewrite(); - DEBUGUPDATE( "\t rewrite update: " << logObj ); - - // It is possible that the entire mod set was a no-op over this - // document. We would have an empty log record in that case. If we - // call logOp, with an empty record, that would be replicated as "clear - // this record", which is not what we want. Therefore, to get a no-op - // in the replica, we simply don't log. - if ( logObj.nFields() ) { - logOp("u", ns, logObj , &pattern, 0, fromMigrate ); - } + collection->details()->paddingFits(); + + // All updates were in place. Apply them via durability and writing pointer. + mutablebson::DamageVector::const_iterator where = damages.begin(); + const mutablebson::DamageVector::const_iterator end = damages.end(); + for( ; where != end; ++where ) { + const char* sourcePtr = source + where->sourceOffset; + void* targetPtr = getDur().writingPtr( + const_cast(oldObj.objdata()) + where->targetOffset, + where->size); + std::memcpy(targetPtr, sourcePtr, where->size); } - numModded++; - if ( ! multi ) - return UpdateResult( 1 , 1 , numModded , BSONObj() ); - if ( willAdvanceCursor ) - c->recoverFromTouchingEarlierIterate(); + docWasModified = true; + opDebug->fastmod = true; + } - getDur().commitIfNeeded(); + newObj = oldObj; + } + else { - continue; + // The updates were not in place. Apply them through the file manager. + newObj = doc.getObject(); + uassert(17419, + str::stream() << "Resulting document after update is larger than " + << BSONObjMaxUserSize, + newObj.objsize() <= BSONObjMaxUserSize); + StatusWith res = collection->updateDocument(loc, + newObj, + true, + opDebug); + uassertStatusOK(res.getStatus()); + DiskLoc newLoc = res.getValue(); + docWasModified = true; + + // If the document moved, we might see it again in a collection scan (maybe it's + // a document after our current document). + // + // If the document is indexed and the mod changes an indexed value, we might see it + // again. For an example, see the comment above near declaration of updatedLocs. + if (updatedLocs && (newLoc != loc || driver->modsAffectIndices())) { + updatedLocs->insert(newLoc); } + } - uassert( 10158 , "multi update only works with $ operators" , ! multi ); + // Restore state after modification + uassert(17278, + "Update could not restore runner state after updating a document.", + runner->restoreState()); + + // Call logOp if requested. + if (request.shouldCallLogOp() && !logObj.isEmpty()) { + BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request.isMulti()); + logOp("u", nsString.ns().c_str(), logObj , &idQuery, + NULL, request.isFromMigration(), &newObj); + } + + // Only record doc modifications if they wrote (exclude no-ops) + if (docWasModified) + opDebug->nModified++; + + if (!request.isMulti()) { + break; + } - BSONElementManipulator::lookForTimestamps( updateobj ); - checkNoMods( updateobj ); - theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, su); - if ( logop ) { - DEV wassert( !su ); // super used doesn't get logged, this would be bad. - logOp("u", ns, updateobj, &pattern, 0, fromMigrate ); + // Opportunity for journaling to write during the update. + getDur().commitIfNeeded(); + } + + // Runner is still alive, so we should still have a database and curop. We use + // these below to get the profiling level and the elapsed milliseconds for this update. + invariant(cc().database()); + invariant(cc().curop()); + + // If profiling is enabled, this update has taken longer than slowMs, or the logLevel + // is at least set to logLevel 1, then get debug information from the query runner. + if (cc().database()->getProfilingLevel() > 0 || + cc().curop()->elapsedMillis() > serverGlobalParams.slowMS || + logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + TypeExplain* rawExplain; + Status infoStatus = runner->getInfo(&rawExplain, NULL); + if (infoStatus.isOK()) { + // We successfully got explain info from the runner. Copy the relevant + // fields into 'opDebug'. + scoped_ptr explain(rawExplain); + if (explain->isNScannedSet()) { + opDebug->nscanned = explain->getNScanned(); + } + if (explain->isNScannedObjectsSet()) { + opDebug->nscannedObjects = explain->getNScannedObjects(); } - return UpdateResult( 1 , 0 , 1 , BSONObj() ); - } while ( c->ok() ); - } // endif - - if ( numModded ) - return UpdateResult( 1 , 1 , numModded , BSONObj() ); - - if ( upsert ) { - if ( updateobj.firstElementFieldName()[0] == '$' ) { - // upsert of an $operation. build a default object - BSONObj newObj = mods->createNewFromQuery( patternOrig ); - checkNoMods( newObj ); - debug.fastmodinsert = true; - theDataFileMgr.insertWithObjMod(ns, newObj, false, su); - if ( logop ) - logOp( "i", ns, newObj, 0, 0, fromMigrate ); - - return UpdateResult( 0 , 1 , 1 , newObj ); } - uassert( 10159 , "multi update only works with $ operators" , ! multi ); - checkNoMods( updateobj ); - debug.upsert = true; - BSONObj no = updateobj; - theDataFileMgr.insertWithObjMod(ns, no, false, su); - if ( logop ) - logOp( "i", ns, no, 0, 0, fromMigrate ); - return UpdateResult( 0 , 0 , 1 , no ); } - return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() ); - } + // TODO: Can this be simplified? + if ((numMatched > 0) || (numMatched == 0 && !request.isUpsert()) ) { + opDebug->nMatched = numMatched; + return UpdateResult(numMatched > 0 /* updated existing object(s) */, + !driver->isDocReplacement() /* $mod or obj replacement */, + opDebug->nModified /* number of modified docs, no no-ops */, + numMatched /* # of docs matched/updated, even no-ops */, + BSONObj()); + } - void validateUpdate( const char* ns , const BSONObj& updateobj, const BSONObj& patternOrig ) { - uassert( 10155 , "cannot update reserved $ collection", strchr(ns, '$') == 0 ); - if ( strstr(ns, ".system.") ) { - /* dm: it's very important that system.indexes is never updated as IndexDetails - has pointers into it */ - uassert( 10156, - str::stream() << "cannot update system collection: " - << ns << " q: " << patternOrig << " u: " << updateobj, - legalClientSystemNS( ns , true ) ); + // + // We haven't found any existing document so an insert is done + // (upsert is true). + // + opDebug->upsert = true; + + // Since this is an insert (no docs found and upsert:true), we will be logging it + // as an insert in the oplog. We don't need the driver's help to build the + // oplog record, then. We also set the context of the update driver to the INSERT_CONTEXT. + // Some mods may only work in that context (e.g. $setOnInsert). + driver->setLogOp(false); + driver->setContext(ModifierInterface::ExecInfo::INSERT_CONTEXT); + + // Reset the document we will be writing to + doc.reset(); + + // This remains the empty object in the case of an object replacement, but in the case + // of an upsert where we are creating a base object from the query and applying mods, + // we capture the query as the original so that we can detect immutable field mutations. + BSONObj original = BSONObj(); + + // Calling createFromQuery will populate the 'doc' with fields from the query which + // creates the base of the update for the inserterd doc (because upsert was true) + if (cq) { + uassertStatusOK(driver->populateDocumentWithQueryFields(cq, doc)); + if (!driver->isDocReplacement()) { + opDebug->fastmodinsert = true; + // We need all the fields from the query to compare against for validation below. + original = doc.getObject(); + } + else { + original = request.getQuery(); + } + } + else { + fassert(17354, CanonicalQuery::isSimpleIdQuery(request.getQuery())); + BSONElement idElt = request.getQuery()["_id"]; + original = idElt.wrap(); + fassert(17352, doc.root().appendElement(idElt)); } - } - UpdateResult updateObjects( const char* ns, - const BSONObj& updateobj, - const BSONObj& patternOrig, - bool upsert, - bool multi, - bool logop , - OpDebug& debug, - bool fromMigrate, - const QueryPlanSelectionPolicy& planPolicy ) { - - validateUpdate( ns , updateobj , patternOrig ); - - UpdateResult ur = _updateObjects(false, ns, updateobj, patternOrig, - upsert, multi, logop, - debug, NULL, fromMigrate, planPolicy ); - debug.nupdated = ur.num; - return ur; - } + // Apply the update modifications and then log the update as an insert manually. + status = driver->update(StringData(), &doc); + if (!status.isOK()) { + uasserted(16836, status.reason()); + } + + // Ensure _id exists and is first + uassertStatusOK(ensureIdAndFirst(doc)); + + // Validate that the object replacement or modifiers resulted in a document + // that contains all the immutable keys and can be stored if it isn't coming + // from a migration or via replication. + if (!(request.isFromReplication() || request.isFromMigration())){ + const std::vector* immutableFields = NULL; + if (lifecycle) + immutableFields = lifecycle->getImmutableFields(); + + FieldRefSet noFields; + // This will only validate the modified fields if not a replacement. + uassertStatusOK(validate(original, + noFields, + doc, + immutableFields, + driver->modOptions()) ); + } + + // Only create the collection if the doc will be inserted. + if (!collection) { + collection = cc().database()->getCollection(request.getNamespaceString().ns()); + if (!collection) { + collection = cc().database()->createCollection(request.getNamespaceString().ns()); + } + } + + // Insert the doc + BSONObj newObj = doc.getObject(); + uassert(17420, + str::stream() << "Document to upsert is larger than " << BSONObjMaxUserSize, + newObj.objsize() <= BSONObjMaxUserSize); + + StatusWith newLoc = collection->insertDocument(newObj, + !request.isGod() /*enforceQuota*/); + uassertStatusOK(newLoc.getStatus()); + if (request.shouldCallLogOp()) { + logOp("i", nsString.ns().c_str(), newObj, + NULL, NULL, request.isFromMigration(), &newObj); + } - UpdateResult updateObjectsForReplication( const char* ns, - const BSONObj& updateobj, - const BSONObj& patternOrig, - bool upsert, - bool multi, - bool logop , - OpDebug& debug, - bool fromMigrate, - const QueryPlanSelectionPolicy& planPolicy ) { - - validateUpdate( ns , updateobj , patternOrig ); - - UpdateResult ur = _updateObjects(false, - ns, - updateobj, - patternOrig, - upsert, - multi, - logop, - debug, - NULL /* no remove saver */, - fromMigrate, - planPolicy, - true /* for replication */ ); - debug.nupdated = ur.num; - return ur; + opDebug->nMatched = 1; + return UpdateResult(false /* updated a non existing document */, + !driver->isDocReplacement() /* $mod or obj replacement? */, + 1 /* docs written*/, + 1 /* count of updated documents */, + newObj /* object that was upserted */ ); } - BSONObj applyUpdateOperators( const BSONObj& from, const BSONObj& operators ) { - ModSet mods( operators ); - return mods.prepare( from, false /* not an insertion */ )->createNewFromMods(); + BSONObj applyUpdateOperators(const BSONObj& from, const BSONObj& operators) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + Status status = driver.parse(operators); + if (!status.isOK()) { + uasserted(16838, status.reason()); + } + + mutablebson::Document doc(from, mutablebson::Document::kInPlaceDisabled); + status = driver.update(StringData(), &doc); + if (!status.isOK()) { + uasserted(16839, status.reason()); + } + + return doc.getObject(); } - + } // namespace mongo diff --git a/src/mongo/db/ops/update.h b/src/mongo/db/ops/update.h index c24f0628091..34d290cfe85 100644 --- a/src/mongo/db/ops/update.h +++ b/src/mongo/db/ops/update.h @@ -14,88 +14,58 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "mongo/pch.h" +#pragma once #include "mongo/db/jsobj.h" #include "mongo/db/curop.h" -#include "mongo/db/queryoptimizercursor.h" +#include "mongo/db/ops/update_request.h" +#include "mongo/db/ops/update_result.h" namespace mongo { - // ---------- public ------------- - - struct UpdateResult { - const bool existing; // if existing objects were modified - const bool mod; // was this a $ mod - const long long num; // how many objects touched - OID upserted; // if something was upserted, the new _id of the object - - UpdateResult( bool e, bool m, unsigned long long n , const BSONObj& upsertedObject ) - : existing(e) , mod(m), num(n) { - upserted.clear(); - BSONElement id = upsertedObject["_id"]; - if ( ! e && n == 1 && id.type() == jstOID ) { - upserted = id.OID(); - } - } - }; + class CanonicalQuery; + class UpdateDriver; - class RemoveSaver; - - /* returns true if an existing object was updated, false if no existing object was found. - multi - update multiple objects - mostly useful with things like $set - su - allow access to system namespaces (super user) - */ - UpdateResult updateObjects(const char* ns, - const BSONObj& updateobj, - const BSONObj& pattern, - bool upsert, - bool multi, - bool logop, - OpDebug& debug, - bool fromMigrate = false, - const QueryPlanSelectionPolicy& planPolicy = QueryPlanSelectionPolicy::any()); - - /* - * Similar to updateObjects but not strict about applying mods that can fail during initial - * replication. + /** + * Utility method to execute an update described by "request". * - * Reference ticket: SERVER-4781 + * Caller must hold the appropriate database locks. */ - UpdateResult updateObjectsForReplication(const char* ns, - const BSONObj& updateobj, - const BSONObj& pattern, - bool upsert, - bool multi, - bool logop, - OpDebug& debug, - bool fromMigrate = false, - const QueryPlanSelectionPolicy& planPolicy = - QueryPlanSelectionPolicy::any()); - - UpdateResult _updateObjects(bool su, - const char* ns, - const BSONObj& updateobj, - const BSONObj& pattern, - bool upsert, - bool multi, - bool logop, - OpDebug& debug, - RemoveSaver* rs = 0, - bool fromMigrate = false, - const QueryPlanSelectionPolicy& planPolicy = QueryPlanSelectionPolicy::any(), - bool forReplication = false); + UpdateResult update(const UpdateRequest& request, OpDebug* opDebug); + /** + * Execute the update described by "request", using the given already-parsed + * driver and canonical query. + * + * NOTE: This function is really a utility method for UpdateExecutor. + * + * TODO: Move this into a private method of UpdateExecutor. + */ + UpdateResult update(const UpdateRequest& request, + OpDebug* opDebug, + UpdateDriver* driver, + CanonicalQuery* cq); /** * takes the from document and returns a new document - * after apply all the operators - * e.g. + * after apply all the operators + * e.g. * applyUpdateOperators( BSON( "x" << 1 ) , BSON( "$inc" << BSON( "x" << 1 ) ) ); * returns: { x : 2 } */ BSONObj applyUpdateOperators( const BSONObj& from, const BSONObj& operators ); - } // namespace mongo diff --git a/src/mongo/db/ops/update_driver.cpp b/src/mongo/db/ops/update_driver.cpp new file mode 100644 index 00000000000..664f11e202a --- /dev/null +++ b/src/mongo/db/ops/update_driver.cpp @@ -0,0 +1,448 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/update_driver.h" + +#include "mongo/base/error_codes.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/ops/log_builder.h" +#include "mongo/db/ops/modifier_object_replace.h" +#include "mongo/db/ops/modifier_table.h" +#include "mongo/db/ops/path_support.h" +#include "mongo/util/embedded_builder.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + namespace mb = mongo::mutablebson; + + UpdateDriver::UpdateDriver(const Options& opts) + : _replacementMode(false) + , _indexedFields(NULL) + , _logOp(opts.logOp) + , _modOptions(opts.modOptions) + , _affectIndices(false) + , _positional(false) { + } + + UpdateDriver::~UpdateDriver() { + clear(); + } + + Status UpdateDriver::parse(const BSONObj& updateExpr, const bool multi) { + clear(); + + // Check if the update expression is a full object replacement. + if (*updateExpr.firstElementFieldName() != '$') { + if (multi) { + return Status(ErrorCodes::FailedToParse, + "multi update only works with $ operators"); + } + + // Modifiers expect BSONElements as input. But the input to object replace is, by + // definition, an object. We wrap the 'updateExpr' as the mod is expecting. Note + // that the wrapper is temporary so the object replace mod should make a copy of + // the object. + auto_ptr mod(new ModifierObjectReplace); + BSONObj wrapper = BSON( "dummy" << updateExpr ); + Status status = mod->init(wrapper.firstElement(), _modOptions); + if (!status.isOK()) { + return status; + } + + _mods.push_back(mod.release()); + + // Register the fact that this driver will only do full object replacements. + _replacementMode = true; + + return Status::OK(); + } + + // The update expression is made of mod operators, that is + // { <$mod>: {...}, <$mod>: {...}, ... } + BSONObjIterator outerIter(updateExpr); + while (outerIter.more()) { + BSONElement outerModElem = outerIter.next(); + + // Check whether this is a valid mod type. + modifiertable::ModifierType modType = modifiertable::getType(outerModElem.fieldName()); + if (modType == modifiertable::MOD_UNKNOWN) { + return Status(ErrorCodes::FailedToParse, + str::stream() << "Unknown modifier: " << outerModElem.fieldName()); + } + + // Check whether there is indeed a list of mods under this modifier. + if (outerModElem.type() != Object) { + return Status(ErrorCodes::FailedToParse, + str::stream() << "Modifiers operate on fields but we found a " + << typeName(outerModElem.type()) + << " instead. For example: {$mod: {: ...}}" + << " not {" << outerModElem.toString() << "}"); + } + + // Check whether there are indeed mods under this modifier. + if (outerModElem.embeddedObject().isEmpty()) { + return Status(ErrorCodes::FailedToParse, + str::stream() << "'" << outerModElem.fieldName() + << "' is empty. You must specify a field like so: " + "{$mod: {: ...}}"); + } + + BSONObjIterator innerIter(outerModElem.embeddedObject()); + while (innerIter.more()) { + BSONElement innerModElem = innerIter.next(); + + Status status = addAndParse(modType, innerModElem); + if (!status.isOK()) { + return status; + } + } + } + + // Register the fact that there will be only $mod's in this driver -- no object + // replacement. + _replacementMode = false; + + return Status::OK(); + } + + inline Status UpdateDriver::addAndParse(const modifiertable::ModifierType type, + const BSONElement& elem) { + if (elem.eoo()) { + return Status(ErrorCodes::FailedToParse, + str::stream() << "'" << elem.fieldName() + << "' has no value in : " << elem + << " which is not allowed for any $" << type << " mod."); + } + + auto_ptr mod(modifiertable::makeUpdateMod(type)); + dassert(mod.get()); + + bool positional = false; + Status status = mod->init(elem, _modOptions, &positional); + if (!status.isOK()) { + return status; + } + + // If any modifier indicates that it requires a positional match, toggle the + // _positional flag to true. + _positional = _positional || positional; + + _mods.push_back(mod.release()); + + return Status::OK(); + } + + Status UpdateDriver::populateDocumentWithQueryFields(const BSONObj& query, + mutablebson::Document& doc) const { + CanonicalQuery* rawCG; + // We canonicalize the query to collapse $and/$or, and the first arg (ns) is not needed + Status s = CanonicalQuery::canonicalize("", query, &rawCG); + if (!s.isOK()) + return s; + scoped_ptr cq(rawCG); + return populateDocumentWithQueryFields(rawCG, doc); + } + + Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery* query, + mutablebson::Document& doc) const { + + MatchExpression* root = query->root(); + + MatchExpression::MatchType rootType = root->matchType(); + + // These copies are needed until we apply the modifiers at the end. + std::vector copies; + + // We only care about equality and "and"ed equality fields, everything else is ignored + if (rootType != MatchExpression::EQ && rootType != MatchExpression::AND) + return Status::OK(); + + if (isDocReplacement()) { + BSONElement idElem = query->getQueryObj().getField("_id"); + + // Replacement mods need the _id field copied explicitly. + if (idElem.ok()) { + mb::Element elem = doc.makeElement(idElem); + return doc.root().pushFront(elem); + } + + return Status::OK(); + } + + // Create a new UpdateDriver to create the base doc from the query + Options opts; + opts.logOp = false; + opts.modOptions = modOptions(); + + UpdateDriver insertDriver(opts); + insertDriver.setContext(ModifierInterface::ExecInfo::INSERT_CONTEXT); + + // If we are a single equality match query + if (root->matchType() == MatchExpression::EQ) { + EqualityMatchExpression* eqMatch = + static_cast(root); + + const BSONElement matchData = eqMatch->getData(); + BSONElement childElem = matchData; + + // Make copy to new path if not the same field name (for cases like $all) + if (!root->path().empty() && matchData.fieldNameStringData() != root->path()) { + BSONObjBuilder copyBuilder; + copyBuilder.appendAs(eqMatch->getData(), root->path()); + const BSONObj copy = copyBuilder.obj(); + copies.push_back(copy); + childElem = copy[root->path()]; + } + + // Add this element as a $set modifier + Status s = insertDriver.addAndParse(modifiertable::MOD_SET, + childElem); + if (!s.isOK()) + return s; + + } + else { + + // parse query $set mods, including only equality stuff + for (size_t i = 0; i < root->numChildren(); ++i) { + MatchExpression* child = root->getChild(i); + if (child->matchType() == MatchExpression::EQ) { + EqualityMatchExpression* eqMatch = + static_cast(child); + + const BSONElement matchData = eqMatch->getData(); + BSONElement childElem = matchData; + + // Make copy to new path if not the same field name (for cases like $all) + if (!child->path().empty() && + matchData.fieldNameStringData() != child->path()) { + BSONObjBuilder copyBuilder; + copyBuilder.appendAs(eqMatch->getData(), child->path()); + const BSONObj copy = copyBuilder.obj(); + copies.push_back(copy); + childElem = copy[child->path()]; + } + + // Add this element as a $set modifier + Status s = insertDriver.addAndParse(modifiertable::MOD_SET, + childElem); + if (!s.isOK()) + return s; + } + } + } + + // update the document with base field + Status s = insertDriver.update(StringData(), &doc); + copies.clear(); + if (!s.isOK()) { + return Status(ErrorCodes::UnsupportedFormat, + str::stream() << "Cannot create base during" + " insert of update. Caused by :" + << s.toString()); + } + + return Status::OK(); + } + + Status UpdateDriver::update(const StringData& matchedField, + mutablebson::Document* doc, + BSONObj* logOpRec, + FieldRefSet* updatedFields) { + // TODO: assert that update() is called at most once in a !_multi case. + + // Use the passed in FieldRefSet + FieldRefSet* targetFields = updatedFields; + + // If we didn't get a FieldRefSet* from the caller, allocate storage and use + // the scoped_ptr for lifecycle management + scoped_ptr targetFieldScopedPtr; + if (!targetFields) { + targetFieldScopedPtr.reset(new FieldRefSet()); + targetFields = targetFieldScopedPtr.get(); + } + + _affectIndices = false; + + _logDoc.reset(); + LogBuilder logBuilder(_logDoc.root()); + + // Ask each of the mods to type check whether they can operate over the current document + // and, if so, to change that document accordingly. + for (vector::iterator it = _mods.begin(); it != _mods.end(); ++it) { + ModifierInterface::ExecInfo execInfo; + Status status = (*it)->prepare(doc->root(), matchedField, &execInfo); + if (!status.isOK()) { + return status; + } + + // If a mod wants to be applied only if this is an upsert (or only if this is a + // strict update), we should respect that. If a mod doesn't care, it would state + // it is fine with ANY update context. + const bool validContext = (execInfo.context == ModifierInterface::ExecInfo::ANY_CONTEXT || + execInfo.context == _context); + + // Nothing to do if not in a valid context. + if (!validContext) { + continue; + } + + + // Gather which fields this mod is interested on and whether these fields were + // "taken" by previous mods. Note that not all mods are multi-field mods. When we + // see an empty field, we may stop looking for others. + for (int i = 0; i < ModifierInterface::ExecInfo::MAX_NUM_FIELDS; i++) { + if (execInfo.fieldRef[i] == 0) { + break; + } + + // Record each field being updated but check for conflicts first + const FieldRef* other; + if (!targetFields->insert(execInfo.fieldRef[i], &other)) { + return Status(ErrorCodes::ConflictingUpdateOperators, + str::stream() << "Cannot update '" + << other->dottedField() + << "' and '" + << execInfo.fieldRef[i]->dottedField() + << "' at the same time"); + } + + // We start with the expectation that a mod will be in-place. But if the mod + // touched an indexed field and the mod will indeed be executed -- that is, it + // is not a no-op and it is in a valid context -- then we switch back to a + // non-in-place mode. + // + // TODO: make mightBeIndexed and fieldRef like each other. + if (!_affectIndices && + !execInfo.noOp && + _indexedFields && + _indexedFields->mightBeIndexed(execInfo.fieldRef[i]->dottedField())) { + _affectIndices = true; + doc->disableInPlaceUpdates(); + } + } + + if (!execInfo.noOp) { + status = (*it)->apply(); + if (!status.isOK()) { + return status; + } + } + + // If we require a replication oplog entry for this update, go ahead and generate one. + if (!execInfo.noOp && _logOp && logOpRec) { + status = (*it)->log(&logBuilder); + if (!status.isOK()) { + return status; + } + } + + } + + if (_logOp && logOpRec) + *logOpRec = _logDoc.getObject(); + + return Status::OK(); + } + + size_t UpdateDriver::numMods() const { + return _mods.size(); + } + + bool UpdateDriver::isDocReplacement() const { + return _replacementMode; + } + + bool UpdateDriver::modsAffectIndices() const { + return _affectIndices; + } + + void UpdateDriver::refreshIndexKeys(const UpdateIndexData* indexedFields) { + _indexedFields = indexedFields; + } + + bool UpdateDriver::logOp() const { + return _logOp; + } + + void UpdateDriver::setLogOp(bool logOp) { + _logOp = logOp; + } + + ModifierInterface::Options UpdateDriver::modOptions() const { + return _modOptions; + } + + void UpdateDriver::setModOptions(ModifierInterface::Options modOpts) { + _modOptions = modOpts; + } + + ModifierInterface::ExecInfo::UpdateContext UpdateDriver::context() const { + return _context; + } + + void UpdateDriver::setContext(ModifierInterface::ExecInfo::UpdateContext context) { + _context = context; + } + + BSONObj UpdateDriver::makeOplogEntryQuery(const BSONObj& doc, bool multi) const { + BSONObjBuilder idPattern; + BSONElement id; + // NOTE: If the matching object lacks an id, we'll log + // with the original pattern. This isn't replay-safe. + // It might make sense to suppress the log instead + // if there's no id. + if ( doc.getObjectID( id ) ) { + idPattern.append( id ); + return idPattern.obj(); + } + else { + uassert( 16980, + str::stream() << "Multi-update operations require all documents to " + "have an '_id' field. " << doc.toString(false, false), + ! multi ); + return doc; + } + } + void UpdateDriver::clear() { + for (vector::iterator it = _mods.begin(); it != _mods.end(); ++it) { + delete *it; + } + _mods.clear(); + _indexedFields = NULL; + _replacementMode = false; + _positional = false; + } + +} // namespace mongo diff --git a/src/mongo/db/ops/update_driver.h b/src/mongo/db/ops/update_driver.h new file mode 100644 index 00000000000..85db733aaf9 --- /dev/null +++ b/src/mongo/db/ops/update_driver.h @@ -0,0 +1,192 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/base/status.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/db/field_ref_set.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/ops/modifier_interface.h" +#include "mongo/db/ops/modifier_table.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/update_index_data.h" + +namespace mongo { + + class UpdateDriver { + public: + + struct Options; + UpdateDriver(const Options& opts); + + ~UpdateDriver(); + + /** + * Returns OK and fills in '_mods' if 'updateExpr' is correct. Otherwise returns an + * error status with a corresponding description. + */ + Status parse(const BSONObj& updateExpr, const bool multi = false); + + /** + * Fills in document with any fields in the query which are valid. + * + * Valid fields include equality matches like "a":1, or "a.b":false + * + * Each valid field will be expanded (from dot notation) and conflicts will be + * checked for all fields added to the underlying document. + * + * Returns Status::OK() if the document can be used. If there are any error or + * conflicts along the way then those errors will be returned. + */ + Status populateDocumentWithQueryFields(const BSONObj& query, + mutablebson::Document& doc) const; + + Status populateDocumentWithQueryFields(const CanonicalQuery* query, + mutablebson::Document& doc) const; + + /** + * return a BSONObj with the _id field of the doc passed in, or the doc itself. + * If no _id and multi, error. + */ + BSONObj makeOplogEntryQuery(const BSONObj& doc, bool multi) const; + + /** + * Returns OK and executes '_mods' over 'doc', generating 'newObj'. If any mod is + * positional, use 'matchedField' (index of the array item matched). If doc allows + * mods to be applied in place and no index updating is involved, then the mods may + * be applied "in place" over 'doc'. + * + * If the driver's '_logOp' mode is turned on, and if 'logOpRec' is not NULL, fills in + * the latter with the oplog entry corresponding to the update. If '_mods' can't be + * applied, returns an error status with a corresponding description. + * + * If a non-NULL updatedField vector* is supplied, + * then all updated fields will be added to it. + */ + Status update(const StringData& matchedField, + mutablebson::Document* doc, + BSONObj* logOpRec = NULL, + FieldRefSet* updatedFields = NULL); + + // + // Accessors + // + + size_t numMods() const; + + bool isDocReplacement() const; + + bool modsAffectIndices() const; + void refreshIndexKeys(const UpdateIndexData* indexedFields); + + bool logOp() const; + void setLogOp(bool logOp); + + ModifierInterface::Options modOptions() const; + void setModOptions(ModifierInterface::Options modOpts); + + ModifierInterface::ExecInfo::UpdateContext context() const; + void setContext(ModifierInterface::ExecInfo::UpdateContext context); + + mutablebson::Document& getDocument() { + return _objDoc; + } + + const mutablebson::Document& getDocument() const { + return _objDoc; + } + + bool needMatchDetails() const { + return _positional; + } + + private: + + /** Resets the state of the class associated with mods (not the error state) */ + void clear(); + + /** Create the modifier and add it to the back of the modifiers vector */ + inline Status addAndParse(const modifiertable::ModifierType type, + const BSONElement& elem); + + // + // immutable properties after parsing + // + + // Is there a list of $mod's on '_mods' or is it just full object replacement? + bool _replacementMode; + + // Collection of update mod instances. Owned here. + vector _mods; + + // What are the list of fields in the collection over which the update is going to be + // applied that participate in indices? + // + // NOTE: Owned by the collection's info cache!. + const UpdateIndexData* _indexedFields; + + // + // mutable properties after parsing + // + + // Should this driver generate an oplog record when it applies the update? + bool _logOp; + + // The options to initiate the mods with + ModifierInterface::Options _modOptions; + + // Are any of the fields mentioned in the mods participating in any index? Is set anew + // at each call to update. + bool _affectIndices; + + // Do any of the mods require positional match details when calling 'prepare'? + bool _positional; + + // Is this update going to be an upsert? + ModifierInterface::ExecInfo::UpdateContext _context; + + // The document used to represent or store the object being updated. + mutablebson::Document _objDoc; + + // The document used to build the oplog entry for the update. + mutablebson::Document _logDoc; + }; + + struct UpdateDriver::Options { + bool logOp; + ModifierInterface::Options modOptions; + + Options() : logOp(false), modOptions() {} + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/update_driver_test.cpp b/src/mongo/db/ops/update_driver_test.cpp new file mode 100644 index 00000000000..27f26d9dd3a --- /dev/null +++ b/src/mongo/db/ops/update_driver_test.cpp @@ -0,0 +1,173 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/update_driver.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/mutable_bson_test_utils.h" +#include "mongo/db/update_index_data.h" +#include "mongo/db/json.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::fromjson; + using mongo::UpdateIndexData; + using mongo::mutablebson::Document; + using mongo::StringData; + using mongo::UpdateDriver; + + TEST(Parse, Normal) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_OK(driver.parse(fromjson("{$set:{a:1}}"))); + ASSERT_EQUALS(driver.numMods(), 1U); + ASSERT_FALSE(driver.isDocReplacement()); + } + + TEST(Parse, MultiMods) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_OK(driver.parse(fromjson("{$set:{a:1, b:1}}"))); + ASSERT_EQUALS(driver.numMods(), 2U); + ASSERT_FALSE(driver.isDocReplacement()); + } + + TEST(Parse, MixingMods) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_OK(driver.parse(fromjson("{$set:{a:1}, $unset:{b:1}}"))); + ASSERT_EQUALS(driver.numMods(), 2U); + ASSERT_FALSE(driver.isDocReplacement()); + } + + TEST(Parse, ObjectReplacment) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_OK(driver.parse(fromjson("{obj: \"obj replacement\"}"))); + ASSERT_TRUE(driver.isDocReplacement()); + } + + TEST(Parse, EmptyMod) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_NOT_OK(driver.parse(fromjson("{$set:{}}"))); + } + + TEST(Parse, WrongMod) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_NOT_OK(driver.parse(fromjson("{$xyz:{a:1}}"))); + } + + TEST(Parse, WrongType) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_NOT_OK(driver.parse(fromjson("{$set:[{a:1}]}"))); + } + + TEST(Parse, ModsWithLaterObjReplacement) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_NOT_OK(driver.parse(fromjson("{$set:{a:1}, obj: \"obj replacement\"}"))); + } + + TEST(Parse, PushAll) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_OK(driver.parse(fromjson("{$pushAll:{a:[1,2,3]}}"))); + ASSERT_EQUALS(driver.numMods(), 1U); + ASSERT_FALSE(driver.isDocReplacement()); + } + + TEST(Parse, SetOnInsert) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + ASSERT_OK(driver.parse(fromjson("{$setOnInsert:{a:1}}"))); + ASSERT_EQUALS(driver.numMods(), 1U); + ASSERT_FALSE(driver.isDocReplacement()); + } + + + // Test the upsert case where we copy the query parts into the new doc + TEST(CreateFromQuery, Basic) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + Document doc; + + BSONObj query = fromjson("{a:1, b:1}"); + ASSERT_OK(driver.populateDocumentWithQueryFields(query, doc)); + ASSERT_EQUALS(query, doc); + } + + TEST(CreateFromQuery, BasicWithId) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + Document doc; + + BSONObj query = fromjson("{_id:1, a:1, b:1}"); + ASSERT_OK(driver.populateDocumentWithQueryFields(query, doc)); + ASSERT_EQUALS(query, doc); + } + + TEST(CreateFromQuery, NestedSharedRoot) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + Document doc; + + ASSERT_OK(driver.populateDocumentWithQueryFields(fromjson("{'a.c':1, 'a.b':1}"), doc)); + } + + TEST(CreateFromQuery, AllArrayDoesntHaveOrdinalName) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + Document doc; + + ASSERT_OK(driver.populateDocumentWithQueryFields(fromjson("{a:{$all:[1]}}"), doc)); + ASSERT_EQUALS(fromjson("{a:1}"), doc); + } + + // Failures + TEST(CreateFromQuery, DupFieldsFail) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + Document doc; + + ASSERT_NOT_OK(driver.populateDocumentWithQueryFields(fromjson("{a:1, 'a.b':1}"), doc)); + } + + TEST(CreateFromQuery, AllArrayMultipleVals) { + UpdateDriver::Options opts; + UpdateDriver driver(opts); + Document doc; + + ASSERT_NOT_OK(driver.populateDocumentWithQueryFields(fromjson("{a:{$all:[1, 2]}}"), doc)); + } +} // unnamed namespace diff --git a/src/mongo/db/ops/update_executor.cpp b/src/mongo/db/ops/update_executor.cpp new file mode 100644 index 00000000000..5ae9d7fbd24 --- /dev/null +++ b/src/mongo/db/ops/update_executor.cpp @@ -0,0 +1,123 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ops/update_executor.h" + +#include "mongo/db/ops/update.h" +#include "mongo/db/ops/update_request.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + UpdateExecutor::UpdateExecutor(const UpdateRequest* request, OpDebug* opDebug) : + _request(request), + _opDebug(opDebug), + _driver(UpdateDriver::Options()), + _canonicalQuery(), + _isQueryParsed(false), + _isUpdateParsed(false) { + } + + UpdateExecutor::~UpdateExecutor() {} + + Status UpdateExecutor::prepare() { + // We parse the update portion before the query portion because the dispostion of the update + // may determine whether or not we need to produce a CanonicalQuery at all. For example, if + // the update involves the positional-dollar operator, we must have a CanonicalQuery even if + // it isn't required for query execution. + Status status = parseUpdate(); + if (!status.isOK()) + return status; + status = parseQuery(); + if (!status.isOK()) + return status; + return Status::OK(); + } + + UpdateResult UpdateExecutor::execute() { + uassertStatusOK(prepare()); + return update(*_request, + _opDebug, + &_driver, + _canonicalQuery.release()); + } + + Status UpdateExecutor::parseQuery() { + if (_isQueryParsed) + return Status::OK(); + + dassert(!_canonicalQuery.get()); + dassert(_isUpdateParsed); + + if (!_driver.needMatchDetails() && CanonicalQuery::isSimpleIdQuery(_request->getQuery())) { + _isQueryParsed = true; + return Status::OK(); + } + + CanonicalQuery* cqRaw; + Status status = CanonicalQuery::canonicalize(_request->getNamespaceString().ns(), + _request->getQuery(), + &cqRaw); + if (status.isOK()) { + _canonicalQuery.reset(cqRaw); + _isQueryParsed = true; + } + else if (status == ErrorCodes::NoClientContext) { + // _isQueryParsed is still false, but execute() will try again under the lock. + return status = Status::OK(); + } + return status; + } + + Status UpdateExecutor::parseUpdate() { + if (_isUpdateParsed) + return Status::OK(); + + const NamespaceString& ns(_request->getNamespaceString()); + + // Should the modifiers validate their embedded docs via okForStorage + // Only user updates should be checked. Any system or replication stuff should pass through. + // Config db docs shouldn't get checked for valid field names since the shard key can have + // a dot (".") in it. + const bool shouldValidate = !(_request->isFromReplication() || + ns.isConfigDB() || + _request->isFromMigration()); + + _driver.setLogOp(true); + _driver.setModOptions(ModifierInterface::Options(_request->isFromReplication(), + shouldValidate)); + Status status = _driver.parse(_request->getUpdates(), _request->isMulti()); + if (status.isOK()) + _isUpdateParsed = true; + return status; + } + +} // namespace mongo diff --git a/src/mongo/db/ops/update_executor.h b/src/mongo/db/ops/update_executor.h new file mode 100644 index 00000000000..eabfbb18519 --- /dev/null +++ b/src/mongo/db/ops/update_executor.h @@ -0,0 +1,125 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/ops/update_driver.h" +#include "mongo/db/ops/update_result.h" + +namespace mongo { + + class CanonicalQuery; + class OpDebug; + class UpdateRequest; + + /** + * Implementation of the processing of an update operation in a mongod. + * + * The executor has two important methods, prepare() and execute(). The prepare() method can + * run without locks, and does whatever parsing and precomputation can be done without access to + * database data. The execute method performs the update, but the caller must already hold the + * appropriate database lock. + * + * Expected usage is approximately: + * UpdateRequest request(...); + * // configure request + * UpdateExecutor executor(&request, opDebug); + * uassertStatusOK(executor.prepare()); + * // Get locks, get ready to execute. + * try { + * UpdateResult res = executor.execute(); + * } + * catch (const DBException& ex) { + * // Error handling. + * } + */ + class UpdateExecutor { + MONGO_DISALLOW_COPYING(UpdateExecutor); + public: + /** + * Constructs an update executor. + * + * The objects pointed to by "request" and "opDebug" must stay in scope for the life of the + * constructed executor. + */ + UpdateExecutor(const UpdateRequest* request, OpDebug* opDebug); + + ~UpdateExecutor(); + + /** + * Performs preparatory work that does not require database locks. + * + * Returns Status::OK() on success. Other results indicate that the executor will not run + * correctly, and should be abandoned. + * + * Calling prepare() is optional. It is available for situations in which the user + * wishes to do as much work as possible before acquiring database locks. + */ + Status prepare(); + + /** + * Execute an update. Requires the caller to hold the database lock on the + * appropriate resources for the request. + */ + UpdateResult execute(); + + private: + /** + * Parses the query portion of the update request. + */ + Status parseQuery(); + + /** + * Parses the update-descriptor portion of the update request. + */ + Status parseUpdate(); + + /// Unowned pointer to the request object that this executor will process. + const UpdateRequest* const _request; + + /// Unowned pointer to the opdebug object that this executor will populate with debug data. + OpDebug* const _opDebug; + + /// Driver for processing updates on matched documents. + UpdateDriver _driver; + + /// Parsed query object, or NULL if the query proves to be an id hack query. + std::auto_ptr _canonicalQuery; + + /// Flag indicating if the query has been successfully parsed. + bool _isQueryParsed; + + /// Flag indicatin gif the update description has been successfully parsed. + bool _isUpdateParsed; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/update_internal.cpp b/src/mongo/db/ops/update_internal.cpp deleted file mode 100644 index 8046adfd492..00000000000 --- a/src/mongo/db/ops/update_internal.cpp +++ /dev/null @@ -1,1496 +0,0 @@ -//@file update_internal.cpp - -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "pch.h" - -#include // for max - -#include "mongo/db/field_ref.h" -#include "mongo/db/jsobjmanipulator.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/oplog.h" -#include "mongo/util/mongoutils/str.h" - -#include "update_internal.h" - -//#define DEBUGUPDATE(x) cout << x << endl; -#define DEBUGUPDATE(x) - -namespace mongo { - - const char* Mod::modNames[] = { "$inc", "$set", "$push", "$pushAll", "$pull", "$pullAll" , "$pop", "$unset" , - "$bitand" , "$bitor" , "$bit" , "$addToSet", "$rename", "$rename" , - "$setOnInsert" - }; - unsigned Mod::modNamesNum = sizeof(Mod::modNames)/sizeof(char*); - - bool Mod::_pullElementMatch( BSONElement& toMatch ) const { - - if ( elt.type() != Object ) { - // if elt isn't an object, then comparison will work - return toMatch.woCompare( elt , false ) == 0; - } - - if ( matcherOnPrimitive ) - return matcher->matches( toMatch.wrap( "" ) ); - - if ( toMatch.type() != Object ) { - // looking for an object, so this can't match - return false; - } - - // now we have an object on both sides - return matcher->matches( toMatch.embeddedObject() ); - } - - void Mod::appendIncremented( BSONBuilderBase& builder , const BSONElement& in, ModState& ms ) const { - BSONType a = in.type(); - BSONType b = elt.type(); - - if ( a == NumberDouble || b == NumberDouble ) { - ms.incType = NumberDouble; - ms.incdouble = elt.numberDouble() + in.numberDouble(); - } - else if ( a == NumberLong || b == NumberLong ) { - ms.incType = NumberLong; - ms.inclong = elt.numberLong() + in.numberLong(); - } - else { - int x = elt.numberInt() + in.numberInt(); - if ( x < 0 && elt.numberInt() > 0 && in.numberInt() > 0 ) { - // overflow - ms.incType = NumberLong; - ms.inclong = elt.numberLong() + in.numberLong(); - } - else { - ms.incType = NumberInt; - ms.incint = elt.numberInt() + in.numberInt(); - } - } - - ms.appendIncValue( builder , false ); - } - - void appendUnset( BSONBuilderBase& builder ) { - if ( builder.isArray() ) { - builder.appendNull(); - } - } - - void Mod::apply( BSONBuilderBase& builder , BSONElement in , ModState& ms ) const { - if ( ms.dontApply ) { - // Pass the original element through unchanged. - builder << in; - return; - } - - switch ( op ) { - - case INC: { - appendIncremented( builder , in , ms ); - // We don't need to "fix" this operation into a $set, for oplog purposes, - // here. ModState::appendForOpLog will do that for us. It relies on the new value - // being in inc{int,long,double} inside the ModState that wraps around this Mod. - break; - } - - case SET_ON_INSERT: - // There is a corner case that would land us here (making a change to an existing - // field with $setOnInsert). If we're in an upsert, and the query portion of the - // update creates a field, we can modify it with $setOnInsert. This degenerates - // into a $set, so we fall through to the next case. - ms.fixedOpName = "$set"; - // Fall through. - - case SET: { - _checkForAppending( elt ); - builder.appendAs( elt , shortFieldName ); - break; - } - - case UNSET: { - appendUnset( builder ); - break; - } - - case PUSH: { - uassert( 10131 , "$push can only be applied to an array" , in.type() == Array ); - - // - // We can be in a single element push case, a "push all" case, or a "push all" case - // with a slice requirement (ie, a "push to size"). In each of these, we decide - // differently how much of the existing- and of the parameter-array to copy to the - // final object. - // - - // Start the resulting array's builder. - BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); - - // If in the single element push case, we'll copy all elements of the existing - // array and add the new one. - if ( ! isEach() ) { - BSONObjIterator i( in.embeddedObject() ); - while ( i.more() ) { - bb.append( i.next() ); - } - bb.append( elt ); - - // We don't want to log a positional $set for which the '_checkForAppending' test - // won't pass. If we're in that case, fall back to non-optimized logging. - if ( (elt.type() == Object && elt.embeddedObject().okForStorage()) || - (elt.type() != Object) ) { - ms.fixedOpName = "$set"; - ms.forcePositional = true; - ms.position = bb.arrSize() - 1; - bb.done(); - } - else { - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( bb.done().getOwned() ); - } - } - - // If we're in the "push all" case, we'll copy all element of both the existing and - // parameter arrays. - else if ( isEach() && ! isSliceOnly() && ! isSliceAndSort() ) { - BSONObjIterator i( in.embeddedObject() ); - while ( i.more() ) { - bb.append( i.next() ); - } - BSONObjIterator j( getEach() ); - while ( j.more() ) { - bb.append( j.next() ); - } - - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( bb.done().getOwned() ); - } - - // If we're in the "push with a $each" case with slice, we have to decide how much - // of each of the existing and parameter arrays to copy to the final object. - else if ( isSliceOnly() ) { - long long slice = getSlice(); - BSONObj eachArray = getEach(); - long long arraySize = in.embeddedObject().nFields(); - long long eachArraySize = eachArray.nFields(); - - // Zero slice is equivalent to resetting the array in the final object, so - // we won't copy anything. - if (slice == 0) { - // no-op - } - - // If the parameter array alone is larger than the slice, then only copy - // object from that array. - else if (slice <= eachArraySize) { - long long skip = eachArraySize - slice; - BSONObjIterator j( getEach() ); - while ( j.more() ) { - if ( skip-- > 0 ) { - j.next(); - continue; - } - bb.append( j.next() ); - } - } - - // If the parameter array is not sufficient to fill the slice, then some (or all) - // the elements from the existing array will be copied too. - else { - long long skip = std::max(0LL, arraySize - (slice - eachArraySize) ); - BSONObjIterator i( in.embeddedObject() ); - while ( i.more() ) { - if (skip-- > 0) { - i.next(); - continue; - } - bb.append( i.next() ); - } - BSONObjIterator j( getEach() ); - while ( j.more() ) { - bb.append( j.next() ); - } - } - - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( bb.done().getOwned() ); - } - - // If we're in the "push all" case ($push with a $each) with sort, we have to - // concatenate the existing array with the $each array, sort the result, and then - // decide how much of each of the resulting work area to copy to the final object. - else { - long long slice = getSlice(); - - // Zero slice is equivalent to resetting the array in the final object, so - // we only go into sorting if there is anything to sort. - if ( slice > 0 ) { - vector workArea; - BSONObjIterator i( in.embeddedObject() ); - while ( i.more() ) { - workArea.push_back( i.next().Obj() ); - } - BSONObjIterator j( getEach() ); - while ( j.more() ) { - workArea.push_back( j.next().Obj() ); - } - ProjectKeyCmp cmp( getSort() ); - sort( workArea.begin(), workArea.end(), cmp ); - - long long skip = std::max( 0LL, - (long long)workArea.size() - slice ); - for ( vector::iterator it = workArea.begin(); - it != workArea.end(); - ++it ) { - if ( skip-- > 0 ) { - continue; - } - bb.append( *it ); - } - } - - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( bb.done().getOwned() ); - } - - break; - } - - case ADDTOSET: { - uassert( 12592 , "$addToSet can only be applied to an array" , in.type() == Array ); - BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); - BSONObjIterator i( in.embeddedObject() ); - - if ( isEach() ) { - - BSONElementSet toadd; - parseEach( toadd ); - - while ( i.more() ) { - BSONElement cur = i.next(); - bb.append( cur ); - toadd.erase( cur ); - } - - { - BSONObjIterator i( getEach() ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( toadd.count(e) ) { - bb.append( e ); - toadd.erase( e ); - } - } - } - - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(bb.done().getOwned()); - } - else { - - bool found = false; - int pos = 0; - int count = 0; - while ( i.more() ) { - BSONElement cur = i.next(); - bb.append( cur ); - if ( elt.woCompare( cur , false ) == 0 ) { - found = true; - pos = count; - } - count++; - } - - if ( !found ) { - bb.append( elt ); - } - - // We don't want to log a positional $set for which the '_checkForAppending' - // test won't pass. If we're in that case, fall back to non-optimized logging. - if ( (elt.type() == Object && elt.embeddedObject().okForStorage()) || - (elt.type() != Object) ) { - ms.fixedOpName = "$set"; - ms.forcePositional = true; - ms.position = found ? pos : bb.arrSize() - 1; - bb.done(); - } - else { - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(bb.done().getOwned()); - } - } - - break; - } - - case PUSH_ALL: { - uassert( 10132 , "$pushAll can only be applied to an array" , in.type() == Array ); - uassert( 10133 , "$pushAll has to be passed an array" , elt.type() ); - - BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); - - BSONObjIterator i( in.embeddedObject() ); - while ( i.more() ) { - bb.append( i.next() ); - } - - i = BSONObjIterator( elt.embeddedObject() ); - while ( i.more() ) { - bb.append( i.next() ); - } - - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(bb.done().getOwned()); - break; - } - - case PULL: - case PULL_ALL: { - uassert( 10134 , "$pull/$pullAll can only be applied to an array" , in.type() == Array ); - BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); - - //temporarily record the things to pull. only use this set while 'elt' in scope. - BSONElementSet toPull; - if ( op == PULL_ALL ) { - BSONObjIterator j( elt.embeddedObject() ); - while ( j.more() ) { - toPull.insert( j.next() ); - } - } - - BSONObjIterator i( in.embeddedObject() ); - while ( i.more() ) { - BSONElement e = i.next(); - bool allowed = true; - - if ( op == PULL ) { - allowed = ! _pullElementMatch( e ); - } - else { - allowed = ( toPull.find( e ) == toPull.end() ); - } - - if ( allowed ) - bb.append( e ); - } - - // If this is the last element of the array, then we want to write the empty array to the - // oplog. - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(bb.done().getOwned()); - break; - } - - case POP: { - uassert( 10135 , "$pop can only be applied to an array" , in.type() == Array ); - BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); - - - BSONObjIterator i( in.embeddedObject() ); - if ( elt.isNumber() && elt.number() < 0 ) { - // pop from front - if ( i.more() ) { - i.next(); - } - - while( i.more() ) { - bb.append( i.next() ); - } - } - else { - // pop from back - while( i.more() ) { - BSONElement arrI = i.next(); - if ( i.more() ) { - bb.append( arrI ); - } - } - } - - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(bb.done().getOwned()); - break; - } - - case BIT: { - uassert( 10136 , "$bit needs an object" , elt.type() == Object ); - uassert( 10137 , "$bit can only be applied to numbers" , in.isNumber() ); - uassert( 10138 , "$bit cannot update a value of type double" , in.type() != NumberDouble ); - - int x = in.numberInt(); - long long y = in.numberLong(); - - BSONObjIterator it( elt.embeddedObject() ); - while ( it.more() ) { - BSONElement e = it.next(); - uassert( 10139 , "$bit field must be number" , e.isNumber() ); - if ( str::equals(e.fieldName(), "and") ) { - switch( in.type() ) { - case NumberInt: x = x&e.numberInt(); break; - case NumberLong: y = y&e.numberLong(); break; - default: verify( 0 ); - } - } - else if ( str::equals(e.fieldName(), "or") ) { - switch( in.type() ) { - case NumberInt: x = x|e.numberInt(); break; - case NumberLong: y = y|e.numberLong(); break; - default: verify( 0 ); - } - } - else { - uasserted(9016, str::stream() << "unknown $bit operation: " << e.fieldName()); - } - } - - switch( in.type() ) { - - case NumberInt: - builder.append( shortFieldName , x ); - // By recording the result of the bit manipulation into the ModSet, we'll be - // set up so that this $bit operation be "fixed" as a $set of the final result - // in the oplog. This will happen in appendForOpLog and what triggers it is - // setting the incType in the ModSet that is around this Mod. - ms.incType = NumberInt; - ms.incint = x; - break; - - case NumberLong: - // Please see comment on fixing this $bit into a $set for logging purposes in - // the NumberInt case. - builder.append( shortFieldName , y ); - ms.incType = NumberLong; - ms.inclong = y; - break; - - default: verify( 0 ); - } - - break; - } - - case RENAME_FROM: { - // We don't need to "fix" this operation into a $set here. ModState::appendForOpLog - // will do that for us. It relies on the field name being stored on this Mod. - break; - } - - case RENAME_TO: { - // We don't need to "fix" this operation into a $set here, for the same reason we - // didn't either with RENAME_FROM. - ms.handleRename( builder, shortFieldName ); - break; - } - - default: - uasserted( 9017 , str::stream() << "Mod::apply can't handle type: " << op ); - } - } - - // -1 inside a non-object (non-object could be array) - // 0 missing - // 1 found - int validRenamePath( BSONObj obj, const char* path ) { - while( const char* p = strchr( path, '.' ) ) { - string left( path, p - path ); - BSONElement e = obj.getField( left ); - if ( e.eoo() ) { - return 0; - } - if ( e.type() != Object ) { - return -1; - } - obj = e.embeddedObject(); - path = p + 1; - } - return !obj.getField( path ).eoo(); - } - - auto_ptr ModSet::prepare(const BSONObj& obj, bool insertion) const { - DEBUGUPDATE( "\t start prepare" ); - auto_ptr mss( new ModSetState( obj, - _numIndexAlwaysUpdated, - _numIndexMaybeUpdated ) ); - - - // Perform this check first, so that we don't leave a partially modified object on uassert. - for ( ModHolder::const_iterator i = _mods.begin(); i != _mods.end(); ++i ) { - DEBUGUPDATE( "\t\t prepare : " << i->first ); - mss->_mods[i->first].reset( new ModState() ); - ModState& ms = *mss->_mods[i->first]; - - const Mod& m = i->second; - - // Check for any positional operators that have not been replaced with a numeric field - // name (from a query match element). - // Only perform this positional operator validation in 'strictApply' mode. When - // replicating from a legacy primary that does not implement this validation, the - // secondary bypasses validation and remains consistent with the primary. - if ( m.strictApply ) { - FieldRef fieldRef; - fieldRef.parse( m.fieldName ); - StringData positionalOpField( "$" ); - for( size_t i = 0; i < fieldRef.numParts(); ++i ) { - uassert( 16650, - "Cannot apply the positional operator without a corresponding query " - "field containing an array.", - fieldRef.getPart( i ).compare( positionalOpField ) != 0 ); - } - } - - BSONElement e = obj.getFieldDotted(m.fieldName); - - ms.m = &m; - ms.old = e; - - if ( m.op == Mod::RENAME_FROM ) { - int source = validRenamePath( obj, m.fieldName ); - uassert( 13489, "$rename source field invalid", source != -1 ); - if ( source != 1 ) { - ms.dontApply = true; - } - continue; - } - - if ( m.op == Mod::RENAME_TO ) { - int source = validRenamePath( obj, m.renameFrom() ); - if ( source == 1 ) { - int target = validRenamePath( obj, m.fieldName ); - uassert( 13490, "$rename target field invalid", target != -1 ); - ms.newVal = obj.getFieldDotted( m.renameFrom() ); - mss->amIInPlacePossible( false ); - } - else { - ms.dontApply = true; - } - continue; - } - - if ( m.op != Mod::SET_ON_INSERT && e.eoo() ) { - mss->amIInPlacePossible( m.op == Mod::UNSET ); - continue; - } - - switch( m.op ) { - case Mod::INC: - uassert( 10140 , "Cannot apply $inc modifier to non-number", e.isNumber() || e.eoo() ); - if ( mss->amIInPlacePossible( e.isNumber() ) ) { - // check more typing info here - if ( m.elt.type() != e.type() ) { - // if i'm incrementing with a double, then the storage has to be a double - mss->amIInPlacePossible( m.elt.type() != NumberDouble ); - } - - // check for overflow - if ( e.type() == NumberInt && e.numberLong() + m.elt.numberLong() > numeric_limits::max() ) { - mss->amIInPlacePossible( false ); - } - } - break; - - case Mod::SET: - mss->amIInPlacePossible( m.elt.type() == e.type() && - m.elt.valuesize() == e.valuesize() ); - break; - - case Mod::SET_ON_INSERT: - // If the document exist (i.e this is an update, not an insert) $setOnInsert - // becomes a no-op. - if ( !insertion ) { - ms.dontApply = true; - mss->amIInPlacePossible( true ); - } - else { - mss->amIInPlacePossible( false ); - } - break; - - case Mod::PUSH: - case Mod::PUSH_ALL: - uassert( 10141, - "Cannot apply $push/$pushAll modifier to non-array", - e.type() == Array || e.eoo() ); - - // Currently, we require the base array of a $sort to be made of - // objects (as opposed to base types). - if ( !e.eoo() && m.isEach() && m.isSliceAndSort() ) { - BSONObjIterator i( e.embeddedObject() ); - while ( i.more() ) { - BSONElement arrayItem = i.next(); - uassert( 16638, - "$sort can only be applied to an array of objects", - arrayItem.type() == Object ); - } - } - mss->amIInPlacePossible( false ); - break; - - case Mod::PULL: - case Mod::PULL_ALL: { - uassert( 10142, - "Cannot apply $pull/$pullAll modifier to non-array", - e.type() == Array || e.eoo() ); - - //temporarily record the things to pull. only use this set while 'm.elt' in scope. - BSONElementSet toPull; - if ( m.op == Mod::PULL_ALL ) { - BSONObjIterator j( m.elt.embeddedObject() ); - while ( j.more() ) { - toPull.insert( j.next() ); - } - } - - BSONObjIterator i( e.embeddedObject() ); - while( mss->_inPlacePossible && i.more() ) { - BSONElement arrI = i.next(); - if ( m.op == Mod::PULL ) { - mss->amIInPlacePossible( ! m._pullElementMatch( arrI ) ); - } - else if ( m.op == Mod::PULL_ALL ) { - mss->amIInPlacePossible( toPull.find( arrI ) == toPull.end() ); - } - } - break; - } - - case Mod::POP: { - uassert( 10143, - "Cannot apply $pop modifier to non-array", - e.type() == Array || e.eoo() ); - mss->amIInPlacePossible( e.embeddedObject().isEmpty() ); - break; - } - - case Mod::ADDTOSET: { - uassert( 12591, - "Cannot apply $addToSet modifier to non-array", - e.type() == Array || e.eoo() ); - - BSONObjIterator i( e.embeddedObject() ); - if ( m.isEach() ) { - BSONElementSet toadd; - m.parseEach( toadd ); - while( i.more() ) { - BSONElement arrI = i.next(); - toadd.erase( arrI ); - } - mss->amIInPlacePossible( toadd.size() == 0 ); - } - else { - bool found = false; - while( i.more() ) { - BSONElement arrI = i.next(); - if ( arrI.woCompare( m.elt , false ) == 0 ) { - found = true; - break; - } - } - mss->amIInPlacePossible( found ); - } - break; - } - - default: - // mods we don't know about shouldn't be done in place - mss->amIInPlacePossible( false ); - } - } - - DEBUGUPDATE( "\t mss\n" << mss->toString() << "\t--" ); - - return mss; - } - - const char* ModState::getOpLogName() const { - if ( dontApply ) { - return NULL; - } - - if ( incType ) { - return "$set"; - } - - if ( m->op == Mod::RENAME_FROM ) { - return "$unset"; - } - - if ( m->op == Mod::RENAME_TO ) { - return "$set"; - } - - return fixedOpName ? fixedOpName : Mod::modNames[op()]; - } - - - void ModState::appendForOpLog( BSONObjBuilder& bb ) const { - // dontApply logic is deprecated for all but $rename. - if ( dontApply ) { - return; - } - - if ( incType ) { - DEBUGUPDATE( "\t\t\t\t\t appendForOpLog inc fieldname: " << m->fieldName - << " short:" << m->shortFieldName ); - appendIncValue( bb , true ); - return; - } - - if ( m->op == Mod::RENAME_FROM ) { - DEBUGUPDATE( "\t\t\t\t\t appendForOpLog RENAME_FROM fieldName:" << m->fieldName ); - bb.append( m->fieldName, 1 ); - return; - } - - if ( m->op == Mod::RENAME_TO ) { - DEBUGUPDATE( "\t\t\t\t\t appendForOpLog RENAME_TO fieldName:" << m->fieldName ); - bb.appendAs( newVal, m->fieldName ); - return; - } - - const char* name = fixedOpName ? fixedOpName : Mod::modNames[op()]; - - DEBUGUPDATE( "\t\t\t\t\t appendForOpLog name:" << name << " fixed: " << fixed - << " fn: " << m->fieldName ); - - if (strcmp(name, "$unset") == 0) { - bb.append(m->fieldName, 1); - return; - } - - if ( fixed ) { - bb.appendAs( *fixed , m->fieldName ); - } - else if ( ! fixedArray.isEmpty() || forceEmptyArray ) { - bb.append( m->fieldName, fixedArray ); - } - else if ( forcePositional ) { - string positionalField = str::stream() << m->fieldName << "." << position; - bb.appendAs( m->elt, positionalField.c_str() ); - } - else { - bb.appendAs( m->elt , m->fieldName ); - } - - } - - typedef map > NamedModMap; - - BSONObj ModSetState::getOpLogRewrite() const { - NamedModMap names; - for ( ModStateHolder::const_iterator i = _mods.begin(); i != _mods.end(); ++i ) { - const char* name = i->second->getOpLogName(); - if ( ! name ) - continue; - names[name].push_back( i->second.get() ); - } - - BSONObjBuilder b; - for ( NamedModMap::const_iterator i = names.begin(); - i != names.end(); - ++i ) { - BSONObjBuilder bb( b.subobjStart( i->first ) ); - const vector& mods = i->second; - for ( unsigned j = 0; j < mods.size(); j++ ) { - mods[j]->appendForOpLog( bb ); - } - bb.doneFast(); - } - return b.obj(); - } - - string ModState::toString() const { - stringstream ss; - if ( fixedOpName ) - ss << " fixedOpName: " << fixedOpName; - if ( fixed ) - ss << " fixed: " << fixed; - return ss.str(); - } - - void ModState::handleRename( BSONBuilderBase& newObjBuilder, const char* shortFieldName ) { - newObjBuilder.appendAs( newVal , shortFieldName ); - BSONObjBuilder b; - b.appendAs( newVal, shortFieldName ); - verify( _objData.isEmpty() ); - _objData = b.obj(); - newVal = _objData.firstElement(); - } - - void ModSetState::applyModsInPlace( bool isOnDisk ) { - // TODO i think this assert means that we can get rid of the isOnDisk param - // and just use isOwned as the determination - DEV verify( isOnDisk == ! _obj.isOwned() ); - - for ( ModStateHolder::iterator i = _mods.begin(); i != _mods.end(); ++i ) { - ModState& m = *i->second; - - if ( m.dontApply ) { - continue; - } - - switch ( m.m->op ) { - case Mod::UNSET: - m.fixedOpName = "$unset"; - break; - - case Mod::ADDTOSET: - m.fixedOpName = "$set"; - m.fixed = &(m.old); - break; - - case Mod::RENAME_FROM: - case Mod::RENAME_TO: - // this should have been handled by prepare - break; - - case Mod::PULL: - case Mod::PULL_ALL: - // this should have been handled by prepare - m.fixedOpName = "$set"; - m.fixed = &(m.old); - break; - - case Mod::POP: - verify( m.old.isABSONObj() && m.old.Obj().isEmpty() ); - m.fixedOpName = "$set"; - m.fixed = &(m.old); - break; - // [dm] the BSONElementManipulator statements below are for replication (correct?) - - case Mod::INC: - if ( isOnDisk ) - m.m->IncrementMe( m.old ); - else - m.m->incrementMe( m.old ); - m.fixedOpName = "$set"; - m.fixed = &(m.old); - break; - - case Mod::SET: - if ( isOnDisk ) - BSONElementManipulator( m.old ).ReplaceTypeAndValue( m.m->elt ); - else - BSONElementManipulator( m.old ).replaceTypeAndValue( m.m->elt ); - break; - - case Mod::SET_ON_INSERT: - // this should have been handled by prepare - break; - - default: - uassert( 13478 , "can't apply mod in place - shouldn't have gotten here" , 0 ); - } - } - } - - void ModSetState::_appendNewFromMods( const string& root, - ModState& modState, - BSONBuilderBase& builder, - set& onedownseen ) { - Mod& m = *((Mod*)(modState.m)); // HACK - switch (m.op) { - // unset/pull/pullAll on nothing does nothing, so don't append anything. Still, - // explicitly log that the target array was reset. - case Mod::POP: - case Mod::PULL: - case Mod::PULL_ALL: - case Mod::UNSET: - modState.fixedOpName = "$unset"; - return; - - // $rename/$setOnInsert may involve dotted path creation, so we want to make sure we're - // not creating a path here for a rename that's a no-op. In other words if we're - // issuing a {$rename: {a.b : c.d} } that's a no-op, we don't want to create the a and - // c paths here. See test NestedNoName in the 'repl' suite. - case Mod::RENAME_FROM: - case Mod::RENAME_TO: - case Mod::SET_ON_INSERT: - if (modState.dontApply) { - return; - } - - default: - ;// fall through - } - const char* temp = modState.fieldName(); - temp += root.size(); - const char* dot = strchr( temp , '.' ); - if ( dot ) { - string nr( modState.fieldName() , 0 , 1 + ( dot - modState.fieldName() ) ); - string nf( temp , 0 , dot - temp ); - if ( onedownseen.count( nf ) ) - return; - onedownseen.insert( nf ); - BSONObjBuilder bb ( builder.subobjStart( nf ) ); - // Always insert an object, even if the field name is numeric. - createNewObjFromMods( nr , bb , BSONObj() ); - bb.done(); - } - else { - appendNewFromMod( modState , builder ); - } - } - - bool ModSetState::duplicateFieldName( const BSONElement& a, const BSONElement& b ) { - return - !a.eoo() && - !b.eoo() && - ( a.rawdata() != b.rawdata() ) && - str::equals( a.fieldName(), b.fieldName() ); - } - - ModSetState::ModStateRange ModSetState::modsForRoot( const string& root ) { - ModStateHolder::iterator mstart = _mods.lower_bound( root ); - StringBuilder buf; - buf << root << (char)255; - ModStateHolder::iterator mend = _mods.lower_bound( buf.str() ); - return make_pair( mstart, mend ); - } - - void ModSetState::createNewObjFromMods( const string& root, - BSONObjBuilder& builder, - const BSONObj& obj ) { - BSONObjIteratorSorted es( obj ); - createNewFromMods( root, builder, es, modsForRoot( root ), LexNumCmp( true ) ); - } - - void ModSetState::createNewArrayFromMods( const string& root, - BSONArrayBuilder& builder, - const BSONArray& arr ) { - BSONArrayIteratorSorted es( arr ); - ModStateRange objectOrderedRange = modsForRoot( root ); - ModStateHolder arrayOrderedMods( LexNumCmp( false ) ); - arrayOrderedMods.insert( objectOrderedRange.first, objectOrderedRange.second ); - ModStateRange arrayOrderedRange( arrayOrderedMods.begin(), arrayOrderedMods.end() ); - createNewFromMods( root, builder, es, arrayOrderedRange, LexNumCmp( false ) ); - } - - void ModSetState::createNewFromMods( const string& root, - BSONBuilderBase& builder, - BSONIteratorSorted& es, - const ModStateRange& modRange, - const LexNumCmp& lexNumCmp ) { - - DEBUGUPDATE( "\t\t createNewFromMods root: " << root ); - ModStateHolder::iterator m = modRange.first; - const ModStateHolder::const_iterator mend = modRange.second; - BSONElement e = es.next(); - - set onedownseen; - BSONElement prevE; - while ( !e.eoo() && m != mend ) { - - if ( duplicateFieldName( prevE, e ) ) { - // Just copy through an element with a duplicate field name. - builder.append( e ); - prevE = e; - e = es.next(); - continue; - } - prevE = e; - - string field = root + e.fieldName(); - FieldCompareResult cmp = compareDottedFieldNames( m->second->m->fieldName , field , - lexNumCmp ); - - DEBUGUPDATE( "\t\t\t field:" << field << "\t mod:" - << m->second->m->fieldName << "\t cmp:" << cmp - << "\t short: " << e.fieldName() ); - - switch ( cmp ) { - - case LEFT_SUBFIELD: { // Mod is embedded under this element - - // SERVER-4781 - bool isObjOrArr = e.type() == Object || e.type() == Array; - if ( ! isObjOrArr ) { - if (m->second->m->strictApply) { - uasserted( 10145, - str::stream() << "LEFT_SUBFIELD only supports Object: " << field - << " not: " << e.type() ); - } - else { - // Since we're not applying the mod, we keep what was there before - builder.append( e ); - - // Skip both as we're not applying this mod. Note that we'll advance - // the iterator on the mod side for all the mods that are under the - // root we are now. - e = es.next(); - m++; - while ( m != mend && - ( compareDottedFieldNames( m->second->m->fieldName, - field, - lexNumCmp ) == LEFT_SUBFIELD ) ) { - m++; - } - continue; - } - } - - if ( onedownseen.count( e.fieldName() ) == 0 ) { - onedownseen.insert( e.fieldName() ); - if ( e.type() == Object ) { - BSONObjBuilder bb( builder.subobjStart( e.fieldName() ) ); - stringstream nr; nr << root << e.fieldName() << "."; - createNewObjFromMods( nr.str() , bb , e.Obj() ); - bb.done(); - } - else { - BSONArrayBuilder ba( builder.subarrayStart( e.fieldName() ) ); - stringstream nr; nr << root << e.fieldName() << "."; - createNewArrayFromMods( nr.str() , ba , BSONArray( e.embeddedObject() ) ); - ba.done(); - } - // inc both as we handled both - e = es.next(); - m++; - while ( m != mend && - ( compareDottedFieldNames( m->second->m->fieldName , field , lexNumCmp ) == - LEFT_SUBFIELD ) ) { - m++; - } - } - else { - massert( 16069 , "ModSet::createNewFromMods - " - "SERVER-4777 unhandled duplicate field" , 0 ); - } - continue; - } - case LEFT_BEFORE: // Mod on a field that doesn't exist - DEBUGUPDATE( "\t\t\t\t creating new field for: " << m->second->m->fieldName ); - _appendNewFromMods( root , *m->second , builder , onedownseen ); - m++; - continue; - case SAME: - DEBUGUPDATE( "\t\t\t\t applying mod on: " << m->second->m->fieldName ); - m->second->apply( builder , e ); - e = es.next(); - m++; - continue; - case RIGHT_BEFORE: // field that doesn't have a MOD - DEBUGUPDATE( "\t\t\t\t just copying" ); - builder.append( e ); // if array, ignore field name - e = es.next(); - continue; - case RIGHT_SUBFIELD: - massert( 10399 , "ModSet::createNewFromMods - RIGHT_SUBFIELD should be impossible" , 0 ); - break; - default: - massert( 10400 , "unhandled case" , 0 ); - } - } - - // finished looping the mods, just adding the rest of the elements - while ( !e.eoo() ) { - DEBUGUPDATE( "\t\t\t copying: " << e.fieldName() ); - builder.append( e ); // if array, ignore field name - e = es.next(); - } - - // do mods that don't have fields already - for ( ; m != mend; m++ ) { - DEBUGUPDATE( "\t\t\t\t appending from mod at end: " << m->second->m->fieldName ); - _appendNewFromMods( root , *m->second , builder , onedownseen ); - } - } - - BSONObj ModSetState::createNewFromMods() { - BSONObjBuilder b( (int)(_obj.objsize() * 1.1) ); - createNewObjFromMods( "" , b , _obj ); - return _newFromMods = b.obj(); - } - - string ModSetState::toString() const { - stringstream ss; - for ( ModStateHolder::const_iterator i=_mods.begin(); i!=_mods.end(); ++i ) { - ss << "\t\t" << i->first << "\t" << i->second->toString() << "\n"; - } - return ss.str(); - } - - bool ModSetState::isUpdateIndexedSlow() const { - // There may be indices over fields for which Mods are no-ops. In other words, if a - // Mod touches an index field but that Mod is a no-op, this update may be - // considered one that does not update indices. - if ( _numIndexMaybeUpdated == 0 ) { - return false; - } - else { - for ( ModStateHolder::const_iterator it = _mods.begin(); - it != _mods.end(); - ++it ) { - const Mod* m = it->second->m; - shared_ptr ms = it->second; - - switch ( m->op ) { - case Mod::SET_ON_INSERT: - case Mod::RENAME_FROM: - case Mod::RENAME_TO: - if ( m->isIndexed && !ms->dontApply ) { - return true; - } - break; - - default: - // no-op - break; - } - } - - return false; - } - } - - - BSONObj ModSet::createNewFromQuery( const BSONObj& query ) { - BSONObj newObj; - - { - BSONObjBuilder bb; - EmbeddedBuilder eb( &bb ); - BSONObjIteratorSorted i( query ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( e.fieldName()[0] == '$' ) // for $atomic and anything else we add - continue; - - if ( e.type() == Object && e.embeddedObject().firstElementFieldName()[0] == '$' ) { - // we have something like { x : { $gt : 5 } } - // this can be a query piece - // or can be a dbref or something - - int op = e.embeddedObject().firstElement().getGtLtOp(); - if ( op > 0 ) { - // This means this is a $gt type filter, so don't make it part of the new - // object. - continue; - } - - if ( str::equals( e.embeddedObject().firstElement().fieldName(), "$not" ) ) { - // A $not filter operator is not detected in getGtLtOp() and should not - // become part of the new object. - continue; - } - } - - eb.appendAs( e , e.fieldName() ); - } - eb.done(); - newObj = bb.obj(); - } - - auto_ptr mss = prepare( newObj, true /* this is an insertion */ ); - - if ( mss->canApplyInPlace() ) - mss->applyModsInPlace( false ); - else - newObj = mss->createNewFromMods(); - - return newObj; - } - - /* get special operations like $inc - { $inc: { a:1, b:1 } } - { $set: { a:77 } } - { $push: { a:55 } } - { $pushAll: { a:[77,88] } } - { $pull: { a:66 } } - { $pullAll : { a:[99,1010] } } - NOTE: MODIFIES source from object! - */ - ModSet::ModSet( const BSONObj& from , - const IndexPathSet& idxKeys, - bool forReplication ) - : _numIndexMaybeUpdated( 0 ) - , _numIndexAlwaysUpdated( 0 ) - , _hasDynamicArray( false ) { - - BSONObjIterator it(from); - - while ( it.more() ) { - BSONElement e = it.next(); - const char* fn = e.fieldName(); - - uassert( 10147 , "Invalid modifier specified: " + string( fn ), e.type() == Object ); - BSONObj j = e.embeddedObject(); - DEBUGUPDATE( "\t" << j ); - - BSONObjIterator jt(j); - Mod::Op op = opFromStr( fn ); - - while ( jt.more() ) { - BSONElement f = jt.next(); // x:44 - - const char* fieldName = f.fieldName(); - - // Allow remove of invalid field name in case it was inserted before this check - // was added (~ version 2.1). - uassert( 15896, - "Modified field name may not start with $", - fieldName[0] != '$' || op == Mod::UNSET ); - uassert( 10148, - "Mod on _id not allowed", - strcmp( fieldName, "_id" ) != 0 ); - uassert( 10149, - "Invalid mod field name, may not end in a period", - fieldName[ strlen( fieldName ) - 1 ] != '.' ); - uassert( 10150, - "Field name duplication not allowed with modifiers", - ! haveModForField( fieldName ) ); - uassert( 10151, - "have conflicting mods in update", - ! haveConflictingMod( fieldName ) ); - uassert( 10152, - "Modifier $inc allowed for numbers only", - f.isNumber() || op != Mod::INC ); - uassert( 10153, - "Modifier $pushAll/pullAll allowed for arrays only", - f.type() == Array || ( op != Mod::PUSH_ALL && op != Mod::PULL_ALL ) ); - - // Check whether $each, $slice, and $sort syntax for $push is correct. - if ( ( op == Mod::PUSH ) && ( f.type() == Object ) ) { - BSONObj pushObj = f.embeddedObject(); - if ( pushObj.nFields() > 0 && - strcmp(pushObj.firstElement().fieldName(), "$each") == 0 ) { - uassert( 16564, - "$each term needs to occur alone (or with $slice/$sort)", - pushObj.nFields() <= 3 ); - uassert( 16565, - "$each requires an array value", - pushObj.firstElement().type() == Array ); - - // If both $slice and $sort are present, they may be switched. - if ( pushObj.nFields() > 1 ) { - BSONObjIterator i( pushObj ); - i.next(); - - bool seenSlice = false; - bool seenSort = false; - while ( i.more() ) { - BSONElement nextElem = i.next(); - - if ( str::equals( nextElem.fieldName(), "$slice" ) ) { - uassert( 16567, "$slice appeared twice", !seenSlice); - seenSlice = true; - uassert( 16568, - "$slice value must be a numeric integer", - nextElem.type() == NumberInt || - nextElem.type() == NumberLong || - (nextElem.type() == NumberDouble && - nextElem.numberDouble() == - (long long)nextElem.numberDouble() ) ); - uassert( 16640, - "$slice value must be negative or zero", - nextElem.number() <= 0 ); - } - else if ( str::equals( nextElem.fieldName(), "$sort" ) ) { - uassert( 16647, "$sort appeared twice", !seenSort ); - seenSort = true; - uassert( 16648, - "$sort component of $push must be an object", - nextElem.type() == Object ); - - BSONObjIterator j( nextElem.embeddedObject() ); - while ( j.more() ) { - BSONElement fieldSortElem = j.next(); - uassert( 16641, - "$sort elements' values must either 1 or -1", - ( fieldSortElem.type() == NumberInt || - fieldSortElem.type() == NumberLong || - ( fieldSortElem.type() == NumberDouble && - fieldSortElem.numberDouble() == - (long long) fieldSortElem.numberDouble() ) ) && - ( fieldSortElem.Number() == 1 || - fieldSortElem.Number() == -1 ) ); - - FieldRef sortField; - sortField.parse( fieldSortElem.fieldName() ); - uassert( 16690, - "$sort field cannot be empty", - sortField.numParts() > 0 ); - - for ( size_t i = 0; i < sortField.numParts(); i++ ) { - uassert( 16691, - "empty field in dotted sort pattern", - sortField.getPart( i ).size() > 0 ); - } - } - - // Finally, check if the $each is made of objects (as opposed - // to basic types). Currently, $sort only supports operating - // on arrays of objects. - BSONObj eachArray = pushObj.firstElement().embeddedObject(); - BSONObjIterator k( eachArray ); - while ( k.more() ) { - BSONElement eachItem = k.next(); - uassert( 16642, - "$sort requires $each to be an array of objects", - eachItem.type() == Object ); - } - - } - else { - uasserted( 16643, - "$each term takes only $slice (and optionally " - "$sort) as complements" ); - } - } - - uassert( 16644, "cannot have a $sort without a $slice", seenSlice ); - } - } - } - - if ( op == Mod::RENAME_TO ) { - uassert( 13494, "$rename target must be a string", f.type() == String ); - const char* target = f.valuestr(); - uassert( 13495, - "$rename source must differ from target", - strcmp( fieldName, target ) != 0 ); - uassert( 13496, - "invalid mod field name, source may not be empty", - fieldName[0] ); - uassert( 13479, - "invalid mod field name, target may not be empty", - target[0] ); - uassert( 13480, - "invalid mod field name, source may not begin or end in period", - fieldName[0] != '.' && fieldName[ strlen( fieldName ) - 1 ] != '.' ); - uassert( 13481, - "invalid mod field name, target may not begin or end in period", - target[0] != '.' && target[ strlen( target ) - 1 ] != '.' ); - uassert( 13482, - "$rename affecting _id not allowed", - !( fieldName[0] == '_' && fieldName[1] == 'i' && fieldName[2] == 'd' - && ( !fieldName[3] || fieldName[3] == '.' ) ) ); - uassert( 13483, - "$rename affecting _id not allowed", - !( target[0] == '_' && target[1] == 'i' && target[2] == 'd' - && ( !target[3] || target[3] == '.' ) ) ); - uassert( 13484, - "field name duplication not allowed with $rename target", - !haveModForField( target ) ); - uassert( 13485, - "conflicting mods not allowed with $rename target", - !haveConflictingMod( target ) ); - uassert( 13486, - "$rename target may not be a parent of source", - !( strncmp( fieldName, target, strlen( target ) ) == 0 - && fieldName[ strlen( target ) ] == '.' ) ); - uassert( 13487, - "$rename source may not be dynamic array", - strstr( fieldName , ".$" ) == 0 ); - uassert( 13488, - "$rename target may not be dynamic array", - strstr( target , ".$" ) == 0 ); - - Mod from; - from.init( Mod::RENAME_FROM, f , forReplication ); - from.setFieldName( fieldName ); - setIndexedStatus( from, idxKeys ); - _mods[ from.fieldName ] = from; - - Mod to; - to.init( Mod::RENAME_TO, f , forReplication ); - to.setFieldName( target ); - setIndexedStatus( to, idxKeys ); - _mods[ to.fieldName ] = to; - - DEBUGUPDATE( "\t\t " << fieldName << "\t" << from.fieldName << "\t" << to.fieldName ); - continue; - } - - _hasDynamicArray = _hasDynamicArray || strstr( fieldName , ".$" ) > 0; - - Mod m; - m.init( op , f , forReplication ); - m.setFieldName( f.fieldName() ); - setIndexedStatus( m, idxKeys ); - _mods[m.fieldName] = m; - - DEBUGUPDATE( "\t\t " << fieldName << "\t" << m.fieldName << "\t" << _hasDynamicArray ); - } - } - - } - - ModSet* ModSet::fixDynamicArray( const string& elemMatchKey ) const { - ModSet* n = new ModSet(); - n->_numIndexMaybeUpdated = _numIndexMaybeUpdated; - n->_numIndexAlwaysUpdated = _numIndexAlwaysUpdated; - n->_hasDynamicArray = _hasDynamicArray; - for ( ModHolder::const_iterator i=_mods.begin(); i!=_mods.end(); i++ ) { - string s = i->first; - size_t idx = s.find( ".$" ); - if ( idx == string::npos ) { - n->_mods[s] = i->second; - continue; - } - StringBuilder buf; - buf << s.substr(0,idx+1) << elemMatchKey << s.substr(idx+2); - string fixed = buf.str(); - DEBUGUPDATE( "fixed dynamic: " << s << " -->> " << fixed ); - n->_mods[fixed] = i->second; - ModHolder::iterator temp = n->_mods.find( fixed ); - temp->second.setFieldName( temp->first.c_str() ); - } - return n; - } - - void ModSet::setIndexedStatus( const IndexPathSet& idxKeys ) { - for ( ModHolder::iterator i = _mods.begin(); i != _mods.end(); ++i ) - setIndexedStatus( i->second, idxKeys ); - } - - void ModSet::setIndexedStatus( Mod& m, const IndexPathSet& idxKeys ) { - if ( idxKeys.mightBeIndexed( m.fieldName ) ) { - m.isIndexed = true; - - // Some mods may be no-ops depending on the document they are applied - // on. Determining how many indices will actually be used can only be - // determined for sure after looking at that target document. - switch ( m.op ) { - - case Mod::SET_ON_INSERT: - case Mod::RENAME_FROM: - case Mod::RENAME_TO: - _numIndexMaybeUpdated++; - break; - - default: - _numIndexAlwaysUpdated++; - - } - } - else { - m.isIndexed = false; - } - } - - -} // namespace mongo diff --git a/src/mongo/db/ops/update_internal.h b/src/mongo/db/ops/update_internal.h deleted file mode 100644 index c8397bf9a49..00000000000 --- a/src/mongo/db/ops/update_internal.h +++ /dev/null @@ -1,780 +0,0 @@ -//@file update_internal.h - -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "mongo/pch.h" - -#include "mongo/bson/bson_builder_base.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/jsobjmanipulator.h" -#include "mongo/db/matcher.h" -#include "mongo/util/embedded_builder.h" -#include "mongo/util/stringutils.h" - -namespace mongo { - - class ModState; - class ModSetState; - - /* Used for modifiers such as $inc, $set, $push, ... - * stores the info about a single operation - * once created should never be modified - */ - struct Mod { - // See opFromStr below - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 - enum Op { INC, SET, PUSH, PUSH_ALL, PULL, PULL_ALL , POP, UNSET, BITAND, BITOR , BIT , ADDTOSET, RENAME_FROM, RENAME_TO, SET_ON_INSERT } op; - - static const char* modNames[]; - static unsigned modNamesNum; - - const char* fieldName; - const char* shortFieldName; - - // Determines if this mod must absoluetly be applied. In some replication scenarios, a - // failed apply of a mod does not constitute an error. In those cases, setting strict - // to off would not throw errors. - bool strictApply; - - // Determines if an index is going to be updated as part of the application of this - // mod. - bool isIndexed; - - BSONElement elt; // x:5 note: this is the actual element from the updateobj - boost::shared_ptr matcher; - bool matcherOnPrimitive; - - void init( Op o , BSONElement& e , bool forReplication ) { - op = o; - elt = e; - strictApply = !forReplication; - isIndexed = false; - if ( op == PULL && e.type() == Object ) { - BSONObj t = e.embeddedObject(); - if ( t.firstElement().getGtLtOp() == 0 ) { - matcher.reset( new Matcher( t ) ); - matcherOnPrimitive = false; - } - else { - matcher.reset( new Matcher( BSON( "" << t ) ) ); - matcherOnPrimitive = true; - } - } - } - - void setFieldName( const char* s ) { - fieldName = s; - shortFieldName = strrchr( fieldName , '.' ); - if ( shortFieldName ) - shortFieldName++; - else - shortFieldName = fieldName; - } - - /** - * @param in increments the actual value inside in - */ - void incrementMe( BSONElement& in ) const { - BSONElementManipulator manip( in ); - switch ( in.type() ) { - case NumberDouble: - manip.setNumber( elt.numberDouble() + in.numberDouble() ); - break; - case NumberLong: - manip.setLong( elt.numberLong() + in.numberLong() ); - break; - case NumberInt: - manip.setInt( elt.numberInt() + in.numberInt() ); - break; - default: - verify(0); - } - } - void IncrementMe( BSONElement& in ) const { - BSONElementManipulator manip( in ); - switch ( in.type() ) { - case NumberDouble: - manip.SetNumber( elt.numberDouble() + in.numberDouble() ); - break; - case NumberLong: - manip.SetLong( elt.numberLong() + in.numberLong() ); - break; - case NumberInt: - manip.SetInt( elt.numberInt() + in.numberInt() ); - break; - default: - verify(0); - } - } - - void appendIncremented( BSONBuilderBase& bb , const BSONElement& in, ModState& ms ) const; - - bool operator<( const Mod& other ) const { - return strcmp( fieldName, other.fieldName ) < 0; - } - - bool arrayDep() const { - switch (op) { - case PUSH: - case PUSH_ALL: - case POP: - return true; - default: - return false; - } - } - - void apply( BSONBuilderBase& b , BSONElement in , ModState& ms ) const; - - /** - * @return true iff toMatch should be removed from the array - */ - bool _pullElementMatch( BSONElement& toMatch ) const; - - void _checkForAppending( const BSONElement& e ) const { - if ( e.type() == Object ) { - // this is a tiny bit slow, but rare and important - // only when setting something TO an object, not setting something in an object - // and it checks for { $set : { x : { 'a.b' : 1 } } } - // which is feel has been common - uassert( 12527 , "not okForStorage" , e.embeddedObject().okForStorage() ); - } - } - - bool isEach() const { - if ( elt.type() != Object ) - return false; - BSONElement e = elt.embeddedObject().firstElement(); - if ( e.type() != Array ) - return false; - return strcmp( e.fieldName() , "$each" ) == 0; - } - - BSONObj getEach() const { - return elt.embeddedObjectUserCheck().firstElement().embeddedObjectUserCheck(); - } - - void parseEach( BSONElementSet& s ) const { - BSONObjIterator i(getEach()); - while ( i.more() ) { - s.insert( i.next() ); - } - } - - bool isSliceOnly() const { - if ( elt.type() != Object ) - return false; - BSONObj obj = elt.embeddedObject(); - if ( obj.nFields() != 2 ) - return false; - BSONObjIterator i( obj ); - i.next(); - BSONElement elemSlice = i.next(); - return strcmp( elemSlice.fieldName(), "$slice" ) == 0; - } - - long long getSlice() const { - // The $slice may be the second or the third element in the field object. - // { : { $each: [], $slice: -N, $sort: } } - // 'elt' here is the BSONElement above. - BSONObj obj = elt.embeddedObject(); - BSONObjIterator i( obj ); - i.next(); - BSONElement elem = i.next(); - if ( ! str::equals( elem.fieldName(), "$slice" ) ) { - elem = i.next(); - } - dassert( elem.isNumber() ); - - // For now, we're only supporting slicing from the back of the array, i.e. - // negative slice. But the code now is wired in the opposite way: trimming from the - // back of the array is positive. - // TODO: fix this. - return -elem.numberLong(); - } - - bool isSliceAndSort() const { - if ( elt.type() != Object ) - return false; - BSONObj obj = elt.embeddedObject(); - if ( obj.nFields() != 3 ) - return false; - BSONObjIterator i( obj ); - i.next(); - - // Slice and sort may be switched. - bool seenSlice = false; - bool seenSort = false; - while ( i.more() ) { - BSONElement elem = i.next(); - if ( str::equals( elem.fieldName(), "$slice" ) ) { - if ( seenSlice ) return false; - seenSlice = true; - } - else if ( str::equals( elem.fieldName(), "$sort" ) ) { - if ( seenSort ) return false; - seenSort = true; - if ( elem.type() != Object ) return false; - } - else { - return false; - } - } - - // If present, the $sort element would have been checked during ModSet construction. - return seenSlice && seenSort; - } - - BSONObj getSort() const { - // The $sort may be the second or the third element in the field object. - // { : { $each: [], $slice: -N, $sort: } } - // 'elt' here is the BSONElement above. - BSONObj obj = elt.embeddedObject(); - BSONObjIterator i( obj ); - i.next(); - BSONElement elem = i.next(); - if ( ! str::equals( elem.fieldName(), "$sort" ) ) { - elem = i.next(); - } - return elem.embeddedObject(); - } - - const char* renameFrom() const { - massert( 13492, "mod must be RENAME_TO type", op == Mod::RENAME_TO ); - return elt.fieldName(); - } - }; - - /** - * stores a set of Mods - * once created, should never be changed - */ - class ModSet : boost::noncopyable { - typedef map ModHolder; - ModHolder _mods; - int _numIndexMaybeUpdated; - int _numIndexAlwaysUpdated; - bool _hasDynamicArray; - - static Mod::Op opFromStr( const char* fn ) { - verify( fn[0] == '$' ); - switch( fn[1] ) { - case 'i': { - if ( fn[2] == 'n' && fn[3] == 'c' && fn[4] == 0 ) - return Mod::INC; - break; - } - case 's': { - if ( fn[2] == 'e' && fn[3] == 't' ) { - if ( fn[4] == 0 ) { - return Mod::SET; - } - else if ( fn[4] == 'O' && fn[5] == 'n' && fn[6] == 'I' && fn[7] == 'n' && - fn[8] == 's' && fn[9] == 'e' && fn[10] == 'r' && fn[11] == 't' && - fn[12] == 0 ) { - return Mod::SET_ON_INSERT; - } - } - break; - } - case 'p': { - if ( fn[2] == 'u' ) { - if ( fn[3] == 's' && fn[4] == 'h' ) { - if ( fn[5] == 0 ) - return Mod::PUSH; - if ( fn[5] == 'A' && fn[6] == 'l' && fn[7] == 'l' && fn[8] == 0 ) - return Mod::PUSH_ALL; - } - else if ( fn[3] == 'l' && fn[4] == 'l' ) { - if ( fn[5] == 0 ) - return Mod::PULL; - if ( fn[5] == 'A' && fn[6] == 'l' && fn[7] == 'l' && fn[8] == 0 ) - return Mod::PULL_ALL; - } - } - else if ( fn[2] == 'o' && fn[3] == 'p' && fn[4] == 0 ) - return Mod::POP; - break; - } - case 'u': { - if ( fn[2] == 'n' && fn[3] == 's' && fn[4] == 'e' && fn[5] == 't' && fn[6] == 0 ) - return Mod::UNSET; - break; - } - case 'b': { - if ( fn[2] == 'i' && fn[3] == 't' ) { - if ( fn[4] == 0 ) - return Mod::BIT; - if ( fn[4] == 'a' && fn[5] == 'n' && fn[6] == 'd' && fn[7] == 0 ) - return Mod::BITAND; - if ( fn[4] == 'o' && fn[5] == 'r' && fn[6] == 0 ) - return Mod::BITOR; - } - break; - } - case 'a': { - if ( fn[2] == 'd' && fn[3] == 'd' ) { - // add - if ( fn[4] == 'T' && fn[5] == 'o' && fn[6] == 'S' && fn[7] == 'e' && fn[8] == 't' && fn[9] == 0 ) - return Mod::ADDTOSET; - - } - break; - } - case 'r': { - if ( fn[2] == 'e' && fn[3] == 'n' && fn[4] == 'a' && fn[5] == 'm' && fn[6] =='e' ) { - return Mod::RENAME_TO; // with this return code we handle both RENAME_TO and RENAME_FROM - } - break; - } - default: break; - } - uassert( 10161 , "Invalid modifier specified " + string( fn ), false ); - return Mod::INC; - } - - ModSet() {} - - /** - * if if applying this mod would require updating an index, set such condition in 'm', - * and update the number of indices touched in 'this' ModSet. - */ - void setIndexedStatus( Mod& m, const IndexPathSet& idxKeys ); - - public: - - ModSet( const BSONObj& from, - const IndexPathSet& idxKeys = IndexPathSet(), - bool forReplication = false ); - - /** - * re-check if this mod is impacted by indexes - */ - void setIndexedStatus( const IndexPathSet& idxKeys ); - - // TODO: this is inefficient - should probably just handle when iterating - ModSet * fixDynamicArray( const string& elemMatchKey ) const; - - bool hasDynamicArray() const { return _hasDynamicArray; } - - /** - * creates a ModSetState suitable for operation on obj - * doesn't change or modify this ModSet or any underlying Mod - * - * flag 'insertion' differentiates between obj existing prior to this update. - */ - auto_ptr prepare( const BSONObj& obj, bool insertion = false ) const; - - /** - * given a query pattern, builds an object suitable for an upsert - * will take the query spec and combine all $ operators - */ - BSONObj createNewFromQuery( const BSONObj& query ); - - int maxNumIndexUpdated() const { return _numIndexMaybeUpdated + _numIndexAlwaysUpdated; } - - unsigned size() const { return _mods.size(); } - - bool haveModForField( const char* fieldName ) const { - return _mods.find( fieldName ) != _mods.end(); - } - - bool haveConflictingMod( const string& fieldName ) { - size_t idx = fieldName.find( '.' ); - if ( idx == string::npos ) - idx = fieldName.size(); - - ModHolder::const_iterator start = _mods.lower_bound(fieldName.substr(0,idx)); - for ( ; start != _mods.end(); start++ ) { - FieldCompareResult r = compareDottedFieldNames( fieldName , start->first , - LexNumCmp( true ) ); - switch ( r ) { - case LEFT_SUBFIELD: return true; - case LEFT_BEFORE: return false; - case SAME: return true; - case RIGHT_BEFORE: return false; - case RIGHT_SUBFIELD: return true; - } - } - return false; - } - - }; - - /** - * Comparator between two BSONObjects that takes in consideration only the keys and - * direction described in the sort pattern. - */ - struct ProjectKeyCmp { - BSONObj sortPattern; - - ProjectKeyCmp( BSONObj pattern ) : sortPattern( pattern) {} - - int operator()( const BSONObj& left, const BSONObj& right ) const { - BSONObj keyLeft = left.extractFields( sortPattern, true ); - BSONObj keyRight = right.extractFields( sortPattern, true ); - return keyLeft.woCompare( keyRight, sortPattern ) < 0; - } - }; - - /** - * stores any information about a single Mod operating on a single Object - */ - class ModState : boost::noncopyable { - public: - const Mod* m; - BSONElement old; - BSONElement newVal; - BSONObj _objData; - - const char* fixedOpName; - BSONElement* fixed; - BSONArray fixedArray; - bool forceEmptyArray; - bool forcePositional; - int position; - int DEPRECATED_pushStartSize; - - BSONType incType; - int incint; - double incdouble; - long long inclong; - - bool dontApply; - - ModState() { - fixedOpName = 0; - fixed = 0; - forceEmptyArray = false; - forcePositional = false; - position = 0; - DEPRECATED_pushStartSize = -1; - incType = EOO; - dontApply = false; - } - - Mod::Op op() const { - return m->op; - } - - const char* fieldName() const { - return m->fieldName; - } - - bool DEPRECATED_needOpLogRewrite() const { - if ( dontApply ) - return false; - - if ( fixed || fixedOpName || incType ) - return true; - - switch( op() ) { - case Mod::RENAME_FROM: - case Mod::RENAME_TO: - return true; - case Mod::BIT: - case Mod::BITAND: - case Mod::BITOR: - return true; - default: - return false; - } - } - - const char* getOpLogName() const; - void appendForOpLog( BSONObjBuilder& b ) const; - - void apply( BSONBuilderBase& b , BSONElement in ) { - m->apply( b , in , *this ); - } - - void appendIncValue( BSONBuilderBase& b , bool useFullName ) const { - const char* n = useFullName ? m->fieldName : m->shortFieldName; - - switch ( incType ) { - case NumberDouble: - b.append( n , incdouble ); break; - case NumberLong: - b.append( n , inclong ); break; - case NumberInt: - b.append( n , incint ); break; - default: - verify(0); - } - } - - string toString() const; - - void handleRename( BSONBuilderBase& newObjBuilder, const char* shortFieldName ); - }; - - /** - * this is used to hold state, meta data while applying a ModSet to a BSONObj - * the goal is to make ModSet const so its re-usable - */ - class ModSetState : boost::noncopyable { - typedef map,LexNumCmp> ModStateHolder; - typedef pair ModStateRange; - const BSONObj& _obj; - ModStateHolder _mods; - bool _inPlacePossible; - BSONObj _newFromMods; // keep this data alive, as oplog generation may depend on it - int _numIndexAlwaysUpdated; - int _numIndexMaybeUpdated; - - ModSetState( const BSONObj& obj , int numIndexAlwaysUpdated , int numIndexMaybeUpdated ) - : _obj( obj ) - , _mods( LexNumCmp( true ) ) - , _inPlacePossible(true) - , _numIndexAlwaysUpdated( numIndexAlwaysUpdated ) - , _numIndexMaybeUpdated( numIndexMaybeUpdated ) { - } - - /** - * @return if in place is still possible - */ - bool amIInPlacePossible( bool inPlacePossible ) { - if ( ! inPlacePossible ) - _inPlacePossible = false; - return _inPlacePossible; - } - - ModStateRange modsForRoot( const string& root ); - - void createNewObjFromMods( const string& root, BSONObjBuilder& b, const BSONObj& obj ); - void createNewArrayFromMods( const string& root, BSONArrayBuilder& b, - const BSONArray& arr ); - - void createNewFromMods( const string& root , BSONBuilderBase& b , BSONIteratorSorted& es , - const ModStateRange& modRange , const LexNumCmp& lexNumCmp ); - - void _appendNewFromMods( const string& root , ModState& m , BSONBuilderBase& b , set& onedownseen ); - - void appendNewFromMod( ModState& ms , BSONBuilderBase& b ) { - if ( ms.dontApply ) { - return; - } - - //const Mod& m = *(ms.m); // HACK - Mod& m = *((Mod*)(ms.m)); // HACK - - switch ( m.op ) { - - case Mod::PUSH: { - ms.fixedOpName = "$set"; - if ( m.isEach() ) { - BSONObj arr = m.getEach(); - if ( !m.isSliceOnly() && !m.isSliceAndSort() ) { - b.appendArray( m.shortFieldName, arr ); - - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( arr.getOwned() ); - } - else if ( m.isSliceOnly() && ( m.getSlice() >= arr.nFields() ) ) { - b.appendArray( m.shortFieldName, arr ); - - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( arr.getOwned() ); - } - else if ( m.isSliceOnly() ) { - BSONArrayBuilder arrBuilder( b.subarrayStart( m.shortFieldName ) ); - long long skip = arr.nFields() - m.getSlice(); - BSONObjIterator j( arr ); - while ( j.more() ) { - if ( skip-- > 0 ) { - j.next(); - continue; - } - arrBuilder.append( j.next() ); - } - - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( arrBuilder.done().getOwned() ); - } - else if ( m.isSliceAndSort() ) { - long long slice = m.getSlice(); - - // Sort the $each array over sortPattern. - vector workArea; - BSONObjIterator j( arr ); - while ( j.more() ) { - workArea.push_back( j.next().Obj() ); - } - ProjectKeyCmp cmp( m.getSort() ); - sort( workArea.begin(), workArea.end(), cmp ); - - // Slice to the appropriate size. If slice is zero, that's equivalent - // to resetting the array, ie, a no-op. - BSONArrayBuilder arrBuilder( b.subarrayStart( m.shortFieldName ) ); - if (slice > 0) { - long long skip = std::max( 0LL, - (long long)workArea.size() - slice ); - for (vector::iterator it = workArea.begin(); - it != workArea.end(); - ++it ) { - if ( skip-- > 0 ) { - continue; - } - arrBuilder.append( *it ); - } - } - - // Log the full resulting array. - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray( arrBuilder.done().getOwned() ); - } - } - else { - BSONObjBuilder arr( b.subarrayStart( m.shortFieldName ) ); - arr.appendAs( m.elt, "0" ); - - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(arr.done().getOwned()); - } - break; - } - - case Mod::ADDTOSET: { - ms.fixedOpName = "$set"; - if ( m.isEach() ) { - // Remove any duplicates in given array - BSONArrayBuilder arr( b.subarrayStart( m.shortFieldName ) ); - BSONElementSet toadd; - m.parseEach( toadd ); - BSONObjIterator i( m.getEach() ); - // int n = 0; - while ( i.more() ) { - BSONElement e = i.next(); - if ( toadd.count(e) ) { - arr.append( e ); - toadd.erase( e ); - } - } - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(arr.done().getOwned()); - } - else { - BSONArrayBuilder arr( b.subarrayStart( m.shortFieldName ) ); - arr.append( m.elt ); - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(arr.done().getOwned()); - } - break; - } - - case Mod::PUSH_ALL: { - b.appendAs( m.elt, m.shortFieldName ); - ms.fixedOpName = "$set"; - ms.forceEmptyArray = true; - ms.fixedArray = BSONArray(m.elt.Obj()); - break; - } - - case Mod::POP: - case Mod::PULL: - case Mod::PULL_ALL: - case Mod::UNSET: - // No-op b/c unset/pull of nothing does nothing. Still, explicilty log that - // the target array was reset. - ms.fixedOpName = "$unset"; - break; - - case Mod::INC: - case Mod::SET_ON_INSERT: - ms.fixedOpName = "$set"; - case Mod::SET: { - m._checkForAppending( m.elt ); - b.appendAs( m.elt, m.shortFieldName ); - break; - } - - // shouldn't see RENAME_FROM here - case Mod::RENAME_TO: - ms.handleRename( b, m.shortFieldName ); - break; - - default: - stringstream ss; - ss << "unknown mod in appendNewFromMod: " << m.op; - throw UserException( 9015, ss.str() ); - } - - } - - /** @return true iff the elements aren't eoo(), are distinct, and share a field name. */ - static bool duplicateFieldName( const BSONElement& a, const BSONElement& b ); - - public: - - bool canApplyInPlace() const { - return _inPlacePossible; - } - - bool isUpdateIndexed() const { - if ( _numIndexAlwaysUpdated != 0 ) { - return true; - } - - return isUpdateIndexedSlow(); - } - - bool isUpdateIndexedSlow() const; - - /** - * modified underlying _obj - * @param isOnDisk - true means this is an on disk object, and this update needs to be made durable - */ - void applyModsInPlace( bool isOnDisk ); - - BSONObj createNewFromMods(); - - // re-writing for oplog - - bool DEPRECATED_needOpLogRewrite() const { - for ( ModStateHolder::const_iterator i = _mods.begin(); i != _mods.end(); i++ ) - if ( i->second->DEPRECATED_needOpLogRewrite() ) - return true; - return false; - } - - BSONObj getOpLogRewrite() const; - - bool DEPRECATED_haveArrayDepMod() const { - for ( ModStateHolder::const_iterator i = _mods.begin(); i != _mods.end(); i++ ) - if ( i->second->m->arrayDep() ) - return true; - return false; - } - - void DEPRECATED_appendSizeSpecForArrayDepMods( BSONObjBuilder& b ) const { - for ( ModStateHolder::const_iterator i = _mods.begin(); i != _mods.end(); i++ ) { - const ModState& m = *i->second; - if ( m.m->arrayDep() ) { - if ( m.DEPRECATED_pushStartSize == -1 ) - b.appendNull( m.fieldName() ); - else - b << m.fieldName() << BSON( "$size" << m.DEPRECATED_pushStartSize ); - } - } - } - - string toString() const; - - friend class ModSet; - }; - -} // namespace mongo diff --git a/src/mongo/db/ops/update_lifecycle.h b/src/mongo/db/ops/update_lifecycle.h new file mode 100644 index 00000000000..ed5b55d3e5d --- /dev/null +++ b/src/mongo/db/ops/update_lifecycle.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/field_ref.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/s/chunk_version.h" + +namespace mongo { + + class UpdateLifecycle { + public: + + virtual ~UpdateLifecycle() {} + + /** + * Update the cached collection pointer that this lifecycle object uses. + */ + virtual void setCollection(Collection* collection) = 0; + + /** + * Can the update continue? + * + * The (only) implementation will check the following: + * 1.) Collection still exists + * 2.) Shard version has not changed (indicating that the query/update is not valid + */ + virtual bool canContinue() const = 0; + + /** + * Return a pointer to any indexes if there is a collection. + */ + virtual const UpdateIndexData* getIndexKeys() const = 0; + + /** + * Returns the shard keys as immutable fields + * Immutable fields in this case mean that they are required to exist, cannot change values + * and must not be multi-valued (in an array, or an array) + */ + virtual const std::vector* getImmutableFields() const = 0; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/update_lifecycle_impl.cpp b/src/mongo/db/ops/update_lifecycle_impl.cpp new file mode 100644 index 00000000000..fbdee094885 --- /dev/null +++ b/src/mongo/db/ops/update_lifecycle_impl.cpp @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/ops/update_lifecycle_impl.h" + +#include "mongo/db/client.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/field_ref.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/s/chunk_version.h" +#include "mongo/s/d_logic.h" + +namespace mongo { + namespace { + CollectionMetadataPtr getMetadata(const NamespaceString& nsString) { + if (shardingState.enabled()) { + return shardingState.getCollectionMetadata(nsString.ns()); + } + + return CollectionMetadataPtr(); + } + } + + UpdateLifecycleImpl::UpdateLifecycleImpl(bool ignoreVersion, const NamespaceString& nsStr) + : _nsString(nsStr) + , _shardVersion((!ignoreVersion && getMetadata(_nsString)) ? + getMetadata(_nsString)->getShardVersion() : + ChunkVersion::IGNORED()) { + } + + void UpdateLifecycleImpl::setCollection(Collection* collection) { + _collection = collection; + } + + bool UpdateLifecycleImpl::canContinue() const { + // Collection needs to exist to continue + return _collection; + } + + const UpdateIndexData* UpdateLifecycleImpl::getIndexKeys() const { + if (_collection) + return &_collection->infoCache()->indexKeys(); + return NULL; + } + + const std::vector* UpdateLifecycleImpl::getImmutableFields() const { + CollectionMetadataPtr metadata = getMetadata(_nsString); + if (metadata) { + const std::vector& fields = metadata->getKeyPatternFields(); + // Return shard-keys as immutable for the update system. + return &fields; + } + return NULL; + } + +} // namespace mongo diff --git a/src/mongo/db/ops/update_lifecycle_impl.h b/src/mongo/db/ops/update_lifecycle_impl.h new file mode 100644 index 00000000000..fa612463757 --- /dev/null +++ b/src/mongo/db/ops/update_lifecycle_impl.h @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/ops/update_lifecycle.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + class UpdateLifecycleImpl : public UpdateLifecycle { + MONGO_DISALLOW_COPYING(UpdateLifecycleImpl); + + public: + + /** + * ignoreVersion is for shard version checking and + * means that version checks will not be done + * + * nsString represents the namespace for the + */ + UpdateLifecycleImpl(bool ignoreVersion, const NamespaceString& nsString); + + virtual void setCollection(Collection* collection); + + virtual bool canContinue() const; + + virtual const UpdateIndexData* getIndexKeys() const; + + virtual const std::vector* getImmutableFields() const; + + private: + Collection* _collection; + const NamespaceString& _nsString; + ChunkVersion _shardVersion; + }; + +} /* namespace mongo */ diff --git a/src/mongo/db/ops/update_request.h b/src/mongo/db/ops/update_request.h new file mode 100644 index 00000000000..db0dcbf1e5e --- /dev/null +++ b/src/mongo/db/ops/update_request.h @@ -0,0 +1,180 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/db/curop.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + class FieldRef; + class UpdateLifecycle; + + class UpdateRequest { + public: + inline UpdateRequest(const NamespaceString& nsString) + : _nsString(nsString) + , _god(false) + , _upsert(false) + , _multi(false) + , _callLogOp(false) + , _fromMigration(false) + , _fromReplication(false) + , _lifecycle(NULL) {} + + const NamespaceString& getNamespaceString() const { + return _nsString; + } + + inline void setQuery(const BSONObj& query) { + _query = query; + } + + inline const BSONObj& getQuery() const { + return _query; + } + + inline void setUpdates(const BSONObj& updates) { + _updates = updates; + } + + inline const BSONObj& getUpdates() const { + return _updates; + } + + // Please see documentation on the private members matching these names for + // explanations of the following fields. + + inline void setGod(bool value = true) { + _god = value; + } + + bool isGod() const { + return _god; + } + + inline void setUpsert(bool value = true) { + _upsert = value; + } + + bool isUpsert() const { + return _upsert; + } + + inline void setMulti(bool value = true) { + _multi = value; + } + + bool isMulti() const { + return _multi; + } + + inline void setUpdateOpLog(bool value = true) { + _callLogOp = value; + } + + bool shouldCallLogOp() const { + return _callLogOp; + } + + inline void setFromMigration(bool value = true) { + _fromMigration = value; + } + + bool isFromMigration() const { + return _fromMigration; + } + + inline void setFromReplication(bool value = true) { + _fromReplication = value; + } + + bool isFromReplication() const { + return _fromReplication; + } + + inline void setLifecycle(UpdateLifecycle* value) { + _lifecycle = value; + } + + inline UpdateLifecycle* getLifecycle() const { + return _lifecycle; + } + + const std::string toString() const { + return str::stream() + << " query: " << _query + << " updated: " << _updates + << " god: " << _god + << " upsert: " << _upsert + << " multi: " << _multi + << " callLogOp: " << _callLogOp + << " fromMigration: " << _fromMigration + << " fromReplications: " << _fromReplication; + } + private: + + const NamespaceString& _nsString; + + // Contains the query that selects documents to update. + BSONObj _query; + + // Contains the modifiers to apply to matched objects, or a replacement document. + BSONObj _updates; + + // Flags controlling the update. + + // God bypasses _id checking and index generation. It is only used on behalf of system + // updates, never user updates. + bool _god; + + // True if this should insert if no matching document is found. + bool _upsert; + + // True if this update is allowed to affect more than one document. + bool _multi; + + // True if the effects of the update should be written to the oplog. + bool _callLogOp; + + // True if this update is on behalf of a chunk migration. + bool _fromMigration; + + // True if this update is being applied during the application for the oplog. + bool _fromReplication; + + // The lifecycle data, and events used during the update request. + UpdateLifecycle* _lifecycle; + }; + +} // namespace mongo diff --git a/src/mongo/db/ops/update_result.h b/src/mongo/db/ops/update_result.h new file mode 100644 index 00000000000..945f3553207 --- /dev/null +++ b/src/mongo/db/ops/update_result.h @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/db/curop.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + namespace str = mongoutils::str; + + struct UpdateResult { + + UpdateResult( bool existing_, + bool modifiers_, + unsigned long long numDocsModified_, + unsigned long long numMatched_, + const BSONObj& upsertedObject_ ) + : existing(existing_) + , modifiers(modifiers_) + , numDocsModified(numDocsModified_) + , numMatched(numMatched_) { + + BSONElement id = upsertedObject_["_id"]; + if ( ! existing && numMatched == 1 && !id.eoo() ) { + upserted = id.wrap(kUpsertedFieldName); + } + + LOG(4) << "UpdateResult -- " << toString(); + } + + + // if existing objects were modified + const bool existing; + + // was this a $ mod + const bool modifiers; + + // how many docs updated + const long long numDocsModified; + + // how many docs seen by update + const long long numMatched; + + // if something was upserted, the new _id of the object + BSONObj upserted; + + const std::string toString() const { + return str::stream() + << " upserted: " << upserted + << " modifiers: " << modifiers + << " existing: " << existing + << " numDocsModified: " << numDocsModified + << " numMatched: " << numMatched; + } + }; + +} // namespace mongo diff --git a/src/mongo/db/pagefault.cpp b/src/mongo/db/pagefault.cpp index 68eb9685752..1c4de1b1142 100644 --- a/src/mongo/db/pagefault.cpp +++ b/src/mongo/db/pagefault.cpp @@ -14,15 +14,29 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "diskloc.h" -#include "pagefault.h" -#include "client.h" -#include "pdfile.h" -#include "server.h" +#include "mongo/pch.h" + +#include "mongo/db/pagefault.h" + +#include "mongo/db/client.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/pdfile.h" +#include "mongo/server.h" namespace mongo { @@ -43,7 +57,7 @@ namespace mongo { if( LockMongoFilesShared::getEra() != era ) { // files opened and closed. we don't try to handle but just bail out; this is much simpler // and less error prone and saves us from taking a dbmutex readlock. - dlog(2) << "era changed" << endl; + MONGO_DLOG(2) << "era changed" << endl; return; } r->touch(); @@ -57,13 +71,12 @@ namespace mongo { verify( cc()._pageFaultRetryableSection == 0 ); if( Lock::isLocked() ) { cc()._pageFaultRetryableSection = 0; - if( debug || logLevel > 2 ) { + if( debug || logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(3)) ) { LOGSOME << "info PageFaultRetryableSection will not yield, already locked upon reaching" << endl; } } else { cc()._pageFaultRetryableSection = this; - cc()._hasWrittenThisPass = false; } } diff --git a/src/mongo/db/pagefault.h b/src/mongo/db/pagefault.h index e361881ab92..d625d593512 100644 --- a/src/mongo/db/pagefault.h +++ b/src/mongo/db/pagefault.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ diff --git a/src/mongo/db/pdfile.cpp b/src/mongo/db/pdfile.cpp index 928b8560ced..82567489193 100644 --- a/src/mongo/db/pdfile.cpp +++ b/src/mongo/db/pdfile.cpp @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* @@ -33,30 +45,38 @@ _ disallow system* manipulations from the database. #include #include "mongo/base/counter.h" +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/db/audit.h" #include "mongo/db/auth/auth_index_d.h" -#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/user_document_parser.h" #include "mongo/db/pdfile_private.h" #include "mongo/db/background.h" -#include "mongo/db/btree.h" +#include "mongo/db/structure/btree/btree.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/cloner.h" #include "mongo/db/commands/server_status.h" #include "mongo/db/curop-inl.h" #include "mongo/db/db.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/extsort.h" -#include "mongo/db/index_update.h" +#include "mongo/db/index_legacy.h" +#include "mongo/db/index_names.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index/index_access_method.h" #include "mongo/db/instance.h" #include "mongo/db/kill_current_op.h" #include "mongo/db/lasterror.h" -#include "mongo/db/memconcept.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/ops/delete.h" -#include "mongo/db/repl.h" -#include "mongo/db/replutil.h" +#include "mongo/db/repair_database.h" +#include "mongo/db/repl/is_master.h" #include "mongo/db/sort_phase_one.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/storage_options.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/util/assert_util.h" #include "mongo/util/file.h" #include "mongo/util/file_allocator.h" -#include "mongo/util/hashtab.h" #include "mongo/util/mmap.h" #include "mongo/util/processinfo.h" #include "mongo/db/stats/timer_stats.h" @@ -64,126 +84,20 @@ _ disallow system* manipulations from the database. namespace mongo { - BOOST_STATIC_ASSERT( sizeof(Extent)-4 == 48+128 ); - BOOST_STATIC_ASSERT( sizeof(DataFileHeader)-4 == 8192 ); - - //The oplog entries inserted - static TimerStats oplogInsertStats; - static ServerStatusMetricField displayInsertedOplogEntries( - "repl.oplog.insert", - &oplogInsertStats ); - static Counter64 oplogInsertBytesStats; - static ServerStatusMetricField displayInsertedOplogEntryBytes( - "repl.oplog.insertBytes", - &oplogInsertBytesStats ); - - bool isValidNS( const StringData& ns ) { - // TODO: should check for invalid characters - - size_t idx = ns.find( '.' ); - if ( idx == string::npos ) - return false; - - if ( idx == ns.size() - 1 ) - return false; - - return true; - } - - // TODO SERVER-4328 - bool inDBRepair = false; - struct doingRepair { - doingRepair() { - verify( ! inDBRepair ); - inDBRepair = true; - } - ~doingRepair() { - inDBRepair = false; - } - }; - - SimpleMutex BackgroundOperation::m("bg"); - map BackgroundOperation::dbsInProg; - set BackgroundOperation::nsInProg; - - bool BackgroundOperation::inProgForDb(const char *db) { - SimpleMutex::scoped_lock lk(m); - return dbsInProg[db] != 0; - } - - bool BackgroundOperation::inProgForNs(const char *ns) { - SimpleMutex::scoped_lock lk(m); - return nsInProg.count(ns) != 0; - } - - void BackgroundOperation::assertNoBgOpInProgForDb(const char *db) { - uassert(12586, "cannot perform operation: a background operation is currently running for this database", - !inProgForDb(db)); - } - - void BackgroundOperation::assertNoBgOpInProgForNs(const char *ns) { - uassert(12587, "cannot perform operation: a background operation is currently running for this collection", - !inProgForNs(ns)); - } - - BackgroundOperation::BackgroundOperation(const char *ns) : _ns(ns) { - SimpleMutex::scoped_lock lk(m); - dbsInProg[_ns.db]++; - nsInProg.insert(_ns.ns()); - } - - BackgroundOperation::~BackgroundOperation() { - SimpleMutex::scoped_lock lk(m); - dbsInProg[_ns.db]--; - nsInProg.erase(_ns.ns()); - } - - void BackgroundOperation::dump(stringstream& ss) { - SimpleMutex::scoped_lock lk(m); - if( nsInProg.size() ) { - ss << "\nBackground Jobs in Progress\n"; - for( set::iterator i = nsInProg.begin(); i != nsInProg.end(); i++ ) - ss << " " << *i << '\n'; - } - for( map::iterator i = dbsInProg.begin(); i != dbsInProg.end(); i++ ) { - if( i->second ) - ss << "database " << i->first << ": " << i->second << '\n'; - } - } - /* ----------------------------------------- */ -#ifdef _WIN32 - string dbpath = "\\data\\db\\"; -#else - string dbpath = "/data/db/"; -#endif - const char FREELIST_NS[] = ".$freelist"; - bool directoryperdb = false; - string repairpath; string pidfilepath; - DataFileMgr theDataFileMgr; DatabaseHolder _dbHolder; - int MAGIC = 0x1000; DatabaseHolder& dbHolderUnchecked() { return _dbHolder; } - void addNewNamespaceToCatalog(const char *ns, const BSONObj *options = 0); - void ensureIdIndexForNewNs(const char *ns) { - if ( ( strstr( ns, ".system." ) == 0 || legalClientSystemNS( ns , false ) ) && - strstr( ns, FREELIST_NS ) == 0 ) { - LOG( 1 ) << "adding _id index for collection " << ns << endl; - ensureHaveIdIndex( ns, false ); - } - } + void ensureIdIndexForNewNs( Collection* collection ) { + if ( collection->ns().isSystem() && !legalClientSystemNS( collection->ns().ns(), false ) ) + return; - static void _ensureSystemIndexes(const char* ns) { - NamespaceString nsstring(ns); - if (StringData(nsstring.coll).substr(0, 7) == "system.") { - authindex::createSystemIndexes(nsstring); - } + uassertStatusOK( collection->getIndexCatalog()->ensureHaveIdIndex() ); } string getDbContext() { @@ -194,7 +108,7 @@ namespace mongo { if ( cx ) { Database *database = cx->db(); if ( database ) { - ss << database->name << ' '; + ss << database->name() << ' '; ss << cx->ns() << ' '; } } @@ -204,1950 +118,91 @@ namespace mongo { /*---------------------------------------------------------------------*/ - // inheritable class to implement an operation that may be applied to all - // files in a database using _applyOpToDataFiles() - class FileOp { - public: - virtual ~FileOp() {} - // Return true if file exists and operation successful - virtual bool apply( const boost::filesystem::path &p ) = 0; - virtual const char * op() const = 0; - }; - - void _applyOpToDataFiles( const char *database, FileOp &fo, bool afterAllocator = false, const string& path = dbpath ); + /** { ..., capped: true, size: ..., max: ... } + * @param createDefaultIndexes - if false, defers id (and other) index creation. + * @return true if successful + */ + bool userCreateNS(const char *ns, BSONObj options, string& err, + bool logForReplication, bool createDefaultIndexes ) { - void _deleteDataFiles(const char *database) { - if ( directoryperdb ) { - FileAllocator::get()->waitUntilFinished(); - MONGO_ASSERT_ON_EXCEPTION_WITH_MSG( boost::filesystem::remove_all( boost::filesystem::path( dbpath ) / database ), "delete data files with a directoryperdb" ); - return; - } - class : public FileOp { - virtual bool apply( const boost::filesystem::path &p ) { - return boost::filesystem::remove( p ); - } - virtual const char * op() const { - return "remove"; - } - } deleter; - _applyOpToDataFiles( database, deleter, true ); - } + LOG(1) << "create collection " << ns << ' ' << options; - int Extent::initialSize(int len) { - long long sz = len * 16; - if ( len < 1000 ) sz = len * 64; - if ( sz > 1000000000 ) - sz = 1000000000; - int z = ((int)sz) & 0xffffff00; - verify( z > len ); - return z; - } + massert(10356 , + str::stream() << "invalid ns: " << ns, + NamespaceString::validCollectionComponent(ns)); - void checkConfigNS(const char *ns) { - if ( cmdLine.configsvr && - !( str::startsWith( ns, "config." ) || - str::startsWith( ns, "local." ) || - str::startsWith( ns, "admin." ) ) ) { - uasserted(14037, "can't create user databases on a --configsvr instance"); - } - } + Database* db = cc().database(); - bool _userCreateNS(const char *ns, const BSONObj& options, string& err, bool *deferIdIndex) { - LOG(1) << "create collection " << ns << ' ' << options << endl; + Collection* collection = db->getCollection( ns ); - if ( nsdetails(ns) ) { + if ( collection ) { err = "collection already exists"; return false; } - checkConfigNS(ns); - - long long size = Extent::initialSize(128); - { - BSONElement e = options.getField("size"); - if ( e.isNumber() ) { - size = e.numberLong(); - uassert( 10083 , "create collection invalid size spec", size > 0 ); - - size += 0xff; - size &= 0xffffffffffffff00LL; - if ( size < Extent::minSize() ) - size = Extent::minSize(); - } - } - - bool newCapped = false; - long long mx = 0; - if( options["capped"].trueValue() ) { - newCapped = true; - BSONElement e = options.getField("max"); - if ( e.isNumber() ) { - mx = e.numberLong(); - uassert( 16495, - "max in a capped collection has to be < 2^31 or not set", - NamespaceDetails::validMaxCappedDocs(&mx) ); - } - } - - // $nExtents just for debug/testing. - BSONElement e = options.getField( "$nExtents" ); - Database *database = cc().database(); - if ( e.type() == Array ) { - // We create one extent per array entry, with size specified - // by the array value. - BSONObjIterator i( e.embeddedObject() ); - while( i.more() ) { - BSONElement e = i.next(); - int size = int( e.number() ); - verify( size <= 0x7fffffff ); - // $nExtents is just for testing - always allocate new extents - // rather than reuse existing extents so we have some predictibility - // in the extent size used by our tests - database->suitableFile( ns, (int) size, false, false )->createExtent( ns, (int) size, newCapped ); - } - } - else if ( int( e.number() ) > 0 ) { - // We create '$nExtents' extents, each of size 'size'. - int nExtents = int( e.number() ); - verify( size <= 0x7fffffff ); - for ( int i = 0; i < nExtents; ++i ) { - verify( size <= 0x7fffffff ); - // $nExtents is just for testing - always allocate new extents - // rather than reuse existing extents so we have some predictibility - // in the extent size used by our tests - database->suitableFile( ns, (int) size, false, false )->createExtent( ns, (int) size, newCapped ); - } - } - else { - // This is the non test case, where we don't have a $nExtents spec. - while ( size > 0 ) { - const int max = Extent::maxSize(); - const int min = Extent::minSize(); - int desiredExtentSize = static_cast (size > max ? max : size); - desiredExtentSize = static_cast (desiredExtentSize < min ? min : desiredExtentSize); - - desiredExtentSize &= 0xffffff00; - Extent *e = database->allocExtent( ns, desiredExtentSize, newCapped, true ); - size -= e->length; - } - } - - NamespaceDetails *d = nsdetails(ns); - verify(d); - - bool ensure = true; - - // respect autoIndexId if set. otherwise, create an _id index for all colls, except for - // capped ones in local w/o autoIndexID (reason for the exception is for the oplog and - // non-replicated capped colls) - if( options.hasField( "autoIndexId" ) || - (newCapped && nsToDatabase( ns ) == "local" ) ) { - ensure = options.getField( "autoIndexId" ).trueValue(); - } - - if( ensure ) { - if( deferIdIndex ) - *deferIdIndex = true; - else - ensureIdIndexForNewNs( ns ); - } - - _ensureSystemIndexes(ns); - - if ( mx > 0 ) - d->setMaxCappedDocs( mx ); - - bool isFreeList = strstr(ns, FREELIST_NS) != 0; - if( !isFreeList ) - addNewNamespaceToCatalog(ns, options.isEmpty() ? 0 : &options); - - if ( options["flags"].numberInt() ) { - d->replaceUserFlags( options["flags"].numberInt() ); + CollectionOptions collectionOptions; + Status status = collectionOptions.parse( options ); + if ( !status.isOK() ) { + err = status.toString(); + return false; } - return true; - } + invariant( db->createCollection( ns, collectionOptions, true, createDefaultIndexes ) ); - /** { ..., capped: true, size: ..., max: ... } - @param deferIdIndex - if not not, defers id index creation. sets the bool value to true if we wanted to create the id index. - @return true if successful - */ - bool userCreateNS(const char *ns, BSONObj options, string& err, bool logForReplication, bool *deferIdIndex) { - const char *coll = strchr( ns, '.' ) + 1; - massert( 10356 , str::stream() << "invalid ns: " << ns , NamespaceString::validCollectionName(ns)); - bool ok = _userCreateNS(ns, options, err, deferIdIndex); - if ( logForReplication && ok ) { + if ( logForReplication ) { if ( options.getField( "create" ).eoo() ) { BSONObjBuilder b; - b << "create" << coll; + b << "create" << nsToCollectionSubstring( ns ); b.appendElements( options ); options = b.obj(); } string logNs = nsToDatabase(ns) + ".$cmd"; logOp("c", logNs.c_str(), options); } - return ok; - } - - /*---------------------------------------------------------------------*/ - - int MongoDataFile::maxSize() { - if ( sizeof( int* ) == 4 ) { - return 512 * 1024 * 1024; - } - else if ( cmdLine.smallfiles ) { - return 0x7ff00000 >> 2; - } - else { - return 0x7ff00000; - } - } - - NOINLINE_DECL void MongoDataFile::badOfs2(int ofs) const { - stringstream ss; - ss << "bad offset:" << ofs << " accessing file: " << mmf.filename() << " - consider repairing database"; - uasserted(13441, ss.str()); - } - - NOINLINE_DECL void MongoDataFile::badOfs(int ofs) const { - stringstream ss; - ss << "bad offset:" << ofs << " accessing file: " << mmf.filename() << " - consider repairing database"; - uasserted(13440, ss.str()); - } - - int MongoDataFile::defaultSize( const char *filename ) const { - int size; - if ( fileNo <= 4 ) - size = (64*1024*1024) << fileNo; - else - size = 0x7ff00000; - if ( cmdLine.smallfiles ) { - size = size >> 2; - } - return size; - } - - static void check(void *_mb) { - if( sizeof(char *) == 4 ) - uassert( 10084 , "can't map file memory - mongo requires 64 bit build for larger datasets", _mb != 0); - else - uassert( 10085 , "can't map file memory", _mb != 0); - } - - /** @return true if found and opened. if uninitialized (prealloc only) does not open. */ - bool MongoDataFile::openExisting( const char *filename ) { - verify( _mb == 0 ); - if( !boost::filesystem::exists(filename) ) - return false; - if( !mmf.open(filename,false) ) { - dlog(2) << "info couldn't open " << filename << " probably end of datafile list" << endl; - return false; - } - _mb = mmf.getView(); verify(_mb); - unsigned long long sz = mmf.length(); - verify( sz <= 0x7fffffff ); - verify( sz % 4096 == 0 ); - if( sz < 64*1024*1024 && !cmdLine.smallfiles ) { - if( sz >= 16*1024*1024 && sz % (1024*1024) == 0 ) { - log() << "info openExisting file size " << sz << " but cmdLine.smallfiles=false" << endl; - } - else { - log() << "openExisting size " << sz << " less then minimum file size expectation " << filename << endl; - verify(false); - } - } - check(_mb); - if( header()->uninitialized() ) - return false; return true; } - void MongoDataFile::open( const char *filename, int minSize, bool preallocateOnly ) { - long size = defaultSize( filename ); - while ( size < minSize ) { - if ( size < maxSize() / 2 ) - size *= 2; - else { - size = maxSize(); - break; - } - } - if ( size > maxSize() ) - size = maxSize(); - - verify( size >= 64*1024*1024 || cmdLine.smallfiles ); - verify( size % 4096 == 0 ); - - if ( preallocateOnly ) { - if ( cmdLine.prealloc ) { - FileAllocator::get()->requestAllocation( filename, size ); - } - return; - } - - { - verify( _mb == 0 ); - unsigned long long sz = size; - if( mmf.create(filename, sz, false) ) - _mb = mmf.getView(); - verify( sz <= 0x7fffffff ); - size = (int) sz; - } - check(_mb); - header()->init(fileNo, size, filename); - } - - void MongoDataFile::flush( bool sync ) { - mmf.flush( sync ); - } - - void addNewExtentToNamespace(const char *ns, Extent *e, DiskLoc eloc, DiskLoc emptyLoc, bool capped) { - NamespaceIndex *ni = nsindex(ns); - NamespaceDetails *details = ni->details(ns); - if ( details ) { - verify( !details->lastExtent.isNull() ); - verify( !details->firstExtent.isNull() ); - getDur().writingDiskLoc(e->xprev) = details->lastExtent; - getDur().writingDiskLoc(details->lastExtent.ext()->xnext) = eloc; - verify( !eloc.isNull() ); - getDur().writingDiskLoc(details->lastExtent) = eloc; - } - else { - ni->add_ns(ns, eloc, capped); - details = ni->details(ns); - } - - { - NamespaceDetails *dw = details->writingWithoutExtra(); - dw->lastExtentSize = e->length; - } - details->addDeletedRec(emptyLoc.drec(), emptyLoc); - } - - Extent* MongoDataFile::createExtent(const char *ns, int approxSize, bool newCapped, int loops) { - verify( approxSize <= Extent::maxSize() ); - { - // make sizes align with VM page size - int newSize = (approxSize + 0xfff) & 0xfffff000; - verify( newSize >= 0 ); - if( newSize < Extent::maxSize() ) - approxSize = newSize; - } - massert( 10357 , "shutdown in progress", ! inShutdown() ); - massert( 10358 , "bad new extent size", approxSize >= Extent::minSize() && approxSize <= Extent::maxSize() ); - massert( 10359 , "header==0 on new extent: 32 bit mmap space exceeded?", header() ); // null if file open failed - int ExtentSize = min(header()->unusedLength, approxSize); - DiskLoc loc; - if ( ExtentSize < Extent::minSize() ) { - /* note there could be a lot of looping here is db just started and - no files are open yet. we might want to do something about that. */ - if ( loops > 8 ) { - verify( loops < 10000 ); - out() << "warning: loops=" << loops << " fileno:" << fileNo << ' ' << ns << '\n'; - } - log() << "newExtent: " << ns << " file " << fileNo << " full, adding a new file" << endl; - return cc().database()->addAFile( 0, true )->createExtent(ns, approxSize, newCapped, loops+1); - } - int offset = header()->unused.getOfs(); - - DataFileHeader *h = header(); - h->unused.writing().set( fileNo, offset + ExtentSize ); - getDur().writingInt(h->unusedLength) = h->unusedLength - ExtentSize; - loc.set(fileNo, offset); - Extent *e = _getExtent(loc); - DiskLoc emptyLoc = getDur().writing(e)->init(ns, ExtentSize, fileNo, offset, newCapped); - - addNewExtentToNamespace(ns, e, loc, emptyLoc, newCapped); - - DEV tlog(1) << "new extent " << ns << " size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset - << " emptyLoc:" << hex << emptyLoc.getOfs() << dec << endl; - return e; - } - - Extent* DataFileMgr::allocFromFreeList(const char *ns, int approxSize, bool capped) { - string s = cc().database()->name + FREELIST_NS; - NamespaceDetails *f = nsdetails(s); - if( f ) { - int low, high; - if( capped ) { - // be strict about the size - low = approxSize; - if( low > 2048 ) low -= 256; - high = (int) (approxSize * 1.05) + 256; - } - else { - low = (int) (approxSize * 0.8); - high = (int) (approxSize * 1.4); - } - if( high <= 0 ) { - // overflowed - high = max(approxSize, Extent::maxSize()); - } - if ( high <= Extent::minSize() ) { - // the minimum extent size is 4097 - high = Extent::minSize() + 1; - } - int n = 0; - Extent *best = 0; - int bestDiff = 0x7fffffff; - { - Timer t; - DiskLoc L = f->firstExtent; - while( !L.isNull() ) { - Extent * e = L.ext(); - if( e->length >= low && e->length <= high ) { - int diff = abs(e->length - approxSize); - if( diff < bestDiff ) { - bestDiff = diff; - best = e; - if( ((double) diff) / approxSize < 0.1 ) { - // close enough - break; - } - if( t.seconds() >= 2 ) { - // have spent lots of time in write lock, and we are in [low,high], so close enough - // could come into play if extent freelist is very long - break; - } - } - else { - OCCASIONALLY { - if( high < 64 * 1024 && t.seconds() >= 2 ) { - // be less picky if it is taking a long time - high = 64 * 1024; - } - } - } - } - L = e->xnext; - ++n; - } - if( t.seconds() >= 10 ) { - log() << "warning: slow scan in allocFromFreeList (in write lock)" << endl; - } - } - - if( n > 128 ) LOG( n < 512 ? 1 : 0 ) << "warning: newExtent " << n << " scanned\n"; - - if( best ) { - Extent *e = best; - // remove from the free list - if( !e->xprev.isNull() ) - e->xprev.ext()->xnext.writing() = e->xnext; - if( !e->xnext.isNull() ) - e->xnext.ext()->xprev.writing() = e->xprev; - if( f->firstExtent == e->myLoc ) - f->firstExtent.writing() = e->xnext; - if( f->lastExtent == e->myLoc ) - f->lastExtent.writing() = e->xprev; - - // use it - OCCASIONALLY if( n > 512 ) log() << "warning: newExtent " << n << " scanned" << endl; - DiskLoc emptyLoc = e->reuse(ns, capped); - addNewExtentToNamespace(ns, e, e->myLoc, emptyLoc, capped); - return e; - } - } - - return 0; - // return createExtent(ns, approxSize, capped); - } - - /*---------------------------------------------------------------------*/ - - void Extent::markEmpty() { - xnext.Null(); - xprev.Null(); - firstRecord.Null(); - lastRecord.Null(); - } - - DiskLoc Extent::reuse(const char *nsname, bool capped) { - return getDur().writing(this)->_reuse(nsname, capped); - } - - void getEmptyLoc(const char *ns, const DiskLoc extentLoc, int extentLength, bool capped, /*out*/DiskLoc& emptyLoc, /*out*/int& delRecLength) { - emptyLoc = extentLoc; - emptyLoc.inc( Extent::HeaderSize() ); - delRecLength = extentLength - Extent::HeaderSize(); - if( delRecLength >= 32*1024 && str::contains(ns, '$') && !capped ) { - // probably an index. so skip forward to keep its records page aligned - int& ofs = emptyLoc.GETOFS(); - int newOfs = (ofs + 0xfff) & ~0xfff; - delRecLength -= (newOfs-ofs); - dassert( delRecLength > 0 ); - ofs = newOfs; - } - } - - DiskLoc Extent::_reuse(const char *nsname, bool capped) { - LOG(3) << "_reuse extent was:" << nsDiagnostic.toString() << " now:" << nsname << endl; - if (magic != extentSignature) { - StringBuilder sb; - sb << "bad extent signature " << toHex(&magic, 4) - << " for namespace '" << nsDiagnostic.toString() - << "' found in Extent::_reuse"; - msgasserted(10360, sb.str()); - } - nsDiagnostic = nsname; - markEmpty(); - - DiskLoc emptyLoc; - int delRecLength; - getEmptyLoc(nsname, myLoc, length, capped, emptyLoc, delRecLength); - - // todo: some dup code here and below in Extent::init - DeletedRecord* empty = getDur().writing(DataFileMgr::getDeletedRecord(emptyLoc)); - empty->lengthWithHeaders() = delRecLength; - empty->extentOfs() = myLoc.getOfs(); - empty->nextDeleted().Null(); - return emptyLoc; - } - - /* assumes already zeroed -- insufficient for block 'reuse' perhaps */ - DiskLoc Extent::init(const char *nsname, int _length, int _fileNo, int _offset, bool capped) { - magic = extentSignature; - myLoc.set(_fileNo, _offset); - xnext.Null(); - xprev.Null(); - nsDiagnostic = nsname; - length = _length; - firstRecord.Null(); - lastRecord.Null(); - - DiskLoc emptyLoc; - int delRecLength; - getEmptyLoc(nsname, myLoc, _length, capped, emptyLoc, delRecLength); - - DeletedRecord* empty = getDur().writing(DataFileMgr::getDeletedRecord(emptyLoc)); - empty->lengthWithHeaders() = delRecLength; - empty->extentOfs() = myLoc.getOfs(); - empty->nextDeleted().Null(); - return emptyLoc; - } - - bool Extent::validates(const DiskLoc diskLoc, BSONArrayBuilder* errors) { - bool extentOk = true; - if (magic != extentSignature) { - if (errors) { - StringBuilder sb; - sb << "bad extent signature " << toHex(&magic, 4) - << " in extent " << diskLoc.toString(); - *errors << sb.str(); - } - extentOk = false; - } - if (myLoc != diskLoc) { - if (errors) { - StringBuilder sb; - sb << "extent " << diskLoc.toString() - << " self-pointer is " << myLoc.toString(); - *errors << sb.str(); - } - extentOk = false; - } - if (firstRecord.isNull() != lastRecord.isNull()) { - if (errors) { - StringBuilder sb; - if (firstRecord.isNull()) { - sb << "in extent " << diskLoc.toString() - << ", firstRecord is null but lastRecord is " - << lastRecord.toString(); - } - else { - sb << "in extent " << diskLoc.toString() - << ", firstRecord is " << firstRecord.toString() - << " but lastRecord is null"; - } - *errors << sb.str(); - } - extentOk = false; - } - if (length < minSize()) { - if (errors) { - StringBuilder sb; - sb << "length of extent " << diskLoc.toString() - << " is " << length - << ", which is less than minimum length of " << minSize(); - *errors << sb.str(); - } - extentOk = false; - } - return extentOk; - } - -/* - Record* Extent::newRecord(int len) { - if( firstEmptyRegion.isNull() )8 - return 0; - - verify(len > 0); - int newRecSize = len + Record::HeaderSize; - DiskLoc newRecordLoc = firstEmptyRegion; - Record *r = getRecord(newRecordLoc); - int left = r->netLength() - len; - if( left < 0 ) { - // - firstEmptyRegion.Null(); - return 0; - } - - DiskLoc nextEmpty = r->next.getNextEmpty(firstEmptyRegion); - r->lengthWithHeaders = newRecSize; - r->next.markAsFirstOrLastInExtent(this); // we're now last in the extent - if( !lastRecord.isNull() ) { - verify(getRecord(lastRecord)->next.lastInExtent()); // it was the last one - getRecord(lastRecord)->next.set(newRecordLoc); // until now - r->prev.set(lastRecord); - } - else { - r->prev.markAsFirstOrLastInExtent(this); // we are the first in the extent - verify( firstRecord.isNull() ); - firstRecord = newRecordLoc; - } - lastRecord = newRecordLoc; - - if( left < Record::HeaderSize + 32 ) { - firstEmptyRegion.Null(); - } - else { - firstEmptyRegion.inc(newRecSize); - Record *empty = getRecord(firstEmptyRegion); - empty->next.set(nextEmpty); // not for empty records, unless in-use records, next and prev can be null. - empty->prev.Null(); - empty->lengthWithHeaders = left; - } - - return r; - } -*/ - - int Extent::maxSize() { - int maxExtentSize = 0x7ff00000; - if ( cmdLine.smallfiles ) { - maxExtentSize >>= 2; - } - return maxExtentSize; - } - - /*---------------------------------------------------------------------*/ - - DataFileMgr::DataFileMgr() : _precalcedMutex("PrecalcedMutex"), _precalced(NULL) { - } - - SortPhaseOne* DataFileMgr::getPrecalced() const { - return _precalced; - } - - void DataFileMgr::setPrecalced(SortPhaseOne* precalced) { - _precalced = precalced; - } - - shared_ptr DataFileMgr::findAll(const StringData& ns, const DiskLoc &startLoc) { - NamespaceDetails * d = nsdetails( ns ); - if ( ! d ) - return shared_ptr(new BasicCursor(DiskLoc())); - - DiskLoc loc = d->firstExtent; - Extent *e = getExtent(loc); - - DEBUGGING { - out() << "listing extents for " << ns << endl; - DiskLoc tmp = loc; - set extents; - - while ( 1 ) { - Extent *f = getExtent(tmp); - out() << "extent: " << tmp.toString() << endl; - extents.insert(tmp); - tmp = f->xnext; - if ( tmp.isNull() ) - break; - f = f->getNextExtent(); - } - - out() << endl; - d->dumpDeleted(&extents); - } - - if ( d->isCapped() ) - return shared_ptr( ForwardCappedCursor::make( d , startLoc ) ); - - if ( !startLoc.isNull() ) - return shared_ptr(new BasicCursor( startLoc )); - - while ( e->firstRecord.isNull() && !e->xnext.isNull() ) { - /* todo: if extent is empty, free it for reuse elsewhere. - that is a bit complicated have to clean up the freelists. - */ - RARELY out() << "info DFM::findAll(): extent " << loc.toString() << " was empty, skipping ahead. ns:" << ns << endl; - // find a nonempty extent - // it might be nice to free the whole extent here! but have to clean up free recs then. - e = e->getNextExtent(); - } - return shared_ptr(new BasicCursor( e->firstRecord )); - } - - /* get a table scan cursor, but can be forward or reverse direction. - order.$natural - if set, > 0 means forward (asc), < 0 backward (desc). - */ - shared_ptr findTableScan(const char *ns, const BSONObj& order, const DiskLoc &startLoc) { - BSONElement el = order.getField("$natural"); // e.g., { $natural : -1 } - - if ( el.number() >= 0 ) - return DataFileMgr::findAll(ns, startLoc); - - // "reverse natural order" - NamespaceDetails *d = nsdetails(ns); - - if ( !d ) - return shared_ptr(new BasicCursor(DiskLoc())); + void dropAllDatabasesExceptLocal() { + Lock::GlobalWrite lk; - if ( !d->isCapped() ) { - if ( !startLoc.isNull() ) - return shared_ptr(new ReverseCursor( startLoc )); - Extent *e = d->lastExtent.ext(); - while ( e->lastRecord.isNull() && !e->xprev.isNull() ) { - OCCASIONALLY out() << " findTableScan: extent empty, skipping ahead" << endl; - e = e->getPrevExtent(); + vector n; + getDatabaseNames(n); + if( n.size() == 0 ) return; + log() << "dropAllDatabasesExceptLocal " << n.size() << endl; + for( vector::iterator i = n.begin(); i != n.end(); i++ ) { + if( *i != "local" ) { + Client::Context ctx(*i); + dropDatabase(*i); } - return shared_ptr(new ReverseCursor( e->lastRecord )); - } - else { - return shared_ptr( new ReverseCappedCursor( d, startLoc ) ); } } - void printFreeList() { - string s = cc().database()->name + FREELIST_NS; - log() << "dump freelist " << s << endl; - NamespaceDetails *freeExtents = nsdetails(s); - if( freeExtents == 0 ) { - log() << " freeExtents==0" << endl; - return; - } - DiskLoc a = freeExtents->firstExtent; - while( !a.isNull() ) { - Extent *e = a.ext(); - log() << " extent " << a.toString() << " len:" << e->length << " prev:" << e->xprev.toString() << endl; - a = e->xnext; - } - - log() << "end freelist" << endl; - } - - /** free a list of extents that are no longer in use. this is a double linked list of extents - (could be just one in the list) - */ - void freeExtents(DiskLoc firstExt, DiskLoc lastExt) { - { - verify( !firstExt.isNull() && !lastExt.isNull() ); - Extent *f = firstExt.ext(); - Extent *l = lastExt.ext(); - verify( f->xprev.isNull() ); - verify( l->xnext.isNull() ); - verify( f==l || !f->xnext.isNull() ); - verify( f==l || !l->xprev.isNull() ); - } - - string s = cc().database()->name + FREELIST_NS; - NamespaceDetails *freeExtents = nsdetails(s); - if( freeExtents == 0 ) { - string err; - _userCreateNS(s.c_str(), BSONObj(), err, 0); // todo: this actually allocates an extent, which is bad! - freeExtents = nsdetails(s); - massert( 10361 , "can't create .$freelist", freeExtents); - } - if( freeExtents->firstExtent.isNull() ) { - freeExtents->firstExtent.writing() = firstExt; - freeExtents->lastExtent.writing() = lastExt; - } - else { - DiskLoc a = freeExtents->firstExtent; - verify( a.ext()->xprev.isNull() ); - getDur().writingDiskLoc( a.ext()->xprev ) = lastExt; - getDur().writingDiskLoc( lastExt.ext()->xnext ) = a; - getDur().writingDiskLoc( freeExtents->firstExtent ) = firstExt; - } - - //printFreeList(); - } - - /* drop a collection/namespace */ - void dropNS(const string& nsToDrop) { - NamespaceDetails* d = nsdetails(nsToDrop); - uassert( 10086 , (string)"ns not found: " + nsToDrop , d ); + void dropDatabase(const std::string& db) { + LOG(1) << "dropDatabase " << db << endl; + Lock::assertWriteLocked(db); + Database *d = cc().database(); + verify( d ); + verify( d->name() == db ); - BackgroundOperation::assertNoBgOpInProgForNs(nsToDrop.c_str()); + BackgroundOperation::assertNoBgOpInProgForDb(d->name().c_str()); - NamespaceString s(nsToDrop); - verify( s.db == cc().database()->name ); - if( s.isSystem() ) { - if( s.coll == "system.profile" ) { - uassert( 10087, - "turn off profiling before dropping system.profile collection", - cc().database()->getProfilingLevel() == 0 ); - } - else { - uasserted( 12502, "can't drop system ns" ); - } - } + audit::logDropDatabase( currentClient.get(), db ); - { - // remove from the system catalog - BSONObj cond = BSON( "name" << nsToDrop ); // { name: "colltodropname" } - string system_namespaces = cc().database()->name + ".system.namespaces"; - /*int n = */ deleteObjects(system_namespaces.c_str(), cond, false, false, true); - // no check of return code as this ns won't exist for some of the new storage engines - } + GeneratorHolder::getInstance()->droppedDatabase( db ); - // free extents - if( !d->firstExtent.isNull() ) { - freeExtents(d->firstExtent, d->lastExtent); - getDur().writingDiskLoc( d->firstExtent ).setInvalid(); - getDur().writingDiskLoc( d->lastExtent ).setInvalid(); - } + // Not sure we need this here, so removed. If we do, we need to move it down + // within other calls both (1) as they could be called from elsewhere and + // (2) to keep the lock order right - groupcommitmutex must be locked before + // mmmutex (if both are locked). + // + // RWLockRecursive::Exclusive lk(MongoFile::mmmutex); - // remove from the catalog hashtable - cc().database()->namespaceIndex.kill_ns(nsToDrop.c_str()); - } + getDur().syncDataAndTruncateJournal(); - void dropCollection( const string &name, string &errmsg, BSONObjBuilder &result ) { - LOG(1) << "dropCollection: " << name << endl; - NamespaceDetails *d = nsdetails(name); - if( d == 0 ) - return; + Database::closeDatabase( d->name(), d->path() ); + d = 0; // d is now deleted - BackgroundOperation::assertNoBgOpInProgForNs(name.c_str()); - - if ( d->nIndexes != 0 ) { - try { - verify( dropIndexes(d, name.c_str(), "*", errmsg, result, true) ); - } - catch( DBException& e ) { - stringstream ss; - ss << "drop: dropIndexes for collection failed - consider trying repair "; - ss << " cause: " << e.what(); - uasserted(12503,ss.str()); - } - verify( d->nIndexes == 0 ); - } - LOG(1) << "\t dropIndexes done" << endl; - result.append("ns", name.c_str()); - ClientCursor::invalidate(name.c_str()); - Top::global.collectionDropped( name ); - NamespaceDetailsTransient::eraseCollection( name ); - dropNS(name); - } - - /* deletes a record, just the pdfile portion -- no index cleanup, no cursor cleanup, etc. - caller must check if capped - */ - void DataFileMgr::_deleteRecord(NamespaceDetails *d, const char *ns, Record *todelete, const DiskLoc& dl) { - /* remove ourself from the record next/prev chain */ - { - if ( todelete->prevOfs() != DiskLoc::NullOfs ) - getDur().writingInt( todelete->getPrev(dl).rec()->nextOfs() ) = todelete->nextOfs(); - if ( todelete->nextOfs() != DiskLoc::NullOfs ) - getDur().writingInt( todelete->getNext(dl).rec()->prevOfs() ) = todelete->prevOfs(); - } - - /* remove ourself from extent pointers */ - { - Extent *e = getDur().writing( todelete->myExtent(dl) ); - if ( e->firstRecord == dl ) { - if ( todelete->nextOfs() == DiskLoc::NullOfs ) - e->firstRecord.Null(); - else - e->firstRecord.set(dl.a(), todelete->nextOfs() ); - } - if ( e->lastRecord == dl ) { - if ( todelete->prevOfs() == DiskLoc::NullOfs ) - e->lastRecord.Null(); - else - e->lastRecord.set(dl.a(), todelete->prevOfs() ); - } - } - - /* add to the free list */ - { - { - NamespaceDetails::Stats *s = getDur().writing(&d->stats); - s->datasize -= todelete->netLength(); - s->nrecords--; - } - - if (NamespaceString(ns).coll == "system.indexes") { - /* temp: if in system.indexes, don't reuse, and zero out: we want to be - careful until validated more, as IndexDetails has pointers - to this disk location. so an incorrectly done remove would cause - a lot of problems. - */ - memset(getDur().writingPtr(todelete, todelete->lengthWithHeaders() ), 0, todelete->lengthWithHeaders() ); - } - else { - DEV { - unsigned long long *p = reinterpret_cast( todelete->data() ); - *getDur().writing(p) = 0; - //DEV memset(todelete->data, 0, todelete->netLength()); // attempt to notice invalid reuse. - } - d->addDeletedRec((DeletedRecord*)todelete, dl); - } - } - } - - void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK, bool noWarn, bool doLog ) { - deleteRecord( nsdetails(ns), ns, todelete, dl, cappedOK, noWarn, doLog ); - } - - void DataFileMgr::deleteRecord(NamespaceDetails* d, const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK, bool noWarn, bool doLog ) { - dassert( todelete == dl.rec() ); - - if ( d->isCapped() && !cappedOK ) { - out() << "failing remove on a capped ns " << ns << endl; - uassert( 10089 , "can't remove from a capped collection" , 0 ); - return; - } - - BSONObj toDelete; - if ( doLog ) { - BSONElement e = dl.obj()["_id"]; - if ( e.type() ) { - toDelete = e.wrap(); - } - } - - /* check if any cursors point to us. if so, advance them. */ - ClientCursor::aboutToDelete(d, dl); - - unindexRecord(d, todelete, dl, noWarn); - - _deleteRecord(d, ns, todelete, dl); - NamespaceDetailsTransient::get( ns ).notifyOfWriteOp(); - - if ( ! toDelete.isEmpty() ) { - logOp( "d" , ns , toDelete ); - } - } - - Counter64 moveCounter; - ServerStatusMetricField moveCounterDisplay( "record.moves", &moveCounter ); - - /** Note: if the object shrinks a lot, we don't free up space, we leave extra at end of the record. - */ - const DiskLoc DataFileMgr::updateRecord( - const char *ns, - NamespaceDetails *d, - NamespaceDetailsTransient *nsdt, - Record *toupdate, const DiskLoc& dl, - const char *_buf, int _len, OpDebug& debug, bool god) { - - dassert( toupdate == dl.rec() ); - - BSONObj objOld = BSONObj::make(toupdate); - BSONObj objNew(_buf); - DEV verify( objNew.objsize() == _len ); - DEV verify( objNew.objdata() == _buf ); - - if( !objNew.hasElement("_id") && objOld.hasElement("_id") ) { - /* add back the old _id value if the update removes it. Note this implementation is slow - (copies entire object multiple times), but this shouldn't happen often, so going for simple - code, not speed. - */ - BSONObjBuilder b; - BSONElement e; - verify( objOld.getObjectID(e) ); - b.append(e); // put _id first, for best performance - b.appendElements(objNew); - objNew = b.obj(); - } - - NamespaceString nsstring(ns); - if (nsstring.coll == "system.users") { - uassertStatusOK(AuthorizationManager::checkValidPrivilegeDocument(nsstring.db, objNew)); - } - - /* duplicate key check. we descend the btree twice - once for this check, and once for the actual inserts, further - below. that is suboptimal, but it's pretty complicated to do it the other way without rollbacks... - */ - vector changes; - bool changedId = false; - getIndexChanges(changes, ns, *d, objNew, objOld, changedId); - uassert( 13596 , str::stream() << "cannot change _id of a document old:" << objOld << " new:" << objNew , ! changedId ); - dupCheck(changes, *d, dl); - - if ( toupdate->netLength() < objNew.objsize() ) { - // doesn't fit. reallocate ----------------------------------------------------- - moveCounter.increment(); - uassert( 10003 , "failing update: objects in a capped ns cannot grow", !(d && d->isCapped())); - d->paddingTooSmall(); - deleteRecord(ns, toupdate, dl); - DiskLoc res = insert(ns, objNew.objdata(), objNew.objsize(), false, god); - - if (debug.nmoved == -1) // default of -1 rather than 0 - debug.nmoved = 1; - else - debug.nmoved += 1; - - return res; - } - - nsdt->notifyOfWriteOp(); - d->paddingFits(); - - /* have any index keys changed? */ - { - int keyUpdates = 0; - int z = d->getTotalIndexCount(); - for ( int x = 0; x < z; x++ ) { - IndexDetails& idx = d->idx(x); - IndexInterface& ii = idx.idxInterface(); - for ( unsigned i = 0; i < changes[x].removed.size(); i++ ) { - try { - bool found = ii.unindex(idx.head, idx, *changes[x].removed[i], dl); - if ( ! found ) { - RARELY warning() << "ns: " << ns << " couldn't unindex key: " << *changes[x].removed[i] - << " for doc: " << objOld["_id"] << endl; - } - } - catch (AssertionException&) { - debug.extra << " exception update unindex "; - problem() << " caught assertion update unindex " << idx.indexNamespace() << endl; - } - } - verify( !dl.isNull() ); - BSONObj idxKey = idx.info.obj().getObjectField("key"); - Ordering ordering = Ordering::make(idxKey); - keyUpdates += changes[x].added.size(); - for ( unsigned i = 0; i < changes[x].added.size(); i++ ) { - try { - /* we did the dupCheck() above. so we don't have to worry about it here. */ - ii.bt_insert( - idx.head, - dl, *changes[x].added[i], ordering, /*dupsAllowed*/true, idx); - } - catch (AssertionException& e) { - debug.extra << " exception update index "; - problem() << " caught assertion update index " << idx.indexNamespace() << " " << e << " " << objNew["_id"] << endl; - } - } - } - - debug.keyUpdates = keyUpdates; - } - - // update in place - int sz = objNew.objsize(); - memcpy(getDur().writingPtr(toupdate->data(), sz), objNew.objdata(), sz); - return dl; - } - - int Extent::followupSize(int len, int lastExtentLen) { - verify( len < Extent::maxSize() ); - int x = initialSize(len); - // changed from 1.20 to 1.35 in v2.1.x to get to larger extent size faster - int y = (int) (lastExtentLen < 4000000 ? lastExtentLen * 4.0 : lastExtentLen * 1.35); - int sz = y > x ? y : x; - - if ( sz < lastExtentLen ) { - // this means there was an int overflow - // so we should turn it into maxSize - sz = Extent::maxSize(); - } - else if ( sz > Extent::maxSize() ) { - sz = Extent::maxSize(); - } - - sz = ((int)sz) & 0xffffff00; - verify( sz > len ); - - return sz; - } - -#if 0 - void testSorting() { - BSONObjBuilder b; - b.appendNull(""); - BSONObj x = b.obj(); - - BSONObjExternalSorter sorter(*IndexDetails::iis[1]); - - sorter.add(x, DiskLoc(3,7)); - sorter.add(x, DiskLoc(4,7)); - sorter.add(x, DiskLoc(2,7)); - sorter.add(x, DiskLoc(1,7)); - sorter.add(x, DiskLoc(3,77)); - - sorter.sort(); - - auto_ptr i = sorter.iterator(); - while( i->more() ) { - BSONObjExternalSorter::Data d = i->next(); - /*cout << d.second.toString() << endl; - cout << d.first.objsize() << endl; - cout<<"SORTER next:" << d.first.toString() << endl;*/ - } - } -#endif -#pragma pack(1) - struct IDToInsert { - char type; - char id[4]; - OID oid; - - IDToInsert() { - type = 0; - } - - bool needed() const { return type > 0; } - - void init() { - type = static_cast(jstOID); - strcpy( id, "_id" ); - oid.init(); - verify( size() == 17 ); - } - - int size() const { return sizeof( IDToInsert ); } - - const char* rawdata() const { return reinterpret_cast( this ); } - }; -#pragma pack() - - void DataFileMgr::insertAndLog( const char *ns, const BSONObj &o, bool god, bool fromMigrate ) { - BSONObj tmp = o; - insertWithObjMod( ns, tmp, false, god ); - logOp( "i", ns, tmp, 0, 0, fromMigrate ); - } - - /** @param o the object to insert. can be modified to add _id and thus be an in/out param - */ - DiskLoc DataFileMgr::insertWithObjMod(const char* ns, BSONObj& o, bool mayInterrupt, bool god) { - bool addedID = false; - DiskLoc loc = insert( ns, o.objdata(), o.objsize(), mayInterrupt, god, true, &addedID ); - if( addedID && !loc.isNull() ) - o = BSONObj::make( loc.rec() ); - return loc; - } - - // We are now doing two btree scans for all unique indexes (one here, and one when we've - // written the record to the collection. This could be made more efficient inserting - // dummy data here, keeping pointers to the btree nodes holding the dummy data and then - // updating the dummy data with the DiskLoc of the real record. - void checkNoIndexConflicts( NamespaceDetails *d, const BSONObj &obj ) { - for ( int idxNo = 0; idxNo < d->nIndexes; idxNo++ ) { - if( d->idx(idxNo).unique() ) { - IndexDetails& idx = d->idx(idxNo); - if (ignoreUniqueIndex(idx)) - continue; - BSONObjSet keys; - idx.getKeysFromObject(obj, keys); - BSONObj order = idx.keyPattern(); - IndexInterface& ii = idx.idxInterface(); - for ( BSONObjSet::iterator i=keys.begin(); i != keys.end(); i++ ) { - // WARNING: findSingle may not be compound index safe. this may need to change. see notes in - // findSingle code. - uassert( 12582, "duplicate key insert for unique index of capped collection", - ii.findSingle(idx, idx.head, *i ).isNull() ); - } - } - } - } - - /** add a record to the end of the linked list chain within this extent. - require: you must have already declared write intent for the record header. - */ - void addRecordToRecListInExtent(Record *r, DiskLoc loc) { - dassert( loc.rec() == r ); - Extent *e = r->myExtent(loc); - if ( e->lastRecord.isNull() ) { - Extent::FL *fl = getDur().writing(e->fl()); - fl->firstRecord = fl->lastRecord = loc; - r->prevOfs() = r->nextOfs() = DiskLoc::NullOfs; - } - else { - Record *oldlast = e->lastRecord.rec(); - r->prevOfs() = e->lastRecord.getOfs(); - r->nextOfs() = DiskLoc::NullOfs; - getDur().writingInt(oldlast->nextOfs()) = loc.getOfs(); - getDur().writingDiskLoc(e->lastRecord) = loc; - } - } - - NOINLINE_DECL DiskLoc outOfSpace(const char* ns, NamespaceDetails* d, int lenWHdr, bool god) { - DiskLoc loc; - if ( ! d->isCapped() ) { // size capped doesn't grow - LOG(1) << "allocating new extent for " << ns << " padding:" << d->paddingFactor() << " lenWHdr: " << lenWHdr << endl; - cc().database()->allocExtent(ns, Extent::followupSize(lenWHdr, d->lastExtentSize), false, !god); - loc = d->alloc(ns, lenWHdr); - if ( loc.isNull() ) { - log() << "warning: alloc() failed after allocating new extent. lenWHdr: " << lenWHdr << " last extent size:" << d->lastExtentSize << "; trying again" << endl; - for ( int z=0; z<10 && lenWHdr > d->lastExtentSize; z++ ) { - log() << "try #" << z << endl; - cc().database()->allocExtent(ns, Extent::followupSize(lenWHdr, d->lastExtentSize), false, !god); - loc = d->alloc(ns, lenWHdr); - if ( ! loc.isNull() ) - break; - } - } - } - return loc; - } - - /** used by insert and also compact - * @return null loc if out of space - */ - DiskLoc allocateSpaceForANewRecord(const char* ns, NamespaceDetails* d, int lenWHdr, bool god) { - DiskLoc loc = d->alloc(ns, lenWHdr); - if ( loc.isNull() ) { - loc = outOfSpace(ns, d, lenWHdr, god); - } - return loc; - } - - bool NOINLINE_DECL insert_checkSys(const char *sys, const char *ns, bool& wouldAddIndex, const void *obuf, bool god) { - uassert( 10095 , "attempt to insert in reserved database name 'system'", sys != ns); - if ( strstr(ns, ".system.") ) { - // later:check for dba-type permissions here if have that at some point separate - if (NamespaceString(ns).coll == "system.indexes") - wouldAddIndex = true; - else if ( legalClientSystemNS( ns , true ) ) { - if ( obuf && strstr( ns , ".system.users" ) ) { - BSONObj t( reinterpret_cast( obuf ) ); - uassertStatusOK(AuthorizationManager::checkValidPrivilegeDocument( - nsToDatabaseSubstring(ns), t)); - } - } - else if ( !god ) { - uasserted(16459, str::stream() << "attempt to insert in system namespace '" - << ns << "'"); - } - } - return true; - } - - NOINLINE_DECL NamespaceDetails* insert_newNamespace(const char *ns, int len, bool god) { - checkConfigNS(ns); - // This may create first file in the database. - int ies = Extent::initialSize(len); - if( str::contains(ns, '$') && len + Record::HeaderSize >= BtreeData_V1::BucketSize - 256 && len + Record::HeaderSize <= BtreeData_V1::BucketSize + 256 ) { - // probably an index. so we pick a value here for the first extent instead of using initialExtentSize() which is more - // for user collections. TODO: we could look at the # of records in the parent collection to be smarter here. - ies = (32+4) * 1024; - } - cc().database()->allocExtent(ns, ies, false, false); - NamespaceDetails *d = nsdetails(ns); - if ( !god ) - ensureIdIndexForNewNs(ns); - _ensureSystemIndexes(ns); - addNewNamespaceToCatalog(ns); - return d; - } - - void NOINLINE_DECL insert_makeIndex(NamespaceDetails* tableToIndex, - const string& tabletoidxns, - const DiskLoc& loc, - bool mayInterrupt) { - uassert(13143, - "can't create index on system.indexes", - NamespaceString(tabletoidxns).coll != "system.indexes"); - - BSONObj info = loc.obj(); - bool background = info["background"].trueValue(); - if (background && !isMasterNs(tabletoidxns.c_str())) { - /* don't do background indexing on slaves. there are nuances. this could be added later - but requires more code. - */ - log() << "info: indexing in foreground on this replica; was a background index build on the primary" << endl; - background = false; - } - - // The total number of indexes right before we write to the collection - int oldNIndexes = -1; - int idxNo = tableToIndex->getTotalIndexCount(); - std::string idxName = info["name"].valuestr(); - - // Set curop description before setting indexBuildInProg, so that there's something - // commands can find and kill as soon as indexBuildInProg is set. Only set this if it's a - // killable index, so we don't overwrite commands in currentOp. - if (mayInterrupt) { - cc().curop()->setQuery(info); - } - - try { - IndexDetails& idx = tableToIndex->getNextIndexDetails(tabletoidxns.c_str()); - // It's important that this is outside the inner try/catch so that we never try to call - // kill_idx on a half-formed disk loc (if this asserts). - getDur().writingDiskLoc(idx.info) = loc; - - try { - getDur().writingInt(tableToIndex->indexBuildsInProgress) += 1; - buildAnIndex(tabletoidxns, tableToIndex, idx, background, mayInterrupt); - } - catch (DBException& e) { - // save our error msg string as an exception or dropIndexes will overwrite our message - LastError *le = lastError.get(); - int savecode = 0; - string saveerrmsg; - if ( le ) { - savecode = le->code; - saveerrmsg = le->msg; - } - else { - savecode = e.getCode(); - saveerrmsg = e.what(); - } - - // Recalculate the index # so we can remove it from the list in the next catch - idxNo = IndexBuildsInProgress::get(tabletoidxns.c_str(), idxName); - // roll back this index - idx.kill_idx(); - - verify(le && !saveerrmsg.empty()); - setLastError(savecode,saveerrmsg.c_str()); - throw; - } - - // Recompute index numbers - tableToIndex = nsdetails(tabletoidxns); - idxNo = IndexBuildsInProgress::get(tabletoidxns.c_str(), idxName); - verify(idxNo > -1); - - // Make sure the newly created index is relocated to nIndexes, if it isn't already there - if (idxNo != tableToIndex->nIndexes) { - log() << "switching indexes at position " << idxNo << " and " - << tableToIndex->nIndexes << endl; - // We cannot use idx here, as it may point to a different index entry if it was - // flipped during building - IndexDetails temp = tableToIndex->idx(idxNo); - *getDur().writing(&tableToIndex->idx(idxNo)) = - tableToIndex->idx(tableToIndex->nIndexes); - *getDur().writing(&tableToIndex->idx(tableToIndex->nIndexes)) = temp; - - // We also have to flip multikey entries - bool tempMultikey = tableToIndex->isMultikey(idxNo); - tableToIndex->setIndexIsMultikey(tabletoidxns.c_str(), idxNo, - tableToIndex->isMultikey(tableToIndex->nIndexes)); - tableToIndex->setIndexIsMultikey(tabletoidxns.c_str(), tableToIndex->nIndexes, - tempMultikey); - - idxNo = tableToIndex->nIndexes; - } - - // Store the current total of indexes in case something goes wrong actually adding the - // index - oldNIndexes = tableToIndex->getTotalIndexCount(); - - // clear transient info caches so they refresh; increments nIndexes - tableToIndex->addIndex(tabletoidxns.c_str()); - getDur().writingInt(tableToIndex->indexBuildsInProgress) -= 1; - - IndexType* indexType = idx.getSpec().getType(); - const IndexPlugin *plugin = indexType ? indexType->getPlugin() : NULL; - if (plugin) { - plugin->postBuildHook( idx.getSpec() ); - } - - } - catch (...) { - // Generally, this will be called as an exception from building the index bubbles up. - // Thus, the index will have already been cleaned up. This catch just ensures that the - // metadata is consistent on any exception. It may leak like a sieve if the index - // successfully finished building and addIndex or kill_idx threw. - - // Check if nIndexes was incremented - if (oldNIndexes != -1 && oldNIndexes != tableToIndex->nIndexes) { - getDur().writingInt(tableToIndex->nIndexes) = oldNIndexes; - } - - // Move any other in prog indexes "back" one. It is important that idxNo is set - // correctly so that the correct index is removed - IndexBuildsInProgress::remove(tabletoidxns.c_str(), idxNo); - getDur().writingInt(tableToIndex->indexBuildsInProgress) -= 1; - - throw; - } - } - - // indexName is passed in because index details may not be pointing to something valid at this - // point - int IndexBuildsInProgress::get(const char* ns, const std::string& indexName) { - Lock::assertWriteLocked(ns); - NamespaceDetails* nsd = nsdetails(ns); - - // Go through unfinished index builds and try to find this index - for (int i=nsd->nIndexes; inIndexes+nsd->indexBuildsInProgress; i++) { - if (indexName == nsd->idx(i).indexName()) { - return i; - } - } - - return -1; - } - - void IndexBuildsInProgress::remove(const char* ns, int offset) { - Lock::assertWriteLocked(ns); - NamespaceDetails* nsd = nsdetails(ns); - - for (int i=offset; igetTotalIndexCount(); i++) { - if (i < NamespaceDetails::NIndexesMax-1) { - *getDur().writing(&nsd->idx(i)) = nsd->idx(i+1); - nsd->setIndexIsMultikey(ns, i, nsd->isMultikey(i+1)); - } - else { - *getDur().writing(&nsd->idx(i)) = IndexDetails(); - nsd->setIndexIsMultikey(ns, i, false); - } - } - } - - DiskLoc DataFileMgr::insert(const char* ns, - const void* obuf, - int32_t len, - bool mayInterrupt, - bool god, - bool mayAddIndex, - bool* addedID) { - bool wouldAddIndex = false; - massert( 10093 , "cannot insert into reserved $ collection", god || NamespaceString::normal( ns ) ); - uassert( 10094 , str::stream() << "invalid ns: " << ns , isValidNS( ns ) ); - { - const char *sys = strstr(ns, "system."); - if ( sys && !insert_checkSys(sys, ns, wouldAddIndex, obuf, god) ) - return DiskLoc(); - } - bool addIndex = wouldAddIndex && mayAddIndex; - - NamespaceDetails *d = nsdetails(ns); - if ( d == 0 ) { - d = insert_newNamespace(ns, len, god); - } - - NamespaceDetails *tableToIndex = 0; - - string tabletoidxns; - BSONObj fixedIndexObject; - if ( addIndex ) { - verify( obuf ); - BSONObj io((const char *) obuf); - if( !prepareToBuildIndex(io, - mayInterrupt, - god, - tabletoidxns, - tableToIndex, - fixedIndexObject) ) { - // prepare creates _id itself, or this indicates to fail the build silently (such - // as if index already exists) - return DiskLoc(); - } - if ( ! fixedIndexObject.isEmpty() ) { - obuf = fixedIndexObject.objdata(); - len = fixedIndexObject.objsize(); - } - } - - IDToInsert idToInsert; // only initialized if needed - - if( !god ) { - /* Check if we have an _id field. If we don't, we'll add it. - Note that btree buckets which we insert aren't BSONObj's, but in that case god==true. - */ - BSONObj io((const char *) obuf); - BSONElement idField = io.getField( "_id" ); - uassert( 10099 , "_id cannot be an array", idField.type() != Array ); - // we don't add _id for capped collections in local as they don't have an _id index - if( idField.eoo() && - !wouldAddIndex && - nsToDatabase( ns ) != "local" && - d->haveIdIndex() ) { - - if( addedID ) - *addedID = true; - - idToInsert.init(); - len += idToInsert.size(); - } - - BSONElementManipulator::lookForTimestamps( io ); - } - - int lenWHdr = d->getRecordAllocationSize( len + Record::HeaderSize ); - fassert( 16440, lenWHdr >= ( len + Record::HeaderSize ) ); - - // If the collection is capped, check if the new object will violate a unique index - // constraint before allocating space. - if (d->nIndexes && - d->isCapped() && - !god) { - checkNoIndexConflicts( d, BSONObj( reinterpret_cast( obuf ) ) ); - } - - bool earlyIndex = true; - DiskLoc loc; - if( idToInsert.needed() || tableToIndex || d->isCapped() ) { - // if need id, we don't do the early indexing. this is not the common case so that is sort of ok - earlyIndex = false; - loc = allocateSpaceForANewRecord(ns, d, lenWHdr, god); - } - else { - loc = d->allocWillBeAt(ns, lenWHdr); - if( loc.isNull() ) { - // need to get a new extent so we have to do the true alloc now (not common case) - earlyIndex = false; - loc = allocateSpaceForANewRecord(ns, d, lenWHdr, god); - } - } - if ( loc.isNull() ) { - log() << "insert: couldn't alloc space for object ns:" << ns << " capped:" << d->isCapped() << endl; - verify(d->isCapped()); - return DiskLoc(); - } - - if( earlyIndex ) { - // add record to indexes using two step method so we can do the reading outside a write lock - if ( d->nIndexes ) { - verify( obuf ); - BSONObj obj((const char *) obuf); - try { - indexRecordUsingTwoSteps(ns, d, obj, loc, true); - } - catch( AssertionException& ) { - // should be a dup key error on _id index - dassert( !tableToIndex && !d->isCapped() ); - // no need to delete/rollback the record as it was not added yet - throw; - } - } - // really allocate now - DiskLoc real = allocateSpaceForANewRecord(ns, d, lenWHdr, god); - verify( real == loc ); - } - - Record *r = loc.rec(); - { - verify( r->lengthWithHeaders() >= lenWHdr ); - r = (Record*) getDur().writingPtr(r, lenWHdr); - if( idToInsert.needed() ) { - /* a little effort was made here to avoid a double copy when we add an ID */ - int originalSize = *((int*) obuf); - ((int&)*r->data()) = originalSize + idToInsert.size(); - memcpy(r->data()+4, idToInsert.rawdata(), idToInsert.size()); - memcpy(r->data()+4+idToInsert.size(), ((char*)obuf)+4, originalSize-4); - } - else { - if( obuf ) // obuf can be null from internal callers - memcpy(r->data(), obuf, len); - } - } - - addRecordToRecListInExtent(r, loc); - - /* durability todo : this could be a bit annoying / slow to record constantly */ - { - NamespaceDetails::Stats *s = getDur().writing(&d->stats); - s->datasize += r->netLength(); - s->nrecords++; - } - - // we don't bother resetting query optimizer stats for the god tables - also god is true when adding a btree bucket - if ( !god ) - NamespaceDetailsTransient::get( ns ).notifyOfWriteOp(); - - if ( tableToIndex ) { - insert_makeIndex(tableToIndex, tabletoidxns, loc, mayInterrupt); - } - - /* add this record to our indexes */ - if ( !earlyIndex && d->nIndexes ) { - try { - BSONObj obj(r->data()); - // not sure which of these is better -- either can be used. oldIndexRecord may be faster, - // but twosteps handles dup key errors more efficiently. - //oldIndexRecord(d, obj, loc); - indexRecordUsingTwoSteps(ns, d, obj, loc, false); - - } - catch( AssertionException& e ) { - // should be a dup key error on _id index - if( tableToIndex || d->isCapped() ) { - massert( 12583, "unexpected index insertion failure on capped collection", !d->isCapped() ); - string s = e.toString(); - s += " : on addIndex/capped - collection and its index will not match"; - setLastError(0, s.c_str()); - error() << s << endl; - } - else { - // normal case -- we can roll back - _deleteRecord(d, ns, r, loc); - throw; - } - } - } - - d->paddingFits(); - - return loc; - } - - /* special version of insert for transaction logging -- streamlined a bit. - assumes ns is capped and no indexes - */ - Record* DataFileMgr::fast_oplog_insert(NamespaceDetails *d, const char *ns, int len) { - verify( d ); - RARELY verify( d == nsdetails(ns) ); - DEV verify( d == nsdetails(ns) ); - - massert( 16509, - str::stream() - << "fast_oplog_insert requires a capped collection " - << " but " << ns << " is not capped", - d->isCapped() ); - - //record timing on oplog inserts - boost::optional insertTimer; - //skip non-oplog collections - if (NamespaceString::oplog(ns)) { - insertTimer = boost::in_place(&oplogInsertStats); - oplogInsertBytesStats.increment(len); //record len of inserted records for oplog - } - - int lenWHdr = len + Record::HeaderSize; - DiskLoc loc = d->alloc(ns, lenWHdr); - verify( !loc.isNull() ); - - Record *r = loc.rec(); - verify( r->lengthWithHeaders() >= lenWHdr ); - - Extent *e = r->myExtent(loc); - if ( e->lastRecord.isNull() ) { - Extent::FL *fl = getDur().writing( e->fl() ); - fl->firstRecord = fl->lastRecord = loc; - - Record::NP *np = getDur().writing(r->np()); - np->nextOfs = np->prevOfs = DiskLoc::NullOfs; - } - else { - Record *oldlast = e->lastRecord.rec(); - Record::NP *np = getDur().writing(r->np()); - np->prevOfs = e->lastRecord.getOfs(); - np->nextOfs = DiskLoc::NullOfs; - getDur().writingInt( oldlast->nextOfs() ) = loc.getOfs(); - e->lastRecord.writing() = loc; - } - - /* todo: don't update for oplog? seems wasteful. */ - { - NamespaceDetails::Stats *s = getDur().writing(&d->stats); - s->datasize += r->netLength(); - s->nrecords++; - } - - return r; - } - -} // namespace mongo - -#include "clientcursor.h" - -namespace mongo { - - void dropAllDatabasesExceptLocal() { - Lock::GlobalWrite lk; - - vector n; - getDatabaseNames(n); - if( n.size() == 0 ) return; - log() << "dropAllDatabasesExceptLocal " << n.size() << endl; - for( vector::iterator i = n.begin(); i != n.end(); i++ ) { - if( *i != "local" ) { - Client::Context ctx(*i); - dropDatabase(*i); - } - } - } - - void dropDatabase(const std::string& db) { - LOG(1) << "dropDatabase " << db << endl; - Lock::assertWriteLocked(db); - Database *d = cc().database(); - verify( d ); - verify( d->name == db ); - - BackgroundOperation::assertNoBgOpInProgForDb(d->name.c_str()); - - // Not sure we need this here, so removed. If we do, we need to move it down - // within other calls both (1) as they could be called from elsewhere and - // (2) to keep the lock order right - groupcommitmutex must be locked before - // mmmutex (if both are locked). - // - // RWLockRecursive::Exclusive lk(MongoFile::mmmutex); - - getDur().syncDataAndTruncateJournal(); - - Database::closeDatabase( d->name.c_str(), d->path ); - d = 0; // d is now deleted - - _deleteDataFiles( db.c_str() ); - } - - typedef boost::filesystem::path Path; - - void boostRenameWrapper( const Path &from, const Path &to ) { - try { - boost::filesystem::rename( from, to ); - } - catch ( const boost::filesystem::filesystem_error & ) { - // boost rename doesn't work across partitions - boost::filesystem::copy_file( from, to); - boost::filesystem::remove( from ); - } - } - - // back up original database files to 'temp' dir - void _renameForBackup( const char *database, const Path &reservedPath ) { - Path newPath( reservedPath ); - if ( directoryperdb ) - newPath /= database; - class Renamer : public FileOp { - public: - Renamer( const Path &newPath ) : newPath_( newPath ) {} - private: - const boost::filesystem::path &newPath_; - virtual bool apply( const Path &p ) { - if ( !boost::filesystem::exists( p ) ) - return false; - boostRenameWrapper( p, newPath_ / ( p.leaf().string() + ".bak" ) ); - return true; - } - virtual const char * op() const { - return "renaming"; - } - } renamer( newPath ); - _applyOpToDataFiles( database, renamer, true ); - } - - // move temp files to standard data dir - void _replaceWithRecovered( const char *database, const char *reservedPathString ) { - Path newPath( dbpath ); - if ( directoryperdb ) - newPath /= database; - class Replacer : public FileOp { - public: - Replacer( const Path &newPath ) : newPath_( newPath ) {} - private: - const boost::filesystem::path &newPath_; - virtual bool apply( const Path &p ) { - if ( !boost::filesystem::exists( p ) ) - return false; - boostRenameWrapper( p, newPath_ / p.leaf() ); - return true; - } - virtual const char * op() const { - return "renaming"; - } - } replacer( newPath ); - _applyOpToDataFiles( database, replacer, true, reservedPathString ); - } - - // generate a directory name for storing temp data files - Path uniqueReservedPath( const char *prefix ) { - Path repairPath = Path( repairpath ); - Path reservedPath; - int i = 0; - bool exists = false; - do { - stringstream ss; - ss << prefix << "_repairDatabase_" << i++; - reservedPath = repairPath / ss.str(); - MONGO_ASSERT_ON_EXCEPTION( exists = boost::filesystem::exists( reservedPath ) ); - } - while ( exists ); - return reservedPath; - } - - boost::intmax_t dbSize( const char *database ) { - class SizeAccumulator : public FileOp { - public: - SizeAccumulator() : totalSize_( 0 ) {} - boost::intmax_t size() const { - return totalSize_; - } - private: - virtual bool apply( const boost::filesystem::path &p ) { - if ( !boost::filesystem::exists( p ) ) - return false; - totalSize_ += boost::filesystem::file_size( p ); - return true; - } - virtual const char *op() const { - return "checking size"; - } - boost::intmax_t totalSize_; - }; - SizeAccumulator sa; - _applyOpToDataFiles( database, sa ); - return sa.size(); - } - - bool repairDatabase( string dbNameS , string &errmsg, - bool preserveClonedFilesOnFailure, bool backupOriginalFiles ) { - doingRepair dr; - dbNameS = nsToDatabase( dbNameS ); - const char * dbName = dbNameS.c_str(); - - stringstream ss; - ss << "localhost:" << cmdLine.port; - string localhost = ss.str(); - - problem() << "repairDatabase " << dbName << endl; - verify( cc().database()->name == dbName ); - verify( cc().database()->path == dbpath ); - - BackgroundOperation::assertNoBgOpInProgForDb(dbName); - - getDur().syncDataAndTruncateJournal(); // Must be done before and after repair - - boost::intmax_t totalSize = dbSize( dbName ); - boost::intmax_t freeSize = File::freeSpace(repairpath); - if ( freeSize > -1 && freeSize < totalSize ) { - stringstream ss; - ss << "Cannot repair database " << dbName << " having size: " << totalSize - << " (bytes) because free disk space is: " << freeSize << " (bytes)"; - errmsg = ss.str(); - problem() << errmsg << endl; - return false; - } - - killCurrentOp.checkForInterrupt(); - - Path reservedPath = - uniqueReservedPath( ( preserveClonedFilesOnFailure || backupOriginalFiles ) ? - "backup" : "_tmp" ); - MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::create_directory( reservedPath ) ); - string reservedPathString = reservedPath.string(); - - bool res; - { - // clone to temp location, which effectively does repair - Client::Context ctx( dbName, reservedPathString ); - verify( ctx.justCreated() ); - - res = Cloner::cloneFrom(localhost.c_str(), errmsg, dbName, - /*logForReplication=*/false, /*slaveOk*/false, - /*replauth*/false, /*snapshot*/false, /*mayYield*/false, - /*mayBeInterrupted*/true); - - Database::closeDatabase( dbName, reservedPathString.c_str() ); - } - - getDur().syncDataAndTruncateJournal(); // Must be done before and after repair - MongoFile::flushAll(true); // need both in case journaling is disabled - - if ( !res ) { - errmsg = str::stream() << "clone failed for " << dbName << " with error: " << errmsg; - problem() << errmsg << endl; - - if ( !preserveClonedFilesOnFailure ) - MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::remove_all( reservedPath ) ); - - return false; - } - - Client::Context ctx( dbName ); - Database::closeDatabase( dbName, dbpath ); - - if ( backupOriginalFiles ) { - _renameForBackup( dbName, reservedPath ); - } - else { - _deleteDataFiles( dbName ); - MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::create_directory( Path( dbpath ) / dbName ) ); - } - - _replaceWithRecovered( dbName, reservedPathString.c_str() ); - - if ( !backupOriginalFiles ) - MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::remove_all( reservedPath ) ); - - return true; - } - - void _applyOpToDataFiles( const char *database, FileOp &fo, bool afterAllocator, const string& path ) { - if ( afterAllocator ) - FileAllocator::get()->waitUntilFinished(); - string c = database; - c += '.'; - boost::filesystem::path p(path); - if ( directoryperdb ) - p /= database; - boost::filesystem::path q; - q = p / (c+"ns"); - bool ok = false; - MONGO_ASSERT_ON_EXCEPTION( ok = fo.apply( q ) ); - if ( ok ) - LOG(2) << fo.op() << " file " << q.string() << endl; - int i = 0; - int extra = 10; // should not be necessary, this is defensive in case there are missing files - while ( 1 ) { - verify( i <= DiskLoc::MaxFiles ); - stringstream ss; - ss << c << i; - q = p / ss.str(); - MONGO_ASSERT_ON_EXCEPTION( ok = fo.apply(q) ); - if ( ok ) { - if ( extra != 10 ) { - LOG(1) << fo.op() << " file " << q.string() << endl; - log() << " _applyOpToDataFiles() warning: extra == " << extra << endl; - } - } - else if ( --extra <= 0 ) - break; - i++; - } - } - - NamespaceDetails* nsdetails_notinline(const char *ns) { return nsdetails(ns); } - - bool DatabaseHolder::closeAll( const string& path , BSONObjBuilder& result , bool force ) { - log() << "DatabaseHolder::closeAll path:" << path << endl; - verify( Lock::isW() ); - getDur().commitNow(); // bad things happen if we close a DB with outstanding writes - - map& m = _paths[path]; - _size -= m.size(); - - set< string > dbs; - for ( map::iterator i = m.begin(); i != m.end(); i++ ) { - wassert( i->second->path == path ); - dbs.insert( i->first ); - } - - currentClient.get()->getContext()->_clear(); - - BSONObjBuilder bb( result.subarrayStart( "dbs" ) ); - int n = 0; - int nNotClosed = 0; - for( set< string >::iterator i = dbs.begin(); i != dbs.end(); ++i ) { - string name = *i; - LOG(2) << "DatabaseHolder::closeAll path:" << path << " name:" << name << endl; - Client::Context ctx( name , path ); - if( !force && BackgroundOperation::inProgForDb(name.c_str()) ) { - log() << "WARNING: can't close database " << name << " because a bg job is in progress - try killOp command" << endl; - nNotClosed++; - } - else { - Database::closeDatabase( name.c_str() , path ); - bb.append( bb.numStr( n++ ) , name ); - } - } - bb.done(); - if( nNotClosed ) - result.append("nNotClosed", nNotClosed); - else { - ClientCursor::assertNoCursors(); - } - - return true; + _deleteDataFiles( db ); } } // namespace mongo diff --git a/src/mongo/db/pdfile.h b/src/mongo/db/pdfile.h index 67343b686b9..ec38a2523d2 100644 --- a/src/mongo/db/pdfile.h +++ b/src/mongo/db/pdfile.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ /* pdfile.h @@ -26,13 +38,13 @@ #pragma once #include "mongo/db/client.h" +#include "mongo/db/catalog/database.h" #include "mongo/db/diskloc.h" -#include "mongo/db/jsobjmanipulator.h" -#include "mongo/db/memconcept.h" -#include "mongo/db/mongommf.h" -#include "mongo/db/namespace-inl.h" -#include "mongo/db/namespace_details-inl.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/storage/data_file.h" +#include "mongo/db/storage/durable_mapped_file.h" +#include "mongo/db/storage/extent.h" +#include "mongo/db/structure/catalog/namespace_details-inl.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/pdfile_version.h" #include "mongo/platform/cstdint.h" #include "mongo/util/log.h" @@ -40,651 +52,42 @@ namespace mongo { - class Cursor; class DataFileHeader; class Extent; class OpDebug; class Record; - struct SortPhaseOne; void dropDatabase(const std::string& db); - bool repairDatabase(string db, string &errmsg, bool preserveClonedFilesOnFailure = false, bool backupOriginalFiles = false); - - /* low level - only drops this ns */ - void dropNS(const string& dropNs); - - /* deletes this ns, indexes and cursors */ - void dropCollection( const string &name, string &errmsg, BSONObjBuilder &result ); - bool userCreateNS(const char *ns, BSONObj j, string& err, bool logForReplication, bool *deferIdIndex = 0); - shared_ptr findTableScan(const char *ns, const BSONObj& order, const DiskLoc &startLoc=DiskLoc()); - bool isValidNS( const StringData& ns ); + bool userCreateNS(const char *ns, BSONObj j, string& err, + bool logForReplication, bool createDefaultIndexes = true ); /*---------------------------------------------------------------------*/ - class MongoDataFile { - friend class DataFileMgr; - friend class BasicCursor; - public: - MongoDataFile(int fn) : _mb(0), fileNo(fn) { } - - /** @return true if found and opened. if uninitialized (prealloc only) does not open. */ - bool openExisting( const char *filename ); - - /** creates if DNE */ - void open(const char *filename, int requestedDataSize = 0, bool preallocateOnly = false); - - /* allocate a new extent from this datafile. - @param capped - true if capped collection - @param loops is our recursion check variable - you want to pass in zero - */ - Extent* createExtent(const char *ns, int approxSize, bool capped = false, int loops = 0); - - DataFileHeader *getHeader() { return header(); } - HANDLE getFd() { return mmf.getFd(); } - unsigned long long length() const { return mmf.length(); } - - /* return max size an extent may be */ - static int maxSize(); - - /** fsync */ - void flush( bool sync ); - - /** only use fore debugging */ - Extent* debug_getExtent(DiskLoc loc) { return _getExtent( loc ); } - private: - void badOfs(int) const; - void badOfs2(int) const; - int defaultSize( const char *filename ) const; - - Extent* getExtent(DiskLoc loc) const; - Extent* _getExtent(DiskLoc loc) const; - Record* recordAt(DiskLoc dl) const; - void grow(DiskLoc dl, int size); - - char* p() const { return (char *) _mb; } - DataFileHeader* header() { return (DataFileHeader*) _mb; } - - MongoMMF mmf; - void *_mb; // the memory mapped view - int fileNo; - }; - - class DataFileMgr { - friend class BasicCursor; - public: - DataFileMgr(); - void init(const string& path ); - - /* see if we can find an extent of the right size in the freelist. */ - static Extent* allocFromFreeList(const char *ns, int approxSize, bool capped = false); - - /** @return DiskLoc where item ends up */ - // changedId should be initialized to false - const DiskLoc updateRecord( - const char *ns, - NamespaceDetails *d, - NamespaceDetailsTransient *nsdt, - Record *toupdate, const DiskLoc& dl, - const char *buf, int len, OpDebug& debug, bool god=false); - - // The object o may be updated if modified on insert. - void insertAndLog( const char *ns, const BSONObj &o, bool god = false, bool fromMigrate = false ); - - /** - * insert() will add an _id to the object if not present. If you would like to see the - * final object after such an addition, use this method. - * note: does NOT put on oplog - * @param o both and in and out param - * @param mayInterrupt When true, killop may interrupt the function call. - */ - DiskLoc insertWithObjMod(const char* ns, - BSONObj& /*out*/o, - bool mayInterrupt = false, - bool god = false); - - /** - * Insert the contents of @param buf with length @param len into namespace @param ns. - * note: does NOT put on oplog - * @param mayInterrupt When true, killop may interrupt the function call. - * @param god if true, you may pass in obuf of NULL and then populate the returned DiskLoc - * after the call -- that will prevent a double buffer copy in some cases (btree.cpp). - * @param mayAddIndex almost always true, except for invocation from rename namespace - * command. - * @param addedID if not null, set to true if adding _id element. You must assure false - * before calling if using. - */ - DiskLoc insert(const char* ns, - const void* buf, - int32_t len, - bool mayInterrupt = false, - bool god = false, - bool mayAddIndex = true, - bool* addedID = 0); - static shared_ptr findAll(const StringData& ns, const DiskLoc &startLoc = DiskLoc()); - - /* special version of insert for transaction logging -- streamlined a bit. - assumes ns is capped and no indexes - no _id field check - */ - Record* fast_oplog_insert(NamespaceDetails *d, const char *ns, int len); - - static Extent* getExtent(const DiskLoc& dl); - static Record* getRecord(const DiskLoc& dl); - static DeletedRecord* getDeletedRecord(const DiskLoc& dl); - - void deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK = false, bool noWarn = false, bool logOp=false); - - void deleteRecord(NamespaceDetails* d, const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK = false, bool noWarn = false, bool logOp=false); - - /* does not clean up indexes, etc. : just deletes the record in the pdfile. use deleteRecord() to unindex */ - void _deleteRecord(NamespaceDetails *d, const char *ns, Record *todelete, const DiskLoc& dl); - - /** - * accessor/mutator for the 'precalced' keys (that is, sorted index keys) - * - * NB: 'precalced' is accessed from fastBuildIndex(), which is called from insert-related - * methods like insertWithObjMod(). It is mutated from various callers of the insert - * methods, which assume 'precalced' will not change while in the insert method. This - * should likely be refactored so theDataFileMgr takes full responsibility. - */ - SortPhaseOne* getPrecalced() const; - void setPrecalced(SortPhaseOne* precalced); - mongo::mutex _precalcedMutex; - - private: - vector files; - SortPhaseOne* _precalced; - }; - - extern DataFileMgr theDataFileMgr; - -#pragma pack(1) - - class DeletedRecord { - public: - - int lengthWithHeaders() const { _accessing(); return _lengthWithHeaders; } - int& lengthWithHeaders() { _accessing(); return _lengthWithHeaders; } - - int extentOfs() const { _accessing(); return _extentOfs; } - int& extentOfs() { _accessing(); return _extentOfs; } - - // TODO: we need to not const_cast here but problem is DiskLoc::writing - DiskLoc& nextDeleted() const { _accessing(); return const_cast(_nextDeleted); } - - DiskLoc myExtentLoc(const DiskLoc& myLoc) const { - _accessing(); - return DiskLoc(myLoc.a(), _extentOfs); - } - Extent* myExtent(const DiskLoc& myLoc) { - _accessing(); - return DataFileMgr::getExtent(DiskLoc(myLoc.a(), _extentOfs)); - } - private: - - void _accessing() const; - - int _lengthWithHeaders; - int _extentOfs; - DiskLoc _nextDeleted; - }; - - /* Record is a record in a datafile. DeletedRecord is similar but for deleted space. - - *11:03:20 AM) dm10gen: regarding extentOfs... - (11:03:42 AM) dm10gen: an extent is a continugous disk area, which contains many Records and DeleteRecords - (11:03:56 AM) dm10gen: a DiskLoc has two pieces, the fileno and ofs. (64 bit total) - (11:04:16 AM) dm10gen: to keep the headesr small, instead of storing a 64 bit ptr to the full extent address, we keep just the offset - (11:04:29 AM) dm10gen: we can do this as we know the record's address, and it has the same fileNo - (11:04:33 AM) dm10gen: see class DiskLoc for more info - (11:04:43 AM) dm10gen: so that is how Record::myExtent() works - (11:04:53 AM) dm10gen: on an alloc(), when we build a new Record, we must populate its extentOfs then - */ - class Record { - public: - enum HeaderSizeValue { HeaderSize = 16 }; - - int lengthWithHeaders() const { _accessing(); return _lengthWithHeaders; } - int& lengthWithHeaders() { _accessing(); return _lengthWithHeaders; } - - int extentOfs() const { _accessing(); return _extentOfs; } - int& extentOfs() { _accessing(); return _extentOfs; } - - int nextOfs() const { _accessing(); return _nextOfs; } - int& nextOfs() { _accessing(); return _nextOfs; } - - int prevOfs() const { _accessing(); return _prevOfs; } - int& prevOfs() { _accessing(); return _prevOfs; } - - const char * data() const { _accessing(); return _data; } - char * data() { _accessing(); return _data; } - - int netLength() const { _accessing(); return _netLength(); } - - /* use this when a record is deleted. basically a union with next/prev fields */ - DeletedRecord& asDeleted() { return *((DeletedRecord*) this); } - - Extent* myExtent(const DiskLoc& myLoc) { return DataFileMgr::getExtent(DiskLoc(myLoc.a(), extentOfs() ) ); } - - /* get the next record in the namespace, traversing extents as necessary */ - DiskLoc getNext(const DiskLoc& myLoc); - DiskLoc getPrev(const DiskLoc& myLoc); - - DiskLoc nextInExtent(const DiskLoc& myLoc) { - _accessing(); - if ( _nextOfs == DiskLoc::NullOfs ) - return DiskLoc(); - verify( _nextOfs ); - return DiskLoc(myLoc.a(), _nextOfs); - } - - struct NP { - int nextOfs; - int prevOfs; - }; - NP* np() { return (NP*) &_nextOfs; } - - // --------------------- - // memory cache - // --------------------- - - /** - * touches the data so that is in physical memory - * @param entireRecrd if false, only the header and first byte is touched - * if true, the entire record is touched - * */ - void touch( bool entireRecrd = false ) const; - - /** - * @return if this record is likely in physical memory - * its not guaranteed because its possible it gets swapped out in a very unlucky windows - */ - bool likelyInPhysicalMemory() const ; - - /** - * tell the cache this Record was accessed - * @return this, for simple chaining - */ - Record* accessed(); - - static bool likelyInPhysicalMemory( const char* data ); - - /** - * this adds stats about page fault exceptions currently - * specically how many times we call _accessing where the record is not in memory - * and how many times we throw a PageFaultException - */ - static void appendStats( BSONObjBuilder& b ); - - static void appendWorkingSetInfo( BSONObjBuilder& b ); - private: - - int _netLength() const { return _lengthWithHeaders - HeaderSize; } - - /** - * call this when accessing a field which could hit disk - */ - void _accessing() const; - - int _lengthWithHeaders; - int _extentOfs; - int _nextOfs; - int _prevOfs; - - /** be careful when referencing this that your write intent was correct */ - char _data[4]; - - public: - - static bool MemoryTrackingEnabled; - }; - - /* extents are datafile regions where all the records within the region - belong to the same namespace. - - (11:12:35 AM) dm10gen: when the extent is allocated, all its empty space is stuck into one big DeletedRecord - (11:12:55 AM) dm10gen: and that is placed on the free list - */ - class Extent { - public: - enum { extentSignature = 0x41424344 }; - unsigned magic; - DiskLoc myLoc; - DiskLoc xnext, xprev; /* next/prev extent for this namespace */ - - /* which namespace this extent is for. this is just for troubleshooting really - and won't even be correct if the collection were renamed! - */ - Namespace nsDiagnostic; - - int length; /* size of the extent, including these fields */ - DiskLoc firstRecord; - DiskLoc lastRecord; - char _extentData[4]; - - static int HeaderSize() { return sizeof(Extent)-4; } - - bool validates(const DiskLoc diskLoc, BSONArrayBuilder* errors = NULL); - - BSONObj dump() { - return BSON( "loc" << myLoc.toString() << "xnext" << xnext.toString() << "xprev" << xprev.toString() - << "nsdiag" << nsDiagnostic.toString() - << "size" << length << "firstRecord" << firstRecord.toString() << "lastRecord" << lastRecord.toString()); - } - - void dump(iostream& s) { - s << " loc:" << myLoc.toString() << " xnext:" << xnext.toString() << " xprev:" << xprev.toString() << '\n'; - s << " nsdiag:" << nsDiagnostic.toString() << '\n'; - s << " size:" << length << " firstRecord:" << firstRecord.toString() << " lastRecord:" << lastRecord.toString() << '\n'; - } - - /* assumes already zeroed -- insufficient for block 'reuse' perhaps - Returns a DeletedRecord location which is the data in the extent ready for us. - Caller will need to add that to the freelist structure in namespacedetail. - */ - DiskLoc init(const char *nsname, int _length, int _fileNo, int _offset, bool capped); - - /* like init(), but for a reuse case */ - DiskLoc reuse(const char *nsname, bool newUseIsAsCapped); - - bool isOk() const { return magic == extentSignature; } - void assertOk() const { verify(isOk()); } - - Record* newRecord(int len); - - Record* getRecord(DiskLoc dl) { - verify( !dl.isNull() ); - verify( dl.sameFile(myLoc) ); - int x = dl.getOfs() - myLoc.getOfs(); - verify( x > 0 ); - return (Record *) (((char *) this) + x); - } - - Extent* getNextExtent() { return xnext.isNull() ? 0 : DataFileMgr::getExtent(xnext); } - Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); } - - static int maxSize(); - static int minSize() { return 0x1000; } - /** - * @param len lengt of record we need - * @param lastRecord size of last extent which is a factor in next extent size - */ - static int followupSize(int len, int lastExtentLen); - - /** get a suggested size for the first extent in a namespace - * @param len length of record we need to insert - */ - static int initialSize(int len); - - struct FL { - DiskLoc firstRecord; - DiskLoc lastRecord; - }; - /** often we want to update just the firstRecord and lastRecord fields. - this helper is for that -- for use with getDur().writing() method - */ - FL* fl() { return (FL*) &firstRecord; } - - /** caller must declare write intent first */ - void markEmpty(); - private: - DiskLoc _reuse(const char *nsname, bool newUseIsAsCapped); // recycle an extent and reuse it for a different ns - }; - - /* a datafile - i.e. the "dbname.<#>" files : - - ---------------------- - DataFileHeader - ---------------------- - Extent (for a particular namespace) - Record - ... - Record (some chained for unused space) - ---------------------- - more Extents... - ---------------------- - */ - class DataFileHeader { - public: - int version; - int versionMinor; - int fileLength; - DiskLoc unused; /* unused is the portion of the file that doesn't belong to any allocated extents. -1 = no more */ - int unusedLength; - char reserved[8192 - 4*4 - 8]; - - char data[4]; // first extent starts here - - enum { HeaderSize = 8192 }; - - bool isCurrentVersion() const { - return version == PDFILE_VERSION && ( versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER - || versionMinor == PDFILE_VERSION_MINOR_24_AND_NEWER - ); - } - - bool uninitialized() const { return version == 0; } - - void init(int fileno, int filelength, const char* filename) { - if ( uninitialized() ) { - DEV log() << "datafileheader::init initializing " << filename << " n:" << fileno << endl; - if( !(filelength > 32768 ) ) { - massert(13640, str::stream() << "DataFileHeader looks corrupt at file open filelength:" << filelength << " fileno:" << fileno, false); - } - - { - // "something" is too vague, but we checked for the right db to be locked higher up the call stack - if( !Lock::somethingWriteLocked() ) { - LockState::Dump(); - log() << "*** TEMP NOT INITIALIZING FILE " << filename << ", not in a write lock." << endl; - log() << "temp bypass until more elaborate change - case that is manifesting is benign anyway" << endl; - return; -/** - log() << "ERROR can't create outside a write lock" << endl; - printStackTrace(); - ::abort(); -**/ - } - } - - getDur().createdFile(filename, filelength); - verify( HeaderSize == 8192 ); - DataFileHeader *h = getDur().writing(this); - h->fileLength = filelength; - h->version = PDFILE_VERSION; - h->versionMinor = PDFILE_VERSION_MINOR_22_AND_OLDER; // All dbs start like this - h->unused.set( fileno, HeaderSize ); - verify( (data-(char*)this) == HeaderSize ); - h->unusedLength = fileLength - HeaderSize - 16; - } - } - - bool isEmpty() const { - return uninitialized() || ( unusedLength == fileLength - HeaderSize - 16 ); - } - }; - -#pragma pack() - - inline Extent* MongoDataFile::_getExtent(DiskLoc loc) const { - loc.assertOk(); - Extent *e = (Extent *) (p()+loc.getOfs()); - return e; - } - - inline Extent* MongoDataFile::getExtent(DiskLoc loc) const { - Extent *e = _getExtent(loc); - e->assertOk(); - memconcept::is(e, memconcept::concept::extent); - return e; - } - -} // namespace mongo - -#include "cursor.h" - -namespace mongo { - - inline Record* MongoDataFile::recordAt(DiskLoc dl) const { - int ofs = dl.getOfs(); - if (ofs < DataFileHeader::HeaderSize) { - badOfs(ofs); // will uassert - external call to keep out of the normal code path - } - return reinterpret_cast(p() + ofs); - } - - inline DiskLoc Record::getNext(const DiskLoc& myLoc) { - _accessing(); - if ( _nextOfs != DiskLoc::NullOfs ) { - /* defensive */ - if ( _nextOfs >= 0 && _nextOfs < 10 ) { - logContext("Assertion failure - Record::getNext() referencing a deleted record?"); - return DiskLoc(); - } - - return DiskLoc(myLoc.a(), _nextOfs); - } - Extent *e = myExtent(myLoc); - while ( 1 ) { - if ( e->xnext.isNull() ) - return DiskLoc(); // end of table. - e = e->xnext.ext(); - if ( !e->firstRecord.isNull() ) - break; - // entire extent could be empty, keep looking - } - return e->firstRecord; - } - - inline DiskLoc Record::getPrev(const DiskLoc& myLoc) { - _accessing(); - - // Check if we still have records on our current extent - if ( _prevOfs != DiskLoc::NullOfs ) { - return DiskLoc(myLoc.a(), _prevOfs); - } - - // Get the current extent - Extent *e = myExtent(myLoc); - while ( 1 ) { - if ( e->xprev.isNull() ) { - // There are no more extents before this one - return DiskLoc(); - } - - // Move to the extent before this one - e = e->xprev.ext(); - - if ( !e->lastRecord.isNull() ) { - // We have found a non empty extent - break; - } - } - - // Return the last record in our new extent - return e->lastRecord; - } - - inline BSONObj DiskLoc::obj() const { - return BSONObj::make(rec()->accessed()); - } - inline DeletedRecord* DiskLoc::drec() const { - verify( _a != -1 ); - DeletedRecord* dr = (DeletedRecord*) rec(); - memconcept::is(dr, memconcept::concept::deletedrecord); - return dr; - } - inline Extent* DiskLoc::ext() const { - return DataFileMgr::getExtent(*this); - } - - template< class V > - inline - const BtreeBucket * DiskLoc::btree() const { - verify( _a != -1 ); - Record *r = rec(); - memconcept::is(r, memconcept::concept::btreebucket, "", 8192); - return (const BtreeBucket *) r->data(); - } - -} // namespace mongo - -#include "database.h" -#include "memconcept.h" - -namespace mongo { - - boost::intmax_t dbSize( const char *database ); - inline NamespaceIndex* nsindex(const StringData& ns) { Database *database = cc().database(); verify( database ); - memconcept::is(database, memconcept::concept::database, ns, sizeof(Database)); DEV { StringData dbname = nsToDatabaseSubstring( ns ); - if ( database->name != dbname ) { + if ( database->name() != dbname ) { out() << "ERROR: attempt to write to wrong database\n"; out() << " ns:" << ns << '\n'; - out() << " database->name:" << database->name << endl; - verify( database->name == dbname ); + out() << " database->name:" << database->name() << endl; + verify( database->name() == dbname ); } } - return &database->namespaceIndex; + return &database->namespaceIndex(); } inline NamespaceDetails* nsdetails(const StringData& ns) { // if this faults, did you set the current db first? (Client::Context + dblock) - NamespaceDetails *d = nsindex(ns)->details(ns); - if( d ) { - memconcept::is(d, memconcept::concept::nsdetails, ns, sizeof(NamespaceDetails)); - } - return d; - } - - inline Extent* DataFileMgr::getExtent(const DiskLoc& dl) { - verify( dl.a() != -1 ); - return cc().database()->getFile(dl.a())->getExtent(dl); - } - - inline Record* DataFileMgr::getRecord(const DiskLoc& dl) { - verify(dl.a() != -1); - return cc().database()->getFile(dl.a())->recordAt(dl); + return nsindex(ns)->details(ns); } BOOST_STATIC_ASSERT( 16 == sizeof(DeletedRecord) ); - inline DeletedRecord* DataFileMgr::getDeletedRecord(const DiskLoc& dl) { - return reinterpret_cast(getRecord(dl)); - } - inline BSONObj BSONObj::make(const Record* r ) { return BSONObj( r->data() ); } - DiskLoc allocateSpaceForANewRecord(const char* ns, - NamespaceDetails* d, - int32_t lenWHdr, - bool god); - - void addRecordToRecListInExtent(Record* r, DiskLoc loc); - - /** - * Static helpers to manipulate the list of unfinished index builds. - */ - class IndexBuildsInProgress { - public: - /** - * Find an unfinished index build by name. Does not search finished index builds. - */ - static int get(const char* ns, const std::string& indexName); - - /** - * Remove an unfinished index build from the list of index builds and move every subsequent - * unfinished index build back one. E.g., if x, y, z, and w are building and someone kills - * y, this method would rearrange the list to be x, z, w, (empty), etc. - */ - static void remove(const char* ns, int offset); - }; - } // namespace mongo diff --git a/src/mongo/db/pdfile_private.h b/src/mongo/db/pdfile_private.h index 3ee8fc541c1..9ce405e29aa 100644 --- a/src/mongo/db/pdfile_private.h +++ b/src/mongo/db/pdfile_private.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ // This file contains declarations that should not be considered part of pdfile's diff --git a/src/mongo/db/pdfile_version.h b/src/mongo/db/pdfile_version.h index 0c64c7935c8..c285a8b0d6c 100644 --- a/src/mongo/db/pdfile_version.h +++ b/src/mongo/db/pdfile_version.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -23,9 +35,12 @@ namespace mongo { const int PDFILE_VERSION_MINOR_22_AND_OLDER = 5; const int PDFILE_VERSION_MINOR_24_AND_NEWER = 6; + const int PDFILE_VERSION_MINOR_INDEX_MASK = 0xf; + const int PDFILE_VERSION_MINOR_28_FREELIST_MASK = (1 << 4); // SERVER-14081 + // For backward compatibility with versions before 2.4.0 all new DBs start // with PDFILE_VERSION_MINOR_22_AND_OLDER and are converted when the first // index using a new plugin is created. See the logic in - // prepareToBuildIndex() and upgradeMinorVersionOrAssert() for details + // IndexCatalog::_upgradeDatabaseMinorVersionIfNeeded for details } // namespace mongo diff --git a/src/mongo/db/pipeline/accumulator.cpp b/src/mongo/db/pipeline/accumulator.cpp deleted file mode 100644 index 4b610bec63b..00000000000 --- a/src/mongo/db/pipeline/accumulator.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright (c) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "pch.h" -#include "db/pipeline/accumulator.h" - -#include "db/jsobj.h" -#include "util/mongoutils/str.h" - -namespace mongo { - using namespace mongoutils; - - void Accumulator::addOperand( - const intrusive_ptr &pExpression) { - uassert(15943, str::stream() << "group accumulator " << - getOpName() << " only accepts one operand", - vpOperand.size() < 1); - - ExpressionNary::addOperand(pExpression); - } - - Accumulator::Accumulator(): - ExpressionNary() { - } - - void Accumulator::opToBson(BSONObjBuilder *pBuilder, StringData opName, - StringData fieldName, bool requireExpression) const { - verify(vpOperand.size() == 1); - BSONObjBuilder builder; - vpOperand[0]->addToBsonObj(&builder, opName, requireExpression); - pBuilder->append(fieldName, builder.done()); - } - - void Accumulator::addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const { - opToBson(pBuilder, getOpName(), fieldName, requireExpression); - } - - void Accumulator::addToBsonArray(BSONArrayBuilder *pBuilder) const { - verify(false); // these can't appear in arrays - } - - void agg_framework_reservedErrors() { - uassert(16030, "reserved error", false); - uassert(16031, "reserved error", false); - uassert(16032, "reserved error", false); - uassert(16033, "reserved error", false); - - uassert(16036, "reserved error", false); - uassert(16037, "reserved error", false); - uassert(16038, "reserved error", false); - uassert(16039, "reserved error", false); - uassert(16040, "reserved error", false); - uassert(16041, "reserved error", false); - uassert(16042, "reserved error", false); - uassert(16043, "reserved error", false); - uassert(16044, "reserved error", false); - uassert(16045, "reserved error", false); - uassert(16046, "reserved error", false); - uassert(16047, "reserved error", false); - uassert(16048, "reserved error", false); - uassert(16049, "reserved error", false); - } -} diff --git a/src/mongo/db/pipeline/accumulator.h b/src/mongo/db/pipeline/accumulator.h index a4b52e90318..61b2dade4e7 100644 --- a/src/mongo/db/pipeline/accumulator.h +++ b/src/mongo/db/pipeline/accumulator.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -19,235 +31,160 @@ #include "mongo/pch.h" #include -#include "db/pipeline/value.h" -#include "db/pipeline/expression.h" -#include "bson/bsontypes.h" -namespace mongo { - class ExpressionContext; +#include "mongo/bson/bsontypes.h" +#include "mongo/db/pipeline/value.h" - class Accumulator : - public ExpressionNary { +namespace mongo { + class Accumulator : public RefCountable { public: - // virtuals from ExpressionNary - virtual void addOperand(const intrusive_ptr &pExpression); - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const; - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const; - - /* - Get the accumulated value. - - @returns the accumulated value + /** Process input and update internal state. + * merging should be true when processing outputs from getValue(true). */ - virtual Value getValue() const = 0; - - protected: - Accumulator(); - - /* - Convenience method for doing this for accumulators. The pattern - is always the same, so a common implementation works, but requires - knowing the operator name. + void process(const Value& input, bool merging) { + processInternal(input, merging); + } - @param pBuilder the builder to add to - @param fieldName the projected name - @param opName the operator name + /** Marks the end of the evaluate() phase and return accumulated result. + * toBeMerged should be true when the outputs will be merged by process(). */ - void opToBson(BSONObjBuilder *pBuilder, StringData opName, - StringData fieldName, bool requireExpression) const; - }; + virtual Value getValue(bool toBeMerged) const = 0; + /// The name of the op as used in a serialization of the pipeline. + virtual const char* getOpName() const = 0; - class AccumulatorAddToSet : - public Accumulator { - public: - // virtuals from Expression - virtual Value evaluate(const Document& pDocument) const; - virtual Value getValue() const; - virtual const char *getOpName() const; + int memUsageForSorter() const { + dassert(_memUsageBytes != 0); // This would mean subclass didn't set it + return _memUsageBytes; + } - /* - Create an appending accumulator. + /// Reset this accumulator to a fresh state ready to receive input. + virtual void reset() = 0; - @param pCtx the expression context - @returns the created accumulator - */ - static intrusive_ptr create( - const intrusive_ptr &pCtx); + protected: + Accumulator() : _memUsageBytes(0) {} - private: - AccumulatorAddToSet(const intrusive_ptr &pTheCtx); - typedef boost::unordered_set SetType; - mutable SetType set; - mutable SetType::iterator itr; - intrusive_ptr pCtx; + /// Update subclass's internal state based on input + virtual void processInternal(const Value& input, bool merging) = 0; + + /// subclasses are expected to update this as necessary + int _memUsageBytes; }; - /* - This isn't a finished accumulator, but rather a convenient base class - for others such as $first, $last, $max, $min, and similar. It just - provides a holder for a single Value, and the getter for that. The - holder is protected so derived classes can manipulate it. - */ - class AccumulatorSingleValue : - public Accumulator { + class AccumulatorAddToSet : public Accumulator { public: - // virtuals from Expression - virtual Value getValue() const; + virtual void processInternal(const Value& input, bool merging); + virtual Value getValue(bool toBeMerged) const; + virtual const char* getOpName() const; + virtual void reset(); - protected: - AccumulatorSingleValue(); + static intrusive_ptr create(); - mutable Value pValue; /* current min/max */ + private: + AccumulatorAddToSet(); + typedef boost::unordered_set SetType; + SetType set; }; - class AccumulatorFirst : - public AccumulatorSingleValue { + class AccumulatorFirst : public Accumulator { public: - // virtuals from Expression - virtual Value evaluate(const Document& pDocument) const; - virtual const char *getOpName() const; + virtual void processInternal(const Value& input, bool merging); + virtual Value getValue(bool toBeMerged) const; + virtual const char* getOpName() const; + virtual void reset(); - /* - Create the accumulator. - - @returns the created accumulator - */ - static intrusive_ptr create( - const intrusive_ptr &pCtx); + static intrusive_ptr create(); private: - mutable bool _haveFirst; AccumulatorFirst(); + + bool _haveFirst; + Value _first; }; - class AccumulatorLast : - public AccumulatorSingleValue { + class AccumulatorLast : public Accumulator { public: - // virtuals from Expression - virtual Value evaluate(const Document& pDocument) const; - virtual const char *getOpName() const; - - /* - Create the accumulator. + virtual void processInternal(const Value& input, bool merging); + virtual Value getValue(bool toBeMerged) const; + virtual const char* getOpName() const; + virtual void reset(); - @returns the created accumulator - */ - static intrusive_ptr create( - const intrusive_ptr &pCtx); + static intrusive_ptr create(); private: AccumulatorLast(); + Value _last; }; - class AccumulatorSum : - public Accumulator { + class AccumulatorSum : public Accumulator { public: - // virtuals from Accumulator - virtual Value evaluate(const Document& pDocument) const; - virtual Value getValue() const; - virtual const char *getOpName() const; - - /* - Create a summing accumulator. + virtual void processInternal(const Value& input, bool merging); + virtual Value getValue(bool toBeMerged) const; + virtual const char* getOpName() const; + virtual void reset(); - @param pCtx the expression context - @returns the created accumulator - */ - static intrusive_ptr create( - const intrusive_ptr &pCtx); + static intrusive_ptr create(); - protected: /* reused by AccumulatorAvg */ + private: AccumulatorSum(); - mutable BSONType totalType; - mutable long long longTotal; - mutable double doubleTotal; - // count is only used by AccumulatorAvg, but lives here to avoid counting non-numeric values - mutable long long count; + BSONType totalType; + long long longTotal; + double doubleTotal; }; - class AccumulatorMinMax : - public AccumulatorSingleValue { + class AccumulatorMinMax : public Accumulator { public: - // virtuals from Expression - virtual Value evaluate(const Document& pDocument) const; - virtual const char *getOpName() const; + virtual void processInternal(const Value& input, bool merging); + virtual Value getValue(bool toBeMerged) const; + virtual const char* getOpName() const; + virtual void reset(); - /* - Create either the max or min accumulator. - - @returns the created accumulator - */ - static intrusive_ptr createMin( - const intrusive_ptr &pCtx); - static intrusive_ptr createMax( - const intrusive_ptr &pCtx); + static intrusive_ptr createMin(); + static intrusive_ptr createMax(); private: AccumulatorMinMax(int theSense); - int sense; /* 1 for min, -1 for max; used to "scale" comparison */ + Value _val; + const int _sense; /* 1 for min, -1 for max; used to "scale" comparison */ }; - class AccumulatorPush : - public Accumulator { + class AccumulatorPush : public Accumulator { public: - // virtuals from Expression - virtual Value evaluate(const Document& pDocument) const; - virtual Value getValue() const; - virtual const char *getOpName() const; + virtual void processInternal(const Value& input, bool merging); + virtual Value getValue(bool toBeMerged) const; + virtual const char* getOpName() const; + virtual void reset(); - /* - Create an appending accumulator. - - @param pCtx the expression context - @returns the created accumulator - */ - static intrusive_ptr create( - const intrusive_ptr &pCtx); + static intrusive_ptr create(); private: - AccumulatorPush(const intrusive_ptr &pTheCtx); + AccumulatorPush(); - mutable vector vpValue; - intrusive_ptr pCtx; + vector vpValue; }; - class AccumulatorAvg : - public AccumulatorSum { - typedef AccumulatorSum Super; + class AccumulatorAvg : public Accumulator { public: - // virtuals from Accumulator - virtual Value evaluate(const Document& pDocument) const; - virtual Value getValue() const; - virtual const char *getOpName() const; + virtual void processInternal(const Value& input, bool merging); + virtual Value getValue(bool toBeMerged) const; + virtual const char* getOpName() const; + virtual void reset(); - /* - Create an averaging accumulator. - - @param pCtx the expression context - @returns the created accumulator - */ - static intrusive_ptr create( - const intrusive_ptr &pCtx); + static intrusive_ptr create(); private: - static const char subTotalName[]; - static const char countName[]; + AccumulatorAvg(); - AccumulatorAvg(const intrusive_ptr &pCtx); - - intrusive_ptr pCtx; + double _total; + long long _count; }; - } diff --git a/src/mongo/db/pipeline/accumulator_add_to_set.cpp b/src/mongo/db/pipeline/accumulator_add_to_set.cpp index d64faf48b8f..086253cf116 100644 --- a/src/mongo/db/pipeline/accumulator_add_to_set.cpp +++ b/src/mongo/db/pipeline/accumulator_add_to_set.cpp @@ -12,61 +12,69 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "accumulator.h" +#include "mongo/pch.h" -#include "db/pipeline/expression_context.h" -#include "db/pipeline/value.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - Value AccumulatorAddToSet::evaluate(const Document& pDocument) const { - verify(vpOperand.size() == 1); - Value prhs(vpOperand[0]->evaluate(pDocument)); - - if (!pCtx->getDoingMerge()) { - if (!prhs.missing()) { - set.insert(prhs); + void AccumulatorAddToSet::processInternal(const Value& input, bool merging) { + if (!merging) { + if (!input.missing()) { + bool inserted = set.insert(input).second; + if (inserted) { + _memUsageBytes += input.getApproximateSize(); + } } - } else { - /* - If we're in the router, we need to take apart the arrays we - receive and put their elements into the array we are collecting. - If we didn't, then we'd get an array of arrays, with one array - from each shard that responds. - */ - verify(prhs.getType() == Array); + } + else { + // If we're merging, we need to take apart the arrays we + // receive and put their elements into the array we are collecting. + // If we didn't, then we'd get an array of arrays, with one array + // from each merge source. + verify(input.getType() == Array); - const vector& array = prhs.getArray(); - set.insert(array.begin(), array.end()); + const vector& array = input.getArray(); + for (size_t i=0; i < array.size(); i++) { + bool inserted = set.insert(array[i]).second; + if (inserted) { + _memUsageBytes += array[i].getApproximateSize(); + } + } } - - return Value(); } - Value AccumulatorAddToSet::getValue() const { - vector valVec; + Value AccumulatorAddToSet::getValue(bool toBeMerged) const { + vector valVec(set.begin(), set.end()); + return Value::consume(valVec); + } - for (itr = set.begin(); itr != set.end(); ++itr) { - valVec.push_back(*itr); - } - /* there is no issue of scope since createArray copy constructs */ - return Value::createArray(valVec); + AccumulatorAddToSet::AccumulatorAddToSet() { + _memUsageBytes = sizeof(*this); } - AccumulatorAddToSet::AccumulatorAddToSet( - const intrusive_ptr &pTheCtx): - Accumulator(), - set(), - pCtx(pTheCtx) { + void AccumulatorAddToSet::reset() { + SetType().swap(set); + _memUsageBytes = sizeof(*this); } - intrusive_ptr AccumulatorAddToSet::create( - const intrusive_ptr &pCtx) { - intrusive_ptr pAccumulator( - new AccumulatorAddToSet(pCtx)); - return pAccumulator; + intrusive_ptr AccumulatorAddToSet::create() { + return new AccumulatorAddToSet(); } const char *AccumulatorAddToSet::getOpName() const { diff --git a/src/mongo/db/pipeline/accumulator_avg.cpp b/src/mongo/db/pipeline/accumulator_avg.cpp index 07787325992..8dbfc816de1 100644 --- a/src/mongo/db/pipeline/accumulator_avg.cpp +++ b/src/mongo/db/pipeline/accumulator_avg.cpp @@ -12,71 +12,80 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "accumulator.h" +#include "mongo/pch.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression_context.h" -#include "db/pipeline/value.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - const char AccumulatorAvg::subTotalName[] = "subTotal"; - const char AccumulatorAvg::countName[] = "count"; +namespace { + const char subTotalName[] = "subTotal"; + const char countName[] = "count"; +} + + void AccumulatorAvg::processInternal(const Value& input, bool merging) { + if (!merging) { + // non numeric types have no impact on average + if (!input.numeric()) + return; - Value AccumulatorAvg::evaluate(const Document& pDocument) const { - if (!pCtx->getDoingMerge()) { - Super::evaluate(pDocument); + _total += input.getDouble(); + _count += 1; } else { - /* - If we're in the router, we expect an object that contains - both a subtotal and a count. This is what getValue() produced - below. - */ - Value shardOut = vpOperand[0]->evaluate(pDocument); - verify(shardOut.getType() == Object); - - Value subTotal = shardOut[subTotalName]; - verify(!subTotal.missing()); - doubleTotal += subTotal.getDouble(); - - Value subCount = shardOut[countName]; - verify(!subCount.missing()); - count += subCount.getLong(); + // We expect an object that contains both a subtotal and a count. + // This is what getValue(true) produced below. + verify(input.getType() == Object); + _total += input[subTotalName].getDouble(); + _count += input[countName].getLong(); } - - return Value(); } - intrusive_ptr AccumulatorAvg::create( - const intrusive_ptr &pCtx) { - intrusive_ptr pA(new AccumulatorAvg(pCtx)); - return pA; + intrusive_ptr AccumulatorAvg::create() { + return new AccumulatorAvg(); } - Value AccumulatorAvg::getValue() const { - if (!pCtx->getInShard()) { - double avg = 0; - if (count) - avg = doubleTotal / static_cast(count); + Value AccumulatorAvg::getValue(bool toBeMerged) const { + if (!toBeMerged) { + if (_count == 0) + return Value(0.0); - return Value::createDouble(avg); + return Value(_total / static_cast(_count)); } + else { + return Value(DOC(subTotalName << _total + << countName << _count)); + } + } - MutableDocument out; - out.addField(subTotalName, Value::createDouble(doubleTotal)); - out.addField(countName, Value::createLong(count)); - - return Value::createDocument(out.freeze()); + AccumulatorAvg::AccumulatorAvg() + : _total(0) + , _count(0) + { + // This is a fixed size Accumulator so we never need to update this + _memUsageBytes = sizeof(*this); } - AccumulatorAvg::AccumulatorAvg( - const intrusive_ptr &pTheCtx): - AccumulatorSum(), - pCtx(pTheCtx) { + void AccumulatorAvg::reset() { + _total = 0; + _count = 0; } const char *AccumulatorAvg::getOpName() const { diff --git a/src/mongo/db/pipeline/accumulator_first.cpp b/src/mongo/db/pipeline/accumulator_first.cpp index ce53cccd365..984f80993d9 100644 --- a/src/mongo/db/pipeline/accumulator_first.cpp +++ b/src/mongo/db/pipeline/accumulator_first.cpp @@ -12,38 +12,56 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "accumulator.h" +#include "mongo/pch.h" -#include "db/pipeline/value.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - Value AccumulatorFirst::evaluate(const Document& pDocument) const { - verify(vpOperand.size() == 1); - + void AccumulatorFirst::processInternal(const Value& input, bool merging) { /* only remember the first value seen */ if (!_haveFirst) { // can't use pValue.missing() since we want the first value even if missing _haveFirst = true; - pValue = vpOperand[0]->evaluate(pDocument); + _first = input; + _memUsageBytes = sizeof(*this) + input.getApproximateSize() - sizeof(Value); } + } - return pValue; + Value AccumulatorFirst::getValue(bool toBeMerged) const { + return _first; } AccumulatorFirst::AccumulatorFirst() - : AccumulatorSingleValue() - , _haveFirst(false) - {} - - intrusive_ptr AccumulatorFirst::create( - const intrusive_ptr &pCtx) { - intrusive_ptr pAccumulator( - new AccumulatorFirst()); - return pAccumulator; + : _haveFirst(false) + { + _memUsageBytes = sizeof(*this); + } + + void AccumulatorFirst::reset() { + _haveFirst = false; + _first = Value(); + _memUsageBytes = sizeof(*this); + } + + + intrusive_ptr AccumulatorFirst::create() { + return new AccumulatorFirst(); } const char *AccumulatorFirst::getOpName() const { diff --git a/src/mongo/db/pipeline/accumulator_last.cpp b/src/mongo/db/pipeline/accumulator_last.cpp index ea65094d04b..1c8cf7cfd62 100644 --- a/src/mongo/db/pipeline/accumulator_last.cpp +++ b/src/mongo/db/pipeline/accumulator_last.cpp @@ -12,33 +12,48 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "accumulator.h" +#include "mongo/pch.h" -#include "db/pipeline/value.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - Value AccumulatorLast::evaluate(const Document& pDocument) const { - verify(vpOperand.size() == 1); - + void AccumulatorLast::processInternal(const Value& input, bool merging) { /* always remember the last value seen */ - pValue = vpOperand[0]->evaluate(pDocument); + _last = input; + _memUsageBytes = sizeof(*this) + _last.getApproximateSize() - sizeof(Value); + } + + Value AccumulatorLast::getValue(bool toBeMerged) const { + return _last; + } - return pValue; + AccumulatorLast::AccumulatorLast() { + _memUsageBytes = sizeof(*this); } - AccumulatorLast::AccumulatorLast(): - AccumulatorSingleValue() { + void AccumulatorLast::reset() { + _memUsageBytes = sizeof(*this); + _last = Value(); } - intrusive_ptr AccumulatorLast::create( - const intrusive_ptr &pCtx) { - intrusive_ptr pAccumulator( - new AccumulatorLast()); - return pAccumulator; + intrusive_ptr AccumulatorLast::create() { + return new AccumulatorLast(); } const char *AccumulatorLast::getOpName() const { diff --git a/src/mongo/db/pipeline/accumulator_min_max.cpp b/src/mongo/db/pipeline/accumulator_min_max.cpp index 86172cb3338..57eddfedbde 100644 --- a/src/mongo/db/pipeline/accumulator_min_max.cpp +++ b/src/mongo/db/pipeline/accumulator_min_max.cpp @@ -12,52 +12,65 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "accumulator.h" +#include "mongo/pch.h" -#include "db/pipeline/value.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - Value AccumulatorMinMax::evaluate(const Document& pDocument) const { - verify(vpOperand.size() == 1); - Value prhs(vpOperand[0]->evaluate(pDocument)); - + void AccumulatorMinMax::processInternal(const Value& input, bool merging) { // nullish values should have no impact on result - if (!prhs.nullish()) { + if (!input.nullish()) { /* compare with the current value; swap if appropriate */ - int cmp = Value::compare(pValue, prhs) * sense; - if (cmp > 0 || pValue.missing()) // missing is lower than all other values - pValue = prhs; + int cmp = Value::compare(_val, input) * _sense; + if (cmp > 0 || _val.missing()) { // missing is lower than all other values + _val = input; + _memUsageBytes = sizeof(*this) + input.getApproximateSize() - sizeof(Value); + } } + } + + Value AccumulatorMinMax::getValue(bool toBeMerged) const { + return _val; + } - return Value(); + AccumulatorMinMax::AccumulatorMinMax(int theSense) + :_sense(theSense) + { + verify((_sense == 1) || (_sense == -1)); + _memUsageBytes = sizeof(*this); } - AccumulatorMinMax::AccumulatorMinMax(int theSense): - AccumulatorSingleValue(), - sense(theSense) { - verify((sense == 1) || (sense == -1)); + void AccumulatorMinMax::reset() { + _val = Value(); + _memUsageBytes = sizeof(*this); } - intrusive_ptr AccumulatorMinMax::createMin( - const intrusive_ptr &pCtx) { - intrusive_ptr pAccumulator( - new AccumulatorMinMax(1)); - return pAccumulator; + intrusive_ptr AccumulatorMinMax::createMin() { + return new AccumulatorMinMax(1); } - intrusive_ptr AccumulatorMinMax::createMax( - const intrusive_ptr &pCtx) { - intrusive_ptr pAccumulator( - new AccumulatorMinMax(-1)); - return pAccumulator; + intrusive_ptr AccumulatorMinMax::createMax() { + return new AccumulatorMinMax(-1); } const char *AccumulatorMinMax::getOpName() const { - if (sense == 1) + if (_sense == 1) return "$min"; return "$max"; } diff --git a/src/mongo/db/pipeline/accumulator_push.cpp b/src/mongo/db/pipeline/accumulator_push.cpp index f5c76323602..8250c462af8 100644 --- a/src/mongo/db/pipeline/accumulator_push.cpp +++ b/src/mongo/db/pipeline/accumulator_push.cpp @@ -12,56 +12,65 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "accumulator.h" +#include "mongo/pch.h" -#include "db/pipeline/expression_context.h" -#include "db/pipeline/value.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - Value AccumulatorPush::evaluate(const Document& pDocument) const { - verify(vpOperand.size() == 1); - Value prhs(vpOperand[0]->evaluate(pDocument)); - - if (!pCtx->getDoingMerge()) { - if (!prhs.missing()) { - vpValue.push_back(prhs); + void AccumulatorPush::processInternal(const Value& input, bool merging) { + if (!merging) { + if (!input.missing()) { + vpValue.push_back(input); + _memUsageBytes += input.getApproximateSize(); } } else { - /* - If we're in the router, we need to take apart the arrays we - receive and put their elements into the array we are collecting. - If we didn't, then we'd get an array of arrays, with one array - from each shard that responds. - */ - verify(prhs.getType() == Array); + // If we're merging, we need to take apart the arrays we + // receive and put their elements into the array we are collecting. + // If we didn't, then we'd get an array of arrays, with one array + // from each merge source. + verify(input.getType() == Array); - const vector& vec = prhs.getArray(); + const vector& vec = input.getArray(); vpValue.insert(vpValue.end(), vec.begin(), vec.end()); + + for (size_t i=0; i < vec.size(); i++) { + _memUsageBytes += vec[i].getApproximateSize(); + } } + } - return Value(); + Value AccumulatorPush::getValue(bool toBeMerged) const { + return Value(vpValue); } - Value AccumulatorPush::getValue() const { - return Value::createArray(vpValue); + AccumulatorPush::AccumulatorPush() { + _memUsageBytes = sizeof(*this); } - AccumulatorPush::AccumulatorPush( - const intrusive_ptr &pTheCtx): - Accumulator(), - vpValue(), - pCtx(pTheCtx) { + void AccumulatorPush::reset() { + vector().swap(vpValue); + _memUsageBytes = sizeof(*this); } - intrusive_ptr AccumulatorPush::create( - const intrusive_ptr &pCtx) { - intrusive_ptr pAccumulator( - new AccumulatorPush(pCtx)); - return pAccumulator; + intrusive_ptr AccumulatorPush::create() { + return new AccumulatorPush(); } const char *AccumulatorPush::getOpName() const { diff --git a/src/mongo/db/pipeline/accumulator_single_value.cpp b/src/mongo/db/pipeline/accumulator_single_value.cpp deleted file mode 100644 index 76187d52222..00000000000 --- a/src/mongo/db/pipeline/accumulator_single_value.cpp +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (c) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "pch.h" -#include "accumulator.h" - -#include "db/pipeline/value.h" - -namespace mongo { - - Value AccumulatorSingleValue::getValue() const { - return pValue; - } - - AccumulatorSingleValue::AccumulatorSingleValue(): - pValue(Value()) { - } - -} diff --git a/src/mongo/db/pipeline/accumulator_sum.cpp b/src/mongo/db/pipeline/accumulator_sum.cpp index c737d7ee4a0..76bddc956fe 100644 --- a/src/mongo/db/pipeline/accumulator_sum.cpp +++ b/src/mongo/db/pipeline/accumulator_sum.cpp @@ -12,57 +12,60 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "accumulator.h" +#include "mongo/pch.h" -#include "db/pipeline/value.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - Value AccumulatorSum::evaluate(const Document& pDocument) const { - verify(vpOperand.size() == 1); - Value rhs = vpOperand[0]->evaluate(pDocument); - + void AccumulatorSum::processInternal(const Value& input, bool merging) { // do nothing with non numeric types - if (!rhs.numeric()) - return Value(); + if (!input.numeric()) + return; // upgrade to the widest type required to hold the result - totalType = Value::getWidestNumeric(totalType, rhs.getType()); + totalType = Value::getWidestNumeric(totalType, input.getType()); if (totalType == NumberInt || totalType == NumberLong) { - long long v = rhs.coerceToLong(); + long long v = input.coerceToLong(); longTotal += v; doubleTotal += v; } else if (totalType == NumberDouble) { - double v = rhs.coerceToDouble(); + double v = input.coerceToDouble(); doubleTotal += v; } else { // non numerics should have returned above so we should never get here verify(false); } - - count++; - - return Value(); } - intrusive_ptr AccumulatorSum::create( - const intrusive_ptr &pCtx) { - intrusive_ptr pSummer(new AccumulatorSum()); - return pSummer; + intrusive_ptr AccumulatorSum::create() { + return new AccumulatorSum(); } - Value AccumulatorSum::getValue() const { + Value AccumulatorSum::getValue(bool toBeMerged) const { if (totalType == NumberLong) { - return Value::createLong(longTotal); + return Value(longTotal); } else if (totalType == NumberDouble) { - return Value::createDouble(doubleTotal); + return Value(doubleTotal); } else if (totalType == NumberInt) { return Value::createIntOrLong(longTotal); @@ -72,12 +75,19 @@ namespace mongo { } } - AccumulatorSum::AccumulatorSum(): - Accumulator(), - totalType(NumberInt), - longTotal(0), - doubleTotal(0), - count(0) { + AccumulatorSum::AccumulatorSum() + : totalType(NumberInt) + , longTotal(0) + , doubleTotal(0) + { + // This is a fixed size Accumulator so we never need to update this + _memUsageBytes = sizeof(*this); + } + + void AccumulatorSum::reset() { + totalType = NumberInt; + longTotal = 0; + doubleTotal = 0; } const char *AccumulatorSum::getOpName() const { diff --git a/src/mongo/db/pipeline/builder.cpp b/src/mongo/db/pipeline/builder.cpp deleted file mode 100644 index 69eddcf9747..00000000000 --- a/src/mongo/db/pipeline/builder.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright (c) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "pch.h" - -#include "db/jsobj.h" -#include "db/pipeline/builder.h" - - -namespace mongo { - - void BuilderObj::append() { - pBuilder->appendNull(fieldName); - } - - void BuilderObj::appendUndefined() { - pBuilder->appendUndefined(fieldName); - } - - void BuilderObj::append(bool b) { - pBuilder->append(fieldName, b); - } - - void BuilderObj::append(int i) { - pBuilder->append(fieldName, i); - } - - void BuilderObj::append(long long ll) { - pBuilder->append(fieldName, ll); - } - - void BuilderObj::append(double d) { - pBuilder->append(fieldName, d); - } - - void BuilderObj::append(StringData s) { - pBuilder->append(fieldName, s); - } - - void BuilderObj::append(const OID &o) { - pBuilder->append(fieldName, o); - } - - void BuilderObj::append(const Date_t &d) { - pBuilder->append(fieldName, d); - } - - void BuilderObj::append(BSONObjBuilder *pDone) { - pBuilder->append(fieldName, pDone->done()); - } - - void BuilderObj::append(BSONArrayBuilder *pDone) { - pBuilder->append(fieldName, pDone->arr()); - } - - void BuilderObj::append(const OpTime& ot) { - pBuilder->appendTimestamp(fieldName, ot.getSecs(), ot.getInc()); - } - - BuilderObj::BuilderObj(BSONObjBuilder *pObjBuilder, StringData theFieldName): - pBuilder(pObjBuilder), - fieldName(theFieldName) { - } - - - void BuilderArray::append() { - pBuilder->appendNull(); - } - - void BuilderArray::appendUndefined() { - pBuilder->appendUndefined(); - } - - void BuilderArray::append(bool b) { - pBuilder->append(b); - } - - void BuilderArray::append(int i) { - pBuilder->append(i); - } - - void BuilderArray::append(long long ll) { - pBuilder->append(ll); - } - - void BuilderArray::append(double d) { - pBuilder->append(d); - } - - void BuilderArray::append(StringData s) { - pBuilder->append(s); - } - - void BuilderArray::append(const OID &o) { - pBuilder->append(o); - } - - void BuilderArray::append(const Date_t &d) { - pBuilder->append(d); - } - - void BuilderArray::append(BSONObjBuilder *pDone) { - pBuilder->append(pDone->done()); - } - - void BuilderArray::append(BSONArrayBuilder *pDone) { - pBuilder->append(pDone->arr()); - } - - void BuilderArray::append(const OpTime& ot) { - pBuilder->appendTimestamp(ot.getSecs(), ot.getInc()); - } - - BuilderArray::BuilderArray( - BSONArrayBuilder *pArrayBuilder): - pBuilder(pArrayBuilder) { - } - -} diff --git a/src/mongo/db/pipeline/builder.h b/src/mongo/db/pipeline/builder.h deleted file mode 100644 index 5e4f5e48e38..00000000000 --- a/src/mongo/db/pipeline/builder.h +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright (c) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "mongo/pch.h" - -namespace mongo { - - class BSONArrayBuilder; - class BSONObjBuilder; - - /* - Generic Builder. - - The methods to append items to an object (on BSONObjBuilder) and an array - (on BSONArrayBuilder) differ only by their inclusion of a field name. - For more complicated implementations of addToBsonObj() and - addToBsonArray(), it makes sense to abstract that out and use - this generic builder that always looks the same, and then implement - addToBsonObj() and addToBsonArray() by using a common method. - */ - class Builder : - boost::noncopyable { - public: - virtual ~Builder() {}; - - virtual void append() = 0; // append a null - virtual void appendUndefined() = 0; - virtual void append(bool b) = 0; - virtual void append(int i) = 0; - virtual void append(long long ll) = 0; - virtual void append(double d) = 0; - virtual void append(StringData s) = 0; - virtual void append(const OID &o) = 0; - virtual void append(const Date_t &d) = 0; - virtual void append(const OpTime& ot) = 0; - virtual void append(BSONObjBuilder *pDone) = 0; - virtual void append(BSONArrayBuilder *pDone) = 0; - }; - - class BuilderObj : - public Builder { - public: - // virtuals from Builder - virtual void append(); - virtual void appendUndefined(); - virtual void append(bool b); - virtual void append(int i); - virtual void append(long long ll); - virtual void append(double d); - virtual void append(StringData s); - virtual void append(const OID &o); - virtual void append(const Date_t &d); - virtual void append(const OpTime& ot); - virtual void append(BSONObjBuilder *pDone); - virtual void append(BSONArrayBuilder *pDone); - - BuilderObj(BSONObjBuilder *pBuilder, StringData fieldName); - - private: - BSONObjBuilder *pBuilder; - StringData fieldName; - }; - - class BuilderArray : - public Builder { - public: - // virtuals from Builder - virtual void append(); - virtual void appendUndefined(); - virtual void append(bool b); - virtual void append(int i); - virtual void append(long long ll); - virtual void append(double d); - virtual void append(StringData s); - virtual void append(const OID &o); - virtual void append(const Date_t &d); - virtual void append(const OpTime& ot); - virtual void append(BSONObjBuilder *pDone); - virtual void append(BSONArrayBuilder *pDone); - - BuilderArray(BSONArrayBuilder *pBuilder); - - private: - BSONArrayBuilder *pBuilder; - }; -} diff --git a/src/mongo/db/pipeline/dependencies.cpp b/src/mongo/db/pipeline/dependencies.cpp new file mode 100644 index 00000000000..bed2332549d --- /dev/null +++ b/src/mongo/db/pipeline/dependencies.cpp @@ -0,0 +1,175 @@ +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/pch.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/dependencies.h" +#include "mongo/db/pipeline/field_path.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + namespace str = mongoutils::str; + + BSONObj DepsTracker::toProjection() const { + BSONObjBuilder bb; + + if (needTextScore) + bb.append(Document::metaFieldTextScore, BSON("$meta" << "textScore")); + + if (needWholeDocument) + return bb.obj(); + + if (fields.empty()) { + // Projection language lacks good a way to say no fields needed. This fakes it. + bb.append("_id", 0); + bb.append("$noFieldsNeeded", 1); + return bb.obj(); + } + + bool needId = false; + string last; + for (set::const_iterator it(fields.begin()), end(fields.end()); it!=end; ++it) { + if (str::startsWith(*it, "_id") && (it->size() == 3 || (*it)[3] == '.')) { + // _id and subfields are handled specially due in part to SERVER-7502 + needId = true; + continue; + } + + if (!last.empty() && str::startsWith(*it, last)) { + // we are including a parent of *it so we don't need to include this field + // explicitly. In fact, due to SERVER-6527 if we included this field, the parent + // wouldn't be fully included. This logic relies on on set iterators going in + // lexicographic order so that a string is always directly before of all fields it + // prefixes. + continue; + } + + last = *it + '.'; + bb.append(*it, 1); + } + + if (needId) // we are explicit either way + bb.append("_id", 1); + else + bb.append("_id", 0); + + return bb.obj(); + } + + // ParsedDeps::_fields is a simple recursive look-up table. For each field: + // If the value has type==Bool, the whole field is needed + // If the value has type==Object, the fields in the subobject are needed + // All other fields should be missing which means not needed + boost::optional DepsTracker::toParsedDeps() const { + MutableDocument md; + + if (needWholeDocument || needTextScore) { + // can't use ParsedDeps in this case + return boost::none; + } + + string last; + for (set::const_iterator it(fields.begin()), end(fields.end()); it!=end; ++it) { + if (!last.empty() && str::startsWith(*it, last)) { + // we are including a parent of *it so we don't need to include this field + // explicitly. In fact, if we included this field, the parent wouldn't be fully + // included. This logic relies on on set iterators going in lexicographic order so + // that a string is always directly before of all fields it prefixes. + continue; + } + last = *it + '.'; + md.setNestedField(*it, Value(true)); + } + + return ParsedDeps(md.freeze()); + } + +namespace { + // Mutually recursive with arrayHelper + Document documentHelper(const BSONObj& bson, const Document& neededFields); + + // Handles array-typed values for ParsedDeps::extractFields + Value arrayHelper(const BSONObj& bson, const Document& neededFields) { + BSONObjIterator it(bson); + + vector values; + while (it.more()) { + BSONElement bsonElement(it.next()); + if (bsonElement.type() == Object) { + Document sub = documentHelper(bsonElement.embeddedObject(), neededFields); + values.push_back(Value(sub)); + } + + if (bsonElement.type() == Array) { + values.push_back(arrayHelper(bsonElement.embeddedObject(), neededFields)); + } + } + + return Value::consume(values); + } + + // Handles object-typed values including the top-level for ParsedDeps::extractFields + Document documentHelper(const BSONObj& bson, const Document& neededFields) { + MutableDocument md(neededFields.size()); + + BSONObjIterator it(bson); + while (it.more()) { + BSONElement bsonElement (it.next()); + StringData fieldName = bsonElement.fieldNameStringData(); + Value isNeeded = neededFields[fieldName]; + + if (isNeeded.missing()) + continue; + + if (isNeeded.getType() == Bool) { + md.addField(fieldName, Value(bsonElement)); + continue; + } + + dassert(isNeeded.getType() == Object); + + if (bsonElement.type() == Object) { + Document sub = documentHelper(bsonElement.embeddedObject(), isNeeded.getDocument()); + md.addField(fieldName, Value(sub)); + } + + if (bsonElement.type() == Array) { + md.addField(fieldName, arrayHelper(bsonElement.embeddedObject(), + isNeeded.getDocument())); + } + } + + return md.freeze(); + } +} // namespace + + Document ParsedDeps::extractFields(const BSONObj& input) const { + return documentHelper(input, _fields); + } +} diff --git a/src/mongo/db/pipeline/dependencies.h b/src/mongo/db/pipeline/dependencies.h new file mode 100644 index 00000000000..47f8f46c432 --- /dev/null +++ b/src/mongo/db/pipeline/dependencies.h @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/db/pipeline/document.h" + +namespace mongo { + class ParsedDeps; + + /** + * This struct allows components in an agg pipeline to report what they need from their input. + */ + struct DepsTracker { + DepsTracker() + : needWholeDocument(false) + , needTextScore(false) + {} + + /** + * Returns a projection object covering the dependencies tracked by this class. + */ + BSONObj toProjection() const; + + boost::optional toParsedDeps() const; + + std::set fields; // names of needed fields in dotted notation + bool needWholeDocument; // if true, ignore fields and assume the whole document is needed + bool needTextScore; + }; + + /** + * This class is designed to quickly extract the needed fields from a BSONObj into a Document. + * It should only be created by a call to DepsTracker::ParsedDeps + */ + class ParsedDeps { + public: + Document extractFields(const BSONObj& input) const; + + private: + friend struct DepsTracker; // so it can call constructor + explicit ParsedDeps(const Document& fields) + : _fields(fields) + {} + + Document _fields; + }; +} diff --git a/src/mongo/db/pipeline/doc_mem_monitor.cpp b/src/mongo/db/pipeline/doc_mem_monitor.cpp deleted file mode 100644 index 3cbe14e8f40..00000000000 --- a/src/mongo/db/pipeline/doc_mem_monitor.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright (c) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "pch.h" -#include "db/pipeline/doc_mem_monitor.h" -#include "util/systeminfo.h" - -namespace mongo { - - DocMemMonitor::DocMemMonitor(StringWriter *pW) { - /* - Use the default values. - - Currently, we warn in log at 5%, and assert at 10%. - */ - size_t errorRam = SystemInfo::getPhysicalRam() / 10; - size_t warnRam = errorRam / 2; - - init(pW, warnRam, errorRam); - } - - DocMemMonitor::DocMemMonitor(StringWriter *pW, - size_t warnLimit, size_t errorLimit) { - init(pW, warnLimit, errorLimit); - } - - void DocMemMonitor::addToTotal(size_t amount) { - totalUsed += amount; - - if (!warned) { - if (warnLimit && (totalUsed > warnLimit)) { - stringstream ss; - ss << "warning, 5% of physical RAM used for "; - pWriter->writeString(ss); - ss << endl; - warning() << ss.str(); - warned = true; - } - } - - if (errorLimit) { - uassert(15944, "terminating request: request heap use exceeded 10% of physical RAM", (totalUsed <= errorLimit)); - } - } - - void DocMemMonitor::init(StringWriter *pW, - size_t warnLimit, size_t errorLimit) { - this->pWriter = pW; - this->warnLimit = warnLimit; - this->errorLimit = errorLimit; - - warned = false; - totalUsed = 0; - } -} diff --git a/src/mongo/db/pipeline/doc_mem_monitor.h b/src/mongo/db/pipeline/doc_mem_monitor.h deleted file mode 100644 index b0f06c32b89..00000000000 --- a/src/mongo/db/pipeline/doc_mem_monitor.h +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright 2011 (c) 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "mongo/pch.h" -#include "util/string_writer.h" - - -namespace mongo { - - /* - This utility class provides an easy way to total up, monitor, warn, and - signal an error when the amount of memory used for an operation exceeds - given thresholds. - - Create a local instance of this class, and then inform it of any memory - that you consume using addToTotal(). - - Warnings or errors are issued as usage exceeds certain fractions of - physical memory on the host, as determined by SystemInfo. - - This class is not guaranteed to warn or signal errors if the host system - does not support the ability to report its memory, as per the warnings - for SystemInfo in systeminfo.h. - */ - class DocMemMonitor { - public: - /* - Constructor. - - Uses default limits for warnings and errors. - - The StringWriter parameter must outlive the DocMemMonitor instance. - - @param pWriter string writer that provides information about the - operation being monitored - */ - DocMemMonitor(StringWriter *pWriter); - - /* - Constructor. - - This variant allows explicit selection of the limits. Note that - limits of zero are treated as infinite. - - The StringWriter parameter must outlive the DocMemMonitor instance. - - @param pWriter string writer that provides information about the - operation being monitored - @param warnLimit the amount of ram to issue (log) a warning for - @param errorLimit the amount of ram to throw an error for - */ - DocMemMonitor(StringWriter *pWriter, size_t warnLimit, - size_t errorLimit); - - /* - Increment the total amount of memory used by the given amount. If - the warning threshold is exceeded, a warning will be logged. If the - error threshold is exceeded, an error will be thrown. - - @param amount the amount of memory to add to the current total - */ - void addToTotal(size_t amount); - - private: - /* - Real constructor body. - - Provides common construction for all the variant constructors. - */ - void init(StringWriter *pW, size_t warnLimit, size_t errorLimit); - - bool warned; - size_t totalUsed; - size_t warnLimit; - size_t errorLimit; - StringWriter *pWriter; - }; - -} diff --git a/src/mongo/db/pipeline/document.cpp b/src/mongo/db/pipeline/document.cpp index bdd5b17c7e2..4f260591fe1 100644 --- a/src/mongo/db/pipeline/document.cpp +++ b/src/mongo/db/pipeline/document.cpp @@ -12,9 +12,21 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/db/pipeline/document.h" @@ -183,6 +195,8 @@ namespace mongo { out->_usedBytes = _usedBytes; out->_numFields = _numFields; out->_hashTabMask = _hashTabMask; + out->_hasTextScore = _hasTextScore; + out->_textScore = _textScore; // Tell values that they have been memcpyed (updates ref counts) for (DocumentStorageIterator it = out->iteratorAll(); !it.atEnd(); it.advance()) { @@ -206,7 +220,7 @@ namespace mongo { BSONObjIterator it(bson); while(it.more()) { BSONElement bsonElement(it.next()); - md.addField(bsonElement.fieldName(), Value(bsonElement)); + md.addField(bsonElement.fieldNameStringData(), Value(bsonElement)); } *this = md.freeze(); @@ -225,6 +239,42 @@ namespace mongo { } } + BSONObj Document::toBson() const { + BSONObjBuilder bb; + toBson(&bb); + return bb.obj(); + } + + const StringData Document::metaFieldTextScore("$textScore", StringData::LiteralTag()); + + BSONObj Document::toBsonWithMetaData() const { + BSONObjBuilder bb; + toBson(&bb); + if (hasTextScore()) + bb.append(metaFieldTextScore, getTextScore()); + return bb.obj(); + } + + Document Document::fromBsonWithMetaData(const BSONObj& bson) { + MutableDocument md; + + BSONObjIterator it(bson); + while(it.more()) { + BSONElement elem(it.next()); + if (elem.fieldName()[0] == '$') { + if (elem.fieldNameStringData() == metaFieldTextScore) { + md.setTextScore(elem.Double()); + continue; + } + } + + // Note: this will not parse out metadata in embedded documents. + md.addField(elem.fieldNameStringData(), Value(elem)); + } + + return md.freeze(); + } + MutableDocument::MutableDocument(size_t expectedFields) : _storageHolder(NULL) , _storage(_storageHolder) @@ -365,4 +415,37 @@ namespace mongo { return out.str(); } + + void Document::serializeForSorter(BufBuilder& buf) const { + const int numElems = size(); + buf.appendNum(numElems); + + for (DocumentStorageIterator it = storage().iterator(); !it.atEnd(); it.advance()) { + buf.appendStr(it->nameSD(), /*NUL byte*/ true); + it->val.serializeForSorter(buf); + } + + if (hasTextScore()) { + buf.appendNum(char(1)); + buf.appendNum(getTextScore()); + } + else { + buf.appendNum(char(0)); + } + } + + Document Document::deserializeForSorter(BufReader& buf, const SorterDeserializeSettings&) { + const int numElems = buf.read(); + MutableDocument doc(numElems); + for (int i = 0; i < numElems; i++) { + StringData name = buf.readCStr(); + doc.addField(name, Value::deserializeForSorter(buf, + Value::SorterDeserializeSettings())); + } + + if (buf.read()) // hasTextScore + doc.setTextScore(buf.read()); + + return doc.freeze(); + } } diff --git a/src/mongo/db/pipeline/document.h b/src/mongo/db/pipeline/document.h index b51efa91497..7f2fde8a26f 100644 --- a/src/mongo/db/pipeline/document.h +++ b/src/mongo/db/pipeline/document.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -103,6 +115,8 @@ namespace mongo { * as strings are compared, but comparing one field at a time instead * of one character at a time. * + * Note: This does not consider metadata when comparing documents. + * * @returns an integer less than zero, zero, or an integer greater than * zero, depending on whether lhs < rhs, lhs == rhs, or lhs > rhs * Warning: may return values other than -1, 0, or 1 @@ -121,8 +135,25 @@ namespace mongo { */ void hash_combine(size_t &seed) const; - /// Add this document to the BSONObj under construction with the given BSONObjBuilder. + /** + * Add this document to the BSONObj under construction with the given BSONObjBuilder. + * Does not include metadata. + */ void toBson(BSONObjBuilder *pBsonObjBuilder) const; + BSONObj toBson() const; + + /** + * Like toBson, but includes metadata at the top-level. + * Output is parseable by fromBsonWithMetaData + */ + BSONObj toBsonWithMetaData() const; + + /** + * Like Document(BSONObj) but treats top-level fields with special names as metadata. + * Special field names are available as static constants on this class with names starting + * with metaField. + */ + static Document fromBsonWithMetaData(const BSONObj& bson); // Support BSONObjBuilder and BSONArrayBuilder "stream" API friend BSONObjBuilder& operator << (BSONObjBuilderValueStream& builder, const Document& d); @@ -142,26 +173,27 @@ namespace mongo { */ Document clone() const { return Document(storage().clone().get()); } - // TEMP for compatibility with legacy intrusive_ptr - Document& operator*() { return *this; } - const Document& operator*() const { return *this; } - Document* operator->() { return this; } - const Document* operator->() const { return this; } - const void* getPtr() const { return _storage.get(); } - void reset() { return _storage.reset(); } - static Document createFromBsonObj(BSONObj* pBsonObj) { return Document(*pBsonObj); } - size_t getFieldCount() const { return size(); } - Value getValue(StringData fieldName) const { return getField(fieldName); } + static const StringData metaFieldTextScore; // "$textScore" + bool hasTextScore() const { return storage().hasTextScore(); } + double getTextScore() const { return storage().getTextScore(); } - // TODO: replace with logical equality once all current usages are fixed - bool operator== (const Document& lhs) const { return _storage == lhs._storage; } + /// members for Sorter + struct SorterDeserializeSettings {}; // unused + void serializeForSorter(BufBuilder& buf) const; + static Document deserializeForSorter(BufReader& buf, const SorterDeserializeSettings&); + int memUsageForSorter() const { return getApproximateSize(); } + Document getOwned() const { return *this; } - explicit Document(const DocumentStorage* ptr) : _storage(ptr) {}; + /// only for testing + const void* getPtr() const { return _storage.get(); } private: friend class FieldIterator; friend class ValueStorage; friend class MutableDocument; + friend class MutableValue; + + explicit Document(const DocumentStorage* ptr) : _storage(ptr) {}; const DocumentStorage& storage() const { return (_storage ? *_storage : DocumentStorage::emptyDoc()); @@ -169,6 +201,26 @@ namespace mongo { intrusive_ptr _storage; }; + inline bool operator== (const Document& l, const Document& r) { + return Document::compare(l, r) == 0; + } + inline bool operator!= (const Document& l, const Document& r) { + return Document::compare(l, r) != 0; + } + inline bool operator< (const Document& l, const Document& r) { + return Document::compare(l, r) < 0; + } + inline bool operator<= (const Document& l, const Document& r) { + return Document::compare(l, r) <= 0; + } + inline bool operator> (const Document& l, const Document& r) { + return Document::compare(l, r) > 0; + } + inline bool operator>= (const Document& l, const Document& r) { + return Document::compare(l, r) >= 0; + } + + /** This class is returned by MutableDocument to allow you to modify its values. * You are not allowed to hold variables of this type (enforced by the type system). */ @@ -194,8 +246,17 @@ namespace mongo { /// Used by MutableDocument(MutableValue) const RefCountable*& getDocPtr() { - if (_val.getType() != Object) - *this = Value(Document()); + if (_val.getType() != Object || _val._storage.genericRCPtr == NULL) { + // If the current value isn't an object we replace it with a Object-typed Value. + // Note that we can't just use Document() here because that is a NULL pointer and + // Value doesn't refcount NULL pointers. This led to a memory leak (SERVER-10554) + // because MutableDocument::newStorage() would set a non-NULL pointer into the Value + // without setting the refCounter bit. While allocating a DocumentStorage here could + // result in an allocation where none is needed, in practice this is only called + // when we are about to add a field to the sub-document so this just changes where + // the allocation is done. + _val = Value(Document(new DocumentStorage())); + } return _val._storage.genericRCPtr; } @@ -300,6 +361,16 @@ namespace mongo { getNestedField(positions) = val; } + /** + * Copies all metadata from source if it has any. + * Note: does not clear metadata from this. + */ + void copyMetaDataFrom(const Document& source) { + storage().copyMetaDataFrom(source.storage()); + } + + void setTextScore(double score) { storage().setTextScore(score); } + /** Convert to a read-only document and release reference. * * Call this to indicate that you are done with this Document and will @@ -316,6 +387,11 @@ namespace mongo { return ret; } + /// Used to simplify the common pattern of creating a value of the document. + Value freezeToValue() { + return Value(freeze()); + } + /** Borrow a readable reference to this Document. * * Note that unlike freeze(), this indicates intention to continue @@ -400,6 +476,74 @@ namespace mongo { Document _doc; DocumentStorageIterator _it; }; + + /// Macro to create Document literals. Syntax is the same as the BSON("name" << 123) macro. +#define DOC(fields) ((DocumentStream() << fields).done()) + + /** Macro to create Array-typed Value literals. + * Syntax is the same as the BSON_ARRAY(123 << "foo") macro. + */ +#define DOC_ARRAY(fields) ((ValueArrayStream() << fields).done()) + + + // These classes are only for the implementation of the DOC and DOC_ARRAY macros. + // They should not be used for any other reason. + class DocumentStream { + // The stream alternates between DocumentStream taking a fieldname + // and ValueStream taking a Value. + class ValueStream { + public: + ValueStream(DocumentStream& builder) :builder(builder) {} + + DocumentStream& operator << (const Value& val) { + builder._md[name] = val; + return builder; + } + + /// support anything directly supported by a value constructor + template + DocumentStream& operator << (const T& val) { + return *this << Value(val); + } + + StringData name; + DocumentStream& builder; + }; + + public: + DocumentStream() :_stream(*this) {} + + ValueStream& operator << (const StringData& name) { + _stream.name = name; + return _stream; + } + + Document done() { return _md.freeze(); } + + private: + ValueStream _stream; + MutableDocument _md; + }; + + class ValueArrayStream { + public: + ValueArrayStream& operator << (const Value& val) { + _array.push_back(val); + return *this; + } + + /// support anything directly supported by a value constructor + template + ValueArrayStream& operator << (const T& val) { + return *this << Value(val); + } + + Value done() { return Value::consume(_array); } + + private: + vector _array; + }; + } namespace std { diff --git a/src/mongo/db/pipeline/document_internal.h b/src/mongo/db/pipeline/document_internal.h index 4ec1858c17c..4ef2a0f0710 100644 --- a/src/mongo/db/pipeline/document_internal.h +++ b/src/mongo/db/pipeline/document_internal.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -154,6 +166,8 @@ namespace mongo { , _usedBytes(0) , _numFields(0) , _hashTabMask(0) + , _hasTextScore(false) + , _textScore(0) {} ~DocumentStorage(); @@ -225,6 +239,23 @@ namespace mongo { return !_buffer ? 0 : (_bufferEnd - _buffer + hashTabBytes()); } + /** + * Copies all metadata from source if it has any. + * Note: does not clear metadata from this. + */ + void copyMetaDataFrom(const DocumentStorage& source) { + if (source.hasTextScore()) { + setTextScore(source.getTextScore()); + } + } + + bool hasTextScore() const { return _hasTextScore; } + double getTextScore() const { return _textScore; } + void setTextScore(double score) { + _hasTextScore = true; + _textScore = score; + } + private: /// Same as lastElement->next() or firstElement() if empty. @@ -293,6 +324,9 @@ namespace mongo { unsigned _usedBytes; // position where next field would start unsigned _numFields; // this includes removed fields unsigned _hashTabMask; // equal to hashTabBuckets()-1 but used more often + + bool _hasTextScore; // When adding more metadata fields, this should become a bitvector + double _textScore; // When adding a field, make sure to update clone() method }; } diff --git a/src/mongo/db/pipeline/document_source.cpp b/src/mongo/db/pipeline/document_source.cpp index 6c46604e1ba..d4c21fca2fd 100644 --- a/src/mongo/db/pipeline/document_source.cpp +++ b/src/mongo/db/pipeline/document_source.cpp @@ -12,25 +12,32 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" -#include "db/pipeline/document_source.h" -#include "db/pipeline/expression_context.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - DocumentSource::DocumentSource( - const intrusive_ptr &pCtx): - pSource(NULL), - step(-1), - pExpCtx(pCtx), - nRowsOut(0) { - } - - DocumentSource::~DocumentSource() { - } + DocumentSource::DocumentSource(const intrusive_ptr &pCtx) + : pSource(NULL) + , pExpCtx(pCtx) + {} const char *DocumentSource::getSourceName() const { static const char unknown[] = "[UNKNOWN]"; @@ -50,158 +57,16 @@ namespace mongo { void DocumentSource::optimize() { } - bool DocumentSource::advance() { - pExpCtx->checkForInterrupt(); // might not return - return false; - } - void DocumentSource::dispose() { if ( pSource ) { - // This is required for the DocumentSourceCursor to release its read lock, see - // SERVER-6123. pSource->dispose(); } } - void DocumentSource::addToBsonArray( - BSONArrayBuilder *pBuilder, bool explain) const { - BSONObjBuilder insides; - sourceToBson(&insides, explain); - -/* No statistics at this time - if (explain) { - insides.append("nOut", nOut); - } -*/ - - pBuilder->append(insides.done()); - } - - void DocumentSource::writeString(stringstream &ss) const { - BSONArrayBuilder bab; - addToBsonArray(&bab); - BSONArray ba(bab.arr()); - ss << ba.toString(/* isArray */true); - // our toString should use standard string types..... - } - - BSONObj DocumentSource::depsToProjection(const set& deps) { - BSONObjBuilder bb; - - bool needId = false; - - string last; - for (set::const_iterator it(deps.begin()), end(deps.end()); it!=end; ++it) { - if (str::startsWith(*it, "_id") && (it->size() == 3 || (*it)[3] == '.')) { - // _id and subfields are handled specially due in part to SERVER-7502 - needId = true; - continue; - } - if (!last.empty() && str::startsWith(*it, last)) { - // we are including a parent of *it so we don't need to include this field - // explicitly. In fact, due to SERVER-6527 if we included this field, the parent - // wouldn't be fully included. This logic relies on on set iterators going in - // lexicographic order so that a string is always directly before of all fields it - // prefixes. - continue; - } - last = *it + '.'; - bb.append(*it, 1); - } - - if (needId) // we are explicit either way - bb.append("_id", 1); - else - bb.append("_id", 0); - - return bb.obj(); - } - - // Taken as a whole, these three functions should produce the same output document given the - // same deps set as mongo::Projection::transform would on the output of depsToProjection. The - // only exceptions are that we correctly handle the case where no fields are needed and we don't - // need to work around the above mentioned bug with subfields of _id (SERVER-7502). This is - // tested in a DEV block in DocumentSourceCursor::findNext(). - // - // Output from this function is input for the next two - // - // ParsedDeps is a simple recursive look-up table. For each field in a ParsedDeps: - // If the value has type==Bool, the whole field is needed - // If the value has type==Object, the fields in the subobject are needed - // All other fields should be missing which means not needed - DocumentSource::ParsedDeps DocumentSource::parseDeps(const set& deps) { - MutableDocument md; - - string last; - for (set::const_iterator it(deps.begin()), end(deps.end()); it!=end; ++it) { - if (!last.empty() && str::startsWith(*it, last)) { - // we are including a parent of *it so we don't need to include this field - // explicitly. In fact, if we included this field, the parent wouldn't be fully - // included. This logic relies on on set iterators going in lexicographic order so - // that a string is always directly before of all fields it prefixes. - continue; - } - last = *it + '.'; - md.setNestedField(*it, Value(true)); - } - - return md.freeze(); - } - - // Helper for next function - static Value arrayHelper(const BSONObj& bson, const DocumentSource::ParsedDeps& neededFields) { - BSONObjIterator it(bson); - - vector values; - while (it.more()) { - BSONElement bsonElement(it.next()); - if (bsonElement.type() == Object) { - Document sub = DocumentSource::documentFromBsonWithDeps( - bsonElement.embeddedObject(), - neededFields); - values.push_back(Value(sub)); - } - - if (bsonElement.type() == Array) { - values.push_back(arrayHelper(bsonElement.embeddedObject(), neededFields)); - } - } - - return Value(values); - } - - Document DocumentSource::documentFromBsonWithDeps(const BSONObj& bson, - const ParsedDeps& neededFields) { - MutableDocument md(neededFields.size()); - - BSONObjIterator it(bson); - while (it.more()) { - BSONElement bsonElement (it.next()); - StringData fieldName (bsonElement.fieldName(), bsonElement.fieldNameSize()-1); - Value isNeeded = neededFields[fieldName]; - - if (isNeeded.missing()) - continue; - - if (isNeeded.getType() == Bool) { - md.addField(fieldName, Value(bsonElement)); - continue; - } - - dassert(isNeeded.getType() == Object); - - if (bsonElement.type() == Object) { - Document sub = documentFromBsonWithDeps(bsonElement.embeddedObject(), - isNeeded.getDocument()); - md.addField(fieldName, Value(sub)); - } - - if (bsonElement.type() == Array) { - md.addField(fieldName, arrayHelper(bsonElement.embeddedObject(), - isNeeded.getDocument())); - } + void DocumentSource::serializeToArray(vector& array, bool explain) const { + Value entry = serialize(explain); + if (!entry.missing()) { + array.push_back(entry); } - - return md.freeze(); } } diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index 4f24aabc360..0671f05f951 100644 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -12,91 +12,59 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once #include "mongo/pch.h" +#include #include -#include "util/intrusive_counter.h" -#include "db/clientcursor.h" -#include "db/jsobj.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression.h" +#include + +#include "mongo/db/clientcursor.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/dependencies.h" #include "mongo/db/pipeline/expression_context.h" -#include "db/pipeline/value.h" -#include "util/string_writer.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/value.h" #include "mongo/db/projection.h" +#include "mongo/db/sorter/sorter.h" #include "mongo/s/shard.h" +#include "mongo/s/strategy.h" +#include "mongo/util/intrusive_counter.h" namespace mongo { class Accumulator; - class Cursor; class Document; class Expression; - class ExpressionContext; class ExpressionFieldPath; class ExpressionObject; class DocumentSourceLimit; - class Matcher; + class Runner; - class DocumentSource : - public IntrusiveCounterUnsigned, - public StringWriter { + class DocumentSource : public IntrusiveCounterUnsigned { public: - virtual ~DocumentSource(); - - // virtuals from StringWriter - virtual void writeString(stringstream &ss) const; - - /** - Set the step for a user-specified pipeline step. - - The step is used for diagnostics. - - @param step step number 0 to n. - */ - void setPipelineStep(int step); - - /** - Get the user-specified pipeline step. + virtual ~DocumentSource() {} - @returns the step number, or -1 if it has never been set - */ - int getPipelineStep() const; - - /** - Is the source at EOF? - - @returns true if the source has no more Documents to return. - */ - virtual bool eof() = 0; - - /** - Advance the state of the DocumentSource so that it will return the - next Document. - - The default implementation returns false, after checking for - interrupts. Derived classes can call the default implementation - in their own implementations in order to check for interrupts. - - @returns whether there is another document to fetch, i.e., whether or - not getCurrent() will succeed. This default implementation always - returns false. - */ - virtual bool advance(); - - /** @returns the current Document without advancing. - * - * It is illegal to call this without first checking eof() == false or advance() == true. - * - * While it is legal to call getCurrent() multiple times between calls to advance, and - * you will get the same Document returned, some DocumentSources do expensive work in - * getCurrent(). You are advised to cache the result if you plan to access it more than - * once. + /** Returns the next Document if there is one or boost::none if at EOF. + * Subclasses must call pExpCtx->checkForInterupt(). */ - virtual Document getCurrent() = 0; + virtual boost::optional getNext() = 0; /** * Inform the source that it is no longer needed and may release its resources. After @@ -161,65 +129,38 @@ namespace mongo { virtual void optimize(); enum GetDepsReturn { - NOT_SUPPORTED, // This means the set should be ignored - EXHAUSTIVE, // This means that everything needed should be in the set - SEE_NEXT, // Add the next Source's deps to the set + NOT_SUPPORTED = 0x0, // The full object and all metadata may be required + SEE_NEXT = 0x1, // Later stages could need either fields or metadata + EXHAUSTIVE_FIELDS = 0x2, // Later stages won't need more fields from input + EXHAUSTIVE_META = 0x4, // Later stages won't need more metadata from input + EXHAUSTIVE_ALL = EXHAUSTIVE_FIELDS | EXHAUSTIVE_META, // Later stages won't need either }; - /** Get the fields this operation needs to do its job. - * Deps should be in "a.b.c" notation - * - * @param deps results are added here. NOT CLEARED + /** + * Get the dependencies this operation needs to do its job. */ - virtual GetDepsReturn getDependencies(set& deps) const { + virtual GetDepsReturn getDependencies(DepsTracker* deps) const { return NOT_SUPPORTED; } - /** This takes dependencies from getDependencies and - * returns a projection that includes all of them - */ - static BSONObj depsToProjection(const set& deps); - - /** These functions take the same input as depsToProjection but are able to - * produce a Document from a BSONObj with the needed fields much faster. - */ - typedef Document ParsedDeps; // See implementation for structure - static ParsedDeps parseDeps(const set& deps); - static Document documentFromBsonWithDeps(const BSONObj& object, const ParsedDeps& deps); - /** - Add the DocumentSource to the array builder. + * In the default case, serializes the DocumentSource and adds it to the vector. + * + * A subclass may choose to overwrite this, rather than serialize, + * if it should output multiple stages (eg, $sort sometimes also outputs a $limit). + */ - The default implementation calls sourceToBson() in order to - convert the inner part of the object which will be added to the - array being built here. + virtual void serializeToArray(vector& array, bool explain = false) const; - A subclass may choose to overwrite this rather than addToBsonArray - if it should output multiple stages. + /// Returns true if doesn't require an input source (most DocumentSources do). + virtual bool isValidInitialSource() const { return false; } - @param pBuilder the array builder to add the operation to. - @param explain create explain output - */ - virtual void addToBsonArray(BSONArrayBuilder *pBuilder, bool explain=false) const; - protected: /** Base constructor. */ DocumentSource(const intrusive_ptr &pExpCtx); - /** - Create an object that represents the document source. The object - will have a single field whose name is the source's name. This - will be used by the default implementation of addToBsonArray() - to add this object to a pipeline being represented in BSON. - - @param pBuilder a blank object builder to write to - @param explain create explain output - */ - virtual void sourceToBson(BSONObjBuilder *pBuilder, - bool explain) const = 0; - /* Most DocumentSources have an underlying source they get their data from. This is a convenience for them. @@ -230,27 +171,23 @@ namespace mongo { */ DocumentSource *pSource; - /* - The zero-based user-specified pipeline step. Used for diagnostics. - Will be set to -1 for artificial pipeline steps that were not part - of the original user specification. - */ - int step; - intrusive_ptr pExpCtx; - /* - for explain: # of rows returned by this source - - This is *not* unsigned so it can be passed to BSONObjBuilder.append(). + private: + /** + * Create a Value that represents the document source. + * + * This is used by the default implementation of serializeToArray() to add this object + * to a pipeline being serialized. Returning a missing() Value results in no entry + * being added to the array for this stage (DocumentSource). */ - long long nRowsOut; + virtual Value serialize(bool explain = false) const = 0; }; - /** This class marks DocumentSources that should be split between the router and the shards - * See Pipeline::splitForSharded() for details + /** This class marks DocumentSources that should be split between the merger and the shards. + * See Pipeline::Optimizations::Sharded::findSplitPoint() for details. */ - class SplittableDocumentSource : public DocumentSource { + class SplittableDocumentSource { public: /** returns a source to be run on the shards. * if NULL, don't run on shards @@ -258,11 +195,48 @@ namespace mongo { virtual intrusive_ptr getShardSource() = 0; /** returns a source that combines results from shards. - * if NULL, don't run on router + * if NULL, don't run on merger */ - virtual intrusive_ptr getRouterSource() = 0; + virtual intrusive_ptr getMergeSource() = 0; protected: - SplittableDocumentSource(intrusive_ptr ctx) :DocumentSource(ctx) {} + // It is invalid to delete through a SplittableDocumentSource-typed pointer. + virtual ~SplittableDocumentSource() {} + }; + + + /** This class marks DocumentSources which need mongod-specific functionality. + * It causes a MongodInterface to be injected when in a mongod and prevents mongos from + * merging pipelines containing this stage. + */ + class DocumentSourceNeedsMongod { + public: + // Wraps mongod-specific functions to allow linking into mongos. + class MongodInterface { + public: + virtual ~MongodInterface() {}; + + virtual DBClientBase* directClient() = 0; // Always returns a DBDirectClient + + // Note that in some rare cases this could return a false negative but will never return + // a false positive. This method will be fixed in the future once it becomes possible to + // avoid false negatives. + virtual bool isSharded(const NamespaceString& ns) = 0; + + virtual bool isCapped(const NamespaceString& ns) = 0; + + // Add new methods as needed. + }; + + void injectMongodInterface(boost::shared_ptr mongod) { + _mongod = mongod; + } + + protected: + // It is invalid to delete through a DocumentSourceNeedsMongod-typed pointer. + virtual ~DocumentSourceNeedsMongod() {} + + // Gives subclasses access to a MongodInterface implementation + shared_ptr _mongod; }; @@ -270,11 +244,10 @@ namespace mongo { public DocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceBsonArray(); - virtual bool eof(); - virtual bool advance(); - virtual Document getCurrent(); + virtual boost::optional getNext(); + virtual Value serialize(bool explain = false) const; virtual void setSource(DocumentSource *pSource); + virtual bool isValidInitialSource() const { return true; } /** Create a document source based on a BSON array. @@ -286,26 +259,21 @@ namespace mongo { elements that appear after these documents must not be read until this source is exhausted. - @param pBsonElement the BSON array to treat as a document source + @param array the BSON array to treat as a document source @param pExpCtx the expression context for the pipeline @returns the newly created document source */ static intrusive_ptr create( - BSONElement *pBsonElement, + const BSONObj& array, const intrusive_ptr &pExpCtx); - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: - DocumentSourceBsonArray(BSONElement *pBsonElement, + DocumentSourceBsonArray( + const BSONObj& embeddedArray, const intrusive_ptr &pExpCtx); BSONObj embeddedObject; BSONObjIterator arrayIterator; - BSONElement currentElement; - bool haveCurrent; }; @@ -313,14 +281,19 @@ namespace mongo { public DocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceCommandShards(); - virtual bool eof(); - virtual bool advance(); - virtual Document getCurrent(); + virtual boost::optional getNext(); + virtual Value serialize(bool explain = false) const; virtual void setSource(DocumentSource *pSource); + virtual bool isValidInitialSource() const { return true; } /* convenient shorthand for a commonly used type */ - typedef map ShardOutput; + typedef vector ShardOutput; + + /** Returns the result arrays from shards using the 2.4 protocol. + * Call this instead of getNext() if you want access to the raw streams. + * This method should only be called at most once. + */ + vector getArrays(); /** Create a DocumentSource that wraps the output of many shards @@ -333,10 +306,6 @@ namespace mongo { const ShardOutput& shardOutput, const intrusive_ptr& pExpCtx); - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: DocumentSourceCommandShards(const ShardOutput& shardOutput, const intrusive_ptr& pExpCtx); @@ -361,61 +330,33 @@ namespace mongo { /** - * Constructs and returns Documents from the BSONObj objects produced by a supplied Cursor. + * Constructs and returns Documents from the BSONObj objects produced by a supplied Runner. * An object of this type may only be used by one thread, see SERVER-6123. */ class DocumentSourceCursor : public DocumentSource { public: - /** - * Holds a Cursor and all associated state required to access the cursor. An object of this - * type may only be used by one thread. - */ - struct CursorWithContext { - /** Takes a read lock that will be held for the lifetime of the object. */ - CursorWithContext( const string& ns ); - - // Must be the first struct member for proper construction and destruction, as other - // members may depend on the read lock it acquires. - Client::ReadContext _readContext; - shared_ptr _chunkMgr; - ClientCursor::Holder _cursor; - }; - // virtuals from DocumentSource virtual ~DocumentSourceCursor(); - virtual bool eof(); - virtual bool advance(); - virtual Document getCurrent(); + virtual boost::optional getNext(); + virtual const char *getSourceName() const; + virtual Value serialize(bool explain = false) const; virtual void setSource(DocumentSource *pSource); - - /** - * Release the Cursor and the read lock it requires, but without changing the other data. - * Releasing the lock is required for proper concurrency, see SERVER-6123. This - * functionality is also used by the explain version of pipeline execution. - */ + virtual bool coalesce(const intrusive_ptr& nextSource); + virtual bool isValidInitialSource() const { return true; } virtual void dispose(); /** - Create a document source based on a cursor. - - This is usually put at the beginning of a chain of document sources - in order to fetch data from the database. - - @param pCursor the cursor to use to fetch data - @param pExpCtx the expression context for the pipeline - */ + * Create a document source based on a passed-in Runner. + * + * This is usually put at the beginning of a chain of document sources + * in order to fetch data from the database. + */ static intrusive_ptr create( - const shared_ptr& cursorWithContext, + const string& ns, + const boost::shared_ptr& runner, const intrusive_ptr &pExpCtx); - /* - Record the namespace. Required for explain. - - @param namespace the namespace - */ - void setNamespace(const string &ns); - /* Record the query that was specified for the cursor this wraps, if any. @@ -427,7 +368,7 @@ namespace mongo { @param pBsonObj the query to record */ - void setQuery(const shared_ptr &pBsonObj); + void setQuery(const BSONObj& query) { _query = query; } /* Record the sort that was specified for the cursor this wraps, if @@ -440,178 +381,53 @@ namespace mongo { @param pBsonObj the sort to record */ - void setSort(const shared_ptr &pBsonObj); - - void setProjection(const BSONObj& projection, const ParsedDeps& deps); - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - - private: - DocumentSourceCursor( - const shared_ptr& cursorWithContext, - const intrusive_ptr &pExpCtx); - - void findNext(); - - bool unstarted; - bool hasCurrent; - Document pCurrent; - - string ns; // namespace - - /* - The bson dependencies must outlive the Cursor wrapped by this - source. Therefore, bson dependencies must appear before pCursor - in order cause its destructor to be called *after* pCursor's. - */ - shared_ptr pQuery; - shared_ptr pSort; - shared_ptr _projection; // shared with pClientCursor - ParsedDeps _dependencies; - - shared_ptr _cursorWithContext; - - ClientCursor::Holder& cursor(); - const ShardChunkManager* chunkMgr() { return _cursorWithContext->_chunkMgr.get(); } - - bool canUseCoveredIndex(); - - /* - Yield the cursor sometimes. - - If the state of the world changed during the yield such that we - are unable to continue execution of the query, this will release the - client cursor, and throw an error. NOTE This differs from the - behavior of most other operations, see SERVER-2454. - */ - void yieldSometimes(); - }; - - - /* - This contains all the basic mechanics for filtering a stream of - Documents, except for the actual predicate evaluation itself. This was - factored out so we could create DocumentSources that use both Matcher - style predicates as well as full Expressions. - */ - class DocumentSourceFilterBase : - public DocumentSource { - public: - // virtuals from DocumentSource - virtual ~DocumentSourceFilterBase(); - virtual bool eof(); - virtual bool advance(); - virtual Document getCurrent(); + void setSort(const BSONObj& sort) { _sort = sort; } /** - Create a BSONObj suitable for Matcher construction. - - This is used after filter analysis has moved as many filters to - as early a point as possible in the document processing pipeline. - See db/Matcher.h and the associated documentation for the format. - This conversion is used to move back to the low-level find() - Cursor mechanism. - - @param pBuilder the builder to write to + * Informs this object of projection and dependency information. + * + * @param projection A projection specification describing the fields needed by the rest of + * the pipeline. + * @param deps The output of DepsTracker::toParsedDeps */ - virtual void toMatcherBson(BSONObjBuilder *pBuilder) const = 0; - - protected: - DocumentSourceFilterBase( - const intrusive_ptr &pExpCtx); + void setProjection(const BSONObj& projection, const boost::optional& deps); - /** - Test the given document against the predicate and report if it - should be accepted or not. - - @param pDocument the document to test - @returns true if the document matches the filter, false otherwise - */ - virtual bool accept(const Document& pDocument) const = 0; + /// returns -1 for no limit + long long getLimit() const; private: - - void findNext(); - - bool unstarted; - bool hasCurrent; - Document pCurrent; - }; - - - class DocumentSourceFilter : - public DocumentSourceFilterBase { - public: - // virtuals from DocumentSource - virtual ~DocumentSourceFilter(); - virtual bool coalesce(const intrusive_ptr &pNextSource); - virtual void optimize(); - virtual const char *getSourceName() const; - - /** - Create a filter. - - @param pBsonElement the raw BSON specification for the filter - @param pExpCtx the expression context for the pipeline - @returns the filter - */ - static intrusive_ptr createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx); - - /** - Create a filter. - - @param pFilter the expression to use to filter - @param pExpCtx the expression context for the pipeline - @returns the filter - */ - static intrusive_ptr create( - const intrusive_ptr &pFilter, + DocumentSourceCursor( + const string& ns, + const boost::shared_ptr& runner, const intrusive_ptr &pExpCtx); - /** - Create a BSONObj suitable for Matcher construction. - - This is used after filter analysis has moved as many filters to - as early a point as possible in the document processing pipeline. - See db/Matcher.h and the associated documentation for the format. - This conversion is used to move back to the low-level find() - Cursor mechanism. - - @param pBuilder the builder to write to - */ - void toMatcherBson(BSONObjBuilder *pBuilder) const; - - static const char filterName[]; + void loadBatch(); - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; + std::deque _currentBatch; - // virtuals from DocumentSourceFilterBase - virtual bool accept(const Document& pDocument) const; + // BSONObj members must outlive _projection and cursor. + BSONObj _query; + BSONObj _sort; + BSONObj _projection; + boost::optional _dependencies; + intrusive_ptr _limit; + long long _docsAddedToBatches; // for _limit enforcement - private: - DocumentSourceFilter(const intrusive_ptr &pFilter, - const intrusive_ptr &pExpCtx); - - intrusive_ptr pFilter; + const string _ns; + boost::shared_ptr _runner; // PipelineRunner holds a weak_ptr to this. }; - class DocumentSourceGroup : - public SplittableDocumentSource { + class DocumentSourceGroup : public DocumentSource + , public SplittableDocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceGroup(); - virtual bool eof(); - virtual bool advance(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; - virtual Document getCurrent(); - virtual GetDepsReturn getDependencies(set& deps) const; + virtual void optimize(); + virtual GetDepsReturn getDependencies(DepsTracker* deps) const; virtual void dispose(); + virtual Value serialize(bool explain = false) const; /** Create a new grouping DocumentSource. @@ -622,17 +438,6 @@ namespace mongo { static intrusive_ptr create( const intrusive_ptr &pExpCtx); - /** - Set the Id Expression. - - Documents that pass through the grouping Document are grouped - according to this key. This will generate the id_ field in the - result documents. - - @param pExpression the group key - */ - void setIdExpression(const intrusive_ptr &pExpression); - /** Add an accumulator. @@ -646,10 +451,12 @@ namespace mongo { group field */ void addAccumulator(const std::string& fieldName, - intrusive_ptr (*pAccumulatorFactory)( - const intrusive_ptr &), + intrusive_ptr (*pAccumulatorFactory)(), const intrusive_ptr &pExpression); + /// Tell this source if it is doing a merge from shards. Defaults to false. + void setDoingMerge(bool doingMerge) { _doingMerge = doingMerge; } + /** Create a grouping DocumentSource from BSON. @@ -662,22 +469,24 @@ namespace mongo { @returns the grouping DocumentSource */ static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pExpCtx); // Virtuals for SplittableDocumentSource virtual intrusive_ptr getShardSource(); - virtual intrusive_ptr getRouterSource(); + virtual intrusive_ptr getMergeSource(); static const char groupName[]; - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: DocumentSourceGroup(const intrusive_ptr &pExpCtx); + /// Spill groups map to disk and returns an iterator to the file. + shared_ptr::Iterator> spill(); + + // Only used by spill. Would be function-local if that were legal in C++03. + class SpillSTLComparator; + /* Before returning anything, this source must fetch everything from the underlying source and group it. populate() is used to do that @@ -687,11 +496,26 @@ namespace mongo { void populate(); bool populated; - intrusive_ptr pIdExpression; + /** + * Parses the raw id expression into _idExpressions and possibly _idFieldNames. + */ + void parseIdExpression(BSONElement groupField, const VariablesParseState& vps); - typedef boost::unordered_map >, Value::Hash> GroupsType; - GroupsType groups; + /** + * Computes the internal representation of the group key. + */ + Value computeId(Variables* vars); + + /** + * Converts the internal representation of the group key to the _id shape specified by the + * user. + */ + Value expandId(const Value& val); + + + typedef vector > Accumulators; + typedef boost::unordered_map GroupsMap; + GroupsMap groups; /* The field names for the result documents and the accumulator @@ -706,23 +530,39 @@ namespace mongo { These three vectors parallel each other. */ vector vFieldName; - vector (*)( - const intrusive_ptr &)> vpAccumulatorFactory; + vector (*)()> vpAccumulatorFactory; vector > vpExpression; - Document makeDocument(const GroupsType::iterator &rIter); + Document makeDocument(const Value& id, const Accumulators& accums, bool mergeableOutput); - GroupsType::iterator groupsIterator; + bool _doingMerge; + bool _spilled; + const bool _extSortAllowed; + const int _maxMemoryUsageBytes; + boost::scoped_ptr _variables; + std::vector _idFieldNames; // used when id is a document + std::vector > _idExpressions; + + // only used when !_spilled + GroupsMap::iterator groupsIterator; + + // only used when _spilled + scoped_ptr::Iterator> _sorterIterator; + pair _firstPartOfNextGroup; + Value _currentId; + Accumulators _currentAccumulators; }; - class DocumentSourceMatch : - public DocumentSourceFilterBase { + class DocumentSourceMatch : public DocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceMatch(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; + virtual bool coalesce(const intrusive_ptr& nextSource); + virtual Value serialize(bool explain = false) const; + virtual void setSource(DocumentSource* Source); /** Create a filter. @@ -731,48 +571,116 @@ namespace mongo { @returns the filter */ static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pCtx); - /** - Create a BSONObj suitable for Matcher construction. + /// Returns the query in Matcher syntax. + BSONObj getQuery() const; - This is used after filter analysis has moved as many filters to - as early a point as possible in the document processing pipeline. - See db/Matcher.h and the associated documentation for the format. - This conversion is used to move back to the low-level find() - Cursor mechanism. + static const char matchName[]; - @param pBuilder the builder to write to + /** Returns the portion of the match that can safely be promoted to before a $redact. + * If this returns an empty BSONObj, no part of this match may safely be promoted. + * + * To be safe to promote, removing a field from a document to be matched must not cause + * that document to be accepted when it would otherwise be rejected. As an example, + * {name: {$ne: "bob smith"}} accepts documents without a name field, which means that + * running this filter before a redact that would remove the name field would leak + * information. On the other hand, {age: {$gt:5}} is ok because it doesn't accept documents + * that have had their age field removed. */ - void toMatcherBson(BSONObjBuilder *pBuilder) const; + BSONObj redactSafePortion() const; - static const char matchName[]; + static bool isTextQuery(const BSONObj& query); + bool isTextQuery() const { return _isTextQuery; } + + private: + DocumentSourceMatch(const BSONObj &query, + const intrusive_ptr &pExpCtx); + + scoped_ptr matcher; + bool _isTextQuery; + }; + + class DocumentSourceMergeCursors : + public DocumentSource { + public: + typedef vector > CursorIds; - protected: // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; + boost::optional getNext(); + virtual void setSource(DocumentSource *pSource); + virtual const char *getSourceName() const; + virtual void dispose(); + virtual Value serialize(bool explain = false) const; + virtual bool isValidInitialSource() const { return true; } + + static intrusive_ptr createFromBson( + BSONElement elem, + const intrusive_ptr &pExpCtx); + + static intrusive_ptr create( + const CursorIds& cursorIds, + const intrusive_ptr &pExpCtx); - // virtuals from DocumentSourceFilterBase - virtual bool accept(const Document& pDocument) const; + static const char name[]; + + /** Returns non-owning pointers to cursors managed by this stage. + * Call this instead of getNext() if you want access to the raw streams. + * This method should only be called at most once. + */ + vector getCursors(); + + /** + * Returns the next object from the cursor, throwing an appropriate exception if the cursor + * reported an error. This is a better form of DBClientCursor::nextSafe. + */ + static Document nextSafeFrom(DBClientCursor* cursor); private: - DocumentSourceMatch(const BSONObj &query, + + struct CursorAndConnection { + CursorAndConnection(ConnectionString host, NamespaceString ns, CursorId id); + ScopedDbConnection connection; + DBClientCursor cursor; + }; + + // using list to enable removing arbitrary elements + typedef list > Cursors; + + DocumentSourceMergeCursors( + const CursorIds& cursorIds, const intrusive_ptr &pExpCtx); - Matcher matcher; - }; + // Converts _cursorIds into active _cursors. + void start(); + // This is the description of cursors to merge. + const CursorIds _cursorIds; - class DocumentSourceOut : - public DocumentSource { + // These are the actual cursors we are merging. Created lazily. + Cursors _cursors; + Cursors::iterator _currentCursor; + + bool _unstarted; + }; + + class DocumentSourceOut : public DocumentSource + , public SplittableDocumentSource + , public DocumentSourceNeedsMongod { public: // virtuals from DocumentSource virtual ~DocumentSourceOut(); - virtual bool eof(); - virtual bool advance(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; - virtual Document getCurrent(); + virtual Value serialize(bool explain = false) const; + virtual GetDepsReturn getDependencies(DepsTracker* deps) const; + + // Virtuals for SplittableDocumentSource + virtual intrusive_ptr getShardSource() { return NULL; } + virtual intrusive_ptr getMergeSource() { return this; } + + const NamespaceString& getOutputNs() const { return _outputNs; } /** Create a document source for output and pass-through. @@ -784,19 +692,25 @@ namespace mongo { @param pExpCtx the expression context for the pipeline @returns the newly created document source */ - static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + static intrusive_ptr createFromBson( + BSONElement elem, const intrusive_ptr &pExpCtx); static const char outName[]; - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: - DocumentSourceOut(BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx); + DocumentSourceOut(const NamespaceString& outputNs, + const intrusive_ptr &pExpCtx); + + // Sets _tempsNs and prepares it to receive data. + void prepTempCollection(); + + void spill(DBClientBase* conn, const vector& toInsert); + + bool _done; + + NamespaceString _tempNs; // output goes here as it is being processed. + const NamespaceString _outputNs; // output will go here after all data is processed. }; @@ -804,14 +718,12 @@ namespace mongo { public DocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceProject(); - virtual bool eof(); - virtual bool advance(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; - virtual Document getCurrent(); virtual void optimize(); + virtual Value serialize(bool explain = false) const; - virtual GetDepsReturn getDependencies(set& deps) const; + virtual GetDepsReturn getDependencies(DepsTracker* deps) const; /** Create a new projection DocumentSource from BSON. @@ -824,7 +736,7 @@ namespace mongo { @returns the created projection */ static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pExpCtx); static const char projectName[]; @@ -832,14 +744,12 @@ namespace mongo { /** projection as specified by the user */ BSONObj getRaw() const { return _raw; } - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: - DocumentSourceProject(const intrusive_ptr &pExpCtx); + DocumentSourceProject(const intrusive_ptr& pExpCtx, + const intrusive_ptr& exprObj); // configuration state + boost::scoped_ptr _variables; intrusive_ptr pEO; BSONObj _raw; @@ -849,28 +759,48 @@ namespace mongo { #endif }; + class DocumentSourceRedact : + public DocumentSource { + public: + virtual boost::optional getNext(); + virtual const char* getSourceName() const; + virtual void optimize(); + + static const char redactName[]; + + static intrusive_ptr createFromBson( + BSONElement elem, + const intrusive_ptr& expCtx); - class DocumentSourceSort : - public SplittableDocumentSource { + virtual Value serialize(bool explain = false) const; + + private: + DocumentSourceRedact(const intrusive_ptr& expCtx, + const intrusive_ptr& previsit); + + // These both work over _variables + boost::optional redactObject(); // redacts CURRENT + Value redactValue(const Value& in); + + Variables::Id _currentId; + boost::scoped_ptr _variables; + intrusive_ptr _expression; + }; + + class DocumentSourceSort : public DocumentSource + , public SplittableDocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceSort(); - virtual bool eof(); - virtual bool advance(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; - virtual Document getCurrent(); - virtual void addToBsonArray(BSONArrayBuilder *pBuilder, bool explain=false) const; + virtual void serializeToArray(vector& array, bool explain = false) const; virtual bool coalesce(const intrusive_ptr &pNextSource); virtual void dispose(); - virtual GetDepsReturn getDependencies(set& deps) const; + virtual GetDepsReturn getDependencies(DepsTracker* deps) const; - // Virtuals for SplittableDocumentSource - // All work for sort is done in router currently if there is no limit. - // If there is a limit, the $sort/$limit combination is performed on the - // shards, then the results are resorted and limited on mongos - virtual intrusive_ptr getShardSource() { return limitSrc ? this : NULL; } - virtual intrusive_ptr getRouterSource() { return this; } + virtual intrusive_ptr getShardSource(); + virtual intrusive_ptr getMergeSource(); /** Add sort key field. @@ -884,13 +814,8 @@ namespace mongo { */ void addKey(const string &fieldPath, bool ascending); - /** - Write out an object whose contents are the sort key. - - @param pBuilder initialized object builder. - @param fieldPrefix specify whether or not to include the field prefix - */ - void sortKeyToBson(BSONObjBuilder *pBuilder, bool usePrefix) const; + /// Write out a Document whose contents are the sort key. + Document serializeSortKey(bool explain) const; /** Create a sorting DocumentSource from BSON. @@ -904,7 +829,7 @@ namespace mongo { @returns the grouping DocumentSource */ static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pExpCtx); /// Create a DocumentSourceSort with a given sort and (optional) limit @@ -919,15 +844,14 @@ namespace mongo { intrusive_ptr getLimitSrc() const { return limitSrc; } static const char sortName[]; - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const { - verify(false); // should call addToBsonArray instead - } private: DocumentSourceSort(const intrusive_ptr &pExpCtx); + virtual Value serialize(bool explain = false) const { + verify(false); // should call addToBsonArray instead + } + /* Before returning anything, this source must fetch everything from the underlying source and group it. populate() is used to do that @@ -937,61 +861,57 @@ namespace mongo { void populate(); bool populated; - // These are called by populate() - void populateAll(); // no limit - void populateOne(); // limit == 1 - void populateTopK(); // limit > 1 + SortOptions makeSortOptions() const; + + // These are used to merge pre-sorted results from a DocumentSourceMergeCursors or a + // DocumentSourceCommandShards depending on whether we have finished upgrading to 2.6 or + // not. + class IteratorFromCursor; + class IteratorFromBsonArray; + void populateFromCursors(const vector& cursors); + void populateFromBsonArrays(const vector& arrays); /* these two parallel each other */ - typedef vector > SortPaths; - SortPaths vSortKey; + typedef vector > SortKey; + SortKey vSortKey; vector vAscending; // used like vector but without specialization - struct KeyAndDoc { - explicit KeyAndDoc(const Document& d, const SortPaths& sp); // extracts sort key - Value key; // array of keys if vSortKey.size() > 1 - Document doc; - }; - friend void swap(KeyAndDoc& l, KeyAndDoc& r); + /// Extracts the fields in vSortKey from the Document; + Value extractKey(const Document& d) const; - /// Compare two KeyAndDocs according to the specified sort key. - int compare(const KeyAndDoc& lhs, const KeyAndDoc& rhs) const; + /// Compare two Values according to the specified sort key. + int compare(const Value& lhs, const Value& rhs) const; - /* - This is a utility class just for the STL sort that is done - inside. - */ + typedef Sorter MySorter; + + // For MySorter class Comparator { public: explicit Comparator(const DocumentSourceSort& source): _source(source) {} - bool operator()(const KeyAndDoc& lhs, const KeyAndDoc& rhs) const { - return (_source.compare(lhs, rhs) < 0); + int operator()(const MySorter::Data& lhs, const MySorter::Data& rhs) const { + return _source.compare(lhs.first, rhs.first); } private: const DocumentSourceSort& _source; }; - deque documents; - intrusive_ptr limitSrc; + + bool _done; + bool _mergingPresorted; + scoped_ptr _output; }; - inline void swap(DocumentSourceSort::KeyAndDoc& l, DocumentSourceSort::KeyAndDoc& r) { - l.key.swap(r.key); - l.doc.swap(r.doc); - } - class DocumentSourceLimit : - public SplittableDocumentSource { + class DocumentSourceLimit : public DocumentSource + , public SplittableDocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceLimit(); - virtual bool eof(); - virtual bool advance(); - virtual Document getCurrent(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; virtual bool coalesce(const intrusive_ptr &pNextSource); + virtual Value serialize(bool explain = false) const; - virtual GetDepsReturn getDependencies(set& deps) const { + virtual GetDepsReturn getDependencies(DepsTracker* deps) const { return SEE_NEXT; // This doesn't affect needed fields } @@ -1008,7 +928,7 @@ namespace mongo { // Virtuals for SplittableDocumentSource // Need to run on rounter. Running on shard as well is an optimization. virtual intrusive_ptr getShardSource() { return this; } - virtual intrusive_ptr getRouterSource() { return this; } + virtual intrusive_ptr getMergeSource() { return this; } long long getLimit() const { return limit; } void setLimit(long long newLimit) { limit = newLimit; } @@ -1025,15 +945,11 @@ namespace mongo { @returns the grouping DocumentSource */ static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pExpCtx); static const char limitName[]; - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: DocumentSourceLimit(const intrusive_ptr &pExpCtx, long long limit); @@ -1042,18 +958,16 @@ namespace mongo { long long count; }; - class DocumentSourceSkip : - public SplittableDocumentSource { + class DocumentSourceSkip : public DocumentSource + , public SplittableDocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceSkip(); - virtual bool eof(); - virtual bool advance(); - virtual Document getCurrent(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; virtual bool coalesce(const intrusive_ptr &pNextSource); + virtual Value serialize(bool explain = false) const; - virtual GetDepsReturn getDependencies(set& deps) const { + virtual GetDepsReturn getDependencies(DepsTracker* deps) const { return SEE_NEXT; // This doesn't affect needed fields } @@ -1069,10 +983,10 @@ namespace mongo { // Virtuals for SplittableDocumentSource // Need to run on rounter. Can't run on shards. virtual intrusive_ptr getShardSource() { return NULL; } - virtual intrusive_ptr getRouterSource() { return this; } + virtual intrusive_ptr getMergeSource() { return this; } - long long getSkip() const { return skip; } - void setSkip(long long newSkip) { skip = newSkip; } + long long getSkip() const { return _skip; } + void setSkip(long long newSkip) { _skip = newSkip; } /** Create a skipping DocumentSource from BSON. @@ -1086,26 +1000,16 @@ namespace mongo { @returns the grouping DocumentSource */ static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pExpCtx); static const char skipName[]; - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: DocumentSourceSkip(const intrusive_ptr &pExpCtx); - /* - Skips initial documents. - */ - void skipper(); - - long long skip; - long long count; - Document pCurrent; + long long _skip; + bool _needToSkip; }; @@ -1113,13 +1017,11 @@ namespace mongo { public DocumentSource { public: // virtuals from DocumentSource - virtual ~DocumentSourceUnwind(); - virtual bool eof(); - virtual bool advance(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; - virtual Document getCurrent(); + virtual Value serialize(bool explain = false) const; - virtual GetDepsReturn getDependencies(set& deps) const; + virtual GetDepsReturn getDependencies(DepsTracker* deps) const; /** Create a new projection DocumentSource from BSON. @@ -1132,30 +1034,14 @@ namespace mongo { @returns the created projection */ static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pExpCtx); static const char unwindName[]; - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: DocumentSourceUnwind(const intrusive_ptr &pExpCtx); - /** - * Lazily construct the _unwinder and initialize the iterator state of this DocumentSource. - * To be called by all members that depend on the iterator state. - */ - void lazyInit(); - - /** - * If the _unwinder is exhausted and the source may be advanced, advance the pSource and - * reset the _unwinder's source document. - */ - void mayAdvanceSource(); - /** Specify the field to unwind. */ void unwindPath(const FieldPath &fieldPath); @@ -1167,23 +1053,24 @@ namespace mongo { scoped_ptr _unwinder; }; - class DocumentSourceGeoNear : public SplittableDocumentSource { + class DocumentSourceGeoNear : public DocumentSource + , public SplittableDocumentSource + , public DocumentSourceNeedsMongod { public: // virtuals from DocumentSource - virtual ~DocumentSourceGeoNear(); - virtual bool eof(); - virtual bool advance(); - virtual Document getCurrent(); + virtual boost::optional getNext(); virtual const char *getSourceName() const; - virtual void setSource(DocumentSource *pSource); // errors out since this must be first + virtual void setSource(DocumentSource *pSource); virtual bool coalesce(const intrusive_ptr &pNextSource); + virtual bool isValidInitialSource() const { return true; } + virtual Value serialize(bool explain = false) const; // Virtuals for SplittableDocumentSource virtual intrusive_ptr getShardSource(); - virtual intrusive_ptr getRouterSource(); + virtual intrusive_ptr getMergeSource(); static intrusive_ptr createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pCtx); static char geoNearName[]; @@ -1194,15 +1081,11 @@ namespace mongo { static intrusive_ptr create( const intrusive_ptr &pCtx); - protected: - // virtuals from DocumentSource - virtual void sourceToBson(BSONObjBuilder *pBuilder, bool explain) const; - private: DocumentSourceGeoNear(const intrusive_ptr &pExpCtx); void parseOptions(BSONObj options); - BSONObj buildGeoNearCmd(const StringData& collection) const; + BSONObj buildGeoNearCmd() const; void runCommand(); // These fields describe the command to run. @@ -1218,37 +1101,8 @@ namespace mongo { scoped_ptr includeLocs; bool uniqueDocs; - // These fields are injected by PipelineD. This division of labor allows the - // DocumentSourceGeoNear class to be linked into both mongos and mongod while - // allowing it to run a command using DBDirectClient when in mongod. - string db; - string collection; - boost::scoped_ptr client; // either NULL or a DBDirectClient - friend class PipelineD; - // these fields are used while processing the results BSONObj cmdOutput; boost::scoped_ptr resultsIterator; // iterator over cmdOutput["results"] - Document currentDoc; - bool hasCurrent; }; } - - -/* ======================= INLINED IMPLEMENTATIONS ========================== */ - -namespace mongo { - - inline void DocumentSource::setPipelineStep(int s) { - step = s; - } - - inline int DocumentSource::getPipelineStep() const { - return step; - } - - inline void DocumentSourceGroup::setIdExpression( - const intrusive_ptr &pExpression) { - pIdExpression = pExpression; - } -} diff --git a/src/mongo/db/pipeline/document_source_bson_array.cpp b/src/mongo/db/pipeline/document_source_bson_array.cpp index 64217cc9716..47c14f22570 100644 --- a/src/mongo/db/pipeline/document_source_bson_array.cpp +++ b/src/mongo/db/pipeline/document_source_bson_array.cpp @@ -12,44 +12,35 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" -#include "db/pipeline/document_source.h" -#include "db/pipeline/document.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" namespace mongo { - DocumentSourceBsonArray::~DocumentSourceBsonArray() { - } - - bool DocumentSourceBsonArray::eof() { - return !haveCurrent; - } - - bool DocumentSourceBsonArray::advance() { - DocumentSource::advance(); // check for interrupts - - if (eof()) - return false; - - if (!arrayIterator.more()) { - haveCurrent = false; - return false; - } + boost::optional DocumentSourceBsonArray::getNext() { + pExpCtx->checkForInterrupt(); - currentElement = arrayIterator.next(); - return true; - } + if (!arrayIterator.more()) + return boost::none; - Document DocumentSourceBsonArray::getCurrent() { - verify(haveCurrent); - BSONObj documentObj(currentElement.Obj()); - Document pDocument( - Document::createFromBsonObj(&documentObj)); - return pDocument; + return Document(arrayIterator.next().Obj()); } void DocumentSourceBsonArray::setSource(DocumentSource *pSource) { @@ -58,36 +49,24 @@ namespace mongo { } DocumentSourceBsonArray::DocumentSourceBsonArray( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx): - DocumentSource(pExpCtx), - embeddedObject(pBsonElement->embeddedObject()), - arrayIterator(embeddedObject), - haveCurrent(false) { - if (arrayIterator.more()) { - currentElement = arrayIterator.next(); - haveCurrent = true; - } - } + const BSONObj& array, + const intrusive_ptr &pExpCtx) + : DocumentSource(pExpCtx) + , embeddedObject(array) + , arrayIterator(embeddedObject) + {} intrusive_ptr DocumentSourceBsonArray::create( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + const BSONObj& array, + const intrusive_ptr &pExpCtx) { - verify(pBsonElement->type() == Array); - intrusive_ptr pSource( - new DocumentSourceBsonArray(pBsonElement, pExpCtx)); - - return pSource; + return new DocumentSourceBsonArray(array, pExpCtx); } - void DocumentSourceBsonArray::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - + Value DocumentSourceBsonArray::serialize(bool explain) const { if (explain) { - BSONObj empty; - - pBuilder->append("bsonArray", empty); + return Value(DOC("bsonArray" << Document())); } + return Value(); } } diff --git a/src/mongo/db/pipeline/document_source_command_shards.cpp b/src/mongo/db/pipeline/document_source_command_shards.cpp index e1f04ae1a50..b1ac58bdf54 100644 --- a/src/mongo/db/pipeline/document_source_command_shards.cpp +++ b/src/mongo/db/pipeline/document_source_command_shards.cpp @@ -12,51 +12,34 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/db/pipeline/document_source.h" #include "mongo/s/shard.h" namespace mongo { - DocumentSourceCommandShards::~DocumentSourceCommandShards() { - } - - bool DocumentSourceCommandShards::eof() { - /* if we haven't even started yet, do so */ - if (unstarted) - getNextDocument(); - - return !hasCurrent; - } - - bool DocumentSourceCommandShards::advance() { - DocumentSource::advance(); // check for interrupts - - if (unstarted) - getNextDocument(); // skip first - - /* advance */ - getNextDocument(); - - return hasCurrent; - } - - Document DocumentSourceCommandShards::getCurrent() { - verify(!eof()); - return pCurrent; - } - void DocumentSourceCommandShards::setSource(DocumentSource *pSource) { /* this doesn't take a source */ verify(false); } - void DocumentSourceCommandShards::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - /* this has no BSON equivalent */ + Value DocumentSourceCommandShards::serialize(bool explain) const { + // this has no BSON equivalent verify(false); } @@ -82,58 +65,62 @@ namespace mongo { return pSource; } - void DocumentSourceCommandShards::getNextDocument() { - if (unstarted) { - unstarted = false; - hasCurrent = true; +namespace { + BSONArray extractResultsArray(const Strategy::CommandResult& result) { + /* grab the next command result */ + BSONObj resultObj = result.result; + + uassert(16390, str::stream() << "sharded pipeline failed on shard " << + result.shardTarget.getName() << ": " << + resultObj.toString(), + resultObj["ok"].trueValue()); + + /* grab the result array out of the shard server's response */ + BSONElement resultArray = resultObj["result"]; + massert(16391, str::stream() << "no result array? shard:" << + result.shardTarget.getName() << ": " << + resultObj.toString(), + resultArray.type() == Array); + + return BSONArray(resultArray.Obj()); + } +} + + vector DocumentSourceCommandShards::getArrays() { + vector out; + for (; iterator != listEnd; ++iterator) { + out.push_back(extractResultsArray(*iterator)); } + return out; + } + + boost::optional DocumentSourceCommandShards::getNext() { + pExpCtx->checkForInterrupt(); while(true) { if (!pBsonSource.get()) { /* if there aren't any more futures, we're done */ - if (iterator == listEnd) { - pCurrent = Document(); - hasCurrent = false; - return; - } + if (iterator == listEnd) + return boost::none; - /* grab the next command result */ - BSONObj resultObj = iterator->second; - - uassert(16390, str::stream() << "sharded pipeline failed on shard " << - iterator->first.getName() << ": " << - resultObj.toString(), - resultObj["ok"].trueValue()); - - /* grab the result array out of the shard server's response */ - BSONElement resultArray = resultObj["result"]; - massert(16391, str::stream() << "no result array? shard:" << - iterator->first.getName() << ": " << - resultObj.toString(), - resultArray.type() == Array); + BSONArray resultArray = extractResultsArray(*iterator); // done with error checking, don't need the shard name anymore ++iterator; - if (resultArray.embeddedObject().isEmpty()){ + if (resultArray.isEmpty()){ // this shard had no results, on to the next one continue; } - pBsonSource = DocumentSourceBsonArray::create(&resultArray, pExpCtx); - newSource = true; + pBsonSource = DocumentSourceBsonArray::create(resultArray, pExpCtx); } - /* if we're done with this shard's results, try the next */ - if (pBsonSource->eof() || - (!newSource && !pBsonSource->advance())) { - pBsonSource.reset(); - continue; - } + if (boost::optional out = pBsonSource->getNext()) + return out; - pCurrent = pBsonSource->getCurrent(); - newSource = false; - return; + // Source exhausted. Try next. + pBsonSource.reset(); } } } diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp index 6d5ed8dec9f..bde1a8f32b2 100644 --- a/src/mongo/db/pipeline/document_source_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_cursor.cpp @@ -12,167 +12,117 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/pch.h" #include "mongo/db/pipeline/document_source.h" -#include "mongo/db/clientcursor.h" #include "mongo/db/instance.h" #include "mongo/db/pipeline/document.h" +#include "mongo/db/query/find_constants.h" +#include "mongo/db/query/type_explain.h" +#include "mongo/db/storage_options.h" #include "mongo/s/d_logic.h" #include "mongo/s/stale_exception.h" // for SendStaleConfigException namespace mongo { - DocumentSourceCursor::CursorWithContext::CursorWithContext( const string& ns ) - : _readContext( ns ) // Take a read lock. - , _chunkMgr(shardingState.needShardChunkManager( ns ) - ? shardingState.getShardChunkManager( ns ) - : ShardChunkManagerPtr()) - {} - DocumentSourceCursor::~DocumentSourceCursor() { + dispose(); } - bool DocumentSourceCursor::eof() { - /* if we haven't gotten the first one yet, do so now */ - if (unstarted) - findNext(); - - return !hasCurrent; + const char *DocumentSourceCursor::getSourceName() const { + return "$cursor"; } - bool DocumentSourceCursor::advance() { - DocumentSource::advance(); // check for interrupts + boost::optional DocumentSourceCursor::getNext() { + pExpCtx->checkForInterrupt(); - /* if we haven't gotten the first one yet, do so now */ - if (unstarted) - findNext(); + if (_currentBatch.empty()) { + loadBatch(); - findNext(); - return hasCurrent; - } + if (_currentBatch.empty()) // exhausted the cursor + return boost::none; + } - Document DocumentSourceCursor::getCurrent() { - verify(hasCurrent); - return pCurrent; + Document out = _currentBatch.front(); + _currentBatch.pop_front(); + return out; } void DocumentSourceCursor::dispose() { - _cursorWithContext.reset(); - } - - ClientCursor::Holder& DocumentSourceCursor::cursor() { - verify( _cursorWithContext ); - verify( _cursorWithContext->_cursor ); - return _cursorWithContext->_cursor; - } - - bool DocumentSourceCursor::canUseCoveredIndex() { - // We can't use a covered index when we have a chunk manager because we - // need to examine the object to see if it belongs on this shard - return (!chunkMgr() && - cursor()->ok() && cursor()->c()->keyFieldsOnly()); - } - - void DocumentSourceCursor::yieldSometimes() { - try { // SERVER-5752 may make this try unnecessary - // if we are index only we don't need the recored - bool cursorOk = cursor()->yieldSometimes(canUseCoveredIndex() - ? ClientCursor::DontNeed - : ClientCursor::WillNeed); - uassert( 16028, "collection or database disappeared when cursor yielded", cursorOk ); - } - catch(SendStaleConfigException& e){ - // We want to ignore this because the migrated documents will be filtered out of the - // cursor anyway and, we don't want to restart the aggregation after every migration. - - log() << "Config changed during aggregation - command will resume" << endl; - // useful for debugging but off by default to avoid looking like a scary error. - LOG(1) << "aggregation stale config exception: " << e.what() << endl; - } + // Can't call in to Runner or ClientCursor registries from this function since it will be + // called when an agg cursor is killed which would cause a deadlock. + _runner.reset(); + _currentBatch.clear(); } - void DocumentSourceCursor::findNext() { - unstarted = false; - - if ( !_cursorWithContext ) { - pCurrent = Document(); - hasCurrent = false; + void DocumentSourceCursor::loadBatch() { + if (!_runner) { + dispose(); return; } - for( ; cursor()->ok(); cursor()->advance() ) { - - yieldSometimes(); - if ( !cursor()->ok() ) { - // The cursor was exhausted during the yield. - break; - } + // We have already validated the sharding version when we constructed the Runner + // so we shouldn't check it again. + Lock::DBRead lk(_ns); + Client::Context ctx(_ns, storageGlobalParams.dbpath, /*doVersion=*/false); - if ( !cursor()->currentMatches() || cursor()->currentIsDup() ) - continue; + _runner->restoreState(); - // grab the matching document - if (canUseCoveredIndex()) { - // Can't have a Chunk Manager if we are here - BSONObj indexKey = cursor()->currKey(); - pCurrent = Document(cursor()->c()->keyFieldsOnly()->hydrate(indexKey)); + int memUsageBytes = 0; + BSONObj obj; + Runner::RunnerState state; + while ((state = _runner->getNext(&obj, NULL)) == Runner::RUNNER_ADVANCED) { + if (_dependencies) { + _currentBatch.push_back(_dependencies->extractFields(obj)); } else { - BSONObj next = cursor()->current(); - - // check to see if this is a new object we don't own yet - // because of a chunk migration - if (chunkMgr() && ! chunkMgr()->belongsToMe(next)) - continue; + _currentBatch.push_back(Document::fromBsonWithMetaData(obj)); + } - if (!_projection) { - pCurrent = Document(next); - } - else { - pCurrent = documentFromBsonWithDeps(next, _dependencies); - - if (debug && !_dependencies.empty()) { - // Make sure we behave the same as Projection. Projection doesn't have a - // way to specify "no fields needed" so we skip the test in that case. - - MutableDocument byAggo(pCurrent); - MutableDocument byProj(Document(_projection->transform(next))); - - if (_dependencies["_id"].getType() == Object) { - // We handle subfields of _id identically to other fields. - // Projection doesn't handle them correctly. - - byAggo.remove("_id"); - byProj.remove("_id"); - } - - if (Document::compare(byAggo.peek(), byProj.peek()) != 0) { - PRINT(next); - PRINT(_dependencies); - PRINT(_projection->getSpec()); - PRINT(byAggo.peek()); - PRINT(byProj.peek()); - verify(false); - } - } + if (_limit) { + if (++_docsAddedToBatches == _limit->getLimit()) { + break; } + verify(_docsAddedToBatches < _limit->getLimit()); } - hasCurrent = true; - cursor()->advance(); + memUsageBytes += _currentBatch.back().getApproximateSize(); - return; + if (memUsageBytes > MaxBytesToReturnToClientAtOnce) { + // End this batch and prepare Runner for yielding. + _runner->saveState(); + cc().curop()->yielded(); + return; + } } - // If we got here, there aren't any more documents. - // The CursorWithContext (and its read lock) must be released, see SERVER-6123. - dispose(); - pCurrent = Document(); - hasCurrent = false; + // If we got here, there won't be any more documents, so destroy the runner. Can't use + // dispose since we want to keep the _currentBatch. + _runner.reset(); + + uassert(16028, "collection or index disappeared when cursor yielded", + state != Runner::RUNNER_DEAD); + + uassert(17285, "cursor encountered an error", + state != Runner::RUNNER_ERROR); + + massert(17286, str::stream() << "Unexpected return from Runner::getNext: " << state, + state == Runner::RUNNER_EOF || state == Runner::RUNNER_ADVANCED); } void DocumentSourceCursor::setSource(DocumentSource *pSource) { @@ -180,81 +130,131 @@ namespace mongo { verify(false); } - void DocumentSourceCursor::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { + long long DocumentSourceCursor::getLimit() const { + return _limit ? _limit->getLimit() : -1; + } - /* this has no analog in the BSON world, so only allow it for explain */ - if (explain) - { - BSONObj bsonObj; - - pBuilder->append("query", *pQuery); + bool DocumentSourceCursor::coalesce(const intrusive_ptr& nextSource) { + // Note: Currently we assume the $limit is logically after any $sort or + // $match. If we ever pull in $match or $sort using this method, we + // will need to keep track of the order of the sub-stages. - if (pSort.get()) - { - pBuilder->append("sort", *pSort); - } + if (!_limit) { + _limit = dynamic_cast(nextSource.get()); + return _limit; // false if next is not a $limit + } + else { + return _limit->coalesce(nextSource); + } + + return false; + } + +namespace { + Document extractInfo(ptr info) { + MutableDocument out; - BSONObj projectionSpec; - if (_projection) { - projectionSpec = _projection->getSpec(); - pBuilder->append("projection", projectionSpec); + if (info->isClausesSet()) { + vector clauses; + for (size_t i = 0; i < info->sizeClauses(); i++) { + clauses.push_back(Value(extractInfo(info->getClausesAt(i)))); } + out[TypeExplain::clauses()] = Value::consume(clauses); + } + + if (info->isCursorSet()) + out[TypeExplain::cursor()] = Value(info->getCursor()); - // construct query for explain - BSONObjBuilder queryBuilder; - queryBuilder.append("$query", *pQuery); - if (pSort.get()) - queryBuilder.append("$orderby", *pSort); - queryBuilder.append("$explain", 1); - Query query(queryBuilder.obj()); + if (info->isIsMultiKeySet()) + out[TypeExplain::isMultiKey()] = Value(info->getIsMultiKey()); - DBDirectClient directClient; - BSONObj explainResult(directClient.findOne(ns, query, _projection - ? &projectionSpec - : NULL)); + if (info->isScanAndOrderSet()) + out[TypeExplain::scanAndOrder()] = Value(info->getScanAndOrder()); - pBuilder->append("cursor", explainResult); +#if 0 // Disabled pending SERVER-12015 since until then no aggs will be index only. + if (info->isIndexOnlySet()) + out[TypeExplain::indexOnly()] = Value(info->getIndexOnly()); +#endif + + if (info->isIndexBoundsSet()) + out[TypeExplain::indexBounds()] = Value(info->getIndexBounds()); + + if (info->isAllPlansSet()) { + vector allPlans; + for (size_t i = 0; i < info->sizeAllPlans(); i++) { + allPlans.push_back(Value(extractInfo(info->getAllPlansAt(i)))); + } + out[TypeExplain::allPlans()] = Value::consume(allPlans); } + + return out.freeze(); } +} // namespace - DocumentSourceCursor::DocumentSourceCursor( - const shared_ptr& cursorWithContext, - const intrusive_ptr &pCtx): - DocumentSource(pCtx), - unstarted(true), - hasCurrent(false), - _cursorWithContext( cursorWithContext ) - {} + Value DocumentSourceCursor::serialize(bool explain) const { + // we never parse a documentSourceCursor, so we only serialize for explain + if (!explain) + return Value(); - intrusive_ptr DocumentSourceCursor::create( - const shared_ptr& cursorWithContext, - const intrusive_ptr &pExpCtx) { - verify( cursorWithContext ); - verify( cursorWithContext->_cursor ); - intrusive_ptr pSource( - new DocumentSourceCursor( cursorWithContext, pExpCtx ) ); - return pSource; - } + Status explainStatus(ErrorCodes::InternalError, ""); + scoped_ptr plan; + { + Lock::DBRead lk(_ns); + Client::Context ctx(_ns, storageGlobalParams.dbpath, /*doVersion=*/false); + massert(17392, "No _runner. Were we disposed before explained?", + _runner); - void DocumentSourceCursor::setNamespace(const string &n) { - ns = n; - } + _runner->restoreState(); - void DocumentSourceCursor::setQuery(const shared_ptr &pBsonObj) { - pQuery = pBsonObj; - } + TypeExplain* explainRaw; + explainStatus = _runner->getInfo(&explainRaw, NULL); + if (explainStatus.isOK()) + plan.reset(explainRaw); + + _runner->saveState(); + } + + MutableDocument out; + out["query"] = Value(_query); + + if (!_sort.isEmpty()) + out["sort"] = Value(_sort); + + if (_limit) + out["limit"] = Value(_limit->getLimit()); - void DocumentSourceCursor::setSort(const shared_ptr &pBsonObj) { - pSort = pBsonObj; + if (!_projection.isEmpty()) + out["fields"] = Value(_projection); + + if (explainStatus.isOK()) { + out["plan"] = Value(extractInfo(plan)); + } else { + out["planError"] = Value(explainStatus.toString()); + } + + return Value(DOC(getSourceName() << out.freezeToValue())); } - void DocumentSourceCursor::setProjection(const BSONObj& projection, const ParsedDeps& deps) { - verify(!_projection); - _projection.reset(new Projection); - _projection->init(projection); - cursor()->fields = _projection; + DocumentSourceCursor::DocumentSourceCursor(const string& ns, + const boost::shared_ptr& runner, + const intrusive_ptr &pCtx) + : DocumentSource(pCtx) + , _docsAddedToBatches(0) + , _ns(ns) + , _runner(runner) + {} + + intrusive_ptr DocumentSourceCursor::create( + const string& ns, + const boost::shared_ptr& runner, + const intrusive_ptr &pExpCtx) { + return new DocumentSourceCursor(ns, runner, pExpCtx); + } + void DocumentSourceCursor::setProjection( + const BSONObj& projection, + const boost::optional& deps) { + _projection = projection; _dependencies = deps; } } diff --git a/src/mongo/db/pipeline/document_source_filter.cpp b/src/mongo/db/pipeline/document_source_filter.cpp deleted file mode 100644 index de0736beee5..00000000000 --- a/src/mongo/db/pipeline/document_source_filter.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/** -* Copyright (C) 2011 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" - -#include "db/pipeline/document_source.h" - -#include "db/jsobj.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/value.h" - -namespace mongo { - - const char DocumentSourceFilter::filterName[] = "$filter"; - - DocumentSourceFilter::~DocumentSourceFilter() { - } - - const char *DocumentSourceFilter::getSourceName() const { - return filterName; - } - - bool DocumentSourceFilter::coalesce( - const intrusive_ptr &pNextSource) { - - /* we only know how to coalesce other filters */ - DocumentSourceFilter *pDocFilter = - dynamic_cast(pNextSource.get()); - if (!pDocFilter) - return false; - - /* - Two adjacent filters can be combined by creating a conjunction of - their predicates. - */ - intrusive_ptr pAnd(ExpressionAnd::create()); - pAnd->addOperand(pFilter); - pAnd->addOperand(pDocFilter->pFilter); - pFilter = pAnd; - - return true; - } - - void DocumentSourceFilter::optimize() { - pFilter = pFilter->optimize(); - } - - void DocumentSourceFilter::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - pFilter->addToBsonObj(pBuilder, filterName, false); - } - - bool DocumentSourceFilter::accept(const Document& pDocument) const { - Value pValue(pFilter->evaluate(pDocument)); - return pValue.coerceToBool(); - } - - intrusive_ptr DocumentSourceFilter::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pCtx) { - uassert(15946, "a document filter expression must be an object", - pBsonElement->type() == Object); - - Expression::ObjectCtx oCtx(0); - intrusive_ptr pExpression( - Expression::parseObject(pBsonElement, &oCtx)); - intrusive_ptr pFilter( - DocumentSourceFilter::create(pExpression, pCtx)); - - return pFilter; - } - - intrusive_ptr DocumentSourceFilter::create( - const intrusive_ptr &pFilter, - const intrusive_ptr &pExpCtx) { - intrusive_ptr pSource( - new DocumentSourceFilter(pFilter, pExpCtx)); - return pSource; - } - - DocumentSourceFilter::DocumentSourceFilter( - const intrusive_ptr &pTheFilter, - const intrusive_ptr &pExpCtx): - DocumentSourceFilterBase(pExpCtx), - pFilter(pTheFilter) { - } - - void DocumentSourceFilter::toMatcherBson(BSONObjBuilder *pBuilder) const { - pFilter->toMatcherBson(pBuilder); - } -} diff --git a/src/mongo/db/pipeline/document_source_filter_base.cpp b/src/mongo/db/pipeline/document_source_filter_base.cpp deleted file mode 100644 index fd42993b23e..00000000000 --- a/src/mongo/db/pipeline/document_source_filter_base.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/** -* Copyright (C) 2011 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" - -#include "db/pipeline/document_source.h" - -#include "db/jsobj.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/value.h" - -namespace mongo { - - DocumentSourceFilterBase::~DocumentSourceFilterBase() { - } - - void DocumentSourceFilterBase::findNext() { - unstarted = false; - - for(bool hasDoc = !pSource->eof(); hasDoc; hasDoc = pSource->advance()) { - pCurrent = pSource->getCurrent(); - if (accept(pCurrent)) { - pSource->advance(); // Start next call at correct position - hasCurrent = true; - return; - } - } - - // Nothing matched - pCurrent = Document(); - hasCurrent = false; - } - - bool DocumentSourceFilterBase::eof() { - if (unstarted) - findNext(); - - return !hasCurrent; - } - - bool DocumentSourceFilterBase::advance() { - DocumentSource::advance(); // check for interrupts - - if (unstarted) - findNext(); - - /* - This looks weird after the above, but is correct. Note that calling - getCurrent() when first starting already yields the first document - in the collection. Calling advance() without using getCurrent() - first will skip over the first item. - */ - findNext(); - - return hasCurrent; - } - - Document DocumentSourceFilterBase::getCurrent() { - verify(hasCurrent); - return pCurrent; - } - - DocumentSourceFilterBase::DocumentSourceFilterBase( - const intrusive_ptr &pExpCtx): - DocumentSource(pExpCtx), - unstarted(true), - hasCurrent(false), - pCurrent() { - } -} diff --git a/src/mongo/db/pipeline/document_source_geo_near.cpp b/src/mongo/db/pipeline/document_source_geo_near.cpp index 15e10174fe5..9b8fca031ea 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/pch.h" @@ -23,39 +35,25 @@ namespace mongo { char DocumentSourceGeoNear::geoNearName[] = "$geoNear"; const char *DocumentSourceGeoNear::getSourceName() const { return geoNearName; } - DocumentSourceGeoNear::~DocumentSourceGeoNear() {} + boost::optional DocumentSourceGeoNear::getNext() { + pExpCtx->checkForInterrupt(); - bool DocumentSourceGeoNear::eof() { if (!resultsIterator) runCommand(); - return !hasCurrent; - } - - bool DocumentSourceGeoNear::advance() { - if (!resultsIterator) - runCommand(); + if (!resultsIterator->more()) + return boost::none; - hasCurrent = resultsIterator->more(); - if (hasCurrent) { - // each result from the geoNear command is wrapped in a wrapper object with "obj", - // "dis" and maybe "loc" fields. We want to take the object from "obj" and inject the - // other fields into it. - - Document result (resultsIterator->next().embeddedObject()); - MutableDocument output (result["obj"].getDocument()); - output.setNestedField(*distanceField, result["dis"]); - if (includeLocs) - output.setNestedField(*includeLocs, result["loc"]); - - currentDoc = output.freeze(); - } - return hasCurrent; - } + // each result from the geoNear command is wrapped in a wrapper object with "obj", + // "dis" and maybe "loc" fields. We want to take the object from "obj" and inject the + // other fields into it. + Document result (resultsIterator->next().embeddedObject()); + MutableDocument output (result["obj"].getDocument()); + output.setNestedField(*distanceField, result["dis"]); + if (includeLocs) + output.setNestedField(*includeLocs, result["loc"]); - Document DocumentSourceGeoNear::getCurrent() { - verify(hasCurrent); - return currentDoc; + return output.freeze(); } void DocumentSourceGeoNear::setSource(DocumentSource*) { @@ -75,47 +73,49 @@ namespace mongo { // This command is sent as-is to the shards. // On router this becomes a sort by distance (nearest-first) with limit. intrusive_ptr DocumentSourceGeoNear::getShardSource() { return this; } - intrusive_ptr DocumentSourceGeoNear::getRouterSource() { + intrusive_ptr DocumentSourceGeoNear::getMergeSource() { return DocumentSourceSort::create(pExpCtx, BSON(distanceField->getPath(false) << 1), limit); } - void DocumentSourceGeoNear::sourceToBson(BSONObjBuilder *pBuilder, bool explain) const { - BSONObjBuilder geoNear (pBuilder->subobjStart("$geoNear")); + Value DocumentSourceGeoNear::serialize(bool explain) const { + MutableDocument result; if (coordsIsArray) { - geoNear.appendArray("near", coords); + result.setField("near", Value(BSONArray(coords))); } else { - geoNear.append("near", coords); + result.setField("near", Value(coords)); } - geoNear.append("distanceField", distanceField->getPath(false)); // not in buildGeoNearCmd - geoNear.append("limit", limit); + // not in buildGeoNearCmd + result.setField("distanceField", Value(distanceField->getPath(false))); + + result.setField("limit", Value(limit)); if (maxDistance > 0) - geoNear.append("maxDistance", maxDistance); + result.setField("maxDistance", Value(maxDistance)); - geoNear.append("query", query); - geoNear.append("spherical", spherical); - geoNear.append("distanceMultiplier", distanceMultiplier); + result.setField("query", Value(query)); + result.setField("spherical", Value(spherical)); + result.setField("distanceMultiplier", Value(distanceMultiplier)); if (includeLocs) - geoNear.append("includeLocs", includeLocs->getPath(false)); + result.setField("includeLocs", Value(includeLocs->getPath(false))); - geoNear.append("uniqueDocs", uniqueDocs); + result.setField("uniqueDocs", Value(uniqueDocs)); - geoNear.doneFast(); + return Value(DOC(getSourceName() << result.freeze())); } - BSONObj DocumentSourceGeoNear::buildGeoNearCmd(const StringData& collection) const { + BSONObj DocumentSourceGeoNear::buildGeoNearCmd() const { // this is very similar to sourceToBson, but slightly different. // differences will be noted. BSONObjBuilder geoNear; // not building a subField - geoNear.append("geoNear", collection); // not in toBson + geoNear.append("geoNear", pExpCtx->ns.coll()); // not in toBson if (coordsIsArray) { geoNear.appendArray("near", coords); @@ -145,12 +145,13 @@ namespace mongo { massert(16603, "Already ran geoNearCommand", !resultsIterator); - bool ok = client->runCommand(db, buildGeoNearCmd(collection), cmdOutput); + bool ok = _mongod->directClient()->runCommand(pExpCtx->ns.db().toString(), + buildGeoNearCmd(), + cmdOutput); uassert(16604, "geoNear command failed: " + cmdOutput.toString(), ok); resultsIterator.reset(new BSONObjIterator(cmdOutput["results"].embeddedObject())); - advance(); // consumes first result into currentDoc } intrusive_ptr DocumentSourceGeoNear::create( @@ -159,10 +160,10 @@ namespace mongo { } intrusive_ptr DocumentSourceGeoNear::createFromBson( - BSONElement *pBsonElement, + BSONElement elem, const intrusive_ptr &pCtx) { intrusive_ptr out = new DocumentSourceGeoNear(pCtx); - out->parseOptions(pBsonElement->embeddedObjectUserCheck()); + out->parseOptions(elem.embeddedObjectUserCheck()); return out; } @@ -207,13 +208,12 @@ namespace mongo { } DocumentSourceGeoNear::DocumentSourceGeoNear(const intrusive_ptr &pExpCtx) - : SplittableDocumentSource(pExpCtx) + : DocumentSource(pExpCtx) , coordsIsArray(false) , limit(100) , maxDistance(-1.0) , spherical(false) , distanceMultiplier(1.0) , uniqueDocs(true) - , hasCurrent(false) {} } diff --git a/src/mongo/db/pipeline/document_source_group.cpp b/src/mongo/db/pipeline/document_source_group.cpp index e4b6ee39008..73f5e7cd11f 100644 --- a/src/mongo/db/pipeline/document_source_group.cpp +++ b/src/mongo/db/pipeline/document_source_group.cpp @@ -12,98 +12,177 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" -#include "db/pipeline/document_source.h" -#include "db/jsobj.h" -#include "db/pipeline/accumulator.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/expression_context.h" -#include "db/pipeline/value.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" namespace mongo { const char DocumentSourceGroup::groupName[] = "$group"; - DocumentSourceGroup::~DocumentSourceGroup() { - } - const char *DocumentSourceGroup::getSourceName() const { return groupName; } - bool DocumentSourceGroup::eof() { + boost::optional DocumentSourceGroup::getNext() { + pExpCtx->checkForInterrupt(); + if (!populated) populate(); - return (groupsIterator == groups.end()); - } + if (_spilled) { + if (!_sorterIterator) + return boost::none; - bool DocumentSourceGroup::advance() { - DocumentSource::advance(); // check for interrupts + const size_t numAccumulators = vpAccumulatorFactory.size(); + for (size_t i=0; i < numAccumulators; i++) { + _currentAccumulators[i]->reset(); // prep accumulators for a new group + } - if (!populated) - populate(); + _currentId = _firstPartOfNextGroup.first; + while (_currentId == _firstPartOfNextGroup.first) { + // Inside of this loop, _firstPartOfNextGroup is the current data being processed. + // At loop exit, it is the first value to be processed in the next group. + + switch (numAccumulators) { // mirrors switch in spill() + case 0: // no Accumulators so no Values + break; + + case 1: // single accumulators serialize as a single Value + _currentAccumulators[0]->process(_firstPartOfNextGroup.second, + /*merging=*/true); + break; + + default: { // multiple accumulators serialize as an array + const vector& accumulatorStates = + _firstPartOfNextGroup.second.getArray(); + for (size_t i=0; i < numAccumulators; i++) { + _currentAccumulators[i]->process(accumulatorStates[i], + /*merging=*/true); + } + break; + } + } - verify(groupsIterator != groups.end()); + if (!_sorterIterator->more()) { + dispose(); + break; + } - ++groupsIterator; - if (groupsIterator == groups.end()) { - dispose(); - return false; - } + _firstPartOfNextGroup = _sorterIterator->next(); + } - return true; - } + return makeDocument(_currentId, _currentAccumulators, pExpCtx->inShard); - Document DocumentSourceGroup::getCurrent() { - if (!populated) - populate(); + } else { + if (groups.empty()) + return boost::none; + + Document out = makeDocument(groupsIterator->first, + groupsIterator->second, + pExpCtx->inShard); + + if (++groupsIterator == groups.end()) + dispose(); - return makeDocument(groupsIterator); + return out; + } } void DocumentSourceGroup::dispose() { - GroupsType().swap(groups); + // free our resources + GroupsMap().swap(groups); + _sorterIterator.reset(); + + // make us look done groupsIterator = groups.end(); + // free our source's resources pSource->dispose(); } - void DocumentSourceGroup::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - BSONObjBuilder insides; + void DocumentSourceGroup::optimize() { + // TODO if all _idExpressions are ExpressionConstants after optimization, then we know there + // will only be one group. We should take advantage of that to avoid going through the hash + // table. + for (size_t i = 0; i < _idExpressions.size(); i++) { + _idExpressions[i] = _idExpressions[i]->optimize(); + } + + for (size_t i = 0; i < vFieldName.size(); i++) { + vpExpression[i] = vpExpression[i]->optimize(); + } + } + + Value DocumentSourceGroup::serialize(bool explain) const { + MutableDocument insides; - /* add the _id */ - pIdExpression->addToBsonObj(&insides, "_id", true); + // add the _id + if (_idFieldNames.empty()) { + invariant(_idExpressions.size() == 1); + insides["_id"] = _idExpressions[0]->serialize(explain); + } + else { + // decomposed document case + invariant(_idExpressions.size() == _idFieldNames.size()); + MutableDocument md; + for (size_t i = 0; i < _idExpressions.size(); i++) { + md[_idFieldNames[i]] = _idExpressions[i]->serialize(explain); + } + insides["_id"] = md.freezeToValue(); + } - /* add the remaining fields */ + // add the remaining fields const size_t n = vFieldName.size(); for(size_t i = 0; i < n; ++i) { - intrusive_ptr pA((*vpAccumulatorFactory[i])(pExpCtx)); - pA->addOperand(vpExpression[i]); - pA->addToBsonObj(&insides, vFieldName[i], true); + intrusive_ptr accum = vpAccumulatorFactory[i](); + insides[vFieldName[i]] = + Value(DOC(accum->getOpName() << vpExpression[i]->serialize(explain))); } - pBuilder->append(groupName, insides.done()); + if (_doingMerge) { + // This makes the output unparsable (with error) on pre 2.6 shards, but it will never + // be sent to old shards when this flag is true since they can't do a merge anyway. + + insides["$doingMerge"] = Value(true); + } + + return Value(DOC(getSourceName() << insides.freeze())); } - DocumentSource::GetDepsReturn DocumentSourceGroup::getDependencies(set& deps) const { + DocumentSource::GetDepsReturn DocumentSourceGroup::getDependencies(DepsTracker* deps) const { // add the _id - pIdExpression->addDependencies(deps); + for (size_t i = 0; i < _idExpressions.size(); i++) { + _idExpressions[i]->addDependencies(deps); + } // add the rest const size_t n = vFieldName.size(); for(size_t i = 0; i < n; ++i) { - intrusive_ptr pA((*vpAccumulatorFactory[i])(pExpCtx)); - pA->addOperand(vpExpression[i]); - pA->addDependencies(deps); + vpExpression[i]->addDependencies(deps); } - return EXHAUSTIVE; + return EXHAUSTIVE_ALL; } intrusive_ptr DocumentSourceGroup::create( @@ -113,22 +192,19 @@ namespace mongo { return pSource; } - DocumentSourceGroup::DocumentSourceGroup( - const intrusive_ptr &pExpCtx): - SplittableDocumentSource(pExpCtx), - populated(false), - pIdExpression(), - groups(), - vFieldName(), - vpAccumulatorFactory(), - vpExpression() { - } + DocumentSourceGroup::DocumentSourceGroup(const intrusive_ptr& pExpCtx) + : DocumentSource(pExpCtx) + , populated(false) + , _doingMerge(false) + , _spilled(false) + , _extSortAllowed(pExpCtx->extSortAllowed && !pExpCtx->inRouter) + , _maxMemoryUsageBytes(100*1024*1024) + {} void DocumentSourceGroup::addAccumulator( - const std::string& fieldName, - intrusive_ptr (*pAccumulatorFactory)( - const intrusive_ptr &), - const intrusive_ptr &pExpression) { + const std::string& fieldName, + intrusive_ptr (*pAccumulatorFactory)(), + const intrusive_ptr &pExpression) { vFieldName.push_back(fieldName); vpAccumulatorFactory.push_back(pAccumulatorFactory); vpExpression.push_back(pExpression); @@ -136,14 +212,13 @@ namespace mongo { struct GroupOpDesc { - const char *pName; - intrusive_ptr (*pFactory)( - const intrusive_ptr &); + const char* name; + intrusive_ptr (*factory)(); }; static int GroupOpDescCmp(const void *pL, const void *pR) { - return strcmp(((const GroupOpDesc *)pL)->pName, - ((const GroupOpDesc *)pR)->pName); + return strcmp(((const GroupOpDesc *)pL)->name, + ((const GroupOpDesc *)pR)->name); } /* @@ -164,54 +239,33 @@ namespace mongo { static const size_t NGroupOp = sizeof(GroupOpTable)/sizeof(GroupOpTable[0]); intrusive_ptr DocumentSourceGroup::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + BSONElement elem, + const intrusive_ptr &pExpCtx) { uassert(15947, "a group's fields must be specified in an object", - pBsonElement->type() == Object); + elem.type() == Object); intrusive_ptr pGroup( DocumentSourceGroup::create(pExpCtx)); - bool idSet = false; - BSONObj groupObj(pBsonElement->Obj()); + BSONObj groupObj(elem.Obj()); BSONObjIterator groupIterator(groupObj); + VariablesIdGenerator idGenerator; + VariablesParseState vps(&idGenerator); while(groupIterator.more()) { BSONElement groupField(groupIterator.next()); const char *pFieldName = groupField.fieldName(); if (str::equals(pFieldName, "_id")) { uassert(15948, "a group's _id may only be specified once", - !idSet); - - BSONType groupType = groupField.type(); - - if (groupType == Object) { - /* - Use the projection-like set of field paths to create the - group-by key. - */ - Expression::ObjectCtx oCtx(Expression::ObjectCtx::DOCUMENT_OK); - intrusive_ptr pId( - Expression::parseObject(&groupField, &oCtx)); - - pGroup->setIdExpression(pId); - idSet = true; - } - else if (groupType == String) { - string groupString(groupField.str()); - const char *pGroupString = groupString.c_str(); - if (pGroupString[0] == '$') { - string pathString = Expression::removeFieldPrefix(groupString); - pGroup->setIdExpression(ExpressionFieldPath::create(pathString)); - idSet = true; - } - } + pGroup->_idExpressions.empty()); + pGroup->parseIdExpression(groupField, vps); + invariant(!pGroup->_idExpressions.empty()); + } + else if (str::equals(pFieldName, "$doingMerge")) { + massert(17030, "$doingMerge should be true if present", + groupField.Bool()); - if (!idSet) { - // constant id - single group - pGroup->setIdExpression(ExpressionConstant::create(Value(groupField))); - idSet = true; - } + pGroup->setDoingMerge(true); } else { /* @@ -241,37 +295,31 @@ namespace mongo { /* look for the specified operator */ GroupOpDesc key; - key.pName = subElement.fieldName(); + key.name = subElement.fieldName(); const GroupOpDesc *pOp = (const GroupOpDesc *)bsearch( &key, GroupOpTable, NGroupOp, sizeof(GroupOpDesc), GroupOpDescCmp); - uassert(15952, str::stream() << - "unknown group operator '" << - key.pName << "'", + uassert(15952, str::stream() << "unknown group operator '" << key.name << "'", pOp); intrusive_ptr pGroupExpr; BSONType elementType = subElement.type(); if (elementType == Object) { - Expression::ObjectCtx oCtx( - Expression::ObjectCtx::DOCUMENT_OK); - pGroupExpr = Expression::parseObject( - &subElement, &oCtx); + Expression::ObjectCtx oCtx(Expression::ObjectCtx::DOCUMENT_OK); + pGroupExpr = Expression::parseObject(subElement.Obj(), &oCtx, vps); } else if (elementType == Array) { - uassert(15953, str::stream() << - "aggregating group operators are unary (" << - key.pName << ")", false); + uasserted(15953, str::stream() + << "aggregating group operators are unary (" << key.name << ")"); } else { /* assume its an atomic single operand */ - pGroupExpr = Expression::parseOperand(&subElement); + pGroupExpr = Expression::parseOperand(subElement, vps); } - pGroup->addAccumulator( - pFieldName, pOp->pFactory, pGroupExpr); + pGroup->addAccumulator(pFieldName, pOp->factory, pGroupExpr); } uassert(15954, str::stream() << @@ -281,20 +329,46 @@ namespace mongo { } } - uassert(15955, "a group specification must include an _id", idSet); + uassert(15955, "a group specification must include an _id", + !pGroup->_idExpressions.empty()); + + pGroup->_variables.reset(new Variables(idGenerator.getIdCount())); return pGroup; } + namespace { + class SorterComparator { + public: + typedef pair Data; + int operator() (const Data& lhs, const Data& rhs) const { + return Value::compare(lhs.first, rhs.first); + } + }; + } + void DocumentSourceGroup::populate() { const size_t numAccumulators = vpAccumulatorFactory.size(); dassert(numAccumulators == vpExpression.size()); - for (bool hasNext = !pSource->eof(); hasNext; hasNext = pSource->advance()) { - Document input = pSource->getCurrent(); + // pushed to on spill() + vector::Iterator> > sortedFiles; + int memoryUsageBytes = 0; + + // This loop consumes all input from pSource and buckets it based on pIdExpression. + while (boost::optional input = pSource->getNext()) { + if (memoryUsageBytes > _maxMemoryUsageBytes) { + uassert(16945, "Exceeded memory limit for $group, but didn't allow external sort." + " Pass allowDiskUse:true to opt in.", + _extSortAllowed); + sortedFiles.push_back(spill()); + memoryUsageBytes = 0; + } + + _variables->setRoot(*input); /* get the _id value */ - Value id = pIdExpression->evaluate(input); + Value id = computeId(_variables.get()); /* treat missing values the same as NULL SERVER-4674 */ if (id.missing()) @@ -304,50 +378,209 @@ namespace mongo { Look for the _id value in the map; if it's not there, add a new entry with a blank accumulator. */ + const size_t oldSize = groups.size(); vector >& group = groups[id]; + const bool inserted = groups.size() != oldSize; - if (numAccumulators == 0) - continue; // we are basically building a set + if (inserted) { + memoryUsageBytes += id.getApproximateSize(); - if (group.empty()) { - /* add the accumulators */ + // Add the accumulators group.reserve(numAccumulators); for (size_t i = 0; i < numAccumulators; i++) { - intrusive_ptr accum = (*vpAccumulatorFactory[i])(pExpCtx); - accum->addOperand(vpExpression[i]); - group.push_back(accum); + group.push_back(vpAccumulatorFactory[i]()); + } + } else { + for (size_t i = 0; i < numAccumulators; i++) { + // subtract old mem usage. New usage added back after processing. + memoryUsageBytes -= group[i]->memUsageForSorter(); } } /* tickle all the accumulators for the group we found */ dassert(numAccumulators == group.size()); - for (size_t i = 0; i < numAccumulators; i++) - group[i]->evaluate(input); + for (size_t i = 0; i < numAccumulators; i++) { + group[i]->process(vpExpression[i]->evaluate(_variables.get()), _doingMerge); + memoryUsageBytes += group[i]->memUsageForSorter(); + } + + // We are done with the ROOT document so release it. + _variables->clearRoot(); + + DEV { + // In debug mode, spill every time we have a duplicate id to stress merge logic. + if (!inserted // is a dup + && !pExpCtx->inRouter // can't spill to disk in router + && !_extSortAllowed // don't change behavior when testing external sort + && sortedFiles.size() < 20 // don't open too many FDs + ) { + sortedFiles.push_back(spill()); + } + } + } + + // These blocks do any final steps necessary to prepare to output results. + if (!sortedFiles.empty()) { + _spilled = true; + if (!groups.empty()) { + sortedFiles.push_back(spill()); + } + + // We won't be using groups again so free its memory. + GroupsMap().swap(groups); + + _sorterIterator.reset( + Sorter::Iterator::merge( + sortedFiles, SortOptions(), SorterComparator())); + + // prepare current to accumulate data + _currentAccumulators.reserve(numAccumulators); + for (size_t i = 0; i < numAccumulators; i++) { + _currentAccumulators.push_back(vpAccumulatorFactory[i]()); + } + + verify(_sorterIterator->more()); // we put data in, we should get something out. + _firstPartOfNextGroup = _sorterIterator->next(); + } else { + // start the group iterator + groupsIterator = groups.begin(); } - /* start the group iterator */ - groupsIterator = groups.begin(); populated = true; } - Document DocumentSourceGroup::makeDocument( - const GroupsType::iterator &rIter) { - vector > *pGroup = &rIter->second; + class DocumentSourceGroup::SpillSTLComparator { + public: + bool operator() (const GroupsMap::value_type* lhs, const GroupsMap::value_type* rhs) const { + return Value::compare(lhs->first, rhs->first) < 0; + } + }; + + shared_ptr::Iterator> DocumentSourceGroup::spill() { + vector ptrs; // using pointers to speed sorting + ptrs.reserve(groups.size()); + for (GroupsMap::const_iterator it=groups.begin(), end=groups.end(); it != end; ++it) { + ptrs.push_back(&*it); + } + + stable_sort(ptrs.begin(), ptrs.end(), SpillSTLComparator()); + + SortedFileWriter writer(SortOptions().TempDir(pExpCtx->tempDir)); + switch (vpAccumulatorFactory.size()) { // same as ptrs[i]->second.size() for all i. + case 0: // no values, essentially a distinct + for (size_t i=0; i < ptrs.size(); i++) { + writer.addAlreadySorted(ptrs[i]->first, Value()); + } + break; + + case 1: // just one value, use optimized serialization as single Value + for (size_t i=0; i < ptrs.size(); i++) { + writer.addAlreadySorted(ptrs[i]->first, + ptrs[i]->second[0]->getValue(/*toBeMerged=*/true)); + } + break; + + default: // multiple values, serialize as array-typed Value + for (size_t i=0; i < ptrs.size(); i++) { + vector accums; + for (size_t j=0; j < ptrs[i]->second.size(); j++) { + accums.push_back(ptrs[i]->second[j]->getValue(/*toBeMerged=*/true)); + } + writer.addAlreadySorted(ptrs[i]->first, Value::consume(accums)); + } + break; + } + + groups.clear(); + + return shared_ptr::Iterator>(writer.done()); + } + + void DocumentSourceGroup::parseIdExpression(BSONElement groupField, + const VariablesParseState& vps) { + if (groupField.type() == Object && !groupField.Obj().isEmpty()) { + // {_id: {}} is treated as grouping on a constant, not an expression + + const BSONObj idKeyObj = groupField.Obj(); + if (idKeyObj.firstElementFieldName()[0] == '$') { + // grouping on a $op expression + Expression::ObjectCtx oCtx(0); + _idExpressions.push_back(Expression::parseObject(idKeyObj, &oCtx, vps)); + } + else { + // grouping on an "artificial" object. Rather than create the object for each input + // in populate(), instead group on the output of the raw expressions. The artificial + // object will be created at the end in makeDocument() while outputting results. + BSONForEach(field, idKeyObj) { + uassert(17390, "$group does not support inclusion-style expressions", + !field.isNumber() && field.type() != Bool); + + _idFieldNames.push_back(field.fieldName()); + _idExpressions.push_back(Expression::parseOperand(field, vps)); + } + } + } + else if (groupField.type() == String && groupField.valuestr()[0] == '$') { + // grouping on a field path. + _idExpressions.push_back(ExpressionFieldPath::parse(groupField.str(), vps)); + } + else { + // constant id - single group + _idExpressions.push_back(ExpressionConstant::create(Value(groupField))); + } + } + + Value DocumentSourceGroup::computeId(Variables* vars) { + // If only one expression return result directly + if (_idExpressions.size() == 1) + return _idExpressions[0]->evaluate(vars); + + // Multiple expressions get results wrapped in a vector + vector vals; + vals.reserve(_idExpressions.size()); + for (size_t i = 0; i < _idExpressions.size(); i++) { + vals.push_back(_idExpressions[i]->evaluate(vars)); + } + return Value::consume(vals); + } + + Value DocumentSourceGroup::expandId(const Value& val) { + // _id doesn't get wrapped in a document + if (_idFieldNames.empty()) + return val; + + // _id is a single-field document containing val + if (_idFieldNames.size() == 1) + return Value(DOC(_idFieldNames[0] << val)); + + // _id is a multi-field document containing the elements of val + const vector& vals = val.getArray(); + invariant(_idFieldNames.size() == vals.size()); + MutableDocument md(vals.size()); + for (size_t i = 0; i < vals.size(); i++) { + md[_idFieldNames[i]] = vals[i]; + } + return md.freezeToValue(); + } + + Document DocumentSourceGroup::makeDocument(const Value& id, + const Accumulators& accums, + bool mergeableOutput) { const size_t n = vFieldName.size(); MutableDocument out (1 + n); /* add the _id field */ - out.addField("_id", rIter->first); + out.addField("_id", expandId(id)); /* add the rest of the fields */ for(size_t i = 0; i < n; ++i) { - Value pValue((*pGroup)[i]->getValue()); - if (pValue.missing()) { + Value val = accums[i]->getValue(mergeableOutput); + if (val.missing()) { // we return null in this case so return objects are predictable out.addField(vFieldName[i], Value(BSONNULL)); } else { - out.addField(vFieldName[i], pValue); + out.addField(vFieldName[i], val); } } @@ -358,13 +591,14 @@ namespace mongo { return this; // No modifications necessary when on shard } - intrusive_ptr DocumentSourceGroup::getRouterSource() { - intrusive_ptr pMergerExpCtx = pExpCtx->clone(); - pMergerExpCtx->setDoingMerge(true); - intrusive_ptr pMerger(DocumentSourceGroup::create(pMergerExpCtx)); + intrusive_ptr DocumentSourceGroup::getMergeSource() { + intrusive_ptr pMerger(DocumentSourceGroup::create(pExpCtx)); + pMerger->setDoingMerge(true); + VariablesIdGenerator idGenerator; + VariablesParseState vps(&idGenerator); /* the merger will use the same grouping key */ - pMerger->setIdExpression(ExpressionFieldPath::create("_id")); + pMerger->_idExpressions.push_back(ExpressionFieldPath::parse("$$ROOT._id", vps)); const size_t n = vFieldName.size(); for(size_t i = 0; i < n; ++i) { @@ -378,9 +612,14 @@ namespace mongo { */ pMerger->addAccumulator( vFieldName[i], vpAccumulatorFactory[i], - ExpressionFieldPath::create(vFieldName[i])); + ExpressionFieldPath::parse("$$ROOT." + vFieldName[i], vps)); } + pMerger->_variables.reset(new Variables(idGenerator.getIdCount())); + return pMerger; } } + +#include "db/sorter/sorter.cpp" +// Explicit instantiation unneeded since we aren't exposing Sorter outside of this file. diff --git a/src/mongo/db/pipeline/document_source_limit.cpp b/src/mongo/db/pipeline/document_source_limit.cpp index 863bc5f16a7..981b930fddc 100644 --- a/src/mongo/db/pipeline/document_source_limit.cpp +++ b/src/mongo/db/pipeline/document_source_limit.cpp @@ -12,31 +12,39 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" - -#include "db/pipeline/document_source.h" +#include "mongo/pch.h" -#include "db/jsobj.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/expression_context.h" -#include "db/pipeline/value.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" namespace mongo { const char DocumentSourceLimit::limitName[] = "$limit"; DocumentSourceLimit::DocumentSourceLimit(const intrusive_ptr &pExpCtx, long long limit) - : SplittableDocumentSource(pExpCtx) + : DocumentSource(pExpCtx) , limit(limit) , count(0) {} - DocumentSourceLimit::~DocumentSourceLimit() { - } - const char *DocumentSourceLimit::getSourceName() const { return limitName; } @@ -56,31 +64,19 @@ namespace mongo { return true; } - bool DocumentSourceLimit::eof() { - return pSource->eof() || count >= limit; - } + boost::optional DocumentSourceLimit::getNext() { + pExpCtx->checkForInterrupt(); - bool DocumentSourceLimit::advance() { - DocumentSource::advance(); // check for interrupts - - ++count; - if (count >= limit) { - // This is required for the DocumentSourceCursor to release its read lock, see - // SERVER-6123. + if (++count > limit) { pSource->dispose(); - - return false; + return boost::none; } - return pSource->advance(); - } - Document DocumentSourceLimit::getCurrent() { - return pSource->getCurrent(); + return pSource->getNext(); } - void DocumentSourceLimit::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - pBuilder->append("$limit", limit); + Value DocumentSourceLimit::serialize(bool explain) const { + return Value(DOC(getSourceName() << limit)); } intrusive_ptr DocumentSourceLimit::create( @@ -92,12 +88,12 @@ namespace mongo { } intrusive_ptr DocumentSourceLimit::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + BSONElement elem, + const intrusive_ptr &pExpCtx) { uassert(15957, "the limit must be specified as a number", - pBsonElement->isNumber()); + elem.isNumber()); - long long limit = pBsonElement->numberLong(); + long long limit = elem.numberLong(); return DocumentSourceLimit::create(pExpCtx, limit); } } diff --git a/src/mongo/db/pipeline/document_source_match.cpp b/src/mongo/db/pipeline/document_source_match.cpp index e9944613b3f..5d3525181e0 100644 --- a/src/mongo/db/pipeline/document_source_match.cpp +++ b/src/mongo/db/pipeline/document_source_match.cpp @@ -12,52 +12,302 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" -#include "db/pipeline/document_source.h" +#include -#include "db/jsobj.h" -#include "db/matcher.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/util/stringutils.h" namespace mongo { const char DocumentSourceMatch::matchName[] = "$match"; - DocumentSourceMatch::~DocumentSourceMatch() { - } - const char *DocumentSourceMatch::getSourceName() const { return matchName; } - void DocumentSourceMatch::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - const BSONObj *pQuery = matcher.getQuery(); - pBuilder->append(matchName, *pQuery); + Value DocumentSourceMatch::serialize(bool explain) const { + return Value(DOC(getSourceName() << Document(getQuery()))); + } + + boost::optional DocumentSourceMatch::getNext() { + pExpCtx->checkForInterrupt(); + + // The user facing error should have been generated earlier. + massert(17309, "Should never call getNext on a $match stage with $text clause", + !_isTextQuery); + + while (boost::optional next = pSource->getNext()) { + // The matcher only takes BSON documents, so we have to make one. + if (matcher->matches(next->toBson())) + return next; + } + + // Nothing matched + return boost::none; + } + + bool DocumentSourceMatch::coalesce(const intrusive_ptr& nextSource) { + DocumentSourceMatch* otherMatch = dynamic_cast(nextSource.get()); + if (!otherMatch) + return false; + + if (otherMatch->_isTextQuery) { + // Non-initial text queries are disallowed (enforced by setSource below). This prevents + // "hiding" a non-initial text query by combining it with another match. + return false; + + // The rest of this block is for once we support non-initial text queries. + + if (_isTextQuery) { + // The score should only come from the last $match. We can't combine since then this + // match's score would impact otherMatch's. + return false; + } + + _isTextQuery = true; + } + + // Replace our matcher with the $and of ours and theirs. + matcher.reset(new Matcher(BSON("$and" << BSON_ARRAY(getQuery() + << otherMatch->getQuery())))); + + return true; + } + +namespace { + // This block contains the functions that make up the implementation of + // DocumentSourceMatch::redactSafePortion(). They will only be called after + // the Match expression has been successfully parsed so they can assume that + // input is well formed. + + bool isAllDigits(const StringData& str) { + if (str.empty()) + return false; + + for (size_t i=0; i < str.size(); i++) { + if (!isdigit(str[i])) + return false; + } + return true; + } + + bool isFieldnameRedactSafe(const StringData& fieldName) { + // Can't have numeric elements in the dotted path since redacting elements from an array + // would change the indexes. + + const size_t dotPos = fieldName.find('.'); + if (dotPos == string::npos) + return !isAllDigits(fieldName); + + const StringData part = fieldName.substr(0, dotPos); + const StringData rest = fieldName.substr(dotPos + 1); + return !isAllDigits(part) && isFieldnameRedactSafe(rest); + } + + bool isTypeRedactSafeInComparison(BSONType type) { + if (type == Array) return false; + if (type == Object) return false; + if (type == jstNULL) return false; + if (type == Undefined) return false; // Currently a Matcher parse error. + + return true; + } + + Document redactSafePortionTopLevel(BSONObj query); // mutually recursive with next function + + // Returns the redact-safe portion of an "inner" match expression. This is the layer like + // {$gt: 5} which does not include the field name. Returns an empty document if none of the + // expression can safely be promoted in front of a $redact. + Document redactSafePortionDollarOps(BSONObj expr) { + MutableDocument output; + BSONForEach(field, expr) { + if (field.fieldName()[0] != '$') + continue; + + switch(BSONObj::MatchType(field.getGtLtOp(BSONObj::Equality))) { + // These are always ok + case BSONObj::opTYPE: + case BSONObj::opREGEX: + case BSONObj::opOPTIONS: + case BSONObj::opMOD: + output[field.fieldNameStringData()] = Value(field); + break; + + // These are ok if the type of the rhs is allowed in comparisons + case BSONObj::LTE: + case BSONObj::GTE: + case BSONObj::LT: + case BSONObj::GT: + if (isTypeRedactSafeInComparison(field.type())) + output[field.fieldNameStringData()] = Value(field); + break; + + // $in must be all-or-nothing (like $or). Can't include subset of elements. + case BSONObj::opIN: { + bool allOk = true; + BSONForEach(elem, field.Obj()) { + if (!isTypeRedactSafeInComparison(elem.type())) { + allOk = false; + break; + } + } + if (allOk) { + output[field.fieldNameStringData()] = Value(field); + } + + break; + } + + case BSONObj::opALL: { + // $all can include subset of elements (like $and). + vector matches; + BSONForEach(elem, field.Obj()) { + // NOTE this currently doesn't allow {$all: [{$elemMatch: {...}}]} + if (isTypeRedactSafeInComparison(elem.type())) { + matches.push_back(Value(elem)); + } + } + if (!matches.empty()) + output[field.fieldNameStringData()] = Value::consume(matches); + + break; + } + + case BSONObj::opELEM_MATCH: { + BSONObj subIn = field.Obj(); + Document subOut; + if (subIn.firstElementFieldName()[0] == '$') { + subOut = redactSafePortionDollarOps(subIn); + } else { + subOut = redactSafePortionTopLevel(subIn); + } + + if (!subOut.empty()) + output[field.fieldNameStringData()] = Value(subOut); + + break; + } + + // These are never allowed + case BSONObj::Equality: // This actually means unknown + case BSONObj::opMAX_DISTANCE: + case BSONObj::opNEAR: + case BSONObj::NE: + case BSONObj::opSIZE: + case BSONObj::NIN: + case BSONObj::opEXISTS: + case BSONObj::opWITHIN: + case BSONObj::opGEO_INTERSECTS: + continue; + } + } + return output.freeze(); } - bool DocumentSourceMatch::accept(const Document& pDocument) const { + // Returns the redact-safe portion of an "outer" match expression. This is the layer like + // {fieldName: {...}} which does include the field name. Returns an empty document if none of + // the expression can safely be promoted in front of a $redact. + Document redactSafePortionTopLevel(BSONObj query) { + MutableDocument output; + BSONForEach(field, query) { + if (field.fieldName()[0] == '$') { + if (str::equals(field.fieldName(), "$or")) { + // $or must be all-or-nothing (line $in). Can't include subset of elements. + vector okClauses; + BSONForEach(elem, field.Obj()) { + Document clause = redactSafePortionTopLevel(elem.Obj()); + if (clause.empty()) { + okClauses.clear(); + break; + } + okClauses.push_back(Value(clause)); + } + + if (!okClauses.empty()) + output["$or"] = Value::consume(okClauses); + } + else if (str::equals(field.fieldName(), "$and")) { + // $and can include subset of elements (like $all). + vector okClauses; + BSONForEach(elem, field.Obj()) { + Document clause = redactSafePortionTopLevel(elem.Obj()); + if (!clause.empty()) + okClauses.push_back(Value(clause)); + } + if (!okClauses.empty()) + output["$and"] = Value::consume(okClauses); + } + + continue; + } + + if (!isFieldnameRedactSafe(field.fieldNameStringData())) + continue; - /* - The matcher only takes BSON documents, so we have to make one. + switch (field.type()) { + case Array: continue; // exact matches on arrays are never allowed + case jstNULL: continue; // can't look for missing fields + case Undefined: continue; // Currently a Matcher parse error. - LATER - We could optimize this by making a document with only the - fields referenced by the Matcher. We could do this by looking inside - the Matcher's BSON before it is created, and recording those. The - easiest implementation might be to hold onto an ExpressionDocument - in here, and give that pDocument to create the created subset of - fields, and then convert that instead. - */ - BSONObjBuilder objBuilder; - pDocument->toBson(&objBuilder); - BSONObj obj(objBuilder.done()); + case Object: { + Document sub = redactSafePortionDollarOps(field.Obj()); + if (!sub.empty()) + output[field.fieldNameStringData()] = Value(sub); - return matcher.matches(obj); + break; + } + + // All other types are ok to pass through + default: + output[field.fieldNameStringData()] = Value(field); + break; + } + } + return output.freeze(); + } +} + + BSONObj DocumentSourceMatch::redactSafePortion() const { + return redactSafePortionTopLevel(getQuery()).toBson(); + } + + void DocumentSourceMatch::setSource(DocumentSource* source) { + uassert(17313, "$match with $text is only allowed as the first pipeline stage", + !_isTextQuery); + + DocumentSource::setSource(source); + } + + bool DocumentSourceMatch::isTextQuery(const BSONObj& query) { + BSONForEach(e, query) { + const StringData fieldName = e.fieldNameStringData(); + if (fieldName == StringData("$text", StringData::LiteralTag())) + return true; + + if (e.isABSONObj() && isTextQuery(e.Obj())) + return true; + } + return false; } static void uassertNoDisallowedClauses(BSONObj query) { @@ -76,28 +326,24 @@ namespace mongo { } intrusive_ptr DocumentSourceMatch::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + BSONElement elem, + const intrusive_ptr &pExpCtx) { uassert(15959, "the match filter must be an expression in an object", - pBsonElement->type() == Object); - - uassertNoDisallowedClauses(pBsonElement->Obj()); + elem.type() == Object); - intrusive_ptr pMatcher( - new DocumentSourceMatch(pBsonElement->Obj(), pExpCtx)); + uassertNoDisallowedClauses(elem.Obj()); - return pMatcher; + return new DocumentSourceMatch(elem.Obj(), pExpCtx); } - void DocumentSourceMatch::toMatcherBson(BSONObjBuilder *pBuilder) const { - const BSONObj *pQuery = matcher.getQuery(); - pBuilder->appendElements(*pQuery); + BSONObj DocumentSourceMatch::getQuery() const { + return *(matcher->getQuery()); } - DocumentSourceMatch::DocumentSourceMatch( - const BSONObj &query, - const intrusive_ptr &pExpCtx): - DocumentSourceFilterBase(pExpCtx), - matcher(query) { - } + DocumentSourceMatch::DocumentSourceMatch(const BSONObj &query, + const intrusive_ptr &pExpCtx) + : DocumentSource(pExpCtx) + , matcher(new Matcher(query.getOwned())) + , _isTextQuery(isTextQuery(query)) + {} } diff --git a/src/mongo/db/pipeline/document_source_merge_cursors.cpp b/src/mongo/db/pipeline/document_source_merge_cursors.cpp new file mode 100644 index 00000000000..46a3b6fa73e --- /dev/null +++ b/src/mongo/db/pipeline/document_source_merge_cursors.cpp @@ -0,0 +1,180 @@ +/** + * Copyright 2013 (c) 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" + +#include "mongo/db/pipeline/document_source.h" + + +namespace mongo { + + const char DocumentSourceMergeCursors::name[] = "$mergeCursors"; + + const char* DocumentSourceMergeCursors::getSourceName() const { + return name; + } + + void DocumentSourceMergeCursors::setSource(DocumentSource *pSource) { + /* this doesn't take a source */ + verify(false); + } + + DocumentSourceMergeCursors::DocumentSourceMergeCursors( + const CursorIds& cursorIds, + const intrusive_ptr &pExpCtx) + : DocumentSource(pExpCtx) + , _cursorIds(cursorIds) + , _unstarted(true) + {} + + intrusive_ptr DocumentSourceMergeCursors::create( + const CursorIds& cursorIds, + const intrusive_ptr &pExpCtx) { + return new DocumentSourceMergeCursors(cursorIds, pExpCtx); + } + + intrusive_ptr DocumentSourceMergeCursors::createFromBson( + BSONElement elem, + const intrusive_ptr& pExpCtx) { + + massert(17026, string("Expected an Array, but got a ") + typeName(elem.type()), + elem.type() == Array); + + CursorIds cursorIds; + BSONObj array = elem.embeddedObject(); + BSONForEach(cursor, array) { + massert(17027, string("Expected an Object, but got a ") + typeName(cursor.type()), + cursor.type() == Object); + + cursorIds.push_back(make_pair(ConnectionString(cursor["host"].String()), + cursor["id"].Long())); + } + + return new DocumentSourceMergeCursors(cursorIds, pExpCtx); + } + + Value DocumentSourceMergeCursors::serialize(bool explain) const { + vector cursors; + for (size_t i = 0; i < _cursorIds.size(); i++) { + cursors.push_back(Value(DOC("host" << Value(_cursorIds[i].first.toString()) + << "id" << _cursorIds[i].second))); + } + return Value(DOC(getSourceName() << Value(cursors))); + } + + DocumentSourceMergeCursors::CursorAndConnection::CursorAndConnection( + ConnectionString host, + NamespaceString ns, + CursorId id) + : connection(host) + , cursor(connection.get(), ns, id, 0, 0) + {} + + vector DocumentSourceMergeCursors::getCursors() { + verify(_unstarted); + start(); + vector out; + for (Cursors::const_iterator it = _cursors.begin(); it !=_cursors.end(); ++it) { + out.push_back(&((*it)->cursor)); + } + + return out; + } + + void DocumentSourceMergeCursors::start() { + _unstarted = false; + + // open each cursor and send message asking for a batch + for (CursorIds::const_iterator it = _cursorIds.begin(); it !=_cursorIds.end(); ++it) { + _cursors.push_back(boost::make_shared( + it->first, pExpCtx->ns, it->second)); + verify(_cursors.back()->connection->lazySupported()); + _cursors.back()->cursor.initLazy(); // shouldn't block + } + + // wait for all cursors to return a batch + // TODO need a way to keep cursors alive if some take longer than 10 minutes. + for (Cursors::const_iterator it = _cursors.begin(); it !=_cursors.end(); ++it) { + bool retry = false; + bool ok = (*it)->cursor.initLazyFinish(retry); // blocks here for first batch + + uassert(17028, + "error reading response from " + _cursors.back()->connection->toString(), + ok); + verify(!retry); + } + + _currentCursor = _cursors.begin(); + } + + Document DocumentSourceMergeCursors::nextSafeFrom(DBClientCursor* cursor) { + const BSONObj next = cursor->next(); + if (next.hasField("$err")) { + const int code = next.hasField("code") ? next["code"].numberInt() : 17029; + uasserted(code, str::stream() << "Received error in response from " + << cursor->originalHost() + << ": " << next); + } + return Document::fromBsonWithMetaData(next); + } + + boost::optional DocumentSourceMergeCursors::getNext() { + if (_unstarted) + start(); + + // purge eof cursors and release their connections + while (!_cursors.empty() && !(*_currentCursor)->cursor.more()) { + (*_currentCursor)->connection.done(); + _cursors.erase(_currentCursor); + _currentCursor = _cursors.begin(); + } + + if (_cursors.empty()) + return boost::none; + + const Document next = nextSafeFrom(&((*_currentCursor)->cursor)); + + // advance _currentCursor, wrapping if needed + if (++_currentCursor == _cursors.end()) + _currentCursor = _cursors.begin(); + + return next; + } + + void DocumentSourceMergeCursors::dispose() { + // Note it is an error to call done() on a connection before consuming the response from a + // request. Therefore it is an error to call dispose() if there are any outstanding + // connections which have not received a reply. + for (_currentCursor = _cursors.begin(); _currentCursor != _cursors.end(); ++_currentCursor) { + (*_currentCursor)->cursor.kill(); + (*_currentCursor)->connection.done(); + } + _cursors.clear(); + _currentCursor = _cursors.end(); + } +} diff --git a/src/mongo/db/pipeline/document_source_out.cpp b/src/mongo/db/pipeline/document_source_out.cpp index bbdaacfa053..fca2e582958 100644 --- a/src/mongo/db/pipeline/document_source_out.cpp +++ b/src/mongo/db/pipeline/document_source_out.cpp @@ -12,56 +12,179 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" - -#include "db/pipeline/document_source.h" +#include "mongo/pch.h" +#include "mongo/db/pipeline/document_source.h" namespace mongo { - const char DocumentSourceOut::outName[] = "$out"; DocumentSourceOut::~DocumentSourceOut() { + DESTRUCTOR_GUARD( + // Make sure we drop the temp collection if anything goes wrong. Errors are ignored + // here because nothing can be done about them. Additionally, if this fails and the + // collection is left behind, it will be cleaned up next time the server is started. + if (_mongod && _tempNs.size()) + _mongod->directClient()->dropCollection(_tempNs.ns()); + ) } const char *DocumentSourceOut::getSourceName() const { return outName; } - bool DocumentSourceOut::eof() { - return pSource->eof(); - } + static AtomicUInt32 aggOutCounter; + void DocumentSourceOut::prepTempCollection() { + verify(_mongod); + verify(_tempNs.size() == 0); + + DBClientBase* conn = _mongod->directClient(); + + // Fail early by checking before we do any work. + uassert(17017, str::stream() << "namespace '" << _outputNs.ns() + << "' is sharded so it can't be used for $out'", + !_mongod->isSharded(_outputNs)); - bool DocumentSourceOut::advance() { - DocumentSource::advance(); // check for interrupts + // cannot $out to capped collection + uassert(17152, str::stream() << "namespace '" << _outputNs.ns() + << "' is capped so it can't be used for $out", + !_mongod->isCapped(_outputNs)); - return pSource->advance(); + _tempNs = StringData(str::stream() << _outputNs.db() + << ".tmp.agg_out." + << aggOutCounter.addAndFetch(1) + ); + + { + BSONObj info; + bool ok =conn->runCommand(_outputNs.db().toString(), + BSON("create" << _tempNs.coll() << "temp" << true), + info); + uassert(16994, str::stream() << "failed to create temporary $out collection '" + << _tempNs.ns() << "': " << info.toString(), + ok); + } + + // copy indexes on _outputNs to _tempNs + scoped_ptr indexes(conn->getIndexes(_outputNs)); + while (indexes->more()) { + MutableDocument index(Document(indexes->nextSafe())); + index.remove("_id"); // indexes shouldn't have _ids but some existing ones do + index["ns"] = Value(_tempNs.ns()); + + BSONObj indexBson = index.freeze().toBson(); + conn->insert(_tempNs.getSystemIndexesCollection(), indexBson); + BSONObj err = conn->getLastErrorDetailed(); + uassert(16995, str::stream() << "copying index for $out failed." + << " index: " << indexBson + << " error: " << err, + DBClientWithCommands::getLastErrorString(err).empty()); + } } - Document DocumentSourceOut::getCurrent() { - return pSource->getCurrent(); + void DocumentSourceOut::spill(DBClientBase* conn, const vector& toInsert) { + conn->insert(_tempNs.ns(), toInsert); + BSONObj err = conn->getLastErrorDetailed(); + uassert(16996, str::stream() << "insert for $out failed: " << err, + DBClientWithCommands::getLastErrorString(err).empty()); } - DocumentSourceOut::DocumentSourceOut( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx): - DocumentSource(pExpCtx) { - verify(false && "unimplemented"); + boost::optional DocumentSourceOut::getNext() { + pExpCtx->checkForInterrupt(); + + // make sure we only write out once + if (_done) + return boost::none; + _done = true; + + verify(_mongod); + DBClientBase* conn = _mongod->directClient(); + + prepTempCollection(); + verify(_tempNs.size() != 0); + + vector bufferedObjects; + int bufferedBytes = 0; + while (boost::optional next = pSource->getNext()) { + BSONObj toInsert = next->toBson(); + bufferedBytes += toInsert.objsize(); + if (!bufferedObjects.empty() && bufferedBytes > BSONObjMaxUserSize) { + spill(conn, bufferedObjects); + bufferedObjects.clear(); + bufferedBytes = toInsert.objsize(); + } + bufferedObjects.push_back(toInsert); + } + + if (!bufferedObjects.empty()) + spill(conn, bufferedObjects); + + // Checking again to make sure we didn't become sharded while running. + uassert(17018, str::stream() << "namespace '" << _outputNs.ns() + << "' became sharded so it can't be used for $out'", + !_mongod->isSharded(_outputNs)); + + BSONObj rename = BSON("renameCollection" << _tempNs.ns() + << "to" << _outputNs.ns() + << "dropTarget" << true + ); + BSONObj info; + bool ok = conn->runCommand("admin", rename, info); + uassert(16997, str::stream() << "renameCollection for $out failed: " << info, + ok); + + // We don't need to drop the temp collection in our destructor if the rename succeeded. + _tempNs = NamespaceString(""); + + // This "DocumentSource" doesn't produce output documents. This can change in the future + // if we support using $out in "tee" mode. + return boost::none; + } + + DocumentSourceOut::DocumentSourceOut(const NamespaceString& outputNs, + const intrusive_ptr& pExpCtx) + : DocumentSource(pExpCtx) + , _done(false) + , _tempNs("") // filled in by prepTempCollection + , _outputNs(outputNs) + {} + + intrusive_ptr DocumentSourceOut::createFromBson( + BSONElement elem, + const intrusive_ptr &pExpCtx) { + uassert(16990, str::stream() << "$out only supports a string argument, not " + << typeName(elem.type()), + elem.type() == String); + + NamespaceString outputNs(pExpCtx->ns.db().toString() + '.' + elem.str()); + uassert(17385, "Can't $out to special collection: " + elem.str(), + !outputNs.isSpecial()); + return new DocumentSourceOut(outputNs, pExpCtx); } - intrusive_ptr DocumentSourceOut::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { - intrusive_ptr pSource( - new DocumentSourceOut(pBsonElement, pExpCtx)); + Value DocumentSourceOut::serialize(bool explain) const { + massert(17000, "$out shouldn't have different db than input", + _outputNs.db() == pExpCtx->ns.db()); - return pSource; + return Value(DOC(getSourceName() << _outputNs.coll())); } - void DocumentSourceOut::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - verify(false); // CW TODO + DocumentSource::GetDepsReturn DocumentSourceOut::getDependencies(DepsTracker* deps) const { + deps->needWholeDocument = true; + return EXHAUSTIVE_ALL; } } diff --git a/src/mongo/db/pipeline/document_source_project.cpp b/src/mongo/db/pipeline/document_source_project.cpp index 17fcb98c42d..fdf9acc5b16 100644 --- a/src/mongo/db/pipeline/document_source_project.cpp +++ b/src/mongo/db/pipeline/document_source_project.cpp @@ -12,48 +12,53 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "db/pipeline/document_source.h" +#include "mongo/pch.h" -#include "db/jsobj.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/value.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/value.h" namespace mongo { const char DocumentSourceProject::projectName[] = "$project"; - DocumentSourceProject::~DocumentSourceProject() { - } - - DocumentSourceProject::DocumentSourceProject(const intrusive_ptr &pExpCtx) + DocumentSourceProject::DocumentSourceProject(const intrusive_ptr& pExpCtx, + const intrusive_ptr& exprObj) : DocumentSource(pExpCtx) - , pEO(ExpressionObject::create()) + , pEO(exprObj) { } const char *DocumentSourceProject::getSourceName() const { return projectName; } - bool DocumentSourceProject::eof() { - return pSource->eof(); - } - - bool DocumentSourceProject::advance() { - DocumentSource::advance(); // check for interrupts - - return pSource->advance(); - } + boost::optional DocumentSourceProject::getNext() { + pExpCtx->checkForInterrupt(); - Document DocumentSourceProject::getCurrent() { - Document pInDocument(pSource->getCurrent()); + boost::optional input = pSource->getNext(); + if (!input) + return boost::none; /* create the result document */ const size_t sizeHint = pEO->getSizeHint(); MutableDocument out (sizeHint); + out.copyMetaDataFrom(*input); /* Use the ExpressionObject to create the base result. @@ -61,26 +66,23 @@ namespace mongo { If we're excluding fields at the top level, leave out the _id if it is found, because we took care of it above. */ - pEO->addToDocument(out, pInDocument, /*root=*/pInDocument); + _variables->setRoot(*input); + pEO->addToDocument(out, *input, _variables.get()); + _variables->clearRoot(); #if defined(_DEBUG) if (!_simpleProjection.getSpec().isEmpty()) { // Make sure we return the same results as Projection class - BSONObjBuilder inputBuilder; - pSource->getCurrent()->toBson(&inputBuilder); - BSONObj input = inputBuilder.done(); - - BSONObjBuilder outputBuilder; - out.peek().toBson(&outputBuilder); - BSONObj output = outputBuilder.done(); + BSONObj inputBson = input->toBson(); + BSONObj outputBson = out.peek().toBson(); - BSONObj projected = _simpleProjection.transform(input); + BSONObj projected = _simpleProjection.transform(inputBson); - if (projected != output) { + if (projected != outputBson) { log() << "$project applied incorrectly: " << getRaw() << endl; - log() << "input: " << input << endl; - log() << "out: " << output << endl; + log() << "input: " << inputBson << endl; + log() << "out: " << outputBson << endl; log() << "projected: " << projected << endl; verify(false); // exits in _DEBUG builds } @@ -95,25 +97,18 @@ namespace mongo { pEO = dynamic_pointer_cast(pE); } - void DocumentSourceProject::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - BSONObjBuilder insides; - pEO->documentToBson(&insides, true); - pBuilder->append(projectName, insides.done()); + Value DocumentSourceProject::serialize(bool explain) const { + return Value(DOC(getSourceName() << pEO->serialize(explain))); } intrusive_ptr DocumentSourceProject::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + BSONElement elem, + const intrusive_ptr &pExpCtx) { + /* validate */ uassert(15969, str::stream() << projectName << " specification must be an object", - pBsonElement->type() == Object); - - intrusive_ptr pProject(new DocumentSourceProject(pExpCtx)); - - BSONObj projectObj(pBsonElement->Obj()); - pProject->_raw = projectObj.getOwned(); // probably not necessary, but better to be safe + elem.type() == Object); Expression::ObjectCtx objectCtx( Expression::ObjectCtx::DOCUMENT_OK @@ -121,28 +116,34 @@ namespace mongo { | Expression::ObjectCtx::INCLUSION_OK ); - intrusive_ptr parsed = Expression::parseObject(pBsonElement, &objectCtx); + VariablesIdGenerator idGenerator; + VariablesParseState vps(&idGenerator); + intrusive_ptr parsed = Expression::parseObject(elem.Obj(), &objectCtx, vps); ExpressionObject* exprObj = dynamic_cast(parsed.get()); massert(16402, "parseObject() returned wrong type of Expression", exprObj); uassert(16403, "$projection requires at least one output field", exprObj->getFieldCount()); - pProject->pEO = exprObj; + intrusive_ptr pProject(new DocumentSourceProject(pExpCtx, exprObj)); + pProject->_variables.reset(new Variables(idGenerator.getIdCount())); + + BSONObj projectObj = elem.Obj(); + pProject->_raw = projectObj.getOwned(); #if defined(_DEBUG) if (exprObj->isSimple()) { - set deps; + DepsTracker deps; vector path; - exprObj->addDependencies(deps, &path); - pProject->_simpleProjection.init(depsToProjection(deps)); + exprObj->addDependencies(&deps, &path); + pProject->_simpleProjection.init(deps.toProjection()); } #endif return pProject; } - DocumentSource::GetDepsReturn DocumentSourceProject::getDependencies(set& deps) const { + DocumentSource::GetDepsReturn DocumentSourceProject::getDependencies(DepsTracker* deps) const { vector path; // empty == top-level pEO->addDependencies(deps, &path); - return EXHAUSTIVE; + return EXHAUSTIVE_FIELDS; } } diff --git a/src/mongo/db/pipeline/document_source_redact.cpp b/src/mongo/db/pipeline/document_source_redact.cpp new file mode 100644 index 00000000000..e7a1548b1e4 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_redact.cpp @@ -0,0 +1,169 @@ +/** + * Copyright 2011 (c) 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/pch.h" + +#include "mongo/db/pipeline/document_source.h" + +#include + +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/value.h" + +namespace mongo { + + const char DocumentSourceRedact::redactName[] = "$redact"; + + DocumentSourceRedact::DocumentSourceRedact(const intrusive_ptr& expCtx, + const intrusive_ptr& expression) + : DocumentSource(expCtx) + , _expression(expression) + { } + + const char *DocumentSourceRedact::getSourceName() const { + return redactName; + } + + static const Value descendVal = Value("descend"); + static const Value pruneVal = Value("prune"); + static const Value keepVal = Value("keep"); + + boost::optional DocumentSourceRedact::getNext() { + while (boost::optional in = pSource->getNext()) { + _variables->setRoot(*in); + _variables->setValue(_currentId, Value(*in)); + if (boost::optional result = redactObject()) { + return result; + } + } + + return boost::none; + } + + Value DocumentSourceRedact::redactValue(const Value& in) { + const BSONType valueType = in.getType(); + if (valueType == Object) { + _variables->setValue(_currentId, in); + const boost::optional result = redactObject(); + if (result) { + return Value(*result); + } + else { + return Value(); + } + } + else if (valueType == Array) { + // TODO dont copy if possible + vector newArr; + const vector& arr = in.getArray(); + for (size_t i = 0; i < arr.size(); i++) { + if (arr[i].getType() == Object || arr[i].getType() == Array) { + const Value toAdd = redactValue(arr[i]) ; + if (!toAdd.missing()) { + newArr.push_back(toAdd); + } + } + else { + newArr.push_back(arr[i]); + } + } + return Value::consume(newArr); + } + else { + return in; + } + } + + boost::optional DocumentSourceRedact::redactObject() { + const Value expressionResult = _expression->evaluate(_variables.get()); + + if (expressionResult == keepVal) { + return _variables->getDocument(_currentId); + } + else if (expressionResult == pruneVal) { + return boost::optional(); + } + else if (expressionResult == descendVal) { + const Document in = _variables->getDocument(_currentId); + MutableDocument out; + out.copyMetaDataFrom(in); + FieldIterator fields(in); + while (fields.more()) { + const Document::FieldPair field(fields.next()); + + // This changes CURRENT so don't read from _variables after this + const Value val = redactValue(field.second); + if (!val.missing()) { + out.addField(field.first, val); + } + } + return out.freeze(); + } + else { + uasserted(17053, str::stream() << "$redact's expression should not return anything " + << "aside from the variables $$KEEP, $$DESCEND, and " + << "$$PRUNE, but returned " + << expressionResult.toString()); + } + } + + void DocumentSourceRedact::optimize() { + _expression = _expression->optimize(); + } + + Value DocumentSourceRedact::serialize(bool explain) const { + return Value(DOC(getSourceName() << _expression.get()->serialize(explain))); + } + + intrusive_ptr DocumentSourceRedact::createFromBson( + BSONElement elem, + const intrusive_ptr& expCtx) { + + VariablesIdGenerator idGenerator; + VariablesParseState vps(&idGenerator); + Variables::Id currentId = vps.defineVariable("CURRENT"); // will differ from ROOT + Variables::Id decendId = vps.defineVariable("DESCEND"); + Variables::Id pruneId = vps.defineVariable("PRUNE"); + Variables::Id keepId = vps.defineVariable("KEEP"); + intrusive_ptr expression = Expression::parseOperand(elem, vps); + intrusive_ptr source = new DocumentSourceRedact(expCtx, expression); + + // TODO figure out how much of this belongs in constructor and how much here. + // Set up variables. Never need to reset DESCEND, PRUNE, or KEEP. + source->_currentId = currentId; + source->_variables.reset(new Variables(idGenerator.getIdCount())); + source->_variables->setValue(decendId, descendVal); + source->_variables->setValue(pruneId, pruneVal); + source->_variables->setValue(keepId, keepVal); + + + return source; + } +} diff --git a/src/mongo/db/pipeline/document_source_skip.cpp b/src/mongo/db/pipeline/document_source_skip.cpp index 833de29061a..86f39735e77 100644 --- a/src/mongo/db/pipeline/document_source_skip.cpp +++ b/src/mongo/db/pipeline/document_source_skip.cpp @@ -12,29 +12,37 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" - -#include "db/pipeline/document_source.h" +#include "mongo/pch.h" -#include "db/jsobj.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/expression_context.h" -#include "db/pipeline/value.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" namespace mongo { const char DocumentSourceSkip::skipName[] = "$skip"; DocumentSourceSkip::DocumentSourceSkip(const intrusive_ptr &pExpCtx): - SplittableDocumentSource(pExpCtx), - skip(0), - count(0) { - } - - DocumentSourceSkip::~DocumentSourceSkip() { + DocumentSource(pExpCtx), + _skip(0), + _needToSkip(true) { } const char *DocumentSourceSkip::getSourceName() const { @@ -51,50 +59,26 @@ namespace mongo { return false; /* we need to skip over the sum of the two consecutive $skips */ - skip += pSkip->skip; + _skip += pSkip->_skip; return true; } - void DocumentSourceSkip::skipper() { - if (count == 0) { - while (!pSource->eof() && count++ < skip) { - pSource->advance(); - } - } - - if (pSource->eof()) { - pCurrent.reset(); - return; - } - - pCurrent = pSource->getCurrent(); - } - - bool DocumentSourceSkip::eof() { - skipper(); - return pSource->eof(); - } + boost::optional DocumentSourceSkip::getNext() { + pExpCtx->checkForInterrupt(); - bool DocumentSourceSkip::advance() { - DocumentSource::advance(); // check for interrupts - - if (eof()) { - pCurrent.reset(); - return false; + if (_needToSkip) { + _needToSkip = false; + for (long long i=0; i < _skip; i++) { + if (!pSource->getNext()) + return boost::none; + } } - pCurrent = pSource->getCurrent(); - return pSource->advance(); - } - - Document DocumentSourceSkip::getCurrent() { - skipper(); - return pCurrent; + return pSource->getNext(); } - void DocumentSourceSkip::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { - pBuilder->append("$skip", skip); + Value DocumentSourceSkip::serialize(bool explain) const { + return Value(DOC(getSourceName() << _skip)); } intrusive_ptr DocumentSourceSkip::create( @@ -105,19 +89,19 @@ namespace mongo { } intrusive_ptr DocumentSourceSkip::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + BSONElement elem, + const intrusive_ptr &pExpCtx) { uassert(15972, str::stream() << DocumentSourceSkip::skipName << ": the value to skip must be a number", - pBsonElement->isNumber()); + elem.isNumber()); intrusive_ptr pSkip( DocumentSourceSkip::create(pExpCtx)); - pSkip->skip = pBsonElement->numberLong(); + pSkip->_skip = elem.numberLong(); uassert(15956, str::stream() << DocumentSourceSkip::skipName << ": the number to skip cannot be negative", - pSkip->skip >= 0); + pSkip->_skip >= 0); return pSkip; } diff --git a/src/mongo/db/pipeline/document_source_sort.cpp b/src/mongo/db/pipeline/document_source_sort.cpp index 20e7948af4c..5b078f21508 100644 --- a/src/mongo/db/pipeline/document_source_sort.cpp +++ b/src/mongo/db/pipeline/document_source_sort.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "pch.h" @@ -19,7 +31,6 @@ #include "db/pipeline/document_source.h" #include "db/jsobj.h" -#include "db/pipeline/doc_mem_monitor.h" #include "db/pipeline/document.h" #include "db/pipeline/expression.h" #include "db/pipeline/expression_context.h" @@ -28,74 +39,57 @@ namespace mongo { const char DocumentSourceSort::sortName[] = "$sort"; - DocumentSourceSort::~DocumentSourceSort() { - } - const char *DocumentSourceSort::getSourceName() const { return sortName; } - bool DocumentSourceSort::eof() { - if (!populated) - populate(); - - return documents.empty(); - } - - bool DocumentSourceSort::advance() { - DocumentSource::advance(); // check for interrupts + boost::optional DocumentSourceSort::getNext() { + pExpCtx->checkForInterrupt(); if (!populated) populate(); - if (!documents.empty()) - documents.pop_front(); // this way we release memory as we go - - return !documents.empty(); - } + if (!_output || !_output->more()) { + // Need to be sure connections are marked as done so they can be returned to the + // connection pool. This only needs to happen in the _mergingPresorted case, but it + // doesn't hurt to always do it. + dispose(); + return boost::none; + } - Document DocumentSourceSort::getCurrent() { - verify(!documents.empty()); - return documents.front().doc; + return _output->next().second; } - void DocumentSourceSort::addToBsonArray(BSONArrayBuilder *pBuilder, bool explain) const { - if (explain) { // always one obj for combined $sort + $limit - BSONObjBuilder sortObj (pBuilder->subobjStart()); - BSONObjBuilder insides (sortObj.subobjStart(sortName)); - BSONObjBuilder sortKey (insides.subobjStart("sortKey")); - sortKeyToBson(&sortKey, false); - sortKey.doneFast(); - - if (explain && limitSrc) { - insides.appendNumber("limit", limitSrc->getLimit()); - } - insides.doneFast(); - sortObj.doneFast(); + void DocumentSourceSort::serializeToArray(vector& array, bool explain) const { + if (explain) { // always one Value for combined $sort + $limit + array.push_back(Value(DOC(getSourceName() << + DOC("sortKey" << serializeSortKey(explain) + << "mergePresorted" << (_mergingPresorted ? Value(true) : Value()) + << "limit" << (limitSrc ? Value(limitSrc->getLimit()) : Value()))))); } - else { // one obj for $sort + maybe one obj for $limit - { - BSONObjBuilder sortObj (pBuilder->subobjStart()); - BSONObjBuilder insides (sortObj.subobjStart(sortName)); - sortKeyToBson(&insides, false); - insides.doneFast(); - sortObj.doneFast(); - } + else { // one Value for $sort and maybe a Value for $limit + MutableDocument inner (serializeSortKey(explain)); + if (_mergingPresorted) + inner["$mergePresorted"] = Value(true); + array.push_back(Value(DOC(getSourceName() << inner.freeze()))); if (limitSrc) { - limitSrc->addToBsonArray(pBuilder, explain); + limitSrc->serializeToArray(array); } } } void DocumentSourceSort::dispose() { - documents.clear(); - pSource->dispose(); + _output.reset(); + if (pSource) { + pSource->dispose(); + } } DocumentSourceSort::DocumentSourceSort(const intrusive_ptr &pExpCtx) - : SplittableDocumentSource(pExpCtx) + : DocumentSource(pExpCtx) , populated(false) + , _mergingPresorted(false) {} long long DocumentSourceSort::getLimit() const { @@ -112,27 +106,37 @@ namespace mongo { } } - void DocumentSourceSort::addKey(const string &fieldPath, bool ascending) { - intrusive_ptr pE( - ExpressionFieldPath::create(fieldPath)); - vSortKey.push_back(pE); + void DocumentSourceSort::addKey(const string& fieldPath, bool ascending) { + VariablesIdGenerator idGenerator; + VariablesParseState vps(&idGenerator); + vSortKey.push_back(ExpressionFieldPath::parse("$$ROOT." + fieldPath, vps)); vAscending.push_back(ascending); } - void DocumentSourceSort::sortKeyToBson( - BSONObjBuilder *pBuilder, bool usePrefix) const { - /* add the key fields */ + Document DocumentSourceSort::serializeSortKey(bool explain) const { + MutableDocument keyObj; + // add the key fields const size_t n = vSortKey.size(); for(size_t i = 0; i < n; ++i) { - /* create the "field name" */ - stringstream ss; - vSortKey[i]->writeFieldPath(ss, usePrefix); - - /* append a named integer based on the sort order */ - pBuilder->append(ss.str(), (vAscending[i] ? 1 : -1)); + if (ExpressionFieldPath* efp = dynamic_cast(vSortKey[i].get())) { + // ExpressionFieldPath gets special syntax that includes direction + const FieldPath& withVariable = efp->getFieldPath(); + verify(withVariable.getPathLength() > 1); + verify(withVariable.getFieldName(0) == "ROOT"); + const string fieldPath = withVariable.tail().getPath(false); + + // append a named integer based on the sort order + keyObj.setField(fieldPath, Value(vAscending[i] ? 1 : -1)); + } + else { + // other expressions use a made-up field name + keyObj[string(str::stream() << "$computed" << i)] = vSortKey[i]->serialize(explain); + } } + return keyObj.freeze(); } - DocumentSource::GetDepsReturn DocumentSourceSort::getDependencies(set& deps) const { + + DocumentSource::GetDepsReturn DocumentSourceSort::getDependencies(DepsTracker* deps) const { for(size_t i = 0; i < vSortKey.size(); ++i) { vSortKey[i]->addDependencies(deps); } @@ -142,14 +146,13 @@ namespace mongo { intrusive_ptr DocumentSourceSort::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + BSONElement elem, + const intrusive_ptr &pExpCtx) { uassert(15973, str::stream() << " the " << sortName << " key specification must be an object", - pBsonElement->type() == Object); - - return create(pExpCtx, pBsonElement->embeddedObject()); + elem.type() == Object); + return create(pExpCtx, elem.embeddedObject()); } intrusive_ptr DocumentSourceSort::create( @@ -160,28 +163,40 @@ namespace mongo { intrusive_ptr pSort = new DocumentSourceSort(pExpCtx); /* check for then iterate over the sort object */ - size_t sortKeys = 0; - for(BSONObjIterator keyIterator(sortOrder); - keyIterator.more();) { - BSONElement keyField(keyIterator.next()); - const char *pKeyFieldName = keyField.fieldName(); - int sortOrder = 0; + BSONForEach(keyField, sortOrder) { + const char* fieldName = keyField.fieldName(); + + if (str::equals(fieldName, "$mergePresorted")) { + verify(keyField.Bool()); + pSort->_mergingPresorted = true; + continue; + } + + if (keyField.type() == Object) { + // this restriction is due to needing to figure out sort direction + uassert(17312, + "the only expression supported by $sort right now is {$meta: 'textScore'}", + keyField.Obj() == BSON("$meta" << "textScore")); + + pSort->vSortKey.push_back(new ExpressionMeta()); + pSort->vAscending.push_back(false); // best scoring documents first + continue; + } - uassert(15974, str::stream() << sortName << - " key ordering must be specified using a number", + uassert(15974, + "$sort key ordering must be specified using a number or {$meta: 'textScore'}", keyField.isNumber()); - sortOrder = (int)keyField.numberInt(); - uassert(15975, str::stream() << sortName << - " key ordering must be 1 (for ascending) or -1 (for descending", + int sortOrder = keyField.numberInt(); + + uassert(15975, "$sort key ordering must be 1 (for ascending) or -1 (for descending)", ((sortOrder == 1) || (sortOrder == -1))); - pSort->addKey(pKeyFieldName, (sortOrder > 0)); - ++sortKeys; + pSort->addKey(fieldName, (sortOrder > 0)); } - uassert(15976, str::stream() << sortName << - " must have at least one sort key", (sortKeys > 0)); + uassert(15976, str::stream() << sortName << " must have at least one sort key", + !pSort->vSortKey.empty()); if (limit > 0) { bool coalesced = pSort->coalesce(DocumentSourceLimit::create(pExpCtx, limit)); @@ -192,104 +207,111 @@ namespace mongo { return pSort; } - void DocumentSourceSort::populate() { + SortOptions DocumentSourceSort::makeSortOptions() const { /* make sure we've got a sort key */ verify(vSortKey.size()); - if (!limitSrc) - populateAll(); - else if (limitSrc->getLimit() == 1) - populateOne(); - else - populateTopK(); + SortOptions opts; + if (limitSrc) + opts.limit = limitSrc->getLimit(); - populated = true; - } - - void DocumentSourceSort::populateAll() { - /* track and warn about how much physical memory has been used */ - DocMemMonitor dmm(this); - - /* pull everything from the underlying source */ - for (bool hasNext = !pSource->eof(); hasNext; hasNext = pSource->advance()) { - documents.push_back(KeyAndDoc(pSource->getCurrent(), vSortKey)); - dmm.addToTotal(documents.back().doc.getApproximateSize()); + opts.maxMemoryUsageBytes = 100*1024*1024; + if (pExpCtx->extSortAllowed && !pExpCtx->inRouter) { + opts.extSortAllowed = true; + opts.tempDir = pExpCtx->tempDir; } - /* sort the list */ - Comparator comparator(*this); - sort(documents.begin(), documents.end(), comparator); + return opts; } - void DocumentSourceSort::populateOne() { - if (pSource->eof()) - return; - - KeyAndDoc best (pSource->getCurrent(), vSortKey); - while (pSource->advance()) { - KeyAndDoc next (pSource->getCurrent(), vSortKey); - if (compare(next, best) < 0) { - // we have a new best - swap(best, next); + void DocumentSourceSort::populate() { + if (_mergingPresorted) { + typedef DocumentSourceMergeCursors DSCursors; + typedef DocumentSourceCommandShards DSCommands; + if (DSCursors* castedSource = dynamic_cast(pSource)) { + populateFromCursors(castedSource->getCursors()); + } else if (DSCommands* castedSource = dynamic_cast(pSource)) { + populateFromBsonArrays(castedSource->getArrays()); + } else { + msgasserted(17196, "can only mergePresorted from MergeCursors and CommandShards"); + } + } else { + scoped_ptr sorter (MySorter::make(makeSortOptions(), Comparator(*this))); + while (boost::optional next = pSource->getNext()) { + sorter->add(extractKey(*next), *next); } + _output.reset(sorter->done()); } - - documents.push_back(best); + populated = true; } - void DocumentSourceSort::populateTopK() { - bool hasNext = !pSource->eof(); - - size_t limit = limitSrc->getLimit(); - - // Pull first K documents unconditionally - vector heap; - heap.reserve(limit); - for (; hasNext && heap.size() < limit; hasNext = pSource->advance()) { - heap.push_back(KeyAndDoc(pSource->getCurrent(), vSortKey)); + class DocumentSourceSort::IteratorFromCursor : public MySorter::Iterator { + public: + IteratorFromCursor(DocumentSourceSort* sorter, DBClientCursor* cursor) + : _sorter(sorter) + , _cursor(cursor) + {} + + bool more() { return _cursor->more(); } + Data next() { + const Document doc = DocumentSourceMergeCursors::nextSafeFrom(_cursor); + return make_pair(_sorter->extractKey(doc), doc); + } + private: + DocumentSourceSort* _sorter; + DBClientCursor* _cursor; + }; + + void DocumentSourceSort::populateFromCursors(const vector& cursors) { + vector > iterators; + for (size_t i = 0; i < cursors.size(); i++) { + iterators.push_back(boost::make_shared(this, cursors[i])); } - // We now maintain a MaxHeap of K items. This means that the least-best - // document is at the top of the heap (heap.front()). If a new - // document is better than the top of the heap, we pop the top and add - // the new document to the heap. - - Comparator comp (*this); - - // after this, heap.front() is least-best document - std::make_heap(heap.begin(), heap.end(), comp); - - for (; hasNext; hasNext = pSource->advance()) { - KeyAndDoc next (pSource->getCurrent(), vSortKey); - if (compare(next, heap.front()) < 0) { - // remove least-best from heap - std::pop_heap(heap.begin(), heap.end(), comp); + _output.reset(MySorter::Iterator::merge(iterators, makeSortOptions(), Comparator(*this))); + } - // add next to heap - swap(heap.back(), next); - std::push_heap(heap.begin(), heap.end(), comp); - } + class DocumentSourceSort::IteratorFromBsonArray : public MySorter::Iterator { + public: + IteratorFromBsonArray(DocumentSourceSort* sorter, const BSONArray& array) + : _sorter(sorter) + , _iterator(array) + {} + + bool more() { return _iterator.more(); } + Data next() { + Document doc(_iterator.next().Obj()); + return make_pair(_sorter->extractKey(doc), doc); + } + private: + DocumentSourceSort* _sorter; + BSONObjIterator _iterator; + }; + + void DocumentSourceSort::populateFromBsonArrays(const vector& arrays) { + vector > iterators; + for (size_t i = 0; i < arrays.size(); i++) { + iterators.push_back(boost::make_shared(this, arrays[i])); } - std::sort_heap(heap.begin(), heap.end(), comp); - documents.insert(documents.begin(), heap.begin(), heap.end()); + _output.reset(MySorter::Iterator::merge(iterators, makeSortOptions(), Comparator(*this))); } - DocumentSourceSort::KeyAndDoc::KeyAndDoc(const Document& d, const SortPaths& sp) :doc(d) { - if (sp.size() == 1) { - key = sp[0]->evaluate(d); - return; + Value DocumentSourceSort::extractKey(const Document& d) const { + Variables vars(0, d); + if (vSortKey.size() == 1) { + return vSortKey[0]->evaluate(&vars); } vector keys; - keys.reserve(sp.size()); - for (size_t i=0; i < sp.size(); i++) { - keys.push_back(sp[i]->evaluate(d)); + keys.reserve(vSortKey.size()); + for (size_t i=0; i < vSortKey.size(); i++) { + keys.push_back(vSortKey[i]->evaluate(&vars)); } - key = Value(keys); + return Value::consume(keys); } - int DocumentSourceSort::compare(const KeyAndDoc & lhs, const KeyAndDoc & rhs) const { + int DocumentSourceSort::compare(const Value& lhs, const Value& rhs) const { /* populate() already checked that there is a non-empty sort key, @@ -301,14 +323,14 @@ namespace mongo { const size_t n = vSortKey.size(); if (n == 1) { // simple fast case if (vAscending[0]) - return Value::compare(lhs.key, rhs.key); + return Value::compare(lhs, rhs); else - return -Value::compare(lhs.key, rhs.key); + return -Value::compare(lhs, rhs); } // compound sort for (size_t i = 0; i < n; i++) { - int cmp = Value::compare(lhs.key[i], rhs.key[i]); + int cmp = Value::compare(lhs[i], rhs[i]); if (cmp) { /* if necessary, adjust the return value by the key ordering */ if (!vAscending[i]) @@ -324,4 +346,22 @@ namespace mongo { */ return 0; } + + intrusive_ptr DocumentSourceSort::getShardSource() { + verify(!_mergingPresorted); + return this; + } + + intrusive_ptr DocumentSourceSort::getMergeSource() { + verify(!_mergingPresorted); + intrusive_ptr other = new DocumentSourceSort(pExpCtx); + other->vAscending = vAscending; + other->vSortKey = vSortKey; + other->limitSrc = limitSrc; + other->_mergingPresorted = true; + return other; + } } + +#include "db/sorter/sorter.cpp" +// Explicit instantiation unneeded since we aren't exposing Sorter outside of this file. diff --git a/src/mongo/db/pipeline/document_source_unwind.cpp b/src/mongo/db/pipeline/document_source_unwind.cpp index 249d3efb593..f7c28fa8ec4 100644 --- a/src/mongo/db/pipeline/document_source_unwind.cpp +++ b/src/mongo/db/pipeline/document_source_unwind.cpp @@ -12,36 +12,46 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "db/pipeline/document_source.h" +#include "mongo/pch.h" -#include "db/jsobj.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/value.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/value.h" namespace mongo { - /** Helper class to unwind arrays within a series of documents. */ + /** Helper class to unwind array from a single document. */ class DocumentSourceUnwind::Unwinder { public: /** @param unwindPath is the field path to the array to unwind. */ Unwinder(const FieldPath& unwindPath); /** Reset the unwinder to unwind a new document. */ void resetDocument(const Document& document); - /** @return true if done unwinding the last document passed to resetDocument(). */ - bool eof() const; - /** Try to advance to the next document unwound from the document passed to resetDocument(). */ - void advance(); + /** - * @return the current document unwound from the document provided to resetDocument(), using - * the current value in the array located at the provided unwindPath. But @return - * Document() if resetDocument() has not been called or the results to unwind - * have been exhausted. + * @return the next document unwound from the document provided to resetDocument(), using + * the current value in the array located at the provided unwindPath. + * + * Returns boost::none if the array is exhausted. */ - Document getCurrent(); + boost::optional getNext(); + private: // Path to the array to unwind. const FieldPath _unwindPath; @@ -74,34 +84,17 @@ namespace mongo { } // The target field must be an array to unwind. - uassert(15978, str::stream() << (string)DocumentSourceUnwind::unwindName - << ": value at end of field path must be an array", + uassert(15978, str::stream() << "Value at end of $unwind field path '" + << _unwindPath.getPath(true) << "' must be an Array, but is a " + << typeName(pathValue.getType()), pathValue.getType() == Array); - if (pathValue.getArray().empty()) { - // there are no values to unwind. - return; - } - _inputArray = pathValue; - verify(!eof()); // Checked above that the array is nonempty. } - bool DocumentSourceUnwind::Unwinder::eof() const { - return (_inputArray.getType() != Array) - || (_index == _inputArray.getArrayLength()); - } - - void DocumentSourceUnwind::Unwinder::advance() { - if (!eof()) { // don't advance past end() - _index++; - } - } - - Document DocumentSourceUnwind::Unwinder::getCurrent() { - if (eof()) { - return Document(); - } + boost::optional DocumentSourceUnwind::Unwinder::getNext() { + if (_inputArray.missing() || _index == _inputArray.getArrayLength()) + return boost::none; // If needed, this will automatically clone all the documents along the // field path so that the end values are not shared across documents @@ -111,80 +104,47 @@ namespace mongo { // that change with any other clones (or the original). _output.setNestedField(_unwindPathFieldIndexes, _inputArray[_index]); - + _index++; return _output.peek(); } const char DocumentSourceUnwind::unwindName[] = "$unwind"; - DocumentSourceUnwind::~DocumentSourceUnwind() { - } - DocumentSourceUnwind::DocumentSourceUnwind( const intrusive_ptr &pExpCtx): DocumentSource(pExpCtx) { } - void DocumentSourceUnwind::lazyInit() { - if (!_unwinder) { - verify(_unwindPath); - _unwinder.reset(new Unwinder(*_unwindPath)); - if (!pSource->eof()) { - // Set up the first source document for unwinding. - _unwinder->resetDocument(pSource->getCurrent()); - } - mayAdvanceSource(); - } - } - - void DocumentSourceUnwind::mayAdvanceSource() { - while(_unwinder->eof()) { - // The _unwinder is exhausted. - - if (pSource->eof()) { - // The source is exhausted. - return; - } - if (!pSource->advance()) { - // The source is exhausted. - return; - } - // Reset the _unwinder with pSource's next document. - _unwinder->resetDocument(pSource->getCurrent()); - } - } - const char *DocumentSourceUnwind::getSourceName() const { return unwindName; } - bool DocumentSourceUnwind::eof() { - lazyInit(); - return _unwinder->eof(); - } + boost::optional DocumentSourceUnwind::getNext() { + pExpCtx->checkForInterrupt(); - bool DocumentSourceUnwind::advance() { - DocumentSource::advance(); // check for interrupts - lazyInit(); - _unwinder->advance(); - mayAdvanceSource(); - return !_unwinder->eof(); - } + boost::optional out = _unwinder->getNext(); + while (!out) { + // No more elements in array currently being unwound. This will loop if the input + // document is missing the unwind field or has an empty array. + boost::optional input = pSource->getNext(); + if (!input) + return boost::none; // input exhausted - Document DocumentSourceUnwind::getCurrent() { - verify(!eof()); - return _unwinder->getCurrent(); + // Try to extract an output document from the new input document. + _unwinder->resetDocument(*input); + out = _unwinder->getNext(); + } + + return out; } - void DocumentSourceUnwind::sourceToBson( - BSONObjBuilder *pBuilder, bool explain) const { + Value DocumentSourceUnwind::serialize(bool explain) const { verify(_unwindPath); - pBuilder->append(unwindName, _unwindPath->getPath(true)); + return Value(DOC(getSourceName() << _unwindPath->getPath(true))); } - DocumentSource::GetDepsReturn DocumentSourceUnwind::getDependencies(set& deps) const { - verify(_unwindPath); - deps.insert(_unwindPath->getPath(false)); + DocumentSource::GetDepsReturn DocumentSourceUnwind::getDependencies(DepsTracker* deps) const { + deps->fields.insert(_unwindPath->getPath(false)); return SEE_NEXT; } @@ -194,19 +154,20 @@ namespace mongo { !_unwindPath); // Record the unwind path. _unwindPath.reset(new FieldPath(fieldPath)); + _unwinder.reset(new Unwinder(fieldPath)); } intrusive_ptr DocumentSourceUnwind::createFromBson( - BSONElement *pBsonElement, - const intrusive_ptr &pExpCtx) { + BSONElement elem, + const intrusive_ptr &pExpCtx) { /* The value of $unwind should just be a field path. */ uassert(15981, str::stream() << "the " << unwindName << " field path must be specified as a string", - pBsonElement->type() == String); + elem.type() == String); - string prefixedPathString(pBsonElement->str()); + string prefixedPathString(elem.str()); string pathString(Expression::removeFieldPrefix(prefixedPathString)); intrusive_ptr pUnwind(new DocumentSourceUnwind(pExpCtx)); pUnwind->unwindPath(FieldPath(pathString)); diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index d4a26ce78e4..82d5c69cbea 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -12,28 +12,155 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "pch.h" + #include "db/pipeline/expression.h" +#include +#include +#include +#include // like the ## operator but works with __LINE__ #include -#include "db/jsobj.h" -#include "db/pipeline/builder.h" -#include "db/pipeline/document.h" -#include "db/pipeline/expression_context.h" -#include "db/pipeline/value.h" -#include "util/mongoutils/str.h" + +#include "mongo/base/init.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/value.h" +#include "mongo/util/string_map.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { using namespace mongoutils; - /* --------------------------- Expression ------------------------------ */ + /// Helper function to easily wrap constants with $const. + static Value serializeConstant(Value val) { + return Value(DOC("$const" << val)); + } + + void Variables::uassertValidNameForUserWrite(StringData varName) { + // System variables users allowed to write to (currently just one) + if (varName == "CURRENT") { + return; + } + + uassert(16866, "empty variable names are not allowed", + !varName.empty()); + + const bool firstCharIsValid = (varName[0] >= 'a' && varName[0] <= 'z') + || (varName[0] & '\x80') // non-ascii + ; + + uassert(16867, str::stream() << + "'" << varName << "' starts with an invalid character for a user variable name", + firstCharIsValid); + + for (size_t i = 1; i < varName.size(); i++) { + const bool charIsValid = (varName[i] >= 'a' && varName[i] <= 'z') + || (varName[i] >= 'A' && varName[i] <= 'Z') + || (varName[i] >= '0' && varName[i] <= '9') + || (varName[i] == '_') + || (varName[i] & '\x80') // non-ascii + ; + + uassert(16868, str::stream() << "'" << varName << "' contains an invalid character " + << "for a variable name: '" << varName[i] << "'", + charIsValid); + } + } + + void Variables::uassertValidNameForUserRead(StringData varName) { + uassert(16869, "empty variable names are not allowed", + !varName.empty()); + + const bool firstCharIsValid = (varName[0] >= 'a' && varName[0] <= 'z') + || (varName[0] >= 'A' && varName[0] <= 'Z') + || (varName[0] & '\x80') // non-ascii + ; + + uassert(16870, str::stream() << + "'" << varName << "' starts with an invalid character for a variable name", + firstCharIsValid); + + for (size_t i = 1; i < varName.size(); i++) { + const bool charIsValid = (varName[i] >= 'a' && varName[i] <= 'z') + || (varName[i] >= 'A' && varName[i] <= 'Z') + || (varName[i] >= '0' && varName[i] <= '9') + || (varName[i] == '_') + || (varName[i] & '\x80') // non-ascii + ; + + uassert(16871, str::stream() << "'" << varName << "' contains an invalid character " + << "for a variable name: '" << varName[i] << "'", + charIsValid); + } + } + + void Variables::setValue(Id id, const Value& value) { + massert(17199, "can't use Variables::setValue to set ROOT", + id != ROOT_ID); + + verify(id < _numVars); + _rest[id] = value; + } + + Value Variables::getValue(Id id) const { + if (id == ROOT_ID) + return Value(_root); + + verify(id < _numVars); + return _rest[id]; + } + + Document Variables::getDocument(Id id) const { + if (id == ROOT_ID) + return _root; + + verify(id < _numVars); + const Value var = _rest[id]; + if (var.getType() == Object) + return var.getDocument(); + + return Document(); + } - void Expression::toMatcherBson(BSONObjBuilder *pBuilder) const { - verify(false && "Expression::toMatcherBson()"); + Variables::Id VariablesParseState::defineVariable(const StringData& name) { + // caller should have validated before hand by using Variables::uassertValidNameForUserWrite + massert(17275, "Can't redefine ROOT", + name != "ROOT"); + + Variables::Id id = _idGenerator->generateId(); + _variables[name] = id; + return id; + } + + Variables::Id VariablesParseState::getVariable(const StringData& name) const { + StringMap::const_iterator it = _variables.find(name); + if (it != _variables.end()) + return it->second; + + uassert(17276, str::stream() << "Use of undefined variable: " << name, + name == "ROOT" || name == "CURRENT"); + + return Variables::ROOT_ID; } + /* --------------------------- Expression ------------------------------ */ + Expression::ObjectCtx::ObjectCtx(int theOptions) : options(theOptions) {} @@ -63,7 +190,9 @@ namespace mongo { } intrusive_ptr Expression::parseObject( - BSONElement *pBsonElement, ObjectCtx *pCtx) { + BSONObj obj, + ObjectCtx* pCtx, + const VariablesParseState& vps) { /* An object expression can take any of the following forms: @@ -75,7 +204,6 @@ namespace mongo { intrusive_ptr pExpressionObject; // alt result enum { UNKNOWN, NOTOPERATOR, OPERATOR } kind = UNKNOWN; - BSONObj obj(pBsonElement->Obj()); if (obj.isEmpty()) return ExpressionObject::create(); BSONObjIterator iter(obj); @@ -96,7 +224,7 @@ namespace mongo { /* we've determined this "object" is an operator expression */ kind = OPERATOR; - pExpression = parseExpression(pFieldName, &fieldElement); + pExpression = parseExpression(fieldElement, vps); } else { uassert(15990, str::stream() << "this object is already an operator expression, and can't be used as a document expression (at '" << @@ -111,7 +239,8 @@ namespace mongo { verify(pCtx->documentOk()); // CW TODO error: document not allowed in this context - pExpressionObject = ExpressionObject::create(); + pExpressionObject = pCtx->topLevel() ? ExpressionObject::createRoot() + : ExpressionObject::create(); pExpression = pExpressionObject; /* this "object" is not an operator expression */ @@ -126,18 +255,17 @@ namespace mongo { ObjectCtx oCtx( (pCtx->documentOk() ? ObjectCtx::DOCUMENT_OK : 0) | (pCtx->inclusionOk() ? ObjectCtx::INCLUSION_OK : 0)); - intrusive_ptr pNested( - parseObject(&fieldElement, &oCtx)); - pExpressionObject->addField(fieldName, pNested); + + pExpressionObject->addField(fieldName, + parseObject(fieldElement.Obj(), &oCtx, vps)); break; } case String: { /* it's a renamed field */ // CW TODO could also be a constant - intrusive_ptr pPath( - ExpressionFieldPath::create( - removeFieldPrefix(fieldElement.str()))); - pExpressionObject->addField(fieldName, pPath); + pExpressionObject->addField(fieldName, + ExpressionFieldPath::parse(fieldElement.str(), + vps)); break; } case Bool: @@ -170,165 +298,82 @@ namespace mongo { return pExpression; } +namespace { + typedef boost::function(BSONElement, const VariablesParseState&)> + ExpressionParser; + StringMap expressionParserMap; +} - struct OpDesc { - const char *pName; - intrusive_ptr (*pFactory)(void); - - unsigned flag; - static const unsigned FIXED_COUNT = 0x0001; - static const unsigned OBJECT_ARG = 0x0002; - - unsigned argCount; - }; - - static int OpDescCmp(const void *pL, const void *pR) { - return strcmp(((const OpDesc *)pL)->pName, ((const OpDesc *)pR)->pName); - } - - /* - Keep these sorted alphabetically so we can bsearch() them using - OpDescCmp() above. - */ - static const OpDesc OpTable[] = { - {"$add", ExpressionAdd::create, 0}, - {"$and", ExpressionAnd::create, 0}, - {"$cmp", ExpressionCompare::createCmp, OpDesc::FIXED_COUNT, 2}, - {"$concat", ExpressionConcat::create, 0}, - {"$cond", ExpressionCond::create, OpDesc::FIXED_COUNT, 3}, - // $const handled specially in parseExpression - {"$dayOfMonth", ExpressionDayOfMonth::create, OpDesc::FIXED_COUNT, 1}, - {"$dayOfWeek", ExpressionDayOfWeek::create, OpDesc::FIXED_COUNT, 1}, - {"$dayOfYear", ExpressionDayOfYear::create, OpDesc::FIXED_COUNT, 1}, - {"$divide", ExpressionDivide::create, OpDesc::FIXED_COUNT, 2}, - {"$eq", ExpressionCompare::createEq, OpDesc::FIXED_COUNT, 2}, - {"$gt", ExpressionCompare::createGt, OpDesc::FIXED_COUNT, 2}, - {"$gte", ExpressionCompare::createGte, OpDesc::FIXED_COUNT, 2}, - {"$hour", ExpressionHour::create, OpDesc::FIXED_COUNT, 1}, - {"$ifNull", ExpressionIfNull::create, OpDesc::FIXED_COUNT, 2}, - {"$lt", ExpressionCompare::createLt, OpDesc::FIXED_COUNT, 2}, - {"$lte", ExpressionCompare::createLte, OpDesc::FIXED_COUNT, 2}, - {"$millisecond", ExpressionMillisecond::create, OpDesc::FIXED_COUNT, 1}, - {"$minute", ExpressionMinute::create, OpDesc::FIXED_COUNT, 1}, - {"$mod", ExpressionMod::create, OpDesc::FIXED_COUNT, 2}, - {"$month", ExpressionMonth::create, OpDesc::FIXED_COUNT, 1}, - {"$multiply", ExpressionMultiply::create, 0}, - {"$ne", ExpressionCompare::createNe, OpDesc::FIXED_COUNT, 2}, - {"$not", ExpressionNot::create, OpDesc::FIXED_COUNT, 1}, - {"$or", ExpressionOr::create, 0}, - {"$second", ExpressionSecond::create, OpDesc::FIXED_COUNT, 1}, - {"$strcasecmp", ExpressionStrcasecmp::create, OpDesc::FIXED_COUNT, 2}, - {"$substr", ExpressionSubstr::create, OpDesc::FIXED_COUNT, 3}, - {"$subtract", ExpressionSubtract::create, OpDesc::FIXED_COUNT, 2}, - {"$toLower", ExpressionToLower::create, OpDesc::FIXED_COUNT, 1}, - {"$toUpper", ExpressionToUpper::create, OpDesc::FIXED_COUNT, 1}, - {"$week", ExpressionWeek::create, OpDesc::FIXED_COUNT, 1}, - {"$year", ExpressionYear::create, OpDesc::FIXED_COUNT, 1}, - }; - - static const size_t NOp = sizeof(OpTable)/sizeof(OpTable[0]); +/** Registers an ExpressionParser so it can be called from parseExpression and friends. + * + * As an example, if your expression looks like {"$foo": [1,2,3]} you would add this line: + * REGISTER_EXPRESSION("$foo", ExpressionFoo::parse); + */ +#define REGISTER_EXPRESSION(key, parserFunc) \ + MONGO_INITIALIZER(BOOST_PP_CAT(addToExpressionParserMap, __LINE__))(InitializerContext*) { \ + /* prevent duplicate expressions */ \ + StringMap::const_iterator op = expressionParserMap.find(key); \ + massert(17064, str::stream() << "Duplicate expression (" << key << ") detected at " \ + << __FILE__ << ":" << __LINE__, \ + op == expressionParserMap.end()); \ + /* register expression */ \ + expressionParserMap[key] = (parserFunc); \ + return Status::OK(); \ + } intrusive_ptr Expression::parseExpression( - const char *pOpName, BSONElement *pBsonElement) { - /* look for the specified operator */ - - if (str::equals(pOpName, "$const")) { - return ExpressionConstant::createFromBsonElement(pBsonElement); - } + BSONElement exprElement, + const VariablesParseState& vps) { - OpDesc key; - key.pName = pOpName; - const OpDesc *pOp = (const OpDesc *)bsearch( - &key, OpTable, NOp, sizeof(OpDesc), OpDescCmp); - - uassert(15999, str::stream() << "invalid operator '" << - pOpName << "'", pOp); + /* look for the specified operator */ + const char* opName = exprElement.fieldName(); + StringMap::const_iterator op = expressionParserMap.find(opName); + uassert(15999, str::stream() << "invalid operator '" << opName << "'", + op != expressionParserMap.end()); /* make the expression node */ - intrusive_ptr pExpression((*pOp->pFactory)()); - - /* add the operands to the expression node */ - BSONType elementType = pBsonElement->type(); - - if (pOp->flag & OpDesc::FIXED_COUNT) { - if (pOp->argCount > 1) - uassert(16019, str::stream() << "the " << pOp->pName << - " operator requires an array of " << pOp->argCount << - " operands", elementType == Array); - } + return op->second(exprElement, vps); + } - if (elementType == Object) { - /* the operator must be unary and accept an object argument */ - uassert(16021, str::stream() << "the " << pOp->pName << - " operator does not accept an object as an operand", - pOp->flag & OpDesc::OBJECT_ARG); + Expression::ExpressionVector ExpressionNary::parseArguments( + BSONElement exprElement, + const VariablesParseState& vps) { - BSONObj objOperand(pBsonElement->Obj()); - ObjectCtx oCtx(ObjectCtx::DOCUMENT_OK); - intrusive_ptr pOperand( - Expression::parseObject(pBsonElement, &oCtx)); - pExpression->addOperand(pOperand); - } - else if (elementType == Array) { - /* multiple operands - an n-ary operator */ - vector bsonArray(pBsonElement->Array()); - const size_t n = bsonArray.size(); - - if (pOp->flag & OpDesc::FIXED_COUNT) - uassert(16020, str::stream() << "the " << pOp->pName << - " operator requires " << pOp->argCount << - " operand(s)", pOp->argCount == n); - - for(size_t i = 0; i < n; ++i) { - BSONElement *pBsonOperand = &bsonArray[i]; - intrusive_ptr pOperand( - Expression::parseOperand(pBsonOperand)); - pExpression->addOperand(pOperand); + ExpressionVector out; + if (exprElement.type() == Array) { + BSONForEach(elem, exprElement.Obj()) { + out.push_back(Expression::parseOperand(elem, vps)); } } - else { - /* assume it's an atomic operand */ - if (pOp->flag & OpDesc::FIXED_COUNT) - uassert(16022, str::stream() << "the " << pOp->pName << - " operator requires an array of " << pOp->argCount << - " operands", pOp->argCount == 1); - - intrusive_ptr pOperand( - Expression::parseOperand(pBsonElement)); - pExpression->addOperand(pOperand); + else { // assume it's an atomic operand + out.push_back(Expression::parseOperand(exprElement, vps)); } - return pExpression; + return out; } - intrusive_ptr Expression::parseOperand(BSONElement *pBsonElement) { - BSONType type = pBsonElement->type(); + intrusive_ptr Expression::parseOperand( + BSONElement exprElement, + const VariablesParseState& vps) { - if (type == String && pBsonElement->valuestr()[0] == '$') { + BSONType type = exprElement.type(); + + if (type == String && exprElement.valuestr()[0] == '$') { /* if we got here, this is a field path expression */ - string fieldPath = removeFieldPrefix(pBsonElement->str()); - return ExpressionFieldPath::create(fieldPath); + return ExpressionFieldPath::parse(exprElement.str(), vps); } else if (type == Object) { ObjectCtx oCtx(ObjectCtx::DOCUMENT_OK); - return Expression::parseObject(pBsonElement, &oCtx); + return Expression::parseObject(exprElement.Obj(), &oCtx, vps); } else { - return ExpressionConstant::createFromBsonElement(pBsonElement); + return ExpressionConstant::parse(exprElement, vps); } } /* ------------------------- ExpressionAdd ----------------------------- */ - ExpressionAdd::~ExpressionAdd() { - } - - intrusive_ptr ExpressionAdd::create() { - intrusive_ptr pExpression(new ExpressionAdd()); - return pExpression; - } - - Value ExpressionAdd::evaluate(const Document& pDocument) const { + Value ExpressionAdd::evaluateInternal(Variables* vars) const { /* We'll try to return the narrowest possible result value. To do that @@ -343,7 +388,7 @@ namespace mongo { const size_t n = vpOperand.size(); for (size_t i = 0; i < n; ++i) { - Value val = vpOperand[i]->evaluate(pDocument); + Value val = vpOperand[i]->evaluateInternal(vars); if (val.numeric()) { totalType = Value::getWidestNumeric(totalType, val.getType()); @@ -373,13 +418,13 @@ namespace mongo { if (haveDate) { if (totalType == NumberDouble) longTotal = static_cast(doubleTotal); - return Value::createDate(longTotal); + return Value(Date_t(longTotal)); } else if (totalType == NumberLong) { - return Value::createLong(longTotal); + return Value(longTotal); } else if (totalType == NumberDouble) { - return Value::createDouble(doubleTotal); + return Value(doubleTotal); } else if (totalType == NumberInt) { return Value::createIntOrLong(longTotal); @@ -389,27 +434,33 @@ namespace mongo { } } + REGISTER_EXPRESSION("$add", ExpressionAdd::parse); const char *ExpressionAdd::getOpName() const { return "$add"; } - intrusive_ptr (*ExpressionAdd::getFactory() const)() { - return ExpressionAdd::create; - } - - /* ------------------------- ExpressionAnd ----------------------------- */ + /* ------------------------- ExpressionAllElementsTrue -------------------------- */ - ExpressionAnd::~ExpressionAnd() { + Value ExpressionAllElementsTrue::evaluateInternal(Variables* vars) const { + const Value arr = vpOperand[0]->evaluateInternal(vars); + uassert(17040, str::stream() << getOpName() << "'s argument must be an array, but is " + << typeName(arr.getType()), + arr.getType() == Array); + const vector& array = arr.getArray(); + for (vector::const_iterator it = array.begin(); it != array.end(); ++it) { + if (!it->coerceToBool()) { + return Value(false); + } + } + return Value(true); } - intrusive_ptr ExpressionAnd::create() { - intrusive_ptr pExpression(new ExpressionAnd()); - return pExpression; + REGISTER_EXPRESSION("$allElementsTrue", ExpressionAllElementsTrue::parse); + const char *ExpressionAllElementsTrue::getOpName() const { + return "$allElementsTrue"; } - ExpressionAnd::ExpressionAnd(): - ExpressionNary() { - } + /* ------------------------- ExpressionAnd ----------------------------- */ intrusive_ptr ExpressionAnd::optimize() { /* optimize the conjunction as much as possible */ @@ -438,7 +489,7 @@ namespace mongo { Evaluate and coerce the last argument to a boolean. If it's false, then we can replace this entire expression. */ - bool last = pLast->evaluate(Document()).coerceToBool(); + bool last = pConst->getValue().coerceToBool(); if (!last) { intrusive_ptr pFinal( ExpressionConstant::create(Value(false))); @@ -468,10 +519,10 @@ namespace mongo { return pE; } - Value ExpressionAnd::evaluate(const Document& pDocument) const { + Value ExpressionAnd::evaluateInternal(Variables* vars) const { const size_t n = vpOperand.size(); for(size_t i = 0; i < n; ++i) { - Value pValue(vpOperand[i]->evaluate(pDocument)); + Value pValue(vpOperand[i]->evaluateInternal(vars)); if (!pValue.coerceToBool()) return Value(false); } @@ -479,31 +530,34 @@ namespace mongo { return Value(true); } + REGISTER_EXPRESSION("$and", ExpressionAnd::parse); const char *ExpressionAnd::getOpName() const { return "$and"; } - void ExpressionAnd::toMatcherBson(BSONObjBuilder *pBuilder) const { - /* - There are two patterns we can handle: - (1) one or two comparisons on the same field: { a:{$gte:3, $lt:7} } - (2) multiple field comparisons: {a:7, b:{$lte:6}, c:2} - This can be recognized as a conjunction of a set of range - expressions. Direct equality is a degenerate range expression; - range expressions can be open-ended. - */ - verify(false && "unimplemented"); + /* ------------------------- ExpressionAnyElementTrue -------------------------- */ + + Value ExpressionAnyElementTrue::evaluateInternal(Variables* vars) const { + const Value arr = vpOperand[0]->evaluateInternal(vars); + uassert(17041, str::stream() << getOpName() << "'s argument must be an array, but is " + << typeName(arr.getType()), + arr.getType() == Array); + const vector& array = arr.getArray(); + for (vector::const_iterator it = array.begin(); it != array.end(); ++it) { + if (it->coerceToBool()) { + return Value(true); + } + } + return Value(false); } - intrusive_ptr (*ExpressionAnd::getFactory() const)() { - return ExpressionAnd::create; + REGISTER_EXPRESSION("$anyElementTrue", ExpressionAnyElementTrue::parse); + const char *ExpressionAnyElementTrue::getOpName() const { + return "$anyElementTrue"; } /* -------------------- ExpressionCoerceToBool ------------------------- */ - ExpressionCoerceToBool::~ExpressionCoerceToBool() { - } - intrusive_ptr ExpressionCoerceToBool::create( const intrusive_ptr &pExpression) { intrusive_ptr pNew( @@ -533,190 +587,95 @@ namespace mongo { return intrusive_ptr(this); } - void ExpressionCoerceToBool::addDependencies(set& deps, vector* path) const { + void ExpressionCoerceToBool::addDependencies(DepsTracker* deps, vector* path) const { pExpression->addDependencies(deps); } - Value ExpressionCoerceToBool::evaluate(const Document& pDocument) const { - Value pResult(pExpression->evaluate(pDocument)); + Value ExpressionCoerceToBool::evaluateInternal(Variables* vars) const { + Value pResult(pExpression->evaluateInternal(vars)); bool b = pResult.coerceToBool(); if (b) return Value(true); return Value(false); } - void ExpressionCoerceToBool::addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const { - // Serializing as an $and expression which will become a CoerceToBool - BSONObjBuilder sub (pBuilder->subobjStart(fieldName)); - BSONArrayBuilder arr (sub.subarrayStart("$and")); - pExpression->addToBsonArray(&arr); - arr.doneFast(); - sub.doneFast(); - } - - void ExpressionCoerceToBool::addToBsonArray( - BSONArrayBuilder *pBuilder) const { - // Serializing as an $and expression which will become a CoerceToBool - BSONObjBuilder sub (pBuilder->subobjStart()); - BSONArrayBuilder arr (sub.subarrayStart("$and")); - pExpression->addToBsonArray(&arr); - arr.doneFast(); - sub.doneFast(); + Value ExpressionCoerceToBool::serialize(bool explain) const { + // When not explaining, serialize to an $and expression. When parsed, the $and expression + // will be optimized back into a ExpressionCoerceToBool. + const char* name = explain ? "$coerceToBool" : "$and"; + return Value(DOC(name << DOC_ARRAY(pExpression->serialize(explain)))); } /* ----------------------- ExpressionCompare --------------------------- */ - ExpressionCompare::~ExpressionCompare() { - } - - intrusive_ptr ExpressionCompare::createEq() { - intrusive_ptr pExpression( - new ExpressionCompare(EQ)); - return pExpression; - } - - intrusive_ptr ExpressionCompare::createNe() { - intrusive_ptr pExpression( - new ExpressionCompare(NE)); - return pExpression; - } - - intrusive_ptr ExpressionCompare::createGt() { - intrusive_ptr pExpression( - new ExpressionCompare(GT)); - return pExpression; - } - - intrusive_ptr ExpressionCompare::createGte() { - intrusive_ptr pExpression( - new ExpressionCompare(GTE)); - return pExpression; - } - - intrusive_ptr ExpressionCompare::createLt() { - intrusive_ptr pExpression( - new ExpressionCompare(LT)); - return pExpression; - } - - intrusive_ptr ExpressionCompare::createLte() { - intrusive_ptr pExpression( - new ExpressionCompare(LTE)); - return pExpression; - } - - intrusive_ptr ExpressionCompare::createCmp() { - intrusive_ptr pExpression( - new ExpressionCompare(CMP)); - return pExpression; - } - - ExpressionCompare::ExpressionCompare(CmpOp theCmpOp): - ExpressionNary(), - cmpOp(theCmpOp) { - } - - void ExpressionCompare::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(2); - ExpressionNary::addOperand(pExpression); - } + REGISTER_EXPRESSION("$cmp", + boost::bind(ExpressionCompare::parse, _1, _2, ExpressionCompare::CMP)); + REGISTER_EXPRESSION("$eq", + boost::bind(ExpressionCompare::parse, _1, _2, ExpressionCompare::EQ)); + REGISTER_EXPRESSION("$gt", + boost::bind(ExpressionCompare::parse, _1, _2, ExpressionCompare::GT)); + REGISTER_EXPRESSION("$gte", + boost::bind(ExpressionCompare::parse, _1, _2, ExpressionCompare::GTE)); + REGISTER_EXPRESSION("$lt", + boost::bind(ExpressionCompare::parse, _1, _2, ExpressionCompare::LT)); + REGISTER_EXPRESSION("$lte", + boost::bind(ExpressionCompare::parse, _1, _2, ExpressionCompare::LTE)); + REGISTER_EXPRESSION("$ne", + boost::bind(ExpressionCompare::parse, _1, _2, ExpressionCompare::NE)); + intrusive_ptr ExpressionCompare::parse( + BSONElement bsonExpr, + const VariablesParseState& vps, + CmpOp op) { + + intrusive_ptr expr = new ExpressionCompare(op); + ExpressionVector args = parseArguments(bsonExpr, vps); + expr->validateArguments(args); + expr->vpOperand = args; + return expr; + } + + ExpressionCompare::ExpressionCompare(CmpOp theCmpOp) + : cmpOp(theCmpOp) + {} - /* - Lookup table for truth value returns - */ +namespace { + // Lookup table for truth value returns struct CmpLookup { - bool truthValue[3]; /* truth value for -1, 0, 1 */ - Expression::CmpOp reverse; /* reverse comparison operator */ - char name[5]; /* string name (w/trailing '\0') */ + const bool truthValue[3]; // truth value for -1, 0, 1 + const ExpressionCompare::CmpOp reverse; // reverse(b,a) returns the same as op(a,b) + const char name[5]; // string name with trailing '\0' }; static const CmpLookup cmpLookup[7] = { - /* -1 0 1 reverse name */ - /* EQ */ { { false, true, false }, Expression::EQ, "$eq" }, - /* NE */ { { true, false, true }, Expression::NE, "$ne" }, - /* GT */ { { false, false, true }, Expression::LT, "$gt" }, - /* GTE */ { { false, true, true }, Expression::LTE, "$gte" }, - /* LT */ { { true, false, false }, Expression::GT, "$lt" }, - /* LTE */ { { true, true, false }, Expression::GTE, "$lte" }, - /* CMP */ { { false, false, false }, Expression::CMP, "$cmp" }, + /* -1 0 1 reverse name */ + /* EQ */ { { false, true, false }, ExpressionCompare::EQ, "$eq" }, + /* NE */ { { true, false, true }, ExpressionCompare::NE, "$ne" }, + /* GT */ { { false, false, true }, ExpressionCompare::LT, "$gt" }, + /* GTE */ { { false, true, true }, ExpressionCompare::LTE, "$gte" }, + /* LT */ { { true, false, false }, ExpressionCompare::GT, "$lt" }, + /* LTE */ { { true, true, false }, ExpressionCompare::GTE, "$lte" }, + + // CMP is special. Only name is used. + /* CMP */ { { false, false, false }, ExpressionCompare::CMP, "$cmp" }, }; +} - intrusive_ptr ExpressionCompare::optimize() { - /* first optimize the comparison operands */ - intrusive_ptr pE(ExpressionNary::optimize()); - - /* - If the result of optimization is no longer a comparison, there's - nothing more we can do. - */ - ExpressionCompare *pCmp = dynamic_cast(pE.get()); - if (!pCmp) - return pE; - - /* check to see if optimizing comparison operator is supported */ - CmpOp newOp = pCmp->cmpOp; - // CMP and NE cannot use ExpressionFieldRange which is what this optimization uses - if (newOp == CMP || newOp == NE) - return pE; - - /* - There's one localized optimization we recognize: a comparison - between a field and a constant. If we recognize that pattern, - replace it with an ExpressionFieldRange. - - When looking for this pattern, note that the operands could appear - in any order. If we need to reverse the sense of the comparison to - put it into the required canonical form, do so. - */ - intrusive_ptr pLeft(pCmp->vpOperand[0]); - intrusive_ptr pRight(pCmp->vpOperand[1]); - intrusive_ptr pFieldPath( - dynamic_pointer_cast(pLeft)); - intrusive_ptr pConstant; - if (pFieldPath.get()) { - pConstant = dynamic_pointer_cast(pRight); - if (!pConstant.get()) - return pE; // there's nothing more we can do - } - else { - /* if the first operand wasn't a path, see if it's a constant */ - pConstant = dynamic_pointer_cast(pLeft); - if (!pConstant.get()) - return pE; // there's nothing more we can do + Value ExpressionCompare::evaluateInternal(Variables* vars) const { + Value pLeft(vpOperand[0]->evaluateInternal(vars)); + Value pRight(vpOperand[1]->evaluateInternal(vars)); - /* the left operand was a constant; see if the right is a path */ - pFieldPath = dynamic_pointer_cast(pRight); - if (!pFieldPath.get()) - return pE; // there's nothing more we can do + int cmp = Value::compare(pLeft, pRight); - /* these were not in canonical order, so reverse the sense */ - newOp = cmpLookup[newOp].reverse; + // Make cmp one of 1, 0, or -1. + if (cmp == 0) { + // leave as 0 + } else if (cmp < 0) { + cmp = -1; + } else if (cmp > 0) { + cmp = 1; } - return ExpressionFieldRange::create( - pFieldPath, newOp, pConstant->getValue()); - } - - Value ExpressionCompare::evaluate(const Document& pDocument) const { - checkArgCount(2); - Value pLeft(vpOperand[0]->evaluate(pDocument)); - Value pRight(vpOperand[1]->evaluate(pDocument)); - - int cmp = signum(Value::compare(pLeft, pRight)); - - if (cmpOp == CMP) { - switch(cmp) { - case -1: - case 0: - case 1: - return Value(cmp); - - default: - verify(false); // CW TODO internal error - } - } + if (cmpOp == CMP) + return Value(cmp); bool returnValue = cmpLookup[cmpOp].truthValue[cmp + 1]; return Value(returnValue); @@ -728,19 +687,12 @@ namespace mongo { /* ------------------------- ExpressionConcat ----------------------------- */ - ExpressionConcat::~ExpressionConcat() { - } - - intrusive_ptr ExpressionConcat::create() { - return new ExpressionConcat(); - } - - Value ExpressionConcat::evaluate(const Document& input) const { + Value ExpressionConcat::evaluateInternal(Variables* vars) const { const size_t n = vpOperand.size(); StringBuilder result; for (size_t i = 0; i < n; ++i) { - Value val = vpOperand[i]->evaluate(input); + Value val = vpOperand[i]->evaluateInternal(vars); if (val.nullish()) return Value(BSONNULL); @@ -751,59 +703,71 @@ namespace mongo { result << val.coerceToString(); } - return Value::createString(result.str()); + return Value(result.str()); } + REGISTER_EXPRESSION("$concat", ExpressionConcat::parse); const char *ExpressionConcat::getOpName() const { return "$concat"; } /* ----------------------- ExpressionCond ------------------------------ */ - ExpressionCond::~ExpressionCond() { + Value ExpressionCond::evaluateInternal(Variables* vars) const { + Value pCond(vpOperand[0]->evaluateInternal(vars)); + int idx = pCond.coerceToBool() ? 1 : 2; + return vpOperand[idx]->evaluateInternal(vars); } - intrusive_ptr ExpressionCond::create() { - intrusive_ptr pExpression(new ExpressionCond()); - return pExpression; - } + intrusive_ptr ExpressionCond::parse( + BSONElement expr, + const VariablesParseState& vps) { - ExpressionCond::ExpressionCond(): - ExpressionNary() { - } + if (expr.type() != Object) { + return Base::parse(expr, vps); + } + verify(str::equals(expr.fieldName(), "$cond")); - void ExpressionCond::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(3); - ExpressionNary::addOperand(pExpression); - } + intrusive_ptr ret = new ExpressionCond(); + ret->vpOperand.resize(3); - Value ExpressionCond::evaluate(const Document& pDocument) const { - checkArgCount(3); - Value pCond(vpOperand[0]->evaluate(pDocument)); - int idx = pCond.coerceToBool() ? 1 : 2; - return vpOperand[idx]->evaluate(pDocument); + const BSONObj args = expr.embeddedObject(); + BSONForEach(arg, args) { + if (str::equals(arg.fieldName(), "if")) { + ret->vpOperand[0] = parseOperand(arg, vps); + } else if (str::equals(arg.fieldName(), "then")) { + ret->vpOperand[1] = parseOperand(arg, vps); + } else if (str::equals(arg.fieldName(), "else")) { + ret->vpOperand[2] = parseOperand(arg, vps); + } else { + uasserted(17083, str::stream() + << "Unrecognized parameter to $cond: " << arg.fieldName()); + } + } + + uassert(17080, "Missing 'if' parameter to $cond", + ret->vpOperand[0]); + uassert(17081, "Missing 'then' parameter to $cond", + ret->vpOperand[1]); + uassert(17082, "Missing 'else' parameter to $cond", + ret->vpOperand[2]); + + return ret; } + REGISTER_EXPRESSION("$cond", ExpressionCond::parse); const char *ExpressionCond::getOpName() const { return "$cond"; } /* ---------------------- ExpressionConstant --------------------------- */ - ExpressionConstant::~ExpressionConstant() { - } - - intrusive_ptr ExpressionConstant::createFromBsonElement( - BSONElement *pBsonElement) { - intrusive_ptr pEC( - new ExpressionConstant(pBsonElement)); - return pEC; + intrusive_ptr ExpressionConstant::parse( + BSONElement exprElement, + const VariablesParseState& vps) { + return new ExpressionConstant(Value(exprElement)); } - ExpressionConstant::ExpressionConstant(BSONElement *pBsonElement): - pValue(Value::createFromBsonElement(pBsonElement)) { - } intrusive_ptr ExpressionConstant::create(const Value& pValue) { intrusive_ptr pEC(new ExpressionConstant(pValue)); @@ -818,176 +782,68 @@ namespace mongo { return intrusive_ptr(this); } - void ExpressionConstant::addDependencies(set& deps, vector* path) const { + void ExpressionConstant::addDependencies(DepsTracker* deps, vector* path) const { /* nothing to do */ } - Value ExpressionConstant::evaluate(const Document& pDocument) const { + Value ExpressionConstant::evaluateInternal(Variables* vars) const { return pValue; } - void ExpressionConstant::addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const { - /* - If we don't need an expression, but can use a naked scalar, - do the regular thing. - - This is geared to handle $project, which uses expressions as a cue - that the field is a new virtual field rather than just an - inclusion (or exclusion): - { $project : { - x : true, // include - y : { $const: true } - }} - - This can happen as a result of optimizations. For example, the - above may have originally been - { $project : { - x : true, // include - y : { $eq:["foo", "foo"] } - }} - When this is optimized, the $eq will be replaced with true. However, - if the pipeline is rematerialized (as happens for a split for - sharding) and sent to another node, it will now have - y : true - which will look like an inclusion rather than a computed field. - */ - if (!requireExpression) { - pValue.addToBsonObj(pBuilder, fieldName); - return; - } - - // We require an expression, so build one here, and use that. - BSONObjBuilder constBuilder (pBuilder->subobjStart(fieldName)); - pValue.addToBsonObj(&constBuilder, getOpName()); - constBuilder.done(); - } - - void ExpressionConstant::addToBsonArray( - BSONArrayBuilder *pBuilder) const { - pValue.addToBsonArray(pBuilder); + Value ExpressionConstant::serialize(bool explain) const { + return serializeConstant(pValue); } + REGISTER_EXPRESSION("$const", ExpressionConstant::parse); + REGISTER_EXPRESSION("$literal", ExpressionConstant::parse); // alias const char *ExpressionConstant::getOpName() const { return "$const"; } /* ---------------------- ExpressionDayOfMonth ------------------------- */ - ExpressionDayOfMonth::~ExpressionDayOfMonth() { - } - - intrusive_ptr ExpressionDayOfMonth::create() { - intrusive_ptr pExpression(new ExpressionDayOfMonth()); - return pExpression; - } - - ExpressionDayOfMonth::ExpressionDayOfMonth(): - ExpressionNary() { - } - - void ExpressionDayOfMonth::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionDayOfMonth::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionDayOfMonth::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_mday); + return Value(date.tm_mday); } + REGISTER_EXPRESSION("$dayOfMonth", ExpressionDayOfMonth::parse); const char *ExpressionDayOfMonth::getOpName() const { return "$dayOfMonth"; } /* ------------------------- ExpressionDayOfWeek ----------------------------- */ - ExpressionDayOfWeek::~ExpressionDayOfWeek() { - } - - intrusive_ptr ExpressionDayOfWeek::create() { - intrusive_ptr pExpression(new ExpressionDayOfWeek()); - return pExpression; - } - - ExpressionDayOfWeek::ExpressionDayOfWeek(): - ExpressionNary() { - } - - void ExpressionDayOfWeek::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionDayOfWeek::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionDayOfWeek::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_wday+1); // MySQL uses 1-7 tm uses 0-6 + return Value(date.tm_wday+1); // MySQL uses 1-7 tm uses 0-6 } + REGISTER_EXPRESSION("$dayOfWeek", ExpressionDayOfWeek::parse); const char *ExpressionDayOfWeek::getOpName() const { return "$dayOfWeek"; } /* ------------------------- ExpressionDayOfYear ----------------------------- */ - ExpressionDayOfYear::~ExpressionDayOfYear() { - } - - intrusive_ptr ExpressionDayOfYear::create() { - intrusive_ptr pExpression(new ExpressionDayOfYear()); - return pExpression; - } - - ExpressionDayOfYear::ExpressionDayOfYear(): - ExpressionNary() { - } - - void ExpressionDayOfYear::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionDayOfYear::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionDayOfYear::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_yday+1); // MySQL uses 1-366 tm uses 0-365 + return Value(date.tm_yday+1); // MySQL uses 1-366 tm uses 0-365 } + REGISTER_EXPRESSION("$dayOfYear", ExpressionDayOfYear::parse); const char *ExpressionDayOfYear::getOpName() const { return "$dayOfYear"; } /* ----------------------- ExpressionDivide ---------------------------- */ - ExpressionDivide::~ExpressionDivide() { - } - - intrusive_ptr ExpressionDivide::create() { - intrusive_ptr pExpression(new ExpressionDivide()); - return pExpression; - } - - ExpressionDivide::ExpressionDivide(): - ExpressionNary() { - } - - void ExpressionDivide::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(2); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionDivide::evaluate(const Document& pDocument) const { - checkArgCount(2); - Value lhs = vpOperand[0]->evaluate(pDocument); - Value rhs = vpOperand[1]->evaluate(pDocument); + Value ExpressionDivide::evaluateInternal(Variables* vars) const { + Value lhs = vpOperand[0]->evaluateInternal(vars); + Value rhs = vpOperand[1]->evaluateInternal(vars); if (lhs.numeric() && rhs.numeric()) { double numer = lhs.coerceToDouble(); @@ -995,7 +851,7 @@ namespace mongo { uassert(16608, "can't $divide by zero", denom != 0); - return Value::createDouble(numer / denom); + return Value(numer / denom); } else if (lhs.nullish() || rhs.nullish()) { return Value(BSONNULL); @@ -1008,25 +864,28 @@ namespace mongo { } } + REGISTER_EXPRESSION("$divide", ExpressionDivide::parse); const char *ExpressionDivide::getOpName() const { return "$divide"; } /* ---------------------- ExpressionObject --------------------------- */ - ExpressionObject::~ExpressionObject() { - } - intrusive_ptr ExpressionObject::create() { - intrusive_ptr pExpression(new ExpressionObject()); - return pExpression; + return new ExpressionObject(false); } - ExpressionObject::ExpressionObject(): _excludeId(false) { + intrusive_ptr ExpressionObject::createRoot() { + return new ExpressionObject(true); } + ExpressionObject::ExpressionObject(bool atRoot) + : _excludeId(false) + , _atRoot(atRoot) + {} + intrusive_ptr ExpressionObject::optimize() { - for (ExpressionMap::iterator it(_expressions.begin()); it!=_expressions.end(); ++it) { + for (FieldMap::iterator it(_expressions.begin()); it!=_expressions.end(); ++it) { if (it->second) it->second = it->second->optimize(); } @@ -1035,20 +894,20 @@ namespace mongo { } bool ExpressionObject::isSimple() { - for (ExpressionMap::iterator it(_expressions.begin()); it!=_expressions.end(); ++it) { + for (FieldMap::iterator it(_expressions.begin()); it!=_expressions.end(); ++it) { if (it->second && !it->second->isSimple()) return false; } return true; } - void ExpressionObject::addDependencies(set& deps, vector* path) const { + void ExpressionObject::addDependencies(DepsTracker* deps, vector* path) const { string pathStr; if (path) { if (path->empty()) { // we are in the top level of a projection so _id is implicit if (!_excludeId) - deps.insert("_id"); + deps->fields.insert("_id"); } else { FieldPath f (*path); @@ -1061,7 +920,7 @@ namespace mongo { } - for (ExpressionMap::const_iterator it(_expressions.begin()); it!=_expressions.end(); ++it) { + for (FieldMap::const_iterator it(_expressions.begin()); it!=_expressions.end(); ++it) { if (it->second) { if (path) path->push_back(it->first); it->second->addDependencies(deps, path); @@ -1071,35 +930,33 @@ namespace mongo { uassert(16407, "inclusion not supported in objects nested in $expressions", path); - deps.insert(pathStr + it->first); + deps->fields.insert(pathStr + it->first); } } } void ExpressionObject::addToDocument( MutableDocument& out, - const Document& pDocument, - const Document& rootDoc + const Document& currentDoc, + Variables* vars ) const { - const bool atRoot = (pDocument == rootDoc); - - ExpressionMap::const_iterator end = _expressions.end(); + FieldMap::const_iterator end = _expressions.end(); // This is used to mark fields we've done so that we can add the ones we haven't set doneFields; - FieldIterator fields(pDocument); + FieldIterator fields(currentDoc); while(fields.more()) { Document::FieldPair field (fields.next()); // TODO don't make a new string here const string fieldName = field.first.toString(); - ExpressionMap::const_iterator exprIter = _expressions.find(fieldName); + FieldMap::const_iterator exprIter = _expressions.find(fieldName); // This field is not supposed to be in the output (unless it is _id) if (exprIter == end) { - if (!_excludeId && atRoot && field.first == "_id") { + if (!_excludeId && _atRoot && field.first == "_id") { // _id from the root doc is always included (until exclusion is supported) // not updating doneFields since "_id" isn't in _expressions out.addField(field.first, field.second); @@ -1123,10 +980,10 @@ namespace mongo { if ((valueType != Object && valueType != Array) || !exprObj ) { // This expression replace the whole field - Value pValue(expr->evaluate(rootDoc)); + Value pValue(expr->evaluateInternal(vars)); // don't add field if nothing was found in the subobject - if (exprObj && pValue.getDocument()->getFieldCount() == 0) + if (exprObj && pValue.getDocument().empty()) continue; /* @@ -1147,8 +1004,8 @@ namespace mongo { */ if (valueType == Object) { MutableDocument sub (exprObj->getSizeHint()); - exprObj->addToDocument(sub, field.second.getDocument(), rootDoc); - out.addField(field.first, Value(sub.freeze())); + exprObj->addToDocument(sub, field.second.getDocument(), vars); + out.addField(field.first, sub.freezeToValue()); } else if (valueType == Array) { /* @@ -1164,11 +1021,11 @@ namespace mongo { continue; MutableDocument doc (exprObj->getSizeHint()); - exprObj->addToDocument(doc, input[i].getDocument(), rootDoc); - result.push_back(Value(doc.freeze())); + exprObj->addToDocument(doc, input[i].getDocument(), vars); + result.push_back(doc.freezeToValue()); } - out.addField(field.first, Value(result)); + out.addField(field.first, Value::consume(result)); } else { verify( false ); @@ -1180,7 +1037,7 @@ namespace mongo { /* add any remaining fields we haven't already taken care of */ for (vector::const_iterator i(_order.begin()); i!=_order.end(); ++i) { - ExpressionMap::const_iterator it = _expressions.find(*i); + FieldMap::const_iterator it = _expressions.find(*i); string fieldName(it->first); /* if we've already dealt with this field, above, do nothing */ @@ -1191,7 +1048,7 @@ namespace mongo { if (!it->second) continue; - Value pValue(it->second->evaluate(rootDoc)); + Value pValue(it->second->evaluateInternal(vars)); /* Don't add non-existent values (note: different from NULL or Undefined); @@ -1203,7 +1060,7 @@ namespace mongo { // don't add field if nothing was found in the subobject if (dynamic_cast(it->second.get()) - && pValue.getDocument()->getFieldCount() == 0) + && pValue.getDocument().empty()) continue; @@ -1216,18 +1073,18 @@ namespace mongo { return _expressions.size() + (_excludeId ? 0 : 1); } - Document ExpressionObject::evaluateDocument(const Document& pDocument) const { + Document ExpressionObject::evaluateDocument(Variables* vars) const { /* create and populate the result */ MutableDocument out (getSizeHint()); addToDocument(out, Document(), // No inclusion field matching. - pDocument); + vars); return out.freeze(); } - Value ExpressionObject::evaluate(const Document& pDocument) const { - return Value::createDocument(evaluateDocument(pDocument)); + Value ExpressionObject::evaluateInternal(Variables* vars) const { + return Value(evaluateDocument(vars)); } void ExpressionObject::addField(const FieldPath &fieldPath, @@ -1288,66 +1145,77 @@ namespace mongo { addField(theFieldPath, NULL); } - void ExpressionObject::documentToBson(BSONObjBuilder *pBuilder, bool requireExpression) const { + Value ExpressionObject::serialize(bool explain) const { + MutableDocument valBuilder; if (_excludeId) - pBuilder->appendBool("_id", false); + valBuilder["_id"] = Value(false); for (vector::const_iterator it(_order.begin()); it!=_order.end(); ++it) { string fieldName = *it; verify(_expressions.find(fieldName) != _expressions.end()); intrusive_ptr expr = _expressions.find(fieldName)->second; - + if (!expr) { // this is inclusion, not an expression - pBuilder->appendBool(fieldName, true); + valBuilder[fieldName] = Value(true); } else { - expr->addToBsonObj(pBuilder, fieldName, requireExpression); + valBuilder[fieldName] = expr->serialize(explain); } } + return valBuilder.freezeToValue(); } - void ExpressionObject::addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const { + /* --------------------- ExpressionFieldPath --------------------------- */ - BSONObjBuilder objBuilder (pBuilder->subobjStart(fieldName)); - documentToBson(&objBuilder, requireExpression); - objBuilder.done(); + // this is the old deprecated version only used by tests not using variables + intrusive_ptr ExpressionFieldPath::create(const string& fieldPath) { + return new ExpressionFieldPath("CURRENT." + fieldPath, Variables::ROOT_ID); } - void ExpressionObject::addToBsonArray( - BSONArrayBuilder *pBuilder) const { + // this is the new version that supports every syntax + intrusive_ptr ExpressionFieldPath::parse( + const string& raw, + const VariablesParseState& vps) { - BSONObjBuilder objBuilder (pBuilder->subobjStart()); - documentToBson(&objBuilder, false); - objBuilder.done(); - } + uassert(16873, str::stream() << "FieldPath '" << raw << "' doesn't start with $", + raw.c_str()[0] == '$'); // c_str()[0] is always a valid reference. - /* --------------------- ExpressionFieldPath --------------------------- */ + uassert(16872, str::stream() << "'$' by itself is not a valid FieldPath", + raw.size() >= 2); // need at least "$" and either "$" or a field name - ExpressionFieldPath::~ExpressionFieldPath() { + if (raw[1] == '$') { + const StringData rawSD = raw; + const StringData fieldPath = rawSD.substr(2); // strip off $$ + const StringData varName = fieldPath.substr(0, fieldPath.find('.')); + Variables::uassertValidNameForUserRead(varName); + return new ExpressionFieldPath(fieldPath.toString(), vps.getVariable(varName)); + } + else { + return new ExpressionFieldPath("CURRENT." + raw.substr(1), // strip the "$" prefix + vps.getVariable("CURRENT")); + } } - intrusive_ptr ExpressionFieldPath::create( - const string &fieldPath) { - intrusive_ptr pExpression( - new ExpressionFieldPath(fieldPath)); - return pExpression; - } - ExpressionFieldPath::ExpressionFieldPath( - const string &theFieldPath): - fieldPath(theFieldPath) { - } + ExpressionFieldPath::ExpressionFieldPath(const string& theFieldPath, Variables::Id variable) + : _fieldPath(theFieldPath) + , _variable(variable) + {} intrusive_ptr ExpressionFieldPath::optimize() { /* nothing can be done for these */ return intrusive_ptr(this); } - void ExpressionFieldPath::addDependencies(set& deps, vector* path) const { - deps.insert(fieldPath.getPath(false)); + void ExpressionFieldPath::addDependencies(DepsTracker* deps, vector* path) const { + if (_variable == Variables::ROOT_ID) { // includes CURRENT when it is equivalent to ROOT. + if (_fieldPath.getPathLength() == 1) { + deps->needWholeDocument = true; // need full doc if just "$$ROOT" + } else { + deps->fields.insert(_fieldPath.tail().getPath(false)); + } + } } Value ExpressionFieldPath::evaluatePathArray(size_t index, const Value& input) const { @@ -1365,18 +1233,18 @@ namespace mongo { result.push_back(nested); } - return Value::createArray(result); + return Value::consume(result); } Value ExpressionFieldPath::evaluatePath(size_t index, const Document& input) const { // Note this function is very hot so it is important that is is well optimized. // In particular, all return paths should support RVO. /* if we've hit the end of the path, stop */ - if (index == fieldPath.getPathLength() - 1) - return input[fieldPath.getFieldName(index)]; + if (index == _fieldPath.getPathLength() - 1) + return input[_fieldPath.getFieldName(index)]; // Try to dive deeper - const Value val = input[fieldPath.getFieldName(index)]; + const Value val = input[_fieldPath.getFieldName(index)]; switch (val.getType()) { case Object: return evaluatePath(index+1, val.getDocument()); @@ -1389,420 +1257,312 @@ namespace mongo { } } - Value ExpressionFieldPath::evaluate(const Document& pDocument) const { - return evaluatePath(0, pDocument); - } + Value ExpressionFieldPath::evaluateInternal(Variables* vars) const { + if (_fieldPath.getPathLength() == 1) // get the whole variable + return vars->getValue(_variable); - void ExpressionFieldPath::addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const { - pBuilder->append(fieldName, fieldPath.getPath(true)); + if (_variable == Variables::ROOT_ID) { + // ROOT is always a document so use optimized code path + return evaluatePath(1, vars->getRoot()); + } + + Value var = vars->getValue(_variable); + switch (var.getType()) { + case Object: return evaluatePath(1, var.getDocument()); + case Array: return evaluatePathArray(1, var); + default: return Value(); + } } - void ExpressionFieldPath::addToBsonArray( - BSONArrayBuilder *pBuilder) const { - pBuilder->append(getFieldPath(true)); + Value ExpressionFieldPath::serialize(bool explain) const { + if (_fieldPath.getFieldName(0) == "CURRENT" && _fieldPath.getPathLength() > 1) { + // use short form for "$$CURRENT.foo" but not just "$$CURRENT" + return Value("$" + _fieldPath.tail().getPath(false)); + } + else { + return Value("$$" + _fieldPath.getPath(false)); + } } - /* --------------------- ExpressionFieldRange -------------------------- */ + /* ------------------------- ExpressionLet ----------------------------- */ - ExpressionFieldRange::~ExpressionFieldRange() { - } + REGISTER_EXPRESSION("$let", ExpressionLet::parse); + intrusive_ptr ExpressionLet::parse( + BSONElement expr, + const VariablesParseState& vpsIn) { - intrusive_ptr ExpressionFieldRange::optimize() { - /* if there is no range to match, this will never evaluate true */ - if (!pRange.get()) - return ExpressionConstant::create(Value(false)); + verify(str::equals(expr.fieldName(), "$let")); - /* - If we ended up with a double un-ended range, anything matches. I - don't know how that can happen, given intersect()'s interface, but - here it is, just in case. - */ - if (pRange->pBottom.missing() && pRange->pTop.missing()) - return ExpressionConstant::create(Value(true)); + uassert(16874, "$let only supports an object as it's argument", + expr.type() == Object); + const BSONObj args = expr.embeddedObject(); - /* - In all other cases, we have to test candidate values. The - intersect() method has already optimized those tests, so there - aren't any more optimizations to look for here. - */ - return intrusive_ptr(this); - } + // varsElem must be parsed before inElem regardless of BSON order. + BSONElement varsElem; + BSONElement inElem; + BSONForEach(arg, args) { + if (str::equals(arg.fieldName(), "vars")) { + varsElem = arg; + } else if (str::equals(arg.fieldName(), "in")) { + inElem = arg; + } else { + uasserted(16875, str::stream() + << "Unrecognized parameter to $let: " << arg.fieldName()); + } + } - void ExpressionFieldRange::addDependencies(set& deps, vector* path) const { - pFieldPath->addDependencies(deps); - } + uassert(16876, "Missing 'vars' parameter to $let", + !varsElem.eoo()); + uassert(16877, "Missing 'in' parameter to $let", + !inElem.eoo()); - Value ExpressionFieldRange::evaluate(const Document& pDocument) const { - /* if there's no range, there can't be a match */ - if (!pRange.get()) - return Value(false); + // parse "vars" + VariablesParseState vpsSub(vpsIn); // vpsSub gets our vars, vpsIn doesn't. + VariableMap vars; + BSONForEach(varElem, varsElem.embeddedObjectUserCheck()) { + const string varName = varElem.fieldName(); + Variables::uassertValidNameForUserWrite(varName); + Variables::Id id = vpsSub.defineVariable(varName); - /* get the value of the specified field */ - Value pValue(pFieldPath->evaluate(pDocument)); + vars[id] = NameAndExpression(varName, + parseOperand(varElem, vpsIn)); // only has outer vars + } - /* see if it fits within any of the ranges */ - if (pRange->contains(pValue)) - return Value(true); + // parse "in" + intrusive_ptr subExpression = parseOperand(inElem, vpsSub); // has our vars - return Value(false); + return new ExpressionLet(vars, subExpression); } - void ExpressionFieldRange::addToBson(Builder *pBuilder) const { - if (!pRange.get()) { - /* nothing will satisfy this predicate */ - pBuilder->append(false); - return; + ExpressionLet::ExpressionLet(const VariableMap& vars, intrusive_ptr subExpression) + : _variables(vars) + , _subExpression(subExpression) + {} + + intrusive_ptr ExpressionLet::optimize() { + if (_variables.empty()) { + // we aren't binding any variables so just return the subexpression + return _subExpression->optimize(); } - if (pRange->pTop.missing() && pRange->pBottom.missing()) { - /* any value will satisfy this predicate */ - pBuilder->append(true); - return; + for (VariableMap::iterator it=_variables.begin(), end=_variables.end(); it != end; ++it) { + it->second.expression = it->second.expression->optimize(); } - // FIXME Append constant values using the $const operator. SERVER-6769 + // TODO be smarter with constant "variables" + _subExpression = _subExpression->optimize(); - // FIXME This checks pointer equality not value equality. - if (pRange->pTop == pRange->pBottom) { - BSONArrayBuilder operands; - pFieldPath->addToBsonArray(&operands); - pRange->pTop.addToBsonArray(&operands); - - BSONObjBuilder equals; - equals.append("$eq", operands.arr()); - pBuilder->append(&equals); - return; + return this; + } + + Value ExpressionLet::serialize(bool explain) const { + MutableDocument vars; + for (VariableMap::const_iterator it=_variables.begin(), end=_variables.end(); + it != end; ++it) { + vars[it->second.name] = it->second.expression->serialize(explain); } - BSONObjBuilder leftOperator; - if (!pRange->pBottom.missing()) { - BSONArrayBuilder leftOperands; - pFieldPath->addToBsonArray(&leftOperands); - pRange->pBottom.addToBsonArray(&leftOperands); - leftOperator.append( - (pRange->bottomOpen ? "$gt" : "$gte"), - leftOperands.arr()); + return Value(DOC("$let" << DOC("vars" << vars.freeze() + << "in" << _subExpression->serialize(explain)) + )); + } - if (pRange->pTop.missing()) { - pBuilder->append(&leftOperator); - return; - } + Value ExpressionLet::evaluateInternal(Variables* vars) const { + for (VariableMap::const_iterator it=_variables.begin(), end=_variables.end(); + it != end; ++it) { + // It is guaranteed at parse-time that these expressions don't use the variable ids we + // are setting + vars->setValue(it->first, + it->second.expression->evaluateInternal(vars)); } - BSONObjBuilder rightOperator; - if (!pRange->pTop.missing()) { - BSONArrayBuilder rightOperands; - pFieldPath->addToBsonArray(&rightOperands); - pRange->pTop.addToBsonArray(&rightOperands); - rightOperator.append( - (pRange->topOpen ? "$lt" : "$lte"), - rightOperands.arr()); + return _subExpression->evaluateInternal(vars); + } - if (pRange->pBottom.missing()) { - pBuilder->append(&rightOperator); - return; - } + void ExpressionLet::addDependencies(DepsTracker* deps, vector* path) const { + for (VariableMap::const_iterator it=_variables.begin(), end=_variables.end(); + it != end; ++it) { + it->second.expression->addDependencies(deps); } - BSONArrayBuilder andOperands; - andOperands.append(leftOperator.done()); - andOperands.append(rightOperator.done()); - BSONObjBuilder andOperator; - andOperator.append("$and", andOperands.arr()); - pBuilder->append(&andOperator); + // TODO be smarter when CURRENT is a bound variable + _subExpression->addDependencies(deps); } - void ExpressionFieldRange::addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const { - BuilderObj builder(pBuilder, fieldName); - addToBson(&builder); - } - void ExpressionFieldRange::addToBsonArray( - BSONArrayBuilder *pBuilder) const { - BuilderArray builder(pBuilder); - addToBson(&builder); - } + /* ------------------------- ExpressionMap ----------------------------- */ - void ExpressionFieldRange::toMatcherBson( - BSONObjBuilder *pBuilder) const { - verify(pRange.get()); // otherwise, we can't do anything + REGISTER_EXPRESSION("$map", ExpressionMap::parse); + intrusive_ptr ExpressionMap::parse( + BSONElement expr, + const VariablesParseState& vpsIn) { - /* if there are no endpoints, then every value is accepted */ - if (pRange->pBottom.missing() && pRange->pTop.missing()) - return; // nothing to add to the predicate + verify(str::equals(expr.fieldName(), "$map")); - /* we're going to need the field path */ - string fieldPath(pFieldPath->getFieldPath(false)); + uassert(16878, "$map only supports an object as it's argument", + expr.type() == Object); - BSONObjBuilder range; - if (!pRange->pBottom.missing()) { - /* the test for equality doesn't generate a subobject */ - if (pRange->pBottom == pRange->pTop) { - pRange->pBottom.addToBsonObj(pBuilder, fieldPath); - return; + // "in" must be parsed after "as" regardless of BSON order + BSONElement inputElem; + BSONElement asElem; + BSONElement inElem; + const BSONObj args = expr.embeddedObject(); + BSONForEach(arg, args) { + if (str::equals(arg.fieldName(), "input")) { + inputElem = arg; + } else if (str::equals(arg.fieldName(), "as")) { + asElem = arg; + } else if (str::equals(arg.fieldName(), "in")) { + inElem = arg; + } else { + uasserted(16879, str::stream() + << "Unrecognized parameter to $map: " << arg.fieldName()); } - - pRange->pBottom.addToBsonObj( - pBuilder, (pRange->bottomOpen ? "$gt" : "$gte")); } - if (!pRange->pTop.missing()) { - pRange->pTop.addToBsonObj( - pBuilder, (pRange->topOpen ? "$lt" : "$lte")); - } + uassert(16880, "Missing 'input' parameter to $map", + !inputElem.eoo()); + uassert(16881, "Missing 'as' parameter to $map", + !asElem.eoo()); + uassert(16882, "Missing 'in' parameter to $map", + !inElem.eoo()); - pBuilder->append(fieldPath, range.done()); - } + // parse "input" + intrusive_ptr input = parseOperand(inputElem, vpsIn); // only has outer vars - intrusive_ptr ExpressionFieldRange::create( - const intrusive_ptr &pFieldPath, CmpOp cmpOp, - const Value& pValue) { - intrusive_ptr pE( - new ExpressionFieldRange(pFieldPath, cmpOp, pValue)); - return pE; - } - - ExpressionFieldRange::ExpressionFieldRange( - const intrusive_ptr &pTheFieldPath, CmpOp cmpOp, - const Value& pValue): - pFieldPath(pTheFieldPath), - pRange(new Range(cmpOp, pValue)) { - } + // parse "as" + VariablesParseState vpsSub(vpsIn); // vpsSub gets our vars, vpsIn doesn't. + string varName = asElem.str(); + Variables::uassertValidNameForUserWrite(varName); + Variables::Id varId = vpsSub.defineVariable(varName); - void ExpressionFieldRange::intersect(CmpOp cmpOp, const Value& pValue) { + // parse "in" + intrusive_ptr in = parseOperand(inElem, vpsSub); // has access to map variable - /* create the new range */ - scoped_ptr pNew(new Range(cmpOp, pValue)); - - /* - Go through the range list. For every range, either add the - intersection of that to the range list, or if there is none, the - original range. This has the effect of restricting overlapping - ranges, but leaving non-overlapping ones as-is. - */ - pRange.reset(pRange->intersect(pNew.get())); + return new ExpressionMap(varName, varId, input, in); } - ExpressionFieldRange::Range::Range(CmpOp cmpOp, const Value& pValue): - bottomOpen(false), - topOpen(false), - pBottom(), - pTop() { - switch(cmpOp) { - - case EQ: - pBottom = pTop = pValue; - break; - - case GT: - bottomOpen = true; - /* FALLTHROUGH */ - case GTE: - topOpen = true; - pBottom = pValue; - break; - - case LT: - topOpen = true; - /* FALLTHROUGH */ - case LTE: - bottomOpen = true; - pTop = pValue; - break; - - case NE: - case CMP: - verify(false); // not allowed - break; - } - } + ExpressionMap::ExpressionMap(const string& varName, + Variables::Id varId, + intrusive_ptr input, + intrusive_ptr each) + : _varName(varName) + , _varId(varId) + , _input(input) + , _each(each) + {} - ExpressionFieldRange::Range::Range(const Range &rRange): - bottomOpen(rRange.bottomOpen), - topOpen(rRange.topOpen), - pBottom(rRange.pBottom), - pTop(rRange.pTop) { + intrusive_ptr ExpressionMap::optimize() { + // TODO handle when _input is constant + _input = _input->optimize(); + _each = _each->optimize(); + return this; } - ExpressionFieldRange::Range::Range( - const Value& pTheBottom, bool theBottomOpen, - const Value& pTheTop, bool theTopOpen): - bottomOpen(theBottomOpen), - topOpen(theTopOpen), - pBottom(pTheBottom), - pTop(pTheTop) { + Value ExpressionMap::serialize(bool explain) const { + return Value(DOC("$map" << DOC("input" << _input->serialize(explain) + << "as" << _varName + << "in" << _each->serialize(explain) + ))); } - ExpressionFieldRange::Range *ExpressionFieldRange::Range::intersect( - const Range *pRange) const { - /* - Find the max of the bottom end of the ranges. + Value ExpressionMap::evaluateInternal(Variables* vars) const { + // guaranteed at parse time that this isn't using our _varId + const Value inputVal = _input->evaluateInternal(vars); + if (inputVal.nullish()) + return Value(BSONNULL); - Start by assuming the maximum is from pRange. Then, if we have - values of our own, see if they're greater. - */ - Value pMaxBottom(pRange->pBottom); - bool maxBottomOpen = pRange->bottomOpen; - if (!pBottom.missing()) { - if (pRange->pBottom.missing()) { - pMaxBottom = pBottom; - maxBottomOpen = bottomOpen; - } - else { - const int cmp = Value::compare(pBottom, pRange->pBottom); - if (cmp == 0) - maxBottomOpen = bottomOpen || pRange->bottomOpen; - else if (cmp > 0) { - pMaxBottom = pBottom; - maxBottomOpen = bottomOpen; - } - } - } + uassert(16883, str::stream() << "input to $map must be an Array not " + << typeName(inputVal.getType()), + inputVal.getType() == Array); - /* - Find the minimum of the tops of the ranges. + const vector& input = inputVal.getArray(); - Start by assuming the minimum is from pRange. Then, if we have - values of our own, see if they are less. - */ - Value pMinTop(pRange->pTop); - bool minTopOpen = pRange->topOpen; - if (!pTop.missing()) { - if (pRange->pTop.missing()) { - pMinTop = pTop; - minTopOpen = topOpen; - } - else { - const int cmp = Value::compare(pTop, pRange->pTop); - if (cmp == 0) - minTopOpen = topOpen || pRange->topOpen; - else if (cmp < 0) { - pMinTop = pTop; - minTopOpen = topOpen; - } - } - } + if (input.empty()) + return inputVal; - /* - If the intersections didn't create a disjoint set, create the - new range. - */ - if (Value::compare(pMaxBottom, pMinTop) <= 0) - return new Range(pMaxBottom, maxBottomOpen, pMinTop, minTopOpen); + vector output; + output.reserve(input.size()); + for (size_t i=0; i < input.size(); i++) { + vars->setValue(_varId, input[i]); - /* if we got here, the intersection is empty */ - return NULL; - } + Value toInsert = _each->evaluateInternal(vars); + if (toInsert.missing()) + toInsert = Value(BSONNULL); // can't insert missing values into array - bool ExpressionFieldRange::Range::contains(const Value& pValue) const { - if (!pBottom.missing()) { - const int cmp = Value::compare(pValue, pBottom); - if (cmp < 0) - return false; - if (bottomOpen && (cmp == 0)) - return false; + output.push_back(toInsert); } - if (!pTop.missing()) { - const int cmp = Value::compare(pValue, pTop); - if (cmp > 0) - return false; - if (topOpen && (cmp == 0)) - return false; - } + return Value::consume(output); + } - return true; + void ExpressionMap::addDependencies(DepsTracker* deps, vector* path) const { + _input->addDependencies(deps); + _each->addDependencies(deps); } - /* ------------------------- ExpressionMillisecond ----------------------------- */ + /* ------------------------- ExpressionMeta ----------------------------- */ - ExpressionMillisecond::~ExpressionMillisecond() { + REGISTER_EXPRESSION("$meta", ExpressionMeta::parse); + intrusive_ptr ExpressionMeta::parse( + BSONElement expr, + const VariablesParseState& vpsIn) { + + uassert(17307, "$meta only supports String arguments", + expr.type() == String); + uassert(17308, "Unsupported argument to $meta: " + expr.String(), + expr.String() == "textScore"); + + return new ExpressionMeta(); } - intrusive_ptr ExpressionMillisecond::create() { - intrusive_ptr pExpression(new ExpressionMillisecond()); - return pExpression; + Value ExpressionMeta::serialize(bool explain) const { + return Value(DOC("$meta" << "textScore")); } - ExpressionMillisecond::ExpressionMillisecond(): - ExpressionNary() { + Value ExpressionMeta::evaluateInternal(Variables* vars) const { + const Document& root = vars->getRoot(); + return root.hasTextScore() + ? Value(root.getTextScore()) + : Value(); } - void ExpressionMillisecond::addOperand(const intrusive_ptr& pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); + void ExpressionMeta::addDependencies(DepsTracker* deps, vector* path) const { + deps->needTextScore = true; } - Value ExpressionMillisecond::evaluate(const Document& document) const { - checkArgCount(1); - Value date(vpOperand[0]->evaluate(document)); + /* ------------------------- ExpressionMillisecond ----------------------------- */ + + Value ExpressionMillisecond::evaluateInternal(Variables* vars) const { + Value date(vpOperand[0]->evaluateInternal(vars)); const int ms = date.coerceToDate() % 1000LL; - return Value::createInt( ms >= 0 ? ms : 1000 + ms ); + // adding 1000 since dates before 1970 would have negative ms + return Value(ms >= 0 ? ms : 1000 + ms); } + REGISTER_EXPRESSION("$millisecond", ExpressionMillisecond::parse); const char *ExpressionMillisecond::getOpName() const { return "$millisecond"; } /* ------------------------- ExpressionMinute -------------------------- */ - ExpressionMinute::~ExpressionMinute() { - } - - intrusive_ptr ExpressionMinute::create() { - intrusive_ptr pExpression(new ExpressionMinute()); - return pExpression; - } - - ExpressionMinute::ExpressionMinute(): - ExpressionNary() { - } - - void ExpressionMinute::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionMinute::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionMinute::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_min); + return Value(date.tm_min); } + REGISTER_EXPRESSION("$minute", ExpressionMinute::parse); const char *ExpressionMinute::getOpName() const { return "$minute"; } /* ----------------------- ExpressionMod ---------------------------- */ - ExpressionMod::~ExpressionMod() { - } - - intrusive_ptr ExpressionMod::create() { - intrusive_ptr pExpression(new ExpressionMod()); - return pExpression; - } - - ExpressionMod::ExpressionMod(): - ExpressionNary() { - } - - void ExpressionMod::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(2); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionMod::evaluate(const Document& pDocument) const { - checkArgCount(2); - Value lhs = vpOperand[0]->evaluate(pDocument); - Value rhs = vpOperand[1]->evaluate(pDocument); + Value ExpressionMod::evaluateInternal(Variables* vars) const { + Value lhs = vpOperand[0]->evaluateInternal(vars); + Value rhs = vpOperand[1]->evaluateInternal(vars); BSONType leftType = lhs.getType(); BSONType rightType = rhs.getType(); @@ -1821,19 +1581,19 @@ namespace mongo { // Integer-valued double case is handled below double left = lhs.coerceToDouble(); - return Value::createDouble(fmod(left, right)); + return Value(fmod(left, right)); } else if (leftType == NumberLong || rightType == NumberLong) { // if either is long, return long long long left = lhs.coerceToLong(); long long rightLong = rhs.coerceToLong(); - return Value::createLong(left % rightLong); + return Value(left % rightLong); } // lastly they must both be ints, return int int left = lhs.coerceToInt(); int rightInt = rhs.coerceToInt(); - return Value::createInt(left % rightInt); + return Value(left % rightInt); } else if (lhs.nullish() || rhs.nullish()) { return Value(BSONNULL); @@ -1846,55 +1606,27 @@ namespace mongo { } } + REGISTER_EXPRESSION("$mod", ExpressionMod::parse); const char *ExpressionMod::getOpName() const { return "$mod"; } /* ------------------------ ExpressionMonth ----------------------------- */ - ExpressionMonth::~ExpressionMonth() { - } - - intrusive_ptr ExpressionMonth::create() { - intrusive_ptr pExpression(new ExpressionMonth()); - return pExpression; - } - - ExpressionMonth::ExpressionMonth(): - ExpressionNary() { - } - - void ExpressionMonth::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionMonth::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionMonth::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_mon + 1); // MySQL uses 1-12 tm uses 0-11 + return Value(date.tm_mon + 1); // MySQL uses 1-12 tm uses 0-11 } + REGISTER_EXPRESSION("$month", ExpressionMonth::parse); const char *ExpressionMonth::getOpName() const { return "$month"; } /* ------------------------- ExpressionMultiply ----------------------------- */ - ExpressionMultiply::~ExpressionMultiply() { - } - - intrusive_ptr ExpressionMultiply::create() { - intrusive_ptr pExpression(new ExpressionMultiply()); - return pExpression; - } - - ExpressionMultiply::ExpressionMultiply(): - ExpressionNary() { - } - - Value ExpressionMultiply::evaluate(const Document& pDocument) const { + Value ExpressionMultiply::evaluateInternal(Variables* vars) const { /* We'll try to return the narrowest possible result value. To do that without creating intermediate Values, do the arithmetic for double @@ -1907,7 +1639,7 @@ namespace mongo { const size_t n = vpOperand.size(); for(size_t i = 0; i < n; ++i) { - Value val = vpOperand[i]->evaluate(pDocument); + Value val = vpOperand[i]->evaluateInternal(vars); if (val.numeric()) { productType = Value::getWidestNumeric(productType, val.getType()); @@ -1925,328 +1657,168 @@ namespace mongo { } if (productType == NumberDouble) - return Value::createDouble(doubleProduct); + return Value(doubleProduct); else if (productType == NumberLong) - return Value::createLong(longProduct); + return Value(longProduct); else if (productType == NumberInt) return Value::createIntOrLong(longProduct); else massert(16418, "$multiply resulted in a non-numeric type", false); } + REGISTER_EXPRESSION("$multiply", ExpressionMultiply::parse); const char *ExpressionMultiply::getOpName() const { - return "$multiply"; - } - - intrusive_ptr (*ExpressionMultiply::getFactory() const)() { - return ExpressionMultiply::create; + return "$multiply"; } /* ------------------------- ExpressionHour ----------------------------- */ - ExpressionHour::~ExpressionHour() { - } - - intrusive_ptr ExpressionHour::create() { - intrusive_ptr pExpression(new ExpressionHour()); - return pExpression; - } - - ExpressionHour::ExpressionHour(): - ExpressionNary() { - } - - void ExpressionHour::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionHour::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionHour::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_hour); + return Value(date.tm_hour); } + REGISTER_EXPRESSION("$hour", ExpressionHour::parse); const char *ExpressionHour::getOpName() const { return "$hour"; } /* ----------------------- ExpressionIfNull ---------------------------- */ - ExpressionIfNull::~ExpressionIfNull() { - } - - intrusive_ptr ExpressionIfNull::create() { - intrusive_ptr pExpression(new ExpressionIfNull()); - return pExpression; - } - - ExpressionIfNull::ExpressionIfNull(): - ExpressionNary() { - } - - void ExpressionIfNull::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(2); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionIfNull::evaluate(const Document& pDocument) const { - checkArgCount(2); - - Value pLeft(vpOperand[0]->evaluate(pDocument)); + Value ExpressionIfNull::evaluateInternal(Variables* vars) const { + Value pLeft(vpOperand[0]->evaluateInternal(vars)); if (!pLeft.nullish()) return pLeft; - Value pRight(vpOperand[1]->evaluate(pDocument)); + Value pRight(vpOperand[1]->evaluateInternal(vars)); return pRight; } + REGISTER_EXPRESSION("$ifNull", ExpressionIfNull::parse); const char *ExpressionIfNull::getOpName() const { return "$ifNull"; } /* ------------------------ ExpressionNary ----------------------------- */ - ExpressionNary::ExpressionNary(): - vpOperand() { - } - intrusive_ptr ExpressionNary::optimize() { - unsigned constCount = 0; // count of constant operands - unsigned stringCount = 0; // count of constant string operands const size_t n = vpOperand.size(); + + // optimize sub-expressions and count constants + unsigned constCount = 0; for(size_t i = 0; i < n; ++i) { - intrusive_ptr pNew(vpOperand[i]->optimize()); - - /* subsitute the optimized expression */ - vpOperand[i] = pNew; - - /* check to see if the result was a constant */ - const ExpressionConstant *pConst = - dynamic_cast(pNew.get()); - if (pConst) { - ++constCount; - if (pConst->getValue().getType() == String) - ++stringCount; + intrusive_ptr optimized = vpOperand[i]->optimize(); + + // substitute the optimized expression + vpOperand[i] = optimized; + + // check to see if the result was a constant + if (dynamic_cast(optimized.get())) { + constCount++; } } - /* - If all the operands are constant, we can replace this expression - with a constant. We can find the value by evaluating this - expression over a NULL Document because evaluating the - ExpressionConstant never refers to the argument Document. - */ + // If all the operands are constant, we can replace this expression with a constant. Using + // an empty Variables since it will never be accessed. if (constCount == n) { - Value pResult(evaluate(Document())); + Variables emptyVars; + Value pResult(evaluateInternal(&emptyVars)); intrusive_ptr pReplacement( ExpressionConstant::create(pResult)); return pReplacement; } - /* - If there are any strings, we can't re-arrange anything, so stop - now. - - LATER: we could concatenate adjacent strings as a special case. - */ - if (stringCount) - return intrusive_ptr(this); + // Remaining optimizations are only for associative and commutative expressions. + if (!isAssociativeAndCommutative()) + return this; - /* - If there's no more than one constant, then we can't do any - constant folding, so don't bother going any further. - */ - if (constCount <= 1) - return intrusive_ptr(this); - - /* - If the operator isn't commutative or associative, there's nothing - more we can do. We test that by seeing if we can get a factory; - if we can, we can use it to construct a temporary expression which - we'll evaluate to collapse as many constants as we can down to - a single one. - */ - intrusive_ptr (*const pFactory)() = getFactory(); - if (!pFactory) - return intrusive_ptr(this); - - /* - Create a new Expression that will be the replacement for this one. - We actually create two: one to hold constant expressions, and - one to hold non-constants. Once we've got these, we evaluate - the constant expression to produce a single value, as above. - We then add this operand to the end of the non-constant expression, - and return that. - */ - intrusive_ptr pNew((*pFactory)()); - intrusive_ptr pConst((*pFactory)()); - for(size_t i = 0; i < n; ++i) { - intrusive_ptr pE(vpOperand[i]); - if (dynamic_cast(pE.get())) - pConst->addOperand(pE); + // Process vpOperand to split it into constant and nonconstant vectors. + // This can leave vpOperand in an invalid state that is cleaned up after the loop. + ExpressionVector constExprs; + ExpressionVector nonConstExprs; + for(size_t i = 0; i < vpOperand.size(); ++i) { // NOTE: vpOperand grows in loop + intrusive_ptr expr = vpOperand[i]; + if (dynamic_cast(expr.get())) { + constExprs.push_back(expr); + } else { - /* - If the child operand is the same type as this, then we can - extract its operands and inline them here because we already - know this is commutative and associative because it has a - factory. We can detect sameness of the child operator by - checking for equality of the factory - - Note we don't have to do this recursively, because we - called optimize() on all the children first thing in - this call to optimize(). - */ - ExpressionNary *pNary = - dynamic_cast(pE.get()); - if (!pNary) - pNew->addOperand(pE); + // If the child operand is the same type as this, then we can + // extract its operands and inline them here because we know + // this is commutative and associative. We detect sameness of + // the child operator by checking for equality of the opNames + ExpressionNary* nary = dynamic_cast(expr.get()); + if (!nary || !str::equals(nary->getOpName(), getOpName())) { + nonConstExprs.push_back(expr); + } else { - intrusive_ptr (*const pChildFactory)() = - pNary->getFactory(); - if (pChildFactory != pFactory) - pNew->addOperand(pE); - else { - /* same factory, so flatten */ - size_t nChild = pNary->vpOperand.size(); - for(size_t iChild = 0; iChild < nChild; ++iChild) { - intrusive_ptr pCE( - pNary->vpOperand[iChild]); - if (dynamic_cast(pCE.get())) - pConst->addOperand(pCE); - else - pNew->addOperand(pCE); - } - } + // same expression, so flatten by adding to vpOperand which + // will be processed later in this loop. + vpOperand.insert(vpOperand.end(), + nary->vpOperand.begin(), + nary->vpOperand.end()); } } } - /* - If there was only one constant, add it to the end of the expression - operand vector. - */ - if (pConst->vpOperand.size() == 1) - pNew->addOperand(pConst->vpOperand[0]); - else if (pConst->vpOperand.size() > 1) { - /* - If there was more than one constant, collapse all the constants - together before adding the result to the end of the expression - operand vector. - */ - Value pResult(pConst->evaluate(Document())); - pNew->addOperand(ExpressionConstant::create(pResult)); + // collapse all constant expressions (if any) + Value constValue; + if (!constExprs.empty()) { + vpOperand = constExprs; + Variables emptyVars; + constValue = evaluateInternal(&emptyVars); } - return pNew; + // now set the final expression list with constant (if any) at the end + vpOperand = nonConstExprs; + if (!constExprs.empty()) { + vpOperand.push_back(ExpressionConstant::create(constValue)); + } + + return this; } - void ExpressionNary::addDependencies(set& deps, vector* path) const { + void ExpressionNary::addDependencies(DepsTracker* deps, vector* path) const { for(ExpressionVector::const_iterator i(vpOperand.begin()); i != vpOperand.end(); ++i) { (*i)->addDependencies(deps); } } - void ExpressionNary::addOperand( - const intrusive_ptr &pExpression) { + void ExpressionNary::addOperand(const intrusive_ptr& pExpression) { vpOperand.push_back(pExpression); } - intrusive_ptr (*ExpressionNary::getFactory() const)() { - return NULL; - } - - void ExpressionNary::toBson(BSONObjBuilder *pBuilder, const char *pOpName) const { + Value ExpressionNary::serialize(bool explain) const { const size_t nOperand = vpOperand.size(); - + vector array; /* build up the array */ - BSONArrayBuilder arrBuilder (pBuilder->subarrayStart(pOpName)); - for(size_t i = 0; i < nOperand; ++i) - vpOperand[i]->addToBsonArray(&arrBuilder); - arrBuilder.doneFast(); - } - - void ExpressionNary::addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const { - BSONObjBuilder exprBuilder; - toBson(&exprBuilder, getOpName()); - pBuilder->append(fieldName, exprBuilder.done()); - } - - void ExpressionNary::addToBsonArray( - BSONArrayBuilder *pBuilder) const { - BSONObjBuilder exprBuilder; - toBson(&exprBuilder, getOpName()); - pBuilder->append(exprBuilder.done()); - } - - void ExpressionNary::checkArgLimit(unsigned maxArgs) const { - uassert(15993, str::stream() << getOpName() << - " only takes " << maxArgs << - " operand" << (maxArgs == 1 ? "" : "s"), - vpOperand.size() < maxArgs); - } + for(size_t i = 0; i < nOperand; i++) + array.push_back(vpOperand[i]->serialize(explain)); - void ExpressionNary::checkArgCount(unsigned reqArgs) const { - uassert(15997, str::stream() << getOpName() << - ": insufficient operands; " << reqArgs << - " required, only got " << vpOperand.size(), - vpOperand.size() == reqArgs); + return Value(DOC(getOpName() << array)); } /* ------------------------- ExpressionNot ----------------------------- */ - ExpressionNot::~ExpressionNot() { - } - - intrusive_ptr ExpressionNot::create() { - intrusive_ptr pExpression(new ExpressionNot()); - return pExpression; - } - - ExpressionNot::ExpressionNot(): - ExpressionNary() { - } - - void ExpressionNot::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionNot::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pOp(vpOperand[0]->evaluate(pDocument)); + Value ExpressionNot::evaluateInternal(Variables* vars) const { + Value pOp(vpOperand[0]->evaluateInternal(vars)); bool b = pOp.coerceToBool(); return Value(!b); } + REGISTER_EXPRESSION("$not", ExpressionNot::parse); const char *ExpressionNot::getOpName() const { return "$not"; } /* -------------------------- ExpressionOr ----------------------------- */ - ExpressionOr::~ExpressionOr() { - } - - intrusive_ptr ExpressionOr::create() { - intrusive_ptr pExpression(new ExpressionOr()); - return pExpression; - } - - ExpressionOr::ExpressionOr(): - ExpressionNary() { - } - - Value ExpressionOr::evaluate(const Document& pDocument) const { + Value ExpressionOr::evaluateInternal(Variables* vars) const { const size_t n = vpOperand.size(); for(size_t i = 0; i < n; ++i) { - Value pValue(vpOperand[i]->evaluate(pDocument)); + Value pValue(vpOperand[i]->evaluateInternal(vars)); if (pValue.coerceToBool()) return Value(true); } @@ -2254,20 +1826,6 @@ namespace mongo { return Value(false); } - void ExpressionOr::toMatcherBson( - BSONObjBuilder *pBuilder) const { - BSONObjBuilder opArray; - const size_t n = vpOperand.size(); - for(size_t i = 0; i < n; ++i) - vpOperand[i]->toMatcherBson(&opArray); - - pBuilder->append("$or", opArray.done()); - } - - intrusive_ptr (*ExpressionOr::getFactory() const)() { - return ExpressionOr::create; - } - intrusive_ptr ExpressionOr::optimize() { /* optimize the disjunction as much as possible */ intrusive_ptr pE(ExpressionNary::optimize()); @@ -2295,7 +1853,7 @@ namespace mongo { Evaluate and coerce the last argument to a boolean. If it's true, then we can replace this entire expression. */ - bool last = pLast->evaluate(Document()).coerceToBool(); + bool last = pConst->getValue().coerceToBool(); if (last) { intrusive_ptr pFinal( ExpressionConstant::create(Value(true))); @@ -2321,64 +1879,286 @@ namespace mongo { return pE; } + REGISTER_EXPRESSION("$or", ExpressionOr::parse); const char *ExpressionOr::getOpName() const { return "$or"; } /* ------------------------- ExpressionSecond ----------------------------- */ - ExpressionSecond::~ExpressionSecond() { + Value ExpressionSecond::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); + tm date = pDate.coerceToTm(); + return Value(date.tm_sec); } - intrusive_ptr ExpressionSecond::create() { - intrusive_ptr pExpression(new ExpressionSecond()); - return pExpression; + REGISTER_EXPRESSION("$second", ExpressionSecond::parse); + const char *ExpressionSecond::getOpName() const { + return "$second"; } - ExpressionSecond::ExpressionSecond(): - ExpressionNary() { + + + namespace { + ValueSet arrayToSet(const Value& val) { + const vector& array = val.getArray(); + return ValueSet(array.begin(), array.end()); + } } - void ExpressionSecond::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); + /* ----------------------- ExpressionSetDifference ---------------------------- */ + + Value ExpressionSetDifference::evaluateInternal(Variables* vars) const { + const Value lhs = vpOperand[0]->evaluateInternal(vars); + const Value rhs = vpOperand[1]->evaluateInternal(vars); + + if (lhs.nullish() || rhs.nullish()) { + return Value(BSONNULL); + } + + uassert(17048, str::stream() << "both operands of $setDifference must be arrays. First " + << "argument is of type: " << typeName(lhs.getType()), + lhs.getType() == Array); + uassert(17049, str::stream() << "both operands of $setDifference must be arrays. Second " + << "argument is of type: " << typeName(rhs.getType()), + rhs.getType() == Array); + + ValueSet rhsSet = arrayToSet(rhs); + const vector& lhsArray = lhs.getArray(); + vector returnVec; + + for (vector::const_iterator it = lhsArray.begin(); it != lhsArray.end(); ++it) { + // rhsSet serves the dual role of filtering out elements that were originally present + // in RHS and of eleminating duplicates from LHS + if (rhsSet.insert(*it).second) { + returnVec.push_back(*it); + } + } + return Value::consume(returnVec); } - Value ExpressionSecond::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); - tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_sec); + REGISTER_EXPRESSION("$setDifference", ExpressionSetDifference::parse); + const char *ExpressionSetDifference::getOpName() const { + return "$setDifference"; } - const char *ExpressionSecond::getOpName() const { - return "$second"; + /* ----------------------- ExpressionSetEquals ---------------------------- */ + + void ExpressionSetEquals::validateArguments(const ExpressionVector& args) const { + uassert(17045, str::stream() << "$setEquals needs at least two arguments had: " + << args.size(), + args.size() >= 2); } - /* ----------------------- ExpressionStrcasecmp ---------------------------- */ + Value ExpressionSetEquals::evaluateInternal(Variables* vars) const { + const size_t n = vpOperand.size(); + std::set lhs; + + for (size_t i = 0; i < n; i++) { + const Value nextEntry = vpOperand[i]->evaluateInternal(vars); + uassert(17044, str::stream() << "All operands of $setEquals must be arrays. One " + << "argument is of type: " + << typeName(nextEntry.getType()), + nextEntry.getType() == Array); + + if (i == 0) { + lhs.insert(nextEntry.getArray().begin(), nextEntry.getArray().end()); + } + else { + const std::set rhs(nextEntry.getArray().begin(), nextEntry.getArray().end()); + if (lhs != rhs) { + return Value(false); + } + } + } + return Value(true); + } - ExpressionStrcasecmp::~ExpressionStrcasecmp() { + REGISTER_EXPRESSION("$setEquals", ExpressionSetEquals::parse); + const char *ExpressionSetEquals::getOpName() const { + return "$setEquals"; } - intrusive_ptr ExpressionStrcasecmp::create() { - intrusive_ptr pExpression(new ExpressionStrcasecmp()); - return pExpression; + /* ----------------------- ExpressionSetIntersection ---------------------------- */ + + Value ExpressionSetIntersection::evaluateInternal(Variables* vars) const { + const size_t n = vpOperand.size(); + ValueSet currentIntersection; + for (size_t i = 0; i < n; i++) { + const Value nextEntry = vpOperand[i]->evaluateInternal(vars); + if (nextEntry.nullish()) { + return Value(BSONNULL); + } + uassert(17047, str::stream() << "All operands of $setIntersection must be arrays. One " + << "argument is of type: " + << typeName(nextEntry.getType()), + nextEntry.getType() == Array); + + if (i == 0) { + currentIntersection.insert(nextEntry.getArray().begin(), + nextEntry.getArray().end()); + } + else { + ValueSet nextSet = arrayToSet(nextEntry); + if (currentIntersection.size() > nextSet.size()) { + // to iterate over whichever is the smaller set + nextSet.swap(currentIntersection); + } + ValueSet::iterator it = currentIntersection.begin(); + while (it != currentIntersection.end()) { + if (!nextSet.count(*it)) { + ValueSet::iterator del = it; + ++it; + currentIntersection.erase(del); + } + else { + ++it; + } + } + } + if (currentIntersection.empty()) { + break; + } + } + vector result = vector(currentIntersection.begin(), + currentIntersection.end()); + return Value::consume(result); } - ExpressionStrcasecmp::ExpressionStrcasecmp(): - ExpressionNary() { + REGISTER_EXPRESSION("$setIntersection", ExpressionSetIntersection::parse); + const char *ExpressionSetIntersection::getOpName() const { + return "$setIntersection"; } - void ExpressionStrcasecmp::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(2); - ExpressionNary::addOperand(pExpression); + /* ----------------------- ExpressionSetIsSubset ---------------------------- */ + +namespace { + Value setIsSubsetHelper(const vector& lhs, const ValueSet& rhs) { + // do not shortcircuit when lhs.size() > rhs.size() + // because lhs can have redundant entries + for (vector::const_iterator it = lhs.begin(); it != lhs.end(); ++it) { + if (!rhs.count(*it)) { + return Value(false); + } + } + return Value(true); + } +} + + Value ExpressionSetIsSubset::evaluateInternal(Variables* vars) const { + const Value lhs = vpOperand[0]->evaluateInternal(vars); + const Value rhs = vpOperand[1]->evaluateInternal(vars); + + uassert(17046, str::stream() << "both operands of $setIsSubset must be arrays. First " + << "argument is of type: " << typeName(lhs.getType()), + lhs.getType() == Array); + uassert(17042, str::stream() << "both operands of $setIsSubset must be arrays. Second " + << "argument is of type: " << typeName(rhs.getType()), + rhs.getType() == Array); + + return setIsSubsetHelper(lhs.getArray(), arrayToSet(rhs)); } - Value ExpressionStrcasecmp::evaluate(const Document& pDocument) const { - checkArgCount(2); - Value pString1(vpOperand[0]->evaluate(pDocument)); - Value pString2(vpOperand[1]->evaluate(pDocument)); + /** + * This class handles the case where the RHS set is constant. + * + * Since it is constant we can construct the hashset once which makes the runtime performance + * effectively constant with respect to the size of RHS. Large, constant RHS is expected to be a + * major use case for $redact and this has been verified to improve performance significantly. + */ + class ExpressionSetIsSubset::Optimized : public ExpressionSetIsSubset { + public: + Optimized(const ValueSet& cachedRhsSet, const ExpressionVector& operands) + : _cachedRhsSet(cachedRhsSet) + { + vpOperand = operands; + } + + virtual Value evaluateInternal(Variables* vars) const { + const Value lhs = vpOperand[0]->evaluateInternal(vars); + + uassert(17310, str::stream() << "both operands of $setIsSubset must be arrays. First " + << "argument is of type: " << typeName(lhs.getType()), + lhs.getType() == Array); + + return setIsSubsetHelper(lhs.getArray(), _cachedRhsSet); + } + + private: + const ValueSet _cachedRhsSet; + }; + + intrusive_ptr ExpressionSetIsSubset::optimize() { + // perfore basic optimizations + intrusive_ptr optimized = ExpressionNary::optimize(); + + // if ExpressionNary::optimize() created a new value, return it directly + if (optimized.get() != this) + return optimized; + + if (ExpressionConstant* ec = dynamic_cast(vpOperand[1].get())) { + const Value rhs = ec->getValue(); + uassert(17311, str::stream() << "both operands of $setIsSubset must be arrays. Second " + << "argument is of type: " << typeName(rhs.getType()), + rhs.getType() == Array); + + return new Optimized(arrayToSet(rhs), vpOperand); + } + + return optimized; + } + + REGISTER_EXPRESSION("$setIsSubset", ExpressionSetIsSubset::parse); + const char *ExpressionSetIsSubset::getOpName() const { + return "$setIsSubset"; + } + + /* ----------------------- ExpressionSetUnion ---------------------------- */ + + Value ExpressionSetUnion::evaluateInternal(Variables* vars) const { + ValueSet unionedSet; + const size_t n = vpOperand.size(); + for (size_t i = 0; i < n; i++) { + const Value newEntries = vpOperand[i]->evaluateInternal(vars); + if (newEntries.nullish()) { + return Value(BSONNULL); + } + uassert(17043, str::stream() << "All operands of $setUnion must be arrays. One argument" + << " is of type: " << typeName(newEntries.getType()), + newEntries.getType() == Array); + + unionedSet.insert(newEntries.getArray().begin(), newEntries.getArray().end()); + } + vector result = vector(unionedSet.begin(), unionedSet.end()); + return Value::consume(result); + } + + REGISTER_EXPRESSION("$setUnion", ExpressionSetUnion::parse); + const char *ExpressionSetUnion::getOpName() const { + return "$setUnion"; + } + + /* ----------------------- ExpressionSize ---------------------------- */ + + Value ExpressionSize::evaluateInternal(Variables* vars) const { + Value array = vpOperand[0]->evaluateInternal(vars); + + uassert(17124, str::stream() << "The argument to $size must be an Array, but was of type: " + << typeName(array.getType()), + array.getType() == Array); + return Value::createIntOrLong(array.getArray().size()); + } + + REGISTER_EXPRESSION("$size", ExpressionSize::parse); + const char *ExpressionSize::getOpName() const { + return "$size"; + } + + /* ----------------------- ExpressionStrcasecmp ---------------------------- */ + + Value ExpressionStrcasecmp::evaluateInternal(Variables* vars) const { + Value pString1(vpOperand[0]->evaluateInternal(vars)); + Value pString2(vpOperand[1]->evaluateInternal(vars)); /* boost::iequals returns a bool not an int so strings must actually be allocated */ string str1 = boost::to_upper_copy( pString1.coerceToString() ); @@ -2393,35 +2173,17 @@ namespace mongo { return Value(-1); } + REGISTER_EXPRESSION("$strcasecmp", ExpressionStrcasecmp::parse); const char *ExpressionStrcasecmp::getOpName() const { return "$strcasecmp"; } /* ----------------------- ExpressionSubstr ---------------------------- */ - ExpressionSubstr::~ExpressionSubstr() { - } - - intrusive_ptr ExpressionSubstr::create() { - intrusive_ptr pExpression(new ExpressionSubstr()); - return pExpression; - } - - ExpressionSubstr::ExpressionSubstr(): - ExpressionNary() { - } - - void ExpressionSubstr::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(3); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionSubstr::evaluate(const Document& pDocument) const { - checkArgCount(3); - Value pString(vpOperand[0]->evaluate(pDocument)); - Value pLower(vpOperand[1]->evaluate(pDocument)); - Value pLength(vpOperand[2]->evaluate(pDocument)); + Value ExpressionSubstr::evaluateInternal(Variables* vars) const { + Value pString(vpOperand[0]->evaluateInternal(vars)); + Value pLower(vpOperand[1]->evaluateInternal(vars)); + Value pLength(vpOperand[2]->evaluateInternal(vars)); string str = pString.coerceToString(); uassert(16034, str::stream() << getOpName() << @@ -2441,51 +2203,33 @@ namespace mongo { if ( lower >= str.length() ) { // If lower > str.length() then string::substr() will throw out_of_range, so return an // empty string if lower is not a valid string index. - return Value::createString( "" ); + return Value(""); } - return Value::createString( str.substr(lower, length) ); + return Value(str.substr(lower, length)); } + REGISTER_EXPRESSION("$substr", ExpressionSubstr::parse); const char *ExpressionSubstr::getOpName() const { return "$substr"; } /* ----------------------- ExpressionSubtract ---------------------------- */ - ExpressionSubtract::~ExpressionSubtract() { - } - - intrusive_ptr ExpressionSubtract::create() { - intrusive_ptr pExpression(new ExpressionSubtract()); - return pExpression; - } - - ExpressionSubtract::ExpressionSubtract(): - ExpressionNary() { - } - - void ExpressionSubtract::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(2); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionSubtract::evaluate(const Document& pDocument) const { - checkArgCount(2); - Value lhs = vpOperand[0]->evaluate(pDocument); - Value rhs = vpOperand[1]->evaluate(pDocument); + Value ExpressionSubtract::evaluateInternal(Variables* vars) const { + Value lhs = vpOperand[0]->evaluateInternal(vars); + Value rhs = vpOperand[1]->evaluateInternal(vars); BSONType diffType = Value::getWidestNumeric(rhs.getType(), lhs.getType()); if (diffType == NumberDouble) { double right = rhs.coerceToDouble(); double left = lhs.coerceToDouble(); - return Value::createDouble(left - right); + return Value(left - right); } else if (diffType == NumberLong) { long long right = rhs.coerceToLong(); long long left = lhs.coerceToLong(); - return Value::createLong(left - right); + return Value(left - right); } else if (diffType == NumberInt) { long long right = rhs.coerceToLong(); @@ -2518,95 +2262,43 @@ namespace mongo { } } + REGISTER_EXPRESSION("$subtract", ExpressionSubtract::parse); const char *ExpressionSubtract::getOpName() const { return "$subtract"; } /* ------------------------- ExpressionToLower ----------------------------- */ - ExpressionToLower::~ExpressionToLower() { - } - - intrusive_ptr ExpressionToLower::create() { - intrusive_ptr pExpression(new ExpressionToLower()); - return pExpression; - } - - ExpressionToLower::ExpressionToLower(): - ExpressionNary() { - } - - void ExpressionToLower::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionToLower::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pString(vpOperand[0]->evaluate(pDocument)); + Value ExpressionToLower::evaluateInternal(Variables* vars) const { + Value pString(vpOperand[0]->evaluateInternal(vars)); string str = pString.coerceToString(); boost::to_lower(str); - return Value::createString(str); + return Value(str); } + REGISTER_EXPRESSION("$toLower", ExpressionToLower::parse); const char *ExpressionToLower::getOpName() const { return "$toLower"; } /* ------------------------- ExpressionToUpper -------------------------- */ - ExpressionToUpper::~ExpressionToUpper() { - } - - intrusive_ptr ExpressionToUpper::create() { - intrusive_ptr pExpression(new ExpressionToUpper()); - return pExpression; - } - - ExpressionToUpper::ExpressionToUpper(): - ExpressionNary() { - } - - void ExpressionToUpper::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionToUpper::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pString(vpOperand[0]->evaluate(pDocument)); + Value ExpressionToUpper::evaluateInternal(Variables* vars) const { + Value pString(vpOperand[0]->evaluateInternal(vars)); string str(pString.coerceToString()); boost::to_upper(str); - return Value::createString(str); + return Value(str); } + REGISTER_EXPRESSION("$toUpper", ExpressionToUpper::parse); const char *ExpressionToUpper::getOpName() const { return "$toUpper"; } /* ------------------------- ExpressionWeek ----------------------------- */ - ExpressionWeek::~ExpressionWeek() { - } - - intrusive_ptr ExpressionWeek::create() { - intrusive_ptr pExpression(new ExpressionWeek()); - return pExpression; - } - - ExpressionWeek::ExpressionWeek(): - ExpressionNary() { - } - - void ExpressionWeek::addOperand(const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionWeek::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionWeek::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); int dayOfWeek = date.tm_wday; int dayOfYear = date.tm_yday; @@ -2624,40 +2316,23 @@ namespace mongo { verify(int(str::toUnsigned(buf))==nextSundayWeek); } - return Value::createInt(nextSundayWeek); + return Value(nextSundayWeek); } + REGISTER_EXPRESSION("$week", ExpressionWeek::parse); const char *ExpressionWeek::getOpName() const { return "$week"; } /* ------------------------- ExpressionYear ----------------------------- */ - ExpressionYear::~ExpressionYear() { - } - - intrusive_ptr ExpressionYear::create() { - intrusive_ptr pExpression(new ExpressionYear()); - return pExpression; - } - - ExpressionYear::ExpressionYear(): - ExpressionNary() { - } - - void ExpressionYear::addOperand( - const intrusive_ptr &pExpression) { - checkArgLimit(1); - ExpressionNary::addOperand(pExpression); - } - - Value ExpressionYear::evaluate(const Document& pDocument) const { - checkArgCount(1); - Value pDate(vpOperand[0]->evaluate(pDocument)); + Value ExpressionYear::evaluateInternal(Variables* vars) const { + Value pDate(vpOperand[0]->evaluateInternal(vars)); tm date = pDate.coerceToTm(); - return Value::createInt(date.tm_year + 1900); // tm_year is years since 1900 + return Value(date.tm_year + 1900); // tm_year is years since 1900 } + REGISTER_EXPRESSION("$year", ExpressionYear::parse); const char *ExpressionYear::getOpName() const { return "$year"; } diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h index 8957300cb3c..31a0171ecc2 100644 --- a/src/mongo/db/pipeline/expression.h +++ b/src/mongo/db/pipeline/expression.h @@ -12,28 +12,137 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once #include "mongo/pch.h" -#include "db/pipeline/field_path.h" -#include "db/pipeline/value.h" -#include "util/intrusive_counter.h" +#include "mongo/db/pipeline/dependencies.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/field_path.h" +#include "mongo/db/pipeline/value.h" +#include "mongo/util/intrusive_counter.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/string_map.h" namespace mongo { class BSONArrayBuilder; class BSONElement; class BSONObjBuilder; - class Builder; - class Document; - class MutableDocument; class DocumentSource; - class ExpressionContext; - class Value; + // TODO: Look into merging with ExpressionContext and possibly ObjectCtx. + /// The state used as input and working space for Expressions. + class Variables { + MONGO_DISALLOW_COPYING(Variables); + public: + /** + * Each unique variable is assigned a unique id of this type + */ + typedef size_t Id; + + // This is only for expressions that use no variables (even ROOT). + Variables() :_numVars(0) {} + + explicit Variables(size_t numVars, const Document& root = Document()) + : _root(root) + , _rest(numVars == 0 ? NULL : new Value[numVars]) + , _numVars(numVars) + {} + + static void uassertValidNameForUserWrite(StringData varName); + static void uassertValidNameForUserRead(StringData varName); + + static const Id ROOT_ID = Id(-1); + + /** + * Use this instead of setValue for setting ROOT + */ + void setRoot(const Document& root) { _root = root; } + void clearRoot() { _root = Document(); } + const Document& getRoot() const { return _root; } + + void setValue(Id id, const Value& value); + Value getValue(Id id) const; + + /** + * returns Document() for non-document values. + */ + Document getDocument(Id id) const; + + private: + Document _root; + const boost::scoped_array _rest; + const size_t _numVars; + }; + + /** + * Generates Variables::Ids and keeps track of the number of Ids handed out. + */ + class VariablesIdGenerator { + public: + VariablesIdGenerator() : _nextId(0) {} + + Variables::Id generateId() { return _nextId++; } + + /** + * Returns the number of Ids handed out by this Generator. + * Return value is intended to be passed to Variables constructor. + */ + Variables::Id getIdCount() const { return _nextId; } + + private: + Variables::Id _nextId; + }; + + /** + * This class represents the Variables that are defined in an Expression tree. + * + * All copies from a given instance share enough information to ensure unique Ids are assigned + * and to propagate back to the original instance enough information to correctly construct a + * Variables instance. + */ + class VariablesParseState { + public: + explicit VariablesParseState(VariablesIdGenerator* idGenerator) + : _idGenerator(idGenerator) + {} + + /** + * Assigns a named variable a unique Id. This differs from all other variables, even + * others with the same name. + * + * The special variables ROOT and CURRENT are always implicitly defined with CURRENT + * equivalent to ROOT. If CURRENT is explicitly defined by a call to this function, it + * breaks that equivalence. + * + * NOTE: Name validation is responsibility of caller. + */ + Variables::Id defineVariable(const StringData& name); + + /** + * Returns the current Id for a variable. uasserts if the variable isn't defined. + */ + Variables::Id getVariable(const StringData& name) const; + + private: + StringMap _variables; + VariablesIdGenerator* _idGenerator; + }; class Expression : public IntrusiveCounterUnsigned { @@ -53,82 +162,46 @@ namespace mongo { @returns the optimized Expression */ - virtual intrusive_ptr optimize() = 0; + virtual intrusive_ptr optimize() { return this; } /** - Add this expression's field dependencies to the set - - Expressions are trees, so this is often recursive. - - @param deps output parameter - @param path path to self if all ancestors are ExpressionObjects. - Top-level ExpressionObject gets pointer to empty vector. - If any other Expression is an ancestor, or in other cases - where {a:1} inclusion objects aren't allowed, they get - NULL. + * Add this expression's field dependencies to the set + * + * Expressions are trees, so this is often recursive. + * + * @param deps Fully qualified paths to depended-on fields are added to this set. + * Empty string means need full document. + * @param path path to self if all ancestors are ExpressionObjects. + * Top-level ExpressionObject gets pointer to empty vector. + * If any other Expression is an ancestor, or in other cases + * where {a:1} inclusion objects aren't allowed, they get + * NULL. */ - virtual void addDependencies(set& deps, vector* path=NULL) const = 0; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const = 0; /** simple expressions are just inclusion exclusion as supported by ExpressionObject */ virtual bool isSimple() { return false; } - /* - Evaluate the Expression using the given document as input. - @returns the computed value - */ - virtual Value evaluate(const Document& pDocument) const = 0; - - /* - Add the Expression (and any descendant Expressions) into a BSON - object that is under construction. - - Unevaluated Expressions always materialize as objects. Evaluation - may produce a scalar or another object, either of which will be - substituted inline. - - @param pBuilder the builder to add the expression to - @param fieldName the name the object should be given - @param requireExpression specify true if the value must appear - as an expression; this is used by DocumentSources like - $project which distinguish between field inclusion and virtual - field specification; See ExpressionConstant. + /** + * Serialize the Expression tree recursively. + * If explain is false, returns a Value parsable by parseOperand(). */ - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const = 0; - - /* - Add the Expression (and any descendant Expressions) into a BSON - array that is under construction. + virtual Value serialize(bool explain) const = 0; - Unevaluated Expressions always materialize as objects. Evaluation - may produce a scalar or another object, either of which will be - substituted inline. - - @param pBuilder the builder to add the expression to - */ - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const = 0; + /// Evaluate expression with specified inputs and return result. (only used by tests) + Value evaluate(const Document& root) const { + Variables vars(0, root); + return evaluate(&vars); + } - /* - Convert the expression into a BSONObj that corresponds to the - db.collection.find() predicate language. This is intended for - use by DocumentSourceFilter. - - This is more limited than the full expression language supported - by all available expressions in a DocumentSource processing - pipeline, and will fail with an assertion if an attempt is made - to go outside the bounds of the recognized patterns, which don't - include full computed expressions. There are other methods available - on DocumentSourceFilter which can be used to analyze a filter - predicate and break it up into appropriate expressions which can - be translated within these constraints. As a result, the default - implementation is to fail with an assertion; only a subset of - operators will be able to fulfill this request. - - @param pBuilder the builder to add the expression to. + /** + * Evaluate expression with specified inputs and return result. + * + * While vars is non-const, if properly constructed, subexpressions modifications to it + * should not effect outer expressions due to unique variable Ids. */ - virtual void toMatcherBson(BSONObjBuilder *pBuilder) const; + Value evaluate(Variables* vars) const { return evaluateInternal(vars); } /* Utility class for parseObject() below. @@ -151,37 +224,45 @@ namespace mongo { int options; }; - /* - Parse a BSONElement Object. The object could represent a functional - expression or a Document expression. + // + // Diagram of relationship between parse functions when parsing a $op: + // + // { someFieldOrArrayIndex: { $op: [ARGS] } } + // ^ parseExpression on inner $op BSONElement + // ^ parseObject on BSONObject + // ^ parseOperand on outer BSONElement wrapping the $op Object + // - @param pBsonElement the element representing the object - @param pCtx a MiniCtx representing the options above - @returns the parsed Expression + /** + * Parses a BSON Object that could represent a functional expression or a Document + * expression. */ static intrusive_ptr parseObject( - BSONElement *pBsonElement, ObjectCtx *pCtx); + BSONObj obj, + ObjectCtx *pCtx, + const VariablesParseState& vps); - /* - Parse a BSONElement Object which has already been determined to be - functional expression. - - @param pOpName the name of the (prefix) operator - @param pBsonElement the BSONElement to parse - @returns the parsed Expression - */ + /** + * Parses a BSONElement which has already been determined to be functional expression. + * + * exprElement should be the only element inside the expression object. That is the + * field name should be the $op for the expression. + */ static intrusive_ptr parseExpression( - const char *pOpName, BSONElement *pBsonElement); + BSONElement exprElement, + const VariablesParseState& vps); - /* - Parse a BSONElement which is an operand in an Expression. - - @param pBsonElement the expected operand's BSONElement - @returns the parsed operand, as an Expression + /** + * Parses a BSONElement which is an operand in an Expression. + * + * This is the most generic parser and can parse ExpressionFieldPath, a literal, or a $op. + * If it is a $op, exprElement should be the outer element whose value is an Object + * containing the $op. */ static intrusive_ptr parseOperand( - BSONElement *pBsonElement); + BSONElement exprElement, + const VariablesParseState& vps); /* Produce a field path string with the field prefix removed. @@ -193,41 +274,26 @@ namespace mongo { */ static string removeFieldPrefix(const string &prefixedField); - /* - Enumeration of comparison operators. These are shared between a - few expression implementations, so they are factored out here. - - Any changes to these values require adjustment of the lookup - table in the implementation. - */ - enum CmpOp { - EQ = 0, // return true for a == b, false otherwise - NE = 1, // return true for a != b, false otherwise - GT = 2, // return true for a > b, false otherwise - GTE = 3, // return true for a >= b, false otherwise - LT = 4, // return true for a < b, false otherwise - LTE = 5, // return true for a <= b, false otherwise - CMP = 6, // return -1, 0, 1 for a < b, a == b, a > b - }; - - static int signum(int i); + /** Evaluate the subclass Expression using the given Variables as context and return result. + * + * Should only be called by subclasses, but can't be protected because they need to call + * this function on each other. + */ + virtual Value evaluateInternal(Variables* vars) const = 0; protected: typedef vector > ExpressionVector; - }; + /// Inherit from ExpressionVariadic or ExpressionFixedArity instead of directly from this class. class ExpressionNary : public Expression { public: // virtuals from Expression virtual intrusive_ptr optimize(); - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const; - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const; - virtual void addDependencies(set& deps, vector* path=NULL) const; + virtual Value serialize(bool explain) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; /* Add an operand to the n-ary expression. @@ -236,23 +302,8 @@ namespace mongo { */ virtual void addOperand(const intrusive_ptr &pExpression); - /* - Return a factory function that will make Expression nodes of - the same type as this. This will be used to create constant - expressions for constant folding for optimize(). Only return - a factory function if this operator is both associative and - commutative. The default implementation returns NULL; optimize() - will recognize that and stop. - - Note that ExpressionNary::optimize() promises that if it uses this - to fold constants, then if optimize() returns an ExpressionNary, - any remaining constant will be the last one in vpOperand. Derived - classes may take advantage of this to do further optimizations in - their optimize(). - - @returns pointer to a factory function or NULL - */ - virtual intrusive_ptr (*getFactory() const)(); + // TODO split this into two functions + virtual bool isAssociativeAndCommutative() const { return false; } /* Get the name of the operator. @@ -263,116 +314,99 @@ namespace mongo { */ virtual const char *getOpName() const = 0; - protected: - ExpressionNary(); - - ExpressionVector vpOperand; - - /* - Add the expression to the builder. - - If there is only one operand (a unary operator), then the operand - is added directly, without an array. For more than one operand, - a named array is created. In both cases, the result is an object. + /// Allow subclasses the opportunity to validate arguments at parse time. + virtual void validateArguments(const ExpressionVector& args) const {} - @param pBuilder the (blank) builder to add the expression to - @param pOpName the name of the operator - */ - virtual void toBson(BSONObjBuilder *pBuilder, - const char *pOpName) const; - - /* - Checks the current size of vpOperand; if the size equal to or - greater than maxArgs, fires a user assertion indicating that this - operator cannot have this many arguments. + static ExpressionVector parseArguments( + BSONElement bsonExpr, + const VariablesParseState& vps); - The equal is there because this is intended to be used in - addOperand() to check for the limit *before* adding the requested - argument. + protected: + ExpressionNary() {} - @param maxArgs the maximum number of arguments the operator accepts - */ - void checkArgLimit(unsigned maxArgs) const; + ExpressionVector vpOperand; + }; - /* - Checks the current size of vpOperand; if the size is not equal to - reqArgs, fires a user assertion indicating that this must have - exactly reqArgs arguments. + /// Inherit from ExpressionVariadic or ExpressionFixedArity instead of directly from this class. + template + class ExpressionNaryBase : public ExpressionNary { + public: + static intrusive_ptr parse(BSONElement bsonExpr, + const VariablesParseState& vps) { + intrusive_ptr expr = new SubClass(); + ExpressionVector args = parseArguments(bsonExpr, vps); + expr->validateArguments(args); + expr->vpOperand = args; + return expr; + } + }; - This is meant to be used in evaluate(), *before* the evaluation - takes place. + /// Inherit from this class if your expression takes a variable number of arguments. + template + class ExpressionVariadic : public ExpressionNaryBase { + }; - @param reqArgs the number of arguments this operator requires - */ - void checkArgCount(unsigned reqArgs) const; + /// Inherit from this class if your expression takes a fixed number of arguments. + template + class ExpressionFixedArity : public ExpressionNaryBase { + public: + virtual void validateArguments(const Expression::ExpressionVector& args) const { + uassert(16020, mongoutils::str::stream() + << "Expression " << this->getOpName() << " takes exactly " << NArgs + << " arguments. " << args.size() << " were passed in.", + args.size() == NArgs); + + } }; - class ExpressionAdd : - public ExpressionNary { + class ExpressionAdd : public ExpressionVariadic { public: // virtuals from Expression - virtual ~ExpressionAdd(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; + virtual bool isAssociativeAndCommutative() const { return true; } + }; - // virtuals from ExpressionNary - virtual intrusive_ptr (*getFactory() const)(); - - /* - Create an expression that finds the sum of n operands. - @returns addition expression - */ - static intrusive_ptr create(); + class ExpressionAllElementsTrue : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; }; - class ExpressionAnd : - public ExpressionNary { + class ExpressionAnd : public ExpressionVariadic { public: // virtuals from Expression - virtual ~ExpressionAnd(); virtual intrusive_ptr optimize(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void toMatcherBson(BSONObjBuilder *pBuilder) const; - - // virtuals from ExpressionNary - virtual intrusive_ptr (*getFactory() const)(); - - /* - Create an expression that finds the conjunction of n operands. - The conjunction uses short-circuit logic; the expressions are - evaluated in the order they were added to the conjunction, and - the evaluation stops and returns false on the first operand that - evaluates to false. + virtual bool isAssociativeAndCommutative() const { return true; } + }; - @returns conjunction expression - */ - static intrusive_ptr create(); - private: - ExpressionAnd(); + class ExpressionAnyElementTrue : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; }; - class ExpressionCoerceToBool : - public Expression { + class ExpressionCoerceToBool : public Expression { public: // virtuals from ExpressionNary - virtual ~ExpressionCoerceToBool(); virtual intrusive_ptr optimize(); - virtual void addDependencies(set& deps, vector* path=NULL) const; - virtual Value evaluate(const Document& pDocument) const; - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const; - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; + virtual Value evaluateInternal(Variables* vars) const; + virtual Value serialize(bool explain) const; static intrusive_ptr create( const intrusive_ptr &pExpression); + private: ExpressionCoerceToBool(const intrusive_ptr &pExpression); @@ -380,83 +414,72 @@ namespace mongo { }; - class ExpressionCompare : - public ExpressionNary { + class ExpressionCompare : public ExpressionFixedArity { public: + + /** Enumeration of comparison operators. Any changes to these values require adjustment of + * the lookup table in the implementation. + */ + enum CmpOp { + EQ = 0, // return true for a == b, false otherwise + NE = 1, // return true for a != b, false otherwise + GT = 2, // return true for a > b, false otherwise + GTE = 3, // return true for a >= b, false otherwise + LT = 4, // return true for a < b, false otherwise + LTE = 5, // return true for a <= b, false otherwise + CMP = 6, // return -1, 0, 1 for a < b, a == b, a > b + }; + // virtuals from ExpressionNary - virtual ~ExpressionCompare(); - virtual intrusive_ptr optimize(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - /* - Shorthands for creating various comparisons expressions. - Provide for conformance with the uniform function pointer signature - required for parsing. - These create a particular comparison operand, without any - operands. Those must be added via ExpressionNary::addOperand(). - */ - static intrusive_ptr createCmp(); - static intrusive_ptr createEq(); - static intrusive_ptr createNe(); - static intrusive_ptr createGt(); - static intrusive_ptr createGte(); - static intrusive_ptr createLt(); - static intrusive_ptr createLte(); + static intrusive_ptr parse( + BSONElement bsonExpr, + const VariablesParseState& vps, + CmpOp cmpOp); - private: - friend class ExpressionFieldRange; ExpressionCompare(CmpOp cmpOp); + private: CmpOp cmpOp; }; - class ExpressionConcat : public ExpressionNary { + class ExpressionConcat : public ExpressionVariadic { public: // virtuals from ExpressionNary - virtual ~ExpressionConcat(); - virtual Value evaluate(const Document& input) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - - static intrusive_ptr create(); }; - class ExpressionCond : public ExpressionNary { + class ExpressionCond : public ExpressionFixedArity { + typedef ExpressionFixedArity Base; public: // virtuals from ExpressionNary - virtual ~ExpressionCond(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - private: - ExpressionCond(); + static intrusive_ptr parse( + BSONElement expr, + const VariablesParseState& vps); }; - class ExpressionConstant : - public Expression { + class ExpressionConstant : public Expression { public: // virtuals from Expression - virtual ~ExpressionConstant(); virtual intrusive_ptr optimize(); - virtual void addDependencies(set& deps, vector* path=NULL) const; - virtual Value evaluate(const Document& pDocument) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const; - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const; + virtual Value serialize(bool explain) const; - static intrusive_ptr createFromBsonElement( - BSONElement *pBsonElement); static intrusive_ptr create(const Value& pValue); + static intrusive_ptr parse( + BSONElement bsonExpr, + const VariablesParseState& vps); /* Get the constant value represented by this Expression. @@ -466,92 +489,57 @@ namespace mongo { Value getValue() const; private: - ExpressionConstant(BSONElement *pBsonElement); ExpressionConstant(const Value& pValue); Value pValue; }; - class ExpressionDayOfMonth : - public ExpressionNary { + class ExpressionDayOfMonth : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionDayOfMonth(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionDayOfMonth(); }; - class ExpressionDayOfWeek : - public ExpressionNary { + class ExpressionDayOfWeek : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionDayOfWeek(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionDayOfWeek(); }; - class ExpressionDayOfYear : - public ExpressionNary { + class ExpressionDayOfYear : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionDayOfYear(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionDayOfYear(); }; - class ExpressionDivide : - public ExpressionNary { + class ExpressionDivide : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionDivide(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionDivide(); }; - class ExpressionFieldPath : - public Expression { + class ExpressionFieldPath : public Expression { public: // virtuals from Expression - virtual ~ExpressionFieldPath(); virtual intrusive_ptr optimize(); - virtual void addDependencies(set& deps, vector* path=NULL) const; - virtual Value evaluate(const Document& pDocument) const; - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const; - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; + virtual Value evaluateInternal(Variables* vars) const; + virtual Value serialize(bool explain) const; /* - Create a field path expression. + Create a field path expression using old semantics (rooted off of CURRENT). + + // NOTE: this method is deprecated and only used by tests + // TODO remove this method in favor of parse() Evaluation will extract the value associated with the given field path from the source document. @@ -560,32 +548,20 @@ namespace mongo { indicator @returns the newly created field path expression */ - static intrusive_ptr create( - const string &fieldPath); + static intrusive_ptr create(const string& fieldPath); - /* - Return a string representation of the field path. + /// Like create(), but works with the raw string from the user with the "$" prefixes. + static intrusive_ptr parse( + const string& raw, + const VariablesParseState& vps); - @param fieldPrefix whether or not to include the document field - indicator prefix - @returns the dot-delimited field path - */ - string getFieldPath(bool fieldPrefix) const; - - /* - Write a string representation of the field path to a stream. - - @param the stream to write to - @param fieldPrefix whether or not to include the document field - indicator prefix - */ - void writeFieldPath(ostream &outStream, bool fieldPrefix) const; + const FieldPath& getFieldPath() const { return _fieldPath; } private: - ExpressionFieldPath(const string &fieldPath); + ExpressionFieldPath(const string& fieldPath, Variables::Id variable); /* - Internal implementation of evaluate(), used recursively. + Internal implementation of evaluateInternal(), used recursively. The internal implementation doesn't just use a loop because of the possibility that we need to skip over an array. If the path @@ -602,283 +578,180 @@ namespace mongo { // Helper for evaluatePath to handle Array case Value evaluatePathArray(size_t index, const Value& input) const; - FieldPath fieldPath; + const FieldPath _fieldPath; + const Variables::Id _variable; }; - class ExpressionFieldRange : - public Expression { + class ExpressionHour : public ExpressionFixedArity { public: - // virtuals from expression - virtual ~ExpressionFieldRange(); - virtual intrusive_ptr optimize(); - virtual void addDependencies(set& deps, vector* path=NULL) const; - virtual Value evaluate(const Document& pDocument) const; - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const; - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const; - virtual void toMatcherBson(BSONObjBuilder *pBuilder) const; - - /* - Create a field range expression. - - Field ranges are meant to match up with classic Matcher semantics, - and therefore are conjunctions. For example, these appear in - mongo shell predicates in one of these forms: - { a : C } -> (a == C) // degenerate "point" range - { a : { $lt : C } } -> (a < C) // open range - { a : { $gt : C1, $lte : C2 } } -> ((a > C1) && (a <= C2)) // closed - - When initially created, a field range only includes one end of - the range. Additional points may be added via intersect(). - - Note that NE and CMP are not supported. - - @param pFieldPath the field path for extracting the field value - @param cmpOp the comparison operator - @param pValue the value to compare against - @returns the newly created field range expression - */ - static intrusive_ptr create( - const intrusive_ptr &pFieldPath, - CmpOp cmpOp, const Value& pValue); - - /* - Add an intersecting range. - - This can be done any number of times after creation. The - range is internally optimized for each new addition. If the new - intersection extends or reduces the values within the range, the - internal representation is adjusted to reflect that. - - Note that NE and CMP are not supported. - - @param cmpOp the comparison operator - @param pValue the value to compare against - */ - void intersect(CmpOp cmpOp, const Value& pValue); - - private: - ExpressionFieldRange(const intrusive_ptr &pFieldPath, - CmpOp cmpOp, - const Value& pValue); - - intrusive_ptr pFieldPath; + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; + }; - class Range { - public: - Range(CmpOp cmpOp, const Value& pValue); - Range(const Range &rRange); - Range *intersect(const Range *pRange) const; - bool contains(const Value& pValue) const; + class ExpressionIfNull : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; + }; - Range(const Value& pBottom, bool bottomOpen, - const Value& pTop, bool topOpen); - bool bottomOpen; - bool topOpen; - Value pBottom; - Value pTop; + class ExpressionLet : public Expression { + public: + // virtuals from Expression + virtual intrusive_ptr optimize(); + virtual Value serialize(bool explain) const; + virtual Value evaluateInternal(Variables* vars) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; + + static intrusive_ptr parse( + BSONElement expr, + const VariablesParseState& vps); + + struct NameAndExpression { + NameAndExpression() {} + NameAndExpression(string name, intrusive_ptr expression) + : name(name) + , expression(expression) + {} + + string name; + intrusive_ptr expression; }; - scoped_ptr pRange; + typedef map VariableMap; - /* - Add to a generic Builder. + private: + ExpressionLet(const VariableMap& vars, + intrusive_ptr subExpression); - The methods to append items to an object and an array differ by - their inclusion of a field name. For more complicated objects, - it makes sense to abstract that out and use a generic builder that - always looks the same, and then implement addToBsonObj() and - addToBsonArray() by using the common method. - */ - void addToBson(Builder *pBuilder) const; + VariableMap _variables; + intrusive_ptr _subExpression; }; - - class ExpressionHour : - public ExpressionNary { + class ExpressionMap : public Expression { public: - // virtuals from ExpressionNary - virtual ~ExpressionHour(); - virtual Value evaluate(const Document& pDocument) const; - virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); + // virtuals from Expression + virtual intrusive_ptr optimize(); + virtual Value serialize(bool explain) const; + virtual Value evaluateInternal(Variables* vars) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; - static intrusive_ptr create(); + static intrusive_ptr parse( + BSONElement expr, + const VariablesParseState& vps); private: - ExpressionHour(); - }; + ExpressionMap(const string& varName, // name of variable to set + Variables::Id varId, // id of variable to set + intrusive_ptr input, // yields array to iterate + intrusive_ptr each); // yields results to be added to output array + string _varName; + Variables::Id _varId; + intrusive_ptr _input; + intrusive_ptr _each; + }; - class ExpressionIfNull : - public ExpressionNary { + class ExpressionMeta : public Expression { public: - // virtuals from ExpressionNary - virtual ~ExpressionIfNull(); - virtual Value evaluate(const Document& pDocument) const; - virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); + // virtuals from Expression + virtual Value serialize(bool explain) const; + virtual Value evaluateInternal(Variables* vars) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; - private: - ExpressionIfNull(); + static intrusive_ptr parse( + BSONElement expr, + const VariablesParseState& vps); }; - - class ExpressionMillisecond : - public ExpressionNary { + class ExpressionMillisecond : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionMillisecond(); - virtual Value evaluate(const Document& document) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char* getOpName() const; - virtual void addOperand(const intrusive_ptr& pExpression); - - static intrusive_ptr create(); - - private: - ExpressionMillisecond(); }; - class ExpressionMinute : - public ExpressionNary { + class ExpressionMinute : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionMinute(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionMinute(); }; - class ExpressionMod : - public ExpressionNary { + class ExpressionMod : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionMod(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionMod(); }; - class ExpressionMultiply : - public ExpressionNary { + class ExpressionMultiply : public ExpressionVariadic { public: // virtuals from Expression - virtual ~ExpressionMultiply(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - - // virtuals from ExpressionNary - virtual intrusive_ptr (*getFactory() const)(); - - /* - Create an expression that finds the product of n operands. - - @returns multiplication expression - */ - static intrusive_ptr create(); - - private: - ExpressionMultiply(); + virtual bool isAssociativeAndCommutative() const { return true; } }; - class ExpressionMonth : - public ExpressionNary { + class ExpressionMonth : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionMonth(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionMonth(); }; - class ExpressionNot : - public ExpressionNary { + class ExpressionNot : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionNot(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionNot(); }; - class ExpressionObject : - public Expression { + class ExpressionObject : public Expression { public: // virtuals from Expression - virtual ~ExpressionObject(); virtual intrusive_ptr optimize(); virtual bool isSimple(); - virtual void addDependencies(set& deps, vector* path=NULL) const; + virtual void addDependencies(DepsTracker* deps, vector* path=NULL) const; /** Only evaluates non inclusion expressions. For inclusions, use addToDocument(). */ - virtual Value evaluate(const Document& pDocument) const; - virtual void addToBsonObj(BSONObjBuilder *pBuilder, - StringData fieldName, - bool requireExpression) const; - virtual void addToBsonArray(BSONArrayBuilder *pBuilder) const; + virtual Value evaluateInternal(Variables* vars) const; + virtual Value serialize(bool explain) const; - /* - evaluate(), but return a Document instead of a Value-wrapped - Document. + /// like evaluate(), but return a Document instead of a Value-wrapped Document. + Document evaluateDocument(Variables* vars) const; - @param pDocument the input Document - @returns the result document + /** Evaluates with inclusions and adds results to passed in Mutable document + * + * @param output the MutableDocument to add the evaluated expressions to + * @param currentDoc the input Document for this level (for inclusions) + * @param vars the variables for use in subexpressions */ - Document evaluateDocument(const Document& pDocument) const; - - /* - evaluate(), but add the evaluated fields to a given document - instead of creating a new one. - - @param pResult the Document to add the evaluated expressions to - @param pDocument the input Document for this level - @param rootDoc the root of the whole input document - */ - void addToDocument(MutableDocument& pResult, - const Document& pDocument, - const Document& rootDoc + void addToDocument(MutableDocument& ouput, + const Document& currentDoc, + Variables* vars ) const; // estimated number of fields that will be output size_t getSizeHint() const; - /* - Create an empty expression. Until fields are added, this - will evaluate to an empty document (object). + /** Create an empty expression. + * Until fields are added, this will evaluate to an empty document. */ static intrusive_ptr create(); + /// Like create but uses special handling of _id for root object of $project. + static intrusive_ptr createRoot(); + /* Add a field to the document expression. @@ -939,174 +812,147 @@ namespace mongo { void excludeId(bool b) { _excludeId = b; } private: - ExpressionObject(); + ExpressionObject(bool atRoot); - // mapping from fieldname to Expression to generate the value - // NULL expression means include from source document - typedef map > ExpressionMap; - ExpressionMap _expressions; + // Mapping from fieldname to the Expression that generates its value. + // NULL expression means inclusion from source document. + typedef map > FieldMap; + FieldMap _expressions; // this is used to maintain order for generated fields not in the source document vector _order; bool _excludeId; + bool _atRoot; }; - class ExpressionOr : - public ExpressionNary { + class ExpressionOr : public ExpressionVariadic { public: // virtuals from Expression - virtual ~ExpressionOr(); virtual intrusive_ptr optimize(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void toMatcherBson(BSONObjBuilder *pBuilder) const; - - // virtuals from ExpressionNary - virtual intrusive_ptr (*getFactory() const)(); - - /* - Create an expression that finds the conjunction of n operands. - The conjunction uses short-circuit logic; the expressions are - evaluated in the order they were added to the conjunction, and - the evaluation stops and returns false on the first operand that - evaluates to false. - - @returns conjunction expression - */ - static intrusive_ptr create(); - - private: - ExpressionOr(); + virtual bool isAssociativeAndCommutative() const { return true; } }; - class ExpressionSecond : - public ExpressionNary { + class ExpressionSecond : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionSecond(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionSecond(); }; - class ExpressionStrcasecmp : - public ExpressionNary { + class ExpressionSetDifference : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionStrcasecmp(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); + }; - static intrusive_ptr create(); - private: - ExpressionStrcasecmp(); + class ExpressionSetEquals : public ExpressionVariadic { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; + virtual void validateArguments(const ExpressionVector& args) const; }; - class ExpressionSubstr : - public ExpressionNary { + class ExpressionSetIntersection : public ExpressionVariadic { public: // virtuals from ExpressionNary - virtual ~ExpressionSubstr(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); + virtual bool isAssociativeAndCommutative() const { return true; } + }; - static intrusive_ptr create(); + class ExpressionSetIsSubset : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual intrusive_ptr optimize(); + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; private: - ExpressionSubstr(); + class Optimized; }; - class ExpressionSubtract : - public ExpressionNary { + class ExpressionSetUnion : public ExpressionVariadic { public: // virtuals from ExpressionNary - virtual ~ExpressionSubtract(); - virtual Value evaluate(const Document& pDocument) const; + // virtual intrusive_ptr optimize(); + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); - - static intrusive_ptr create(); - - private: - ExpressionSubtract(); + virtual bool isAssociativeAndCommutative() const { return true; } }; - class ExpressionToLower : - public ExpressionNary { + class ExpressionSize : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionToLower(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); + }; - static intrusive_ptr create(); - private: - ExpressionToLower(); + class ExpressionStrcasecmp : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; }; - class ExpressionToUpper : - public ExpressionNary { + class ExpressionSubstr : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionToUpper(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); + }; - static intrusive_ptr create(); - private: - ExpressionToUpper(); + class ExpressionSubtract : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; }; - class ExpressionWeek : - public ExpressionNary { + class ExpressionToLower : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionWeek(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); + }; - static intrusive_ptr create(); - private: - ExpressionWeek(); + class ExpressionToUpper : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; }; - class ExpressionYear : - public ExpressionNary { + class ExpressionWeek : public ExpressionFixedArity { public: // virtuals from ExpressionNary - virtual ~ExpressionYear(); - virtual Value evaluate(const Document& pDocument) const; + virtual Value evaluateInternal(Variables* vars) const; virtual const char *getOpName() const; - virtual void addOperand(const intrusive_ptr &pExpression); + }; - static intrusive_ptr create(); - private: - ExpressionYear(); + class ExpressionYear : public ExpressionFixedArity { + public: + // virtuals from ExpressionNary + virtual Value evaluateInternal(Variables* vars) const; + virtual const char *getOpName() const; }; } @@ -1115,27 +961,10 @@ namespace mongo { namespace mongo { - inline int Expression::signum(int i) { - if (i < 0) - return -1; - if (i > 0) - return 1; - return 0; - } - inline Value ExpressionConstant::getValue() const { return pValue; } - inline string ExpressionFieldPath::getFieldPath(bool fieldPrefix) const { - return fieldPath.getPath(fieldPrefix); - } - - inline void ExpressionFieldPath::writeFieldPath( - ostream &outStream, bool fieldPrefix) const { - return fieldPath.writePath(outStream, fieldPrefix); - } - inline size_t ExpressionObject::getFieldCount() const { return _expressions.size(); } diff --git a/src/mongo/db/pipeline/expression_context.cpp b/src/mongo/db/pipeline/expression_context.cpp deleted file mode 100644 index 0533fc63e7d..00000000000 --- a/src/mongo/db/pipeline/expression_context.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright (c) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "pch.h" - -#include "db/interrupt_status.h" -#include "db/pipeline/expression_context.h" - -namespace mongo { - - ExpressionContext::~ExpressionContext() { - } - - inline ExpressionContext::ExpressionContext(InterruptStatus *pS): - doingMerge(false), - inShard(false), - inRouter(false), - intCheckCounter(1), - pStatus(pS) { - } - - void ExpressionContext::checkForInterrupt() { - /* - Only really check periodically; the check gets a mutex, and could - be expensive, at least in relative terms. - */ - if ((++intCheckCounter % 128) == 0) { - pStatus->checkForInterrupt(); - } - } - - ExpressionContext* ExpressionContext::clone() { - ExpressionContext* newContext = create(pStatus); - newContext->setDoingMerge(getDoingMerge()); - newContext->setInShard(getInShard()); - newContext->setInRouter(getInRouter()); - return newContext; - } - - ExpressionContext *ExpressionContext::create(InterruptStatus *pStatus) { - return new ExpressionContext(pStatus); - } - -} diff --git a/src/mongo/db/pipeline/expression_context.h b/src/mongo/db/pipeline/expression_context.h index b0a1260b22c..c7d8aaa5c48 100644 --- a/src/mongo/db/pipeline/expression_context.h +++ b/src/mongo/db/pipeline/expression_context.h @@ -12,80 +12,60 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once -#include "mongo/pch.h" +#include -#include "util/intrusive_counter.h" +#include "mongo/db/interrupt_status.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/intrusive_counter.h" namespace mongo { - class InterruptStatus; - - class ExpressionContext : - public IntrusiveCounterUnsigned { + struct ExpressionContext : public IntrusiveCounterUnsigned { public: - virtual ~ExpressionContext(); - - void setDoingMerge(bool b); - void setInShard(bool b); - void setInRouter(bool b); - - bool getDoingMerge() const; - bool getInShard() const; - bool getInRouter() const; - - /** - Used by a pipeline to check for interrupts so that killOp() works. - - @throws if the operation has been interrupted + ExpressionContext(const InterruptStatus& status, const NamespaceString& ns) + : inShard(false) + , inRouter(false) + , extSortAllowed(false) + , ns(ns) + , interruptStatus(status) + , interruptCounter(interruptCheckPeriod) + {} + + /** Used by a pipeline to check for interrupts so that killOp() works. + * @throws if the operation has been interrupted */ - void checkForInterrupt(); - - ExpressionContext* clone(); - - static ExpressionContext *create(InterruptStatus *pStatus); - - private: - ExpressionContext(InterruptStatus *pStatus); + void checkForInterrupt() { + if (--interruptCounter == 0) { + // The checkForInterrupt could be expensive, at least in relative terms. + interruptStatus.checkForInterrupt(); + interruptCounter = interruptCheckPeriod; + } + } - bool doingMerge; bool inShard; bool inRouter; - unsigned intCheckCounter; // interrupt check counter - InterruptStatus *const pStatus; + bool extSortAllowed; + NamespaceString ns; + std::string tempDir; // Defaults to empty to prevent external sorting in mongos. + + const InterruptStatus& interruptStatus; + static const int interruptCheckPeriod = 128; + int interruptCounter; // when 0, check interruptStatus }; } - - -/* ======================= INLINED IMPLEMENTATIONS ========================== */ - -namespace mongo { - - inline void ExpressionContext::setDoingMerge(bool b) { - doingMerge = b; - } - - inline void ExpressionContext::setInShard(bool b) { - inShard = b; - } - - inline void ExpressionContext::setInRouter(bool b) { - inRouter = b; - } - - inline bool ExpressionContext::getDoingMerge() const { - return doingMerge; - } - - inline bool ExpressionContext::getInShard() const { - return inShard; - } - - inline bool ExpressionContext::getInRouter() const { - return inRouter; - } - -}; diff --git a/src/mongo/db/pipeline/field_path.cpp b/src/mongo/db/pipeline/field_path.cpp index a21010bb95c..f91a2ff00cb 100644 --- a/src/mongo/db/pipeline/field_path.cpp +++ b/src/mongo/db/pipeline/field_path.cpp @@ -12,11 +12,25 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "db/pipeline/field_path.h" -#include "util/mongoutils/str.h" +#include "mongo/pch.h" + +#include "mongo/db/pipeline/field_path.h" + +#include "mongo/util/mongoutils/str.h" namespace mongo { diff --git a/src/mongo/db/pipeline/field_path.h b/src/mongo/db/pipeline/field_path.h index f3bd9a48a48..d8a8984b9f0 100644 --- a/src/mongo/db/pipeline/field_path.h +++ b/src/mongo/db/pipeline/field_path.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index 6ec1771d2e6..db4ba8af517 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -12,18 +12,36 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "db/pipeline/pipeline.h" +#include "mongo/pch.h" + +// This file defines functions from both of these headers +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/pipeline/pipeline_optimizations.h" -#include "db/jsobj.h" -#include "db/pipeline/accumulator.h" -#include "db/pipeline/document.h" -#include "db/pipeline/document_source.h" -#include "db/pipeline/expression.h" -#include "db/pipeline/expression_context.h" -#include "util/mongoutils/str.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pipeline/accumulator.h" +#include "mongo/db/pipeline/document.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { @@ -31,17 +49,11 @@ namespace mongo { const char Pipeline::pipelineName[] = "pipeline"; const char Pipeline::explainName[] = "explain"; const char Pipeline::fromRouterName[] = "fromRouter"; - const char Pipeline::splitMongodPipelineName[] = "splitMongodPipeline"; const char Pipeline::serverPipelineName[] = "serverPipeline"; const char Pipeline::mongosPipelineName[] = "mongosPipeline"; - Pipeline::~Pipeline() { - } - Pipeline::Pipeline(const intrusive_ptr &pTheCtx): - collectionName(), explain(false), - splitMongodPipeline(false), pCtx(pTheCtx) { } @@ -50,15 +62,11 @@ namespace mongo { struct StageDesc { const char *pName; intrusive_ptr (*pFactory)( - BSONElement *, const intrusive_ptr &); + BSONElement, const intrusive_ptr &); }; /* this table must be in alphabetical order by name for bsearch() */ static const StageDesc stageDesc[] = { -#ifdef NEVER /* disabled for now in favor of $match */ - {DocumentSourceFilter::filterName, - DocumentSourceFilter::createFromBson}, -#endif {DocumentSourceGeoNear::geoNearName, DocumentSourceGeoNear::createFromBson}, {DocumentSourceGroup::groupName, @@ -67,12 +75,14 @@ namespace mongo { DocumentSourceLimit::createFromBson}, {DocumentSourceMatch::matchName, DocumentSourceMatch::createFromBson}, -#ifdef LATER // https://jira.mongodb.org/browse/SERVER-3253 + {DocumentSourceMergeCursors::name, + DocumentSourceMergeCursors::createFromBson}, {DocumentSourceOut::outName, DocumentSourceOut::createFromBson}, -#endif {DocumentSourceProject::projectName, DocumentSourceProject::createFromBson}, + {DocumentSourceRedact::redactName, + DocumentSourceRedact::createFromBson}, {DocumentSourceSkip::skipName, DocumentSourceSkip::createFromBson}, {DocumentSourceSort::sortName, @@ -87,9 +97,9 @@ namespace mongo { ((const StageDesc *)pR)->pName); } - intrusive_ptr Pipeline::parseCommand( - string &errmsg, BSONObj &cmdObj, - const intrusive_ptr &pCtx) { + intrusive_ptr Pipeline::parseCommand(string& errmsg, + const BSONObj& cmdObj, + const intrusive_ptr& pCtx) { intrusive_ptr pPipeline(new Pipeline(pCtx)); vector pipeline; @@ -104,9 +114,18 @@ namespace mongo { continue; } + // maxTimeMS is also for the command processor. + if (pFieldName == LiteParsedQuery::cmdOptionMaxTimeMS) { + continue; + } + + // ignore cursor options since they are handled externally. + if (str::equals(pFieldName, "cursor")) { + continue; + } + /* look for the aggregation command */ if (!strcmp(pFieldName, commandName)) { - pPipeline->collectionName = cmdElement.String(); continue; } @@ -124,21 +143,22 @@ namespace mongo { /* if the request came from the router, we're in a shard */ if (!strcmp(pFieldName, fromRouterName)) { - pCtx->setInShard(cmdElement.Bool()); + pCtx->inShard = cmdElement.Bool(); continue; } - /* check for debug options */ - if (!strcmp(pFieldName, splitMongodPipelineName)) { - pPipeline->splitMongodPipeline = true; + if (str::equals(pFieldName, "allowDiskUse")) { + uassert(16949, + str::stream() << "allowDiskUse must be a bool, not a " + << typeName(cmdElement.type()), + cmdElement.type() == Bool); + pCtx->extSortAllowed = cmdElement.Bool(); continue; } /* we didn't recognize a field in the command */ ostringstream sb; - sb << - "unrecognized field \"" << - cmdElement.fieldName(); + sb << "unrecognized field '" << cmdElement.fieldName() << "'"; errmsg = sb.str(); return intrusive_ptr(); } @@ -176,34 +196,38 @@ namespace mongo { uassert(16436, str::stream() << "Unrecognized pipeline stage name: '" << stageName << "'", pDesc); - intrusive_ptr stage = (*pDesc->pFactory)(&stageSpec, pCtx); + intrusive_ptr stage = pDesc->pFactory(stageSpec, pCtx); verify(stage); - stage->setPipelineStep(iStep); sources.push_back(stage); - } - /* if there aren't any pipeline stages, there's nothing more to do */ - if (sources.empty()) - return pPipeline; + // TODO find a good general way to check stages that must be first syntactically - /* - Move filters up where possible. + if (dynamic_cast(stage.get())) { + uassert(16991, "$out can only be the final stage in the pipeline", + iStep == nSteps - 1); + } + } - CW TODO -- move filter past projections where possible, and noting - corresponding field renaming. - */ + // The order in which optimizations are applied can have significant impact on the + // efficiency of the final pipeline. Be Careful! + Optimizations::Local::moveMatchBeforeSort(pPipeline.get()); + Optimizations::Local::moveLimitBeforeSkip(pPipeline.get()); + Optimizations::Local::coalesceAdjacent(pPipeline.get()); + Optimizations::Local::optimizeEachDocumentSource(pPipeline.get()); + Optimizations::Local::duplicateMatchBeforeInitalRedact(pPipeline.get()); - /* - Wherever there is a match immediately following a sort, swap them. - This means we sort fewer items. Neither changes the documents in - the stream, so this transformation shouldn't affect the result. + return pPipeline; + } - We do this first, because then when we coalesce operators below, - any adjacent matches will be combined. - */ + void Pipeline::Optimizations::Local::moveMatchBeforeSort(Pipeline* pipeline) { + // TODO Keep moving matches across multiple sorts as moveLimitBeforeSkip does below. + // TODO Check sort for limit. Not an issue currently due to order optimizations are applied, + // but should be fixed. + SourceContainer& sources = pipeline->sources; for (size_t srcn = sources.size(), srci = 1; srci < srcn; ++srci) { intrusive_ptr &pSource = sources[srci]; - if (dynamic_cast(pSource.get())) { + DocumentSourceMatch* match = dynamic_cast(pSource.get()); + if (match && !match->isTextQuery()) { intrusive_ptr &pPrevious = sources[srci - 1]; if (dynamic_cast(pPrevious.get())) { /* swap this item with the previous */ @@ -213,11 +237,13 @@ namespace mongo { } } } + } + + void Pipeline::Optimizations::Local::moveLimitBeforeSkip(Pipeline* pipeline) { + SourceContainer& sources = pipeline->sources; + if (sources.empty()) + return; - /* Move limits in front of skips. This is more optimal for sharding - * since currently, we can only split the pipeline at a single source - * and it is better to limit the results coming from each shard - */ for(int i = sources.size() - 1; i >= 1 /* not looking at 0 */; i--) { DocumentSourceLimit* limit = dynamic_cast(sources[i].get()); @@ -242,144 +268,213 @@ namespace mongo { i = sources.size(); // decremented before next pass } } + } - /* - Coalesce adjacent filters where possible. Two adjacent filters - are equivalent to one filter whose predicate is the conjunction of - the two original filters' predicates. For now, capture this by - giving any DocumentSource the option to absorb it's successor; this - will also allow adjacent projections to coalesce when possible. - - Run through the DocumentSources, and give each one the opportunity - to coalesce with its successor. If successful, remove the - successor. + void Pipeline::Optimizations::Local::coalesceAdjacent(Pipeline* pipeline) { + SourceContainer& sources = pipeline->sources; + if (sources.empty()) + return; - Move all document sources to a temporary list. - */ + // move all sources to a temporary list SourceContainer tempSources; sources.swap(tempSources); - /* move the first one to the final list */ + // move the first one to the final list sources.push_back(tempSources[0]); - /* run through the sources, coalescing them or keeping them */ + // run through the sources, coalescing them or keeping them for (size_t tempn = tempSources.size(), tempi = 1; tempi < tempn; ++tempi) { - /* - If we can't coalesce the source with the last, then move it - to the final list, and make it the new last. (If we succeeded, - then we're still on the same last, and there's no need to move - or do anything with the source -- the destruction of tempSources - will take care of the rest.) - */ + // If we can't coalesce the source with the last, then move it + // to the final list, and make it the new last. (If we succeeded, + // then we're still on the same last, and there's no need to move + // or do anything with the source -- the destruction of tempSources + // will take care of the rest.) intrusive_ptr &pLastSource = sources.back(); intrusive_ptr &pTemp = tempSources[tempi]; verify(pTemp && pLastSource); if (!pLastSource->coalesce(pTemp)) sources.push_back(pTemp); } + } - /* optimize the elements in the pipeline */ - for(SourceContainer::iterator iter(sources.begin()), - listEnd(sources.end()); - iter != listEnd; - ++iter) { - if (!*iter) { - errmsg = "Pipeline received empty document as argument"; - return intrusive_ptr(); - } + void Pipeline::Optimizations::Local::optimizeEachDocumentSource(Pipeline* pipeline) { + SourceContainer& sources = pipeline->sources; + for (SourceContainer::iterator it(sources.begin()); it != sources.end(); ++it) { + (*it)->optimize(); + } + } - (*iter)->optimize(); + void Pipeline::Optimizations::Local::duplicateMatchBeforeInitalRedact(Pipeline* pipeline) { + SourceContainer& sources = pipeline->sources; + if (sources.size() >= 2 && dynamic_cast(sources[0].get())) { + if (DocumentSourceMatch* match = dynamic_cast(sources[1].get())) { + const BSONObj redactSafePortion = match->redactSafePortion(); + if (!redactSafePortion.isEmpty()) { + sources.push_front( + DocumentSourceMatch::createFromBson( + BSON("$match" << redactSafePortion).firstElement(), + pipeline->pCtx)); + } + } } + } - return pPipeline; + void Pipeline::addRequiredPrivileges(Command* commandTemplate, + const string& db, + BSONObj cmdObj, + vector* out) { + ResourcePattern inputResource(commandTemplate->parseResourcePattern(db, cmdObj)); + uassert(17138, + mongoutils::str::stream() << "Invalid input resource, " << inputResource.toString(), + inputResource.isExactNamespacePattern()); + + out->push_back(Privilege(inputResource, ActionType::find)); + + BSONObj pipeline = cmdObj.getObjectField("pipeline"); + BSONForEach(stageElem, pipeline) { + BSONObj stage = stageElem.embeddedObjectUserCheck(); + if (str::equals(stage.firstElementFieldName(), "$out")) { + NamespaceString outputNs(db, stage.firstElement().str()); + uassert(17139, + mongoutils::str::stream() << "Invalid $out target namespace, " << + outputNs.ns(), + outputNs.isValid()); + + ActionSet actions; + actions.addAction(ActionType::remove); + actions.addAction(ActionType::insert); + out->push_back(Privilege(ResourcePattern::forExactNamespace(outputNs), actions)); + } + } } intrusive_ptr Pipeline::splitForSharded() { - /* create an initialize the shard spec we'll return */ - intrusive_ptr pShardPipeline(new Pipeline(pCtx)); - pShardPipeline->collectionName = collectionName; - pShardPipeline->explain = explain; + // Create and initialize the shard spec we'll return. We start with an empty pipeline on the + // shards and all work being done in the merger. Optimizations can move operations between + // the pipelines to be more efficient. + intrusive_ptr shardPipeline(new Pipeline(pCtx)); + shardPipeline->explain = explain; + + // The order in which optimizations are applied can have significant impact on the + // efficiency of the final pipeline. Be Careful! + Optimizations::Sharded::findSplitPoint(shardPipeline.get(), this); + Optimizations::Sharded::moveFinalUnwindFromShardsToMerger(shardPipeline.get(), this); + Optimizations::Sharded::limitFieldsSentFromShardsToMerger(shardPipeline.get(), this); + + return shardPipeline; + } - /* - Run through the pipeline, looking for points to split it into - shard pipelines, and the rest. - */ - while (!sources.empty()) { - // pop the first source - intrusive_ptr pSource = sources.front(); - sources.pop_front(); + void Pipeline::Optimizations::Sharded::findSplitPoint(Pipeline* shardPipe, + Pipeline* mergePipe) { + while (!mergePipe->sources.empty()) { + intrusive_ptr current = mergePipe->sources.front(); + mergePipe->sources.pop_front(); // Check if this source is splittable - SplittableDocumentSource* splittable= - dynamic_cast(pSource.get()); + SplittableDocumentSource* splittable = + dynamic_cast(current.get()); if (!splittable){ - // move the source from the router sources to the shard sources - pShardPipeline->sources.push_back(pSource); + // move the source from the merger sources to the shard sources + shardPipe->sources.push_back(current); } else { - // split into Router and Shard sources - intrusive_ptr shardSource = splittable->getShardSource(); - intrusive_ptr routerSource = splittable->getRouterSource(); - if (shardSource) pShardPipeline->sources.push_back(shardSource); - if (routerSource) this->sources.push_front(routerSource); + // split this source into Merge and Shard sources + intrusive_ptr shardSource = splittable->getShardSource(); + intrusive_ptr mergeSource = splittable->getMergeSource(); + if (shardSource) shardPipe->sources.push_back(shardSource); + if (mergeSource) mergePipe->sources.push_front(mergeSource); break; } } + } - return pShardPipeline; + void Pipeline::Optimizations::Sharded::moveFinalUnwindFromShardsToMerger(Pipeline* shardPipe, + Pipeline* mergePipe) { + while (!shardPipe->sources.empty() + && dynamic_cast(shardPipe->sources.back().get())) { + mergePipe->sources.push_front(shardPipe->sources.back()); + shardPipe->sources.pop_back(); + } } - bool Pipeline::getInitialQuery(BSONObjBuilder *pQueryBuilder) const - { - if (sources.empty()) - return false; + void Pipeline::Optimizations::Sharded::limitFieldsSentFromShardsToMerger(Pipeline* shardPipe, + Pipeline* mergePipe) { + DepsTracker mergeDeps = mergePipe->getDependencies(shardPipe->getInitialQuery()); + if (mergeDeps.needWholeDocument) + return; // the merge needs all fields, so nothing we can do. + + // Empty project is "special" so if no fields are needed, we just ask for _id instead. + if (mergeDeps.fields.empty()) + mergeDeps.fields.insert("_id"); + + // Remove metadata from dependencies since it automatically flows through projection and we + // don't want to project it in to the document. + mergeDeps.needTextScore = false; + + // HEURISTIC: only apply optimization if none of the shard stages have an exhaustive list of + // field dependencies. While this may not be 100% ideal in all cases, it is simple and + // avoids the worst cases by ensuring that: + // 1) Optimization IS applied when the shards wouldn't have known their exhaustive list of + // dependencies. This situation can happen when a $sort is before the first $project or + // $group. Without the optimization, the shards would have to reify and transmit full + // objects even though only a subset of fields are needed. + // 2) Optimization IS NOT applied immediately following a $project or $group since it would + // add an unnecessary project (and therefore a deep-copy). + for (size_t i = 0; i < shardPipe->sources.size(); i++) { + DepsTracker dt; // ignored + if (shardPipe->sources[i]->getDependencies(&dt) & DocumentSource::EXHAUSTIVE_FIELDS) + return; + } - /* look for an initial $match */ - const intrusive_ptr &pMC = sources.front(); - const DocumentSourceMatch *pMatch = - dynamic_cast(pMC.get()); + // if we get here, add the project. + shardPipe->sources.push_back( + DocumentSourceProject::createFromBson( + BSON("$project" << mergeDeps.toProjection()).firstElement(), + shardPipe->pCtx)); + } - if (!pMatch) - return false; + BSONObj Pipeline::getInitialQuery() const { + if (sources.empty()) + return BSONObj(); - /* build the query */ - pMatch->toMatcherBson(pQueryBuilder); + /* look for an initial $match */ + DocumentSourceMatch* match = dynamic_cast(sources.front().get()); + if (!match) + return BSONObj(); - return true; + return match->getQuery(); } - void Pipeline::toBson(BSONObjBuilder *pBuilder) const { - /* create an array out of the pipeline operations */ - BSONArrayBuilder arrayBuilder; + Document Pipeline::serialize() const { + MutableDocument serialized; + // create an array out of the pipeline operations + vector array; for(SourceContainer::const_iterator iter(sources.begin()), listEnd(sources.end()); iter != listEnd; ++iter) { intrusive_ptr pSource(*iter); - pSource->addToBsonArray(&arrayBuilder); + pSource->serializeToArray(array); } - /* add the top-level items to the command */ - pBuilder->append(commandName, getCollectionName()); - pBuilder->append(pipelineName, arrayBuilder.arr()); + // add the top-level items to the command + serialized.setField(commandName, Value(pCtx->ns.coll())); + serialized.setField(pipelineName, Value(array)); if (explain) { - pBuilder->append(explainName, explain); + serialized.setField(explainName, Value(explain)); } - bool btemp; - if ((btemp = getSplitMongodPipeline())) { - pBuilder->append(splitMongodPipelineName, btemp); + if (pCtx->extSortAllowed) { + serialized.setField("allowDiskUse", Value(true)); } - if ((btemp = pCtx->getInRouter())) { - pBuilder->append(fromRouterName, btemp); - } + return serialized.freeze(); } - bool Pipeline::run(BSONObjBuilder &result, string &errmsg) { + void Pipeline::stitch() { massert(16600, "should not have an empty pipeline", !sources.empty()); @@ -393,101 +488,102 @@ namespace mongo { pTemp->setSource(prevSource); prevSource = pTemp.get(); } + } - /* - Iterate through the resulting documents, and add them to the result. - We do this even if we're doing an explain, in order to capture - the document counts and other stats. However, we don't capture - the result documents for explain. - */ - if (explain) { - if (!pCtx->getInRouter()) - writeExplainShard(result); - else { - writeExplainMongos(result); - } + void Pipeline::run(BSONObjBuilder& result) { + // should not get here in the explain case + verify(!explain); + + // the array in which the aggregation results reside + // cant use subArrayStart() due to error handling + BSONArrayBuilder resultArray; + DocumentSource* finalSource = sources.back().get(); + while (boost::optional next = finalSource->getNext()) { + // add the document to the result set + BSONObjBuilder documentBuilder (resultArray.subobjStart()); + next->toBson(&documentBuilder); + documentBuilder.doneFast(); + // object will be too large, assert. the extra 1KB is for headers + uassert(16389, + str::stream() << "aggregation result exceeds maximum document size (" + << BSONObjMaxUserSize / (1024 * 1024) << "MB)", + resultArray.len() < BSONObjMaxUserSize - 1024); } - else { - // the array in which the aggregation results reside - // cant use subArrayStart() due to error handling - BSONArrayBuilder resultArray; - DocumentSource* finalSource = sources.back().get(); - for(bool hasDoc = !finalSource->eof(); hasDoc; hasDoc = finalSource->advance()) { - Document pDocument(finalSource->getCurrent()); - - /* add the document to the result set */ - BSONObjBuilder documentBuilder (resultArray.subobjStart()); - pDocument->toBson(&documentBuilder); - documentBuilder.doneFast(); - // object will be too large, assert. the extra 1KB is for headers - uassert(16389, - str::stream() << "aggregation result exceeds maximum document size (" - << BSONObjMaxUserSize / (1024 * 1024) << "MB)", - resultArray.len() < BSONObjMaxUserSize - 1024); - } - resultArray.done(); - result.appendArray("result", resultArray.arr()); + resultArray.done(); + result.appendArray("result", resultArray.arr()); + } + + vector Pipeline::writeExplainOps() const { + vector array; + for(SourceContainer::const_iterator it = sources.begin(); it != sources.end(); ++it) { + (*it)->serializeToArray(array, /*explain=*/true); } + return array; + } - return true; + void Pipeline::addInitialSource(intrusive_ptr source) { + sources.push_front(source); } - void Pipeline::writeExplainOps(BSONArrayBuilder *pArrayBuilder) const { - for(SourceContainer::const_iterator iter(sources.begin()), - listEnd(sources.end()); - iter != listEnd; - ++iter) { - intrusive_ptr pSource(*iter); + bool Pipeline::canRunInMongos() const { + if (pCtx->extSortAllowed) + return false; - // handled in writeExplainMongos - if (dynamic_cast(pSource.get())) - continue; + if (explain) + return false; - pSource->addToBsonArray(pArrayBuilder, true); - } - } + if (!sources.empty() && dynamic_cast(sources.back().get())) + return false; - void Pipeline::writeExplainShard(BSONObjBuilder &result) const { - BSONArrayBuilder opArray; // where we'll put the pipeline ops + return true; + } - // next, add the pipeline operators - writeExplainOps(&opArray); + DepsTracker Pipeline::getDependencies(const BSONObj& initialQuery) const { + DepsTracker deps; + bool knowAllFields = false; + bool knowAllMeta = false; + for (size_t i=0; i < sources.size() && !(knowAllFields && knowAllMeta); i++) { + DepsTracker localDeps; + DocumentSource::GetDepsReturn status = sources[i]->getDependencies(&localDeps); + + if (status == DocumentSource::NOT_SUPPORTED) { + // Assume this stage needs everything. We may still know something about our + // dependencies if an earlier stage returned either EXHAUSTIVE_FIELDS or + // EXHAUSTIVE_META. + break; + } - result.appendArray(serverPipelineName, opArray.arr()); - } + if (!knowAllFields) { + deps.fields.insert(localDeps.fields.begin(), localDeps.fields.end()); + if (localDeps.needWholeDocument) + deps.needWholeDocument = true; + knowAllFields = status & DocumentSource::EXHAUSTIVE_FIELDS; + } - void Pipeline::writeExplainMongos(BSONObjBuilder &result) const { + if (!knowAllMeta) { + if (localDeps.needTextScore) + deps.needTextScore = true; - /* - For now, this should be a BSON source array. - In future, we might have a more clever way of getting this, when - we have more interleaved fetching between shards. The DocumentSource - interface will have to change to accommodate that. - */ - DocumentSourceBsonArray *pSourceBsonArray = - dynamic_cast(sources.front().get()); - verify(pSourceBsonArray); - - BSONArrayBuilder shardOpArray; // where we'll put the pipeline ops - for(bool hasDocument = !pSourceBsonArray->eof(); hasDocument; - hasDocument = pSourceBsonArray->advance()) { - Document pDocument = pSourceBsonArray->getCurrent(); - BSONObjBuilder opBuilder; - pDocument->toBson(&opBuilder); - shardOpArray.append(opBuilder.obj()); + knowAllMeta = status & DocumentSource::EXHAUSTIVE_META; + } } - BSONArrayBuilder mongosOpArray; // where we'll put the pipeline ops - writeExplainOps(&mongosOpArray); + if (!knowAllFields) + deps.needWholeDocument = true; // don't know all fields we need - // now we combine the shard pipelines with the one here - result.append(serverPipelineName, shardOpArray.arr()); - result.append(mongosPipelineName, mongosOpArray.arr()); - } + // NOTE This code assumes that textScore can only be generated by the initial query. + if (DocumentSourceMatch::isTextQuery(initialQuery)) { + // If doing a text query, assume we need the score if we can't prove we don't. + if (!knowAllMeta) + deps.needTextScore = true; + } + else { + // If we aren't doing a text query, then we don't need to ask for the textScore since we + // know it will be missing anyway. + deps.needTextScore = false; + } - void Pipeline::addInitialSource(intrusive_ptr source) { - sources.push_front(source); + return deps; } - } // namespace mongo diff --git a/src/mongo/db/pipeline/pipeline.h b/src/mongo/db/pipeline/pipeline.h index 151e1d8fcc1..4d11f8eed29 100644 --- a/src/mongo/db/pipeline/pipeline.h +++ b/src/mongo/db/pipeline/pipeline.h @@ -12,25 +12,36 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once -#include "mongo/pch.h" +#include -#include "util/intrusive_counter.h" -#include "util/timer.h" +#include "mongo/db/pipeline/value.h" +#include "mongo/util/intrusive_counter.h" +#include "mongo/util/timer.h" namespace mongo { class BSONObj; class BSONObjBuilder; - class BSONArrayBuilder; + class Command; + struct DepsTracker; class DocumentSource; - class DocumentSourceProject; - class Expression; - class ExpressionContext; - class ExpressionNary; - struct OpDesc; // local private struct + struct ExpressionContext; + class Privilege; /** mongodb "commands" (sent via db.$cmd.findOne(...)) subclass to make a command. define a singleton object for it. @@ -38,25 +49,25 @@ namespace mongo { class Pipeline : public IntrusiveCounterUnsigned { public: - virtual ~Pipeline(); - /** - Create a pipeline from the command. - - @param errmsg where to write errors, if there are any - @param cmdObj the command object sent from the client - @returns the pipeline, if created, otherwise a NULL reference + * Create a pipeline from the command. + * + * @param errmsg where to write errors, if there are any + * @param cmdObj the command object sent from the client + * @returns the pipeline, if created, otherwise a NULL reference */ static intrusive_ptr parseCommand( - string &errmsg, BSONObj &cmdObj, - const intrusive_ptr &pCtx); + string& errmsg, + const BSONObj& cmdObj, + const intrusive_ptr& pCtx); - /** - Get the collection name from the command. + /// Helper to implement Command::addRequiredPrivileges + static void addRequiredPrivileges(Command* commandTemplate, + const string& dbname, + BSONObj cmdObj, + vector* out); - @returns the collection name - */ - string getCollectionName() const; + intrusive_ptr getContext() const { return pCtx; } /** Split the current Pipeline into a Pipeline for each shard, and @@ -69,15 +80,10 @@ namespace mongo { */ intrusive_ptr splitForSharded(); - /** - If the pipeline starts with a $match, dump its BSON predicate - specification to the supplied builder and return true. - - @param pQueryBuilder the builder to put the match BSON into - @returns true if a match was found and dumped to pQueryBuilder, - false otherwise + /** If the pipeline starts with a $match, return its BSON predicate. + * Returns empty BSON if the first stage isn't $match. */ - bool getInitialQuery(BSONObjBuilder *pQueryBuilder) const; + BSONObj getInitialQuery() const; /** Write the Pipeline as a BSONObj command. This should be the @@ -90,39 +96,44 @@ namespace mongo { @param the builder to write the command to */ - void toBson(BSONObjBuilder *pBuilder) const; + Document serialize() const; + + /** Stitch together the source pointers (by calling setSource) for each source in sources. + * Must be called after optimize and addInitialSource but before trying to get results. + */ + void stitch(); /** Run the Pipeline on the given source. @param result builder to write the result to - @param errmsg place to put error messages, if any - @returns true on success, false if an error occurs */ - bool run(BSONObjBuilder &result, string &errmsg); + void run(BSONObjBuilder& result); - /** - Debugging: should the processing pipeline be split within - mongod, simulating the real mongos/mongod split? This is determined - by setting the splitMongodPipeline field in an "aggregate" - command. + bool isExplain() const { return explain; } - The split itself is handled by the caller, which is currently - pipeline_command.cpp. + /// The initial source is special since it varies between mongos and mongod. + void addInitialSource(intrusive_ptr source); - @returns true if the pipeline is to be split - */ - bool getSplitMongodPipeline() const; + /// The source that represents the output. Returns a non-owning pointer. + DocumentSource* output() { invariant( !sources.empty() ); return sources.back().get(); } - /** - Ask if this is for an explain request. + /// Returns true if this pipeline only uses features that work in mongos. + bool canRunInMongos() const; - @returns true if this is an explain + /** + * Write the pipeline's operators to a vector, with the + * explain flag true (for DocumentSource::serializeToArray()). */ - bool isExplain() const; - - /// The initial source is special since it varies between mongos and mongod. - void addInitialSource(intrusive_ptr source); + vector writeExplainOps() const; + + /** + * Returns the dependencies needed by this pipeline. + * + * initialQuery is used as a fallback for metadata dependency detection. The assumption is + * that any metadata produced by the query is needed unless we can prove it isn't. + */ + DepsTracker getDependencies(const BSONObj& initialQuery) const; /** The aggregation command name. @@ -141,81 +152,30 @@ namespace mongo { friend class PipelineD; private: + class Optimizations { + public: + // These contain static functions that optimize pipelines in various ways. + // They are classes rather than namespaces so that they can be friends of Pipeline. + // Classes are defined in pipeline_optimizations.h. + class Local; + class Sharded; + }; + + friend class Optimizations::Local; + friend class Optimizations::Sharded; + static const char pipelineName[]; static const char explainName[]; static const char fromRouterName[]; - static const char splitMongodPipelineName[]; static const char serverPipelineName[]; static const char mongosPipelineName[]; Pipeline(const intrusive_ptr &pCtx); - /* - Write the pipeline's operators to the given array, with the - explain flag true (for DocumentSource::addToBsonArray()). - - @param pArrayBuilder where to write the ops to - */ - void writeExplainOps(BSONArrayBuilder *pArrayBuilder) const; - - /* - Write the pipeline's operators to the given result document, - for a shard server (or regular server, in an unsharded setup). - - This uses writeExplainOps() and adds that array to the result - with the serverPipelineName. That will be preceded by explain - information for the input source. - - @param result the object to add the explain information to - */ - void writeExplainShard(BSONObjBuilder &result) const; - - /* - Write the pipeline's operators to the given result document, - for a mongos instance. - - This first adds the serverPipeline obtained from the input - source. - - Then this uses writeExplainOps() and adds that array to the result - with the serverPipelineName. That will be preceded by explain - information for the input source. - - @param result the object to add the explain information to - */ - void writeExplainMongos(BSONObjBuilder &result) const; - - string collectionName; - typedef deque > SourceContainer; + typedef std::deque > SourceContainer; SourceContainer sources; bool explain; - bool splitMongodPipeline; - intrusive_ptr pCtx; + boost::intrusive_ptr pCtx; }; - } // namespace mongo - - -/* ======================= INLINED IMPLEMENTATIONS ========================== */ - -namespace mongo { - - inline string Pipeline::getCollectionName() const { - return collectionName; - } - - inline bool Pipeline::getSplitMongodPipeline() const { - if (!DEBUG_BUILD) - return false; - - return splitMongodPipeline; - } - - inline bool Pipeline::isExplain() const { - return explain; - } - -} // namespace mongo - - diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp index ab4f0ef784f..ae1b42ce887 100644 --- a/src/mongo/db/pipeline/pipeline_d.cpp +++ b/src/mongo/db/pipeline/pipeline_d.cpp @@ -12,84 +12,105 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ -#include "pch.h" -#include "db/pipeline/pipeline.h" -#include "db/pipeline/pipeline_d.h" +#include "mongo/pch.h" + +#include "mongo/db/pipeline/pipeline_d.h" -#include "db/cursor.h" -#include "db/queryutil.h" -#include "db/pipeline/document_source.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/db/catalog/collection.h" #include "mongo/db/instance.h" - +#include "mongo/db/pdfile.h" +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/s/d_logic.h" namespace mongo { - void PipelineD::prepareCursorSource( - const intrusive_ptr &pPipeline, - const string &dbName, - const intrusive_ptr &pExpCtx) { +namespace { + class MongodImplementation : public DocumentSourceNeedsMongod::MongodInterface { + public: + DBClientBase* directClient() { return &_client; } + + bool isSharded(const NamespaceString& ns) { + const ChunkVersion unsharded(0, 0, OID()); + return !(shardingState.getVersion(ns.ns()).isWriteCompatibleWith(unsharded)); + } + + bool isCapped(const NamespaceString& ns) { + Client::ReadContext ctx(ns.ns()); + Collection* collection = ctx.ctx().db()->getCollection(ns); + return collection && collection->isCapped(); + } + + private: + DBDirectClient _client; + }; +} + + boost::shared_ptr PipelineD::prepareCursorSource( + const intrusive_ptr& pPipeline, + const intrusive_ptr& pExpCtx) { + // get the full "namespace" name + const string& fullName = pExpCtx->ns.ns(); + Lock::assertAtLeastReadLocked(fullName); // We will be modifying the source vector as we go Pipeline::SourceContainer& sources = pPipeline->sources; - if (!sources.empty()) { - DocumentSource* first = sources.front().get(); - DocumentSourceGeoNear* geoNear = dynamic_cast(first); - if (geoNear) { - geoNear->client.reset(new DBDirectClient); - geoNear->db = dbName; - geoNear->collection = pPipeline->collectionName; - return; // we don't need a DocumentSourceCursor in this case + // Inject a MongodImplementation to sources that need them. + for (size_t i = 0; i < sources.size(); i++) { + DocumentSourceNeedsMongod* needsMongod = + dynamic_cast(sources[i].get()); + if (needsMongod) { + needsMongod->injectMongodInterface(boost::make_shared()); } } - /* look for an initial match */ - BSONObjBuilder queryBuilder; - bool initQuery = pPipeline->getInitialQuery(&queryBuilder); - if (initQuery) { - /* - This will get built in to the Cursor we'll create, so - remove the match from the pipeline - */ - sources.pop_front(); + if (!sources.empty() && sources.front()->isValidInitialSource()) { + if (dynamic_cast(sources.front().get())) { + // Enable the hooks for setting up authentication on the subsequent internal + // connections we are going to create. This would normally have been done + // when SetShardVersion was called, but since SetShardVersion is never called + // on secondaries, this is needed. + ShardedConnectionInfo::addHook(); + } + return boost::shared_ptr(); // don't need a cursor } - /* - Create a query object. - - This works whether we got an initial query above or not; if not, - it results in a "{}" query, which will be what we want in that case. - - We create a pointer to a shared object instead of a local - object so that we can preserve it for the Cursor we're going to - create below. - */ - shared_ptr pQueryObj(new BSONObj(queryBuilder.obj())); - - /* Look for an initial simple project; we'll avoid constructing Values - * for fields that won't make it through the projection. - */ - - bool haveProjection = false; - BSONObj projection; - DocumentSource::ParsedDeps dependencies; - { - set deps; - DocumentSource::GetDepsReturn status = DocumentSource::SEE_NEXT; - for (size_t i=0; i < sources.size() && status == DocumentSource::SEE_NEXT; i++) { - status = sources[i]->getDependencies(deps); - } - if (status == DocumentSource::EXHAUSTIVE) { - projection = DocumentSource::depsToProjection(deps); - dependencies = DocumentSource::parseDeps(deps); - haveProjection = true; - } + // Look for an initial match. This works whether we got an initial query or not. + // If not, it results in a "{}" query, which will be what we want in that case. + const BSONObj queryObj = pPipeline->getInitialQuery(); + if (!queryObj.isEmpty()) { + // This will get built in to the Cursor we'll create, so + // remove the match from the pipeline + sources.pop_front(); } + // Find the set of fields in the source documents depended on by this pipeline. + const DepsTracker deps = pPipeline->getDependencies(queryObj); + + // Passing query an empty projection since it is faster to use ParsedDeps::extractFields(). + // This will need to change to support covering indexes (SERVER-12015). There is an + // exception for textScore since that can only be retrieved by a query projection. + const BSONObj projectionForQuery = deps.needTextScore ? deps.toProjection() : BSONObj(); + /* Look for an initial sort; we'll try to add this to the Cursor we create. If we're successful in doing that (further down), @@ -97,132 +118,100 @@ namespace mongo { will already come sorted in the specified order as a result of the index scan. */ - intrusive_ptr pSort; - BSONObjBuilder sortBuilder; + intrusive_ptr sortStage; + BSONObj sortObj; if (!sources.empty()) { - const intrusive_ptr &pSC = sources.front(); - pSort = dynamic_cast(pSC.get()); - - if (pSort) { - /* build the sort key */ - pSort->sortKeyToBson(&sortBuilder, false); + sortStage = dynamic_cast(sources.front().get()); + if (sortStage) { + // build the sort key + sortObj = sortStage->serializeSortKey(/*explain*/false).toBson(); } } - /* Create the sort object; see comments on the query object above */ - shared_ptr pSortObj(new BSONObj(sortBuilder.obj())); + // Create the Runner. + // + // If we try to create a Runner that includes both the match and the + // sort, and the two are incompatible wrt the available indexes, then + // we don't get a Runner back. + // + // So we try to use both first. If that fails, try again, without the + // sort. + // + // If we don't have a sort, jump straight to just creating a Runner + // without the sort. + // + // If we are able to incorporate the sort into the Runner, remove it + // from the head of the pipeline. + // + // LATER - we should be able to find this out before we create the + // cursor. Either way, we can then apply other optimizations there + // are tickets for, such as SERVER-4507. + const size_t runnerOptions = QueryPlannerParams::DEFAULT + | QueryPlannerParams::INCLUDE_SHARD_FILTER + | QueryPlannerParams::NO_BLOCKING_SORT + ; + boost::shared_ptr runner; + bool sortInRunner = false; + if (sortStage) { + CanonicalQuery* cq; + Status status = + CanonicalQuery::canonicalize(pExpCtx->ns, + queryObj, + sortObj, + projectionForQuery, + &cq); + Runner* rawRunner; + if (status.isOK() && getRunner(cq, &rawRunner, runnerOptions).isOK()) { + // success: The Runner will handle sorting for us using an index. + runner.reset(rawRunner); + sortInRunner = true; - /* get the full "namespace" name */ - string fullName(dbName + "." + pPipeline->getCollectionName()); - - /* for debugging purposes, show what the query and sort are */ - DEV { - (log() << "\n---- query BSON\n" << - pQueryObj->jsonString(Strict, 1) << "\n----\n").flush(); - (log() << "\n---- sort BSON\n" << - pSortObj->jsonString(Strict, 1) << "\n----\n").flush(); - (log() << "\n---- fullName\n" << - fullName << "\n----\n").flush(); - } - - // Create the necessary context to use a Cursor, including taking a namespace read lock, - // see SERVER-6123. - // Note: this may throw if the sharding version for this connection is out of date. - shared_ptr cursorWithContext - ( new DocumentSourceCursor::CursorWithContext( fullName ) ); - - /* - Create the cursor. - - If we try to create a cursor that includes both the match and the - sort, and the two are incompatible wrt the available indexes, then - we don't get a cursor back. - - So we try to use both first. If that fails, try again, without the - sort. - - If we don't have a sort, jump straight to just creating a cursor - without the sort. - - If we are able to incorporate the sort into the cursor, remove it - from the head of the pipeline. - - LATER - we should be able to find this out before we create the - cursor. Either way, we can then apply other optimizations there - are tickets for, such as SERVER-4507. - */ - - shared_ptr pCursor; - bool initSort = false; - if (pSort) { - const BSONObj queryAndSort = BSON("$query" << *pQueryObj << "$orderby" << *pSortObj); - shared_ptr pq (new ParsedQuery( - fullName.c_str(), 0, 0, QueryOption_NoCursorTimeout, queryAndSort, projection)); - - /* try to create the cursor with the query and the sort */ - shared_ptr pSortedCursor( - pCursor = NamespaceDetailsTransient::getCursor( - fullName.c_str(), *pQueryObj, *pSortObj, - QueryPlanSelectionPolicy::any(), pq)); - - if (pSortedCursor.get()) { - /* success: remove the sort from the pipeline */ sources.pop_front(); - - if (pSort->getLimitSrc()) { + if (sortStage->getLimitSrc()) { // need to reinsert coalesced $limit after removing $sort - sources.push_front(pSort->getLimitSrc()); + sources.push_front(sortStage->getLimitSrc()); } - - pCursor = pSortedCursor; - initSort = true; } } - if (!pCursor.get()) { - shared_ptr pq (new ParsedQuery( - fullName.c_str(), 0, 0, QueryOption_NoCursorTimeout, *pQueryObj, projection)); - - /* try to create the cursor without the sort */ - shared_ptr pUnsortedCursor( - pCursor = NamespaceDetailsTransient::getCursor( - fullName.c_str(), *pQueryObj, BSONObj(), - QueryPlanSelectionPolicy::any(), pq)); - - pCursor = pUnsortedCursor; + if (!runner.get()) { + const BSONObj noSort; + CanonicalQuery* cq; + uassertStatusOK( + CanonicalQuery::canonicalize(pExpCtx->ns, + queryObj, + noSort, + projectionForQuery, + &cq)); + + Runner* rawRunner; + uassertStatusOK(getRunner(cq, &rawRunner, runnerOptions)); + runner.reset(rawRunner); } - // Now add the Cursor to cursorWithContext. - cursorWithContext->_cursor.reset - ( new ClientCursor( QueryOption_NoCursorTimeout, pCursor, fullName ) ); - /* wrap the cursor with a DocumentSource and return that */ - intrusive_ptr pSource( - DocumentSourceCursor::create( cursorWithContext, pExpCtx ) ); + // DocumentSourceCursor expects a yielding Runner that has had its state saved. + runner->setYieldPolicy(Runner::YIELD_AUTO); + runner->saveState(); - pSource->setNamespace(fullName); + // Put the Runner into a DocumentSourceCursor and add it to the front of the pipeline. + intrusive_ptr pSource = + DocumentSourceCursor::create(fullName, runner, pExpCtx); - /* - Note the query and sort + // Note the query, sort, and projection for explain. + pSource->setQuery(queryObj); + if (sortInRunner) + pSource->setSort(sortObj); - This records them for explain, and keeps them alive; they are - referenced (by reference) by the cursor, which doesn't make its - own copies of them. - */ - pSource->setQuery(pQueryObj); - if (initSort) - pSource->setSort(pSortObj); + pSource->setProjection(deps.toProjection(), deps.toParsedDeps()); - if (haveProjection) { - pSource->setProjection(projection, dependencies); + while (!sources.empty() && pSource->coalesce(sources.front())) { + sources.pop_front(); } - // If we are in an explain, we won't actually use the created cursor so release it. - // This is important to avoid double locking when we use DBDirectClient to run explain. - if (pPipeline->isExplain()) - pSource->dispose(); - pPipeline->addInitialSource(pSource); + + return runner; } } // namespace mongo diff --git a/src/mongo/db/pipeline/pipeline_d.h b/src/mongo/db/pipeline/pipeline_d.h index 154ef36a824..98ea7537be3 100644 --- a/src/mongo/db/pipeline/pipeline_d.h +++ b/src/mongo/db/pipeline/pipeline_d.h @@ -12,15 +12,30 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once -#include "mongo/pch.h" +#include namespace mongo { + class Collection; class DocumentSourceCursor; + struct ExpressionContext; class Pipeline; + class Runner; /* PipelineD is an extension of the Pipeline class, but with additional @@ -35,25 +50,28 @@ namespace mongo { public: /** - Create a Cursor wrapped in a DocumentSourceCursor, which is suitable - to be the first source for a pipeline to begin with. This source - will feed the execution of the pipeline. - - This method looks for early pipeline stages that can be folded into - the underlying cursor, and when a cursor can absorb those, they - are removed from the head of the pipeline. For example, an - early match can be removed and replaced with a Cursor that will - do an index scan. - - The cursor is added to the front of the pipeline's sources. - - @param pPipeline the logical "this" for this operation - @param dbName the name of the database - @param pExpCtx the expression context for this pipeline + * Create a Cursor wrapped in a DocumentSourceCursor, which is suitable + * to be the first source for a pipeline to begin with. This source + * will feed the execution of the pipeline. + * + * This method looks for early pipeline stages that can be folded into + * the underlying cursor, and when a cursor can absorb those, they + * are removed from the head of the pipeline. For example, an + * early match can be removed and replaced with a Cursor that will + * do an index scan. + * + * The cursor is added to the front of the pipeline's sources. + * + * Must have a ReadContext before entering. + * + * If the returned Runner is non-null, you are responsible for ensuring + * it receives appropriate invalidate and kill messages. + * + * @param pPipeline the logical "this" for this operation + * @param pExpCtx the expression context for this pipeline */ - static void prepareCursorSource( + static boost::shared_ptr prepareCursorSource( const intrusive_ptr &pPipeline, - const string &dbName, const intrusive_ptr &pExpCtx); private: diff --git a/src/mongo/db/pipeline/pipeline_optimizations.h b/src/mongo/db/pipeline/pipeline_optimizations.h new file mode 100644 index 00000000000..d8b89350c28 --- /dev/null +++ b/src/mongo/db/pipeline/pipeline_optimizations.h @@ -0,0 +1,124 @@ +/** + * Copyright 2013 (c) 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file declares optimizations available on Pipelines. For now they should be considered part + * of Pipeline's implementation rather than it's interface. + */ + +#pragma once + +#include "mongo/db/pipeline/pipeline.h" + +namespace mongo { + /** + * This class holds optimizations applied to a single Pipeline. + * + * Each function has the same signature and takes a Pipeline as an in/out parameter. + */ + class Pipeline::Optimizations::Local { + public: + /** + * Moves matches before any adjacent sort phases. + * + * This means we sort fewer items. Neither sorts, nor matches (excluding $text) + * change the documents in the stream, so this transformation shouldn't affect + * the result. + */ + static void moveMatchBeforeSort(Pipeline* pipeline); + + /** + * Moves limits before any adjacent skip phases. + * + * This is more optimal for sharding since currently, we can only split + * the pipeline at a single source and it is better to limit the results + * coming from each shard. This also enables other optimizations like + * coalescing the limit into a sort. + */ + static void moveLimitBeforeSkip(Pipeline* pipeline); + + /** + * Runs through the DocumentSources, and give each one the opportunity + * to coalesce with its successor. If successful, remove the successor. + * + * This should generally be run after optimizations that reorder stages + * to be most effective. + * + * NOTE: uses the DocumentSource::coalesce() method + */ + static void coalesceAdjacent(Pipeline* pipeline); + + /** + * Gives each DocumentSource the opportunity to optimize itself. + * + * NOTE: uses the DocumentSource::optimize() method + */ + static void optimizeEachDocumentSource(Pipeline* pipeline); + + /** + * Optimizes [$redact, $match] to [$match, $redact, $match] if possible. + * + * This gives us the ability to use indexes and reduce the number of + * BSONObjs converted to Documents. + */ + static void duplicateMatchBeforeInitalRedact(Pipeline* pipeline); + }; + + /** + * This class holds optimizations applied to a shard Pipeline and a merger Pipeline. + * + * Each function has the same signature and takes two Pipelines, both as an in/out parameters. + */ + class Pipeline::Optimizations::Sharded { + public: + /** + * Moves everything before a splittable stage to the shards. If there + * are no splittable stages, moves everything to the shards. + * + * It is not safe to call this optimization multiple times. + * + * NOTE: looks for SplittableDocumentSources and uses that API + */ + static void findSplitPoint(Pipeline* shardPipe, Pipeline* mergePipe); + + /** + * If the final stage on shards is to unwind an array, move that stage to the merger. This + * cuts down on network traffic and allows us to take advantage of reduced copying in + * unwind. + */ + static void moveFinalUnwindFromShardsToMerger(Pipeline* shardPipe, Pipeline* mergePipe); + + /** + * Adds a stage to the end of shardPipe explicitly requesting all fields that mergePipe + * needs. This is only done if it heuristically determines that it is needed. This + * optimization can reduce the amount of network traffic and can also enable the shards to + * convert less source BSON into Documents. + */ + static void limitFieldsSentFromShardsToMerger(Pipeline* shardPipe, Pipeline* mergePipe); + }; +} // namespace mongo diff --git a/src/mongo/db/pipeline/value.cpp b/src/mongo/db/pipeline/value.cpp index 5561cb41a59..4b91603ce18 100644 --- a/src/mongo/db/pipeline/value.cpp +++ b/src/mongo/db/pipeline/value.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/pch.h" @@ -21,13 +33,54 @@ #include #include "mongo/db/jsobj.h" -#include "mongo/db/pipeline/builder.h" #include "mongo/db/pipeline/document.h" #include "mongo/util/mongoutils/str.h" namespace mongo { using namespace mongoutils; + void ValueStorage::verifyRefCountingIfShould() const { + switch (type) { + case MinKey: + case MaxKey: + case jstOID: + case Date: + case Timestamp: + case EOO: + case jstNULL: + case Undefined: + case Bool: + case NumberInt: + case NumberLong: + case NumberDouble: + // the above types never reference external data + verify(!refCounter); + break; + + case String: + case RegEx: + case Code: + case Symbol: + // the above types reference data when not using short-string optimization + verify(refCounter == !shortStr); + break; + + case BinData: // TODO this should probably support short-string optimization + case Array: // TODO this should probably support empty-is-NULL optimization + case DBRef: + case CodeWScope: + // the above types always reference external data. + verify(refCounter); + verify(bool(genericRCPtr)); + break; + + case Object: + // Objects either hold a NULL ptr or should be ref-counting + verify(refCounter == bool(genericRCPtr)); + break; + } + } + void ValueStorage::putString(const StringData& s) { // Note: this also stores data portion of BinData const size_t sizeNoNUL = s.size(); @@ -78,55 +131,6 @@ namespace mongo { // not in header because document is fwd declared Value::Value(const BSONObj& obj) : _storage(Object, Document(obj)) {} - Value::Value(BSONType theType): _storage(theType) { - switch(theType) { - case EOO: - case Undefined: - case jstNULL: - case Object: // empty - break; - - case Array: // empty - _storage.putVector(new RCVector()); - break; - - case Bool: - _storage.boolValue = false; - break; - - case NumberDouble: - _storage.doubleValue = 0; - break; - - case NumberInt: - _storage.intValue = 0; - break; - - case NumberLong: - _storage.longValue = 0; - break; - - case Date: - _storage.dateValue = 0; - break; - - case Timestamp: - _storage.timestampValue = 0; - break; - - default: - // nothing else is allowed - uassert(16001, str::stream() << - "can't create empty Value of type " << typeName(getType()), false); - break; - } - } - - - Value Value::createFromBsonElement(const BSONElement* pBsonElement) { - return Value(*pBsonElement); - } - Value::Value(const BSONElement& elem) : _storage(elem.type()) { switch(elem.type()) { // These are all type-only, no data @@ -212,21 +216,23 @@ namespace mongo { } } - Value Value::createIntOrLong(long long value) { - if (value > numeric_limits::max() || value < numeric_limits::min()) { + Value::Value(const BSONArray& arr) : _storage(Array) { + intrusive_ptr vec (new RCVector); + BSONForEach(sub, arr) { + vec->vec.push_back(Value(sub)); + } + _storage.putVector(vec.get()); + } + + Value Value::createIntOrLong(long long longValue) { + int intValue = longValue; + if (intValue != longValue) { // it is too large to be an int and should remain a long - return Value(value); + return Value(longValue); } // should be an int since all arguments were int and it fits - return createInt(value); - } - - Value Value::createDate(const long long value) { - // Can't directly construct because constructor would clash with createLong - Value val (Date); - val._storage.dateValue = value; - return val; + return Value(intValue); } double Value::getDouble() const { @@ -471,19 +477,15 @@ namespace mongo { } string Value::coerceToString() const { - stringstream ss; switch(getType()) { case NumberDouble: - ss << _storage.doubleValue; - return ss.str(); + return str::stream() << _storage.doubleValue; case NumberInt: - ss << _storage.intValue; - return ss.str(); + return str::stream() << _storage.intValue; case NumberLong: - ss << _storage.longValue; - return ss.str(); + return str::stream() << _storage.longValue; case Code: case Symbol: @@ -491,8 +493,7 @@ namespace mongo { return getStringData().toString(); case Timestamp: - ss << getTimestamp().toStringPretty(); - return ss.str(); + return getTimestamp().toStringPretty(); case Date: return tmToISODateString(coerceToTm()); @@ -736,7 +737,7 @@ namespace mongo { } case Object: - getDocument()->hash_combine(seed); + getDocument().hash_combine(seed); break; case Array: { @@ -829,11 +830,11 @@ namespace mongo { case BinData: case String: return sizeof(Value) + (_storage.shortStr - ? sizeof(RCString) + _storage.getString().size() - : 0); + ? 0 // string stored inline, so no extra mem usage + : sizeof(RCString) + _storage.getString().size()); case Object: - return sizeof(Value) + getDocument()->getApproximateSize(); + return sizeof(Value) + getDocument().getApproximateSize(); case Array: { size_t size = sizeof(Value); @@ -895,7 +896,7 @@ namespace mongo { case Undefined: return out << "undefined"; case Date: return out << tmToISODateString(val.coerceToTm()); case Timestamp: return out << val.getTimestamp().toString(); - case Object: return out << val.getDocument()->toString(); + case Object: return out << val.getDocument().toString(); case Array: { out << "["; const size_t n = val.getArray().size(); @@ -926,4 +927,146 @@ namespace mongo { // Not in default case to trigger better warning if a case is missing verify(false); } + + void Value::serializeForSorter(BufBuilder& buf) const { + buf.appendChar(getType()); + switch(getType()) { + // type-only types + case EOO: + case MinKey: + case MaxKey: + case jstNULL: + case Undefined: + break; + + // simple types + case jstOID: buf.appendStruct(_storage.oid); break; + case NumberInt: buf.appendNum(_storage.intValue); break; + case NumberLong: buf.appendNum(_storage.longValue); break; + case NumberDouble: buf.appendNum(_storage.doubleValue); break; + case Bool: buf.appendChar(_storage.boolValue); break; + case Date: buf.appendNum(_storage.dateValue); break; + case Timestamp: buf.appendStruct(getTimestamp()); break; + + // types that are like strings + case String: + case Symbol: + case Code: { + StringData str = getStringData(); + buf.appendNum(int(str.size())); + buf.appendStr(str, /*NUL byte*/ false); + break; + } + + case BinData: { + StringData str = getStringData(); + buf.appendChar(_storage.binDataType()); + buf.appendNum(int(str.size())); + buf.appendStr(str, /*NUL byte*/ false); + break; + } + + case RegEx: + buf.appendStr(getRegex(), /*NUL byte*/ true); + buf.appendStr(getRegexFlags(), /*NUL byte*/ true); + break; + + case Object: + getDocument().serializeForSorter(buf); + break; + + case DBRef: + buf.appendStruct(_storage.getDBRef()->oid); + buf.appendStr(_storage.getDBRef()->ns, /*NUL byte*/ true); + break; + + case CodeWScope: { + intrusive_ptr cws = _storage.getCodeWScope(); + buf.appendNum(int(cws->code.size())); + buf.appendStr(cws->code, /*NUL byte*/ false); + cws->scope.serializeForSorter(buf); + break; + } + + case Array: { + const vector& array = getArray(); + const int numElems = array.size(); + buf.appendNum(numElems); + for (int i = 0; i < numElems; i++) + array[i].serializeForSorter(buf); + break; + } + } + } + + Value Value::deserializeForSorter(BufReader& buf, const SorterDeserializeSettings& settings) { + const BSONType type = BSONType(buf.read()); // need sign extension for MinKey + switch(type) { + // type-only types + case EOO: + case MinKey: + case MaxKey: + case jstNULL: + case Undefined: + return Value(ValueStorage(type)); + + // simple types + case jstOID: return Value(buf.read()); + case NumberInt: return Value(buf.read()); + case NumberLong: return Value(buf.read()); + case NumberDouble: return Value(buf.read()); + case Bool: return Value(bool(buf.read())); + case Date: return Value(Date_t(buf.read())); + case Timestamp: return Value(buf.read()); + + // types that are like strings + case String: + case Symbol: + case Code: { + int size = buf.read(); + const char* str = static_cast(buf.skip(size)); + return Value(ValueStorage(type, StringData(str, size))); + } + + case BinData: { + BinDataType bdt = BinDataType(buf.read()); + int size = buf.read(); + const void* data = buf.skip(size); + return Value(BSONBinData(data, size, bdt)); + } + + case RegEx: { + StringData regex = buf.readCStr(); + StringData flags = buf.readCStr(); + return Value(BSONRegEx(regex, flags)); + } + + case Object: + return Value(Document::deserializeForSorter(buf, + Document::SorterDeserializeSettings())); + + case DBRef: { + OID oid = buf.read(); + StringData ns = buf.readCStr(); + return Value(BSONDBRef(ns, oid)); + } + + case CodeWScope: { + int size = buf.read(); + const char* str = static_cast(buf.skip(size)); + BSONObj bson = BSONObj::deserializeForSorter(buf, BSONObj::SorterDeserializeSettings()); + return Value(BSONCodeWScope(StringData(str, size), bson)); + } + + case Array: { + const int numElems = buf.read(); + vector array; + array.reserve(numElems); + for (int i = 0; i < numElems; i++) + array.push_back(deserializeForSorter(buf, settings)); + return Value::consume(array); + } + } + verify(false); + } } diff --git a/src/mongo/db/pipeline/value.h b/src/mongo/db/pipeline/value.h index e1772ca8dff..2a70a94733d 100644 --- a/src/mongo/db/pipeline/value.h +++ b/src/mongo/db/pipeline/value.h @@ -12,15 +12,27 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once #include "mongo/db/pipeline/value_internal.h" +#include "mongo/platform/unordered_set.h" namespace mongo { class BSONElement; - class Builder; /** A variant type that can hold any type of data representable in BSON * @@ -64,7 +76,8 @@ namespace mongo { explicit Value(const string& value) : _storage(String, StringData(value)) {} explicit Value(const char* value) : _storage(String, StringData(value)) {} explicit Value(const Document& doc) : _storage(Object, doc) {} - explicit Value(const BSONObj& obj);// : _storage(Object, Document(obj)) {} // in cpp + explicit Value(const BSONObj& obj); + explicit Value(const BSONArray& arr); explicit Value(const vector& vec) : _storage(Array, new RCVector(vec)) {} explicit Value(const BSONBinData& bd) : _storage(BinData, bd) {} explicit Value(const BSONRegEx& re) : _storage(RegEx, re) {} @@ -80,11 +93,6 @@ namespace mongo { : _storage(Date, static_cast(date.millis)) // millis really signed {} - /** Creates an empty or zero value of specified type. - * This is currently the only way to create Undefined or Null Values. - */ - explicit Value(BSONType type); - // TODO: add an unsafe version that can share storage with the BSONElement /// Deep-convert from BSONElement to Value explicit Value(const BSONElement& elem); @@ -97,6 +105,16 @@ namespace mongo { */ static Value createIntOrLong(long long value); + /** Construct an Array-typed Value from consumed without copying the vector. + * consumed is replaced with an empty vector. + * In C++11 this would be spelled Value(std::move(consumed)). + */ + static Value consume(vector& consumed) { + RCVector* vec = new RCVector(); + std::swap(vec->vec, consumed); + return Value(ValueStorage(Array, vec)); + } + /** A "missing" value indicates the lack of a Value. * This is similar to undefined/null but should not appear in output to BSON. * Missing Values are returned by Document when accessing non-existent fields. @@ -190,6 +208,14 @@ namespace mongo { } return (Value::compare(v1, v2) == 0); } + + friend bool operator!=(const Value& v1, const Value& v2) { + return !(v1 == v2); + } + + friend bool operator<(const Value& lhs, const Value& rhs) { + return (Value::compare(lhs, rhs) < 0); + } /// This is for debugging, logging, etc. See getString() for how to extract a string. string toString() const; @@ -224,16 +250,12 @@ namespace mongo { /// Call this after memcpying to update ref counts if needed void memcpyed() const { _storage.memcpyed(); } - // LEGACY creation functions - static Value createFromBsonElement(const BSONElement* pBsonElement); - static Value createInt(int value) { return Value(value); } - static Value createLong(long long value) { return Value(value); } - static Value createDouble(double value) { return Value(value); } - static Value createTimestamp(const OpTime& value) { return Value(value); } - static Value createString(const string& value) { return Value(value); } - static Value createDocument(const Document& doc) { return Value(doc); } - static Value createArray(const vector& vec) { return Value(vec); } - static Value createDate(const long long value); + /// members for Sorter + struct SorterDeserializeSettings {}; // unused + void serializeForSorter(BufBuilder& buf) const; + static Value deserializeForSorter(BufReader& buf, const SorterDeserializeSettings&); + int memUsageForSorter() const { return getApproximateSize(); } + Value getOwned() const { return *this; } private: /** This is a "honeypot" to prevent unexpected implicit conversions to the accepted argument @@ -244,6 +266,8 @@ namespace mongo { template explicit Value(const InvalidArgumentType& invalidArgument); + explicit Value(const ValueStorage& storage) :_storage(storage) {} + // does no type checking StringData getStringData() const; // May contain embedded NUL bytes @@ -251,6 +275,8 @@ namespace mongo { friend class MutableValue; // gets and sets _storage.genericRCPtr }; BOOST_STATIC_ASSERT(sizeof(Value) == 16); + + typedef unordered_set ValueSet; } namespace std { diff --git a/src/mongo/db/pipeline/value_internal.h b/src/mongo/db/pipeline/value_internal.h index e3481951f9e..7565e318bd5 100644 --- a/src/mongo/db/pipeline/value_internal.h +++ b/src/mongo/db/pipeline/value_internal.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -22,7 +34,7 @@ #include "bson/bsonmisc.h" #include "bson/oid.h" #include "util/intrusive_counter.h" -#include "util/optime.h" +#include "mongo/bson/optime.h" namespace mongo { @@ -89,6 +101,7 @@ namespace mongo { } ~ValueStorage() { + DEV verifyRefCountingIfShould(); if (refCounter) intrusive_ptr_release(genericRCPtr); DEV memset(this, 0xee, sizeof(*this)); @@ -109,6 +122,7 @@ namespace mongo { /// Call this after memcpying to update ref counts if needed void memcpyed() const { + DEV verifyRefCountingIfShould(); if (refCounter) intrusive_ptr_add_ref(genericRCPtr); } @@ -140,6 +154,7 @@ namespace mongo { intrusive_ptr_add_ref(genericRCPtr); refCounter = true; } + DEV verifyRefCountingIfShould(); } StringData getString() const { @@ -191,6 +206,8 @@ namespace mongo { && i64[1] == other.i64[1]); } + void verifyRefCountingIfShould() const; + // This data is public because this should only be used by Value which would be a friend union { struct { diff --git a/src/mongo/db/prefetch.cpp b/src/mongo/db/prefetch.cpp index 73bc3047757..218a5e6ddcc 100644 --- a/src/mongo/db/prefetch.cpp +++ b/src/mongo/db/prefetch.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -20,10 +32,11 @@ #include "mongo/db/dbhelpers.h" #include "mongo/db/diskloc.h" -#include "mongo/db/index.h" -#include "mongo/db/index_update.h" +#include "mongo/db/index/index_access_method.h" +#include "mongo/db/structure/catalog/index_details.h" +#include "mongo/db/catalog/collection.h" #include "mongo/db/jsobj.h" -#include "mongo/db/namespace_details.h" +#include "mongo/db/structure/catalog/namespace_details.h" #include "mongo/db/repl/rs.h" #include "mongo/db/stats/timer_stats.h" #include "mongo/db/commands/server_status.h" @@ -60,12 +73,18 @@ namespace mongo { // prefetch ignores other ops return; } - + BSONObj obj = op.getObjectField(opField); const char *ns = op.getStringField("ns"); - NamespaceDetails *nsd = nsdetails(ns); - if (!nsd) return; // maybe not opened yet - + + Database* db = cc().database(); + if ( !db ) + return; + + Collection* collection = db->getCollection( ns ); + if ( !collection ) + return; + LOG(4) << "index prefetch for op " << *opType << endl; DEV Lock::assertAtLeastReadLocked(ns); @@ -85,7 +104,7 @@ namespace mongo { // a way to achieve that would be to prefetch the record first, and then afterwards do // this part. // - prefetchIndexPages(nsd, obj); + prefetchIndexPages(collection, obj); // do not prefetch the data for inserts; it doesn't exist yet // @@ -96,14 +115,13 @@ namespace mongo { if ((*opType == 'u') && // do not prefetch the data for capped collections because // they typically do not have an _id index for findById() to use. - !nsd->isCapped()) { + !collection->details()->isCapped()) { prefetchRecordPages(ns, obj); } } - void prefetchIndexPages(NamespaceDetails *nsd, const BSONObj& obj) { + void prefetchIndexPages(Collection* collection, const BSONObj& obj) { DiskLoc unusedDl; // unused - IndexInterface::IndexInserter inserter; BSONObjSet unusedKeys; ReplSetImpl::IndexPrefetchConfig prefetchConfig = theReplSet->getIndexPrefetchConfig(); @@ -118,16 +136,13 @@ namespace mongo { TimerHolder timer( &prefetchIndexStats); // on the update op case, the call to prefetchRecordPages will touch the _id index. // thus perhaps this option isn't very useful? - int indexNo = nsd->findIdIndex(); - if (indexNo == -1) return; try { - fetchIndexInserters(/*out*/unusedKeys, - inserter, - nsd, - indexNo, - obj, - unusedDl, - /*allowDups*/true); + IndexDescriptor* desc = collection->getIndexCatalog()->findIdIndex(); + if ( !desc ) + return; + IndexAccessMethod* iam = collection->getIndexCatalog()->getIndex( desc ); + verify( iam ); + iam->touch(obj); } catch (const DBException& e) { LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl; @@ -138,18 +153,15 @@ namespace mongo { { // indexCount includes all indexes, including ones // in the process of being built - int indexCount = nsd->getTotalIndexCount(); - for ( int indexNo = 0; indexNo < indexCount; indexNo++ ) { + IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator( true ); + while ( ii.more() ) { TimerHolder timer( &prefetchIndexStats); // This will page in all index pages for the given object. try { - fetchIndexInserters(/*out*/unusedKeys, - inserter, - nsd, - indexNo, - obj, - unusedDl, - /*allowDups*/true); + IndexDescriptor* desc = ii.next(); + IndexAccessMethod* iam = collection->getIndexCatalog()->getIndex( desc ); + verify( iam ); + iam->touch(obj); } catch (const DBException& e) { LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl; diff --git a/src/mongo/db/prefetch.h b/src/mongo/db/prefetch.h index a5935fe9fac..d70a662a754 100644 --- a/src/mongo/db/prefetch.h +++ b/src/mongo/db/prefetch.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #pragma once @@ -19,12 +31,12 @@ #include "mongo/db/diskloc.h" namespace mongo { - class NamespaceDetails; + class Collection; // page in both index and data pages for an op from the oplog void prefetchPagesForReplicatedOp(const BSONObj& op); // page in pages needed for all index lookups on a given object - void prefetchIndexPages(NamespaceDetails *nsd, const BSONObj& obj); + void prefetchIndexPages(Collection *nsd, const BSONObj& obj); // page in the data pages for a record associated with an object void prefetchRecordPages(const char *ns, const BSONObj& obj); diff --git a/src/mongo/db/projection.cpp b/src/mongo/db/projection.cpp index 5e2f204de56..31315048e4e 100644 --- a/src/mongo/db/projection.cpp +++ b/src/mongo/db/projection.cpp @@ -15,8 +15,10 @@ * limitations under the License. */ -#include "pch.h" -#include "projection.h" +#include "mongo/pch.h" + +#include "mongo/db/projection.h" + #include "mongo/db/matcher.h" #include "mongo/util/mongoutils/str.h" diff --git a/src/mongo/db/projection.h b/src/mongo/db/projection.h index 418c6fe5ca8..cc807553419 100644 --- a/src/mongo/db/projection.h +++ b/src/mongo/db/projection.h @@ -19,14 +19,11 @@ #include "mongo/pch.h" #include "mongo/util/string_map.h" -#include "jsobj.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher.h" namespace mongo { - // fwd decls - class Matcher; - class MatchDetails; - /** * given a document and a projection specification * can transform the document diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript new file mode 100644 index 00000000000..5fef6b04fa0 --- /dev/null +++ b/src/mongo/db/query/SConscript @@ -0,0 +1,242 @@ +# -*- mode: python -*- + +Import("env") + +env.Library( + target='query_planner', + source=[ + "canonical_query.cpp", + "query_settings.cpp", + "index_tag.cpp", + "parsed_projection.cpp", + "plan_cache.cpp", + "plan_enumerator.cpp", + "planner_access.cpp", + "planner_analysis.cpp", + "planner_ixselect.cpp", + "query_knobs.cpp", + "query_planner.cpp", + "query_solution.cpp", + ], + LIBDEPS=[ + "index_bounds", + "lite_parsed_query", + "$BUILD_DIR/mongo/bson", + "$BUILD_DIR/mongo/expressions", + "$BUILD_DIR/mongo/index_names", + "$BUILD_DIR/mongo/server_parameters", + ], +) + +env.Library( + target='query', + source=[ + "cached_plan_runner.cpp", + "eof_runner.cpp", + "explain_plan.cpp", + "get_runner.cpp", + "idhack_runner.cpp", + "internal_runner.cpp", + "multi_plan_runner.cpp", + "new_find.cpp", + "plan_executor.cpp", + "plan_ranker.cpp", + "single_solution_runner.cpp", + "stage_builder.cpp", + "subplan_runner.cpp", + "type_explain.cpp", + ], + LIBDEPS=[ + "query_planner", + "query_planner_test_lib", + "$BUILD_DIR/mongo/db/exec/exec" + ], +) + +env.CppUnitTest( + target="get_runner_test", + source=[ + "get_runner_test.cpp" + ], + LIBDEPS=[ + "query", + "$BUILD_DIR/mongo/serveronly", + "$BUILD_DIR/mongo/coreserver", + "$BUILD_DIR/mongo/coredb", + "$BUILD_DIR/mongo/mocklib", + ], + NO_CRUTCH = True, +) + +env.Library( + target="index_bounds", + source=[ + "index_bounds.cpp", + "index_bounds_builder.cpp", + "interval.cpp", + ], + LIBDEPS=[ + "query_logger", + "$BUILD_DIR/mongo/bson", + "$BUILD_DIR/mongo/expressions_geo", + "$BUILD_DIR/mongo/index_names", + "$BUILD_DIR/mongo/mongohasher", + ], +) + +env.Library( + target="lite_parsed_query", + source=[ + "lite_parsed_query.cpp" + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/bson", + ], +) + +env.Library( + target="query_planner_test_lib", + source=[ + "query_planner_test_lib.cpp", + ], + LIBDEPS=[ + "query_planner", + ], +) + +env.Library( + target="query_logger", + source=[ + "qlog.cpp" + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/server_parameters", + ], +) + +env.CppUnitTest( + target="canonical_query_test", + source=[ + "canonical_query_test.cpp" + ], + LIBDEPS=[ + "query_planner", + "$BUILD_DIR/mongo/expressions_text", + ], +) + +env.CppUnitTest( + target="index_bounds_test", + source=[ + "index_bounds_test.cpp" + ], + LIBDEPS=[ + "index_bounds", + ], +) + +env.CppUnitTest( + target="index_bounds_builder_test", + source=[ + "index_bounds_builder_test.cpp" + ], + LIBDEPS=[ + "index_bounds", + ], +) + +env.CppUnitTest( + target="interval_test", + source=[ + "interval_test.cpp" + ], + LIBDEPS=[ + "index_bounds", + ], +) + +env.CppUnitTest( + target="lite_parsed_query_test", + source=[ + "lite_parsed_query_test.cpp" + ], + LIBDEPS=[ + "lite_parsed_query", + ], +) + +env.CppUnitTest( + target="lru_key_value_test", + source=[ + "lru_key_value_test.cpp", + ], + LIBDEPS=[ + ], +) + +env.CppUnitTest( + target="parsed_projection_test", + source=[ + "parsed_projection_test.cpp" + ], + LIBDEPS=[ + "query_planner", + ], +) + +env.CppUnitTest( + target="plan_cache_test", + source=[ + "plan_cache_test.cpp" + ], + LIBDEPS=[ + "query_planner_test_lib", + ], +) + +env.CppUnitTest( + target="planner_analysis_test", + source=[ + "planner_analysis_test.cpp" + ], + LIBDEPS=[ + "query_planner", + ], +) + +env.CppUnitTest( + target="planner_ixselect_test", + source=[ + "planner_ixselect_test.cpp" + ], + LIBDEPS=[ + "query_planner", + ], +) + +env.CppUnitTest( + target="query_planner_test", + source=[ + "query_planner_test.cpp" + ], + LIBDEPS=[ + "query_planner_test_lib", + ], +) + +# $text pulls in a lot of stuff so we test it here. +env.CppUnitTest( + target="query_planner_text_test", + source=[ + "query_planner_text_test.cpp" + ], + LIBDEPS=[ + "query", + "query_planner_test_lib", + "$BUILD_DIR/mongo/serveronly", + "$BUILD_DIR/mongo/coreserver", + "$BUILD_DIR/mongo/coredb", + "$BUILD_DIR/mongo/mocklib", + ], + NO_CRUTCH = True, +) diff --git a/src/mongo/db/query/cached_plan_runner.cpp b/src/mongo/db/query/cached_plan_runner.cpp new file mode 100644 index 00000000000..18cc3d3af71 --- /dev/null +++ b/src/mongo/db/query/cached_plan_runner.cpp @@ -0,0 +1,215 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/cached_plan_runner.h" + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/plan_executor.h" +#include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/db/query/type_explain.h" + +namespace mongo { + + CachedPlanRunner::CachedPlanRunner(const Collection* collection, + CanonicalQuery* canonicalQuery, + QuerySolution* solution, + PlanStage* root, + WorkingSet* ws) + : _collection(collection), + _canonicalQuery(canonicalQuery), + _solution(solution), + _exec(new PlanExecutor(ws, root)), + _alreadyProduced(false), + _updatedCache(false), + _killed(false) { } + + CachedPlanRunner::~CachedPlanRunner() { + // The runner may produce all necessary results without hitting EOF. In this case, we still + // want to update the cache with feedback. + if (!_updatedCache) { + updateCache(); + } + } + + Runner::RunnerState CachedPlanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { + Runner::RunnerState state = _exec->getNext(objOut, dlOut); + + if (Runner::RUNNER_ADVANCED == state) { + // Indicate that the plan executor already produced results. + _alreadyProduced = true; + } + + // If the plan executor errors before producing any results, + // and we have a backup plan available, then fall back on the + // backup plan. This can happen if '_exec' has a blocking sort. + if (Runner::RUNNER_ERROR == state && !_alreadyProduced && NULL != _backupPlan.get()) { + _exec.reset(_backupPlan.release()); + state = _exec->getNext(objOut, dlOut); + } + + // This could be called several times and we don't want to update the cache every time. + if (Runner::RUNNER_EOF == state && !_updatedCache) { + updateCache(); + } + + return state; + } + + bool CachedPlanRunner::isEOF() { + return _exec->isEOF(); + } + + void CachedPlanRunner::saveState() { + _exec->saveState(); + if (NULL != _backupPlan.get()) { + _backupPlan->saveState(); + } + } + + bool CachedPlanRunner::restoreState() { + if (NULL != _backupPlan.get()) { + _backupPlan->restoreState(); + } + return _exec->restoreState(); + } + + void CachedPlanRunner::invalidate(const DiskLoc& dl, InvalidationType type) { + _exec->invalidate(dl, type); + if (NULL != _backupPlan.get()) { + _backupPlan->invalidate(dl, type); + } + } + + void CachedPlanRunner::setYieldPolicy(Runner::YieldPolicy policy) { + _exec->setYieldPolicy(policy); + if (NULL != _backupPlan.get()) { + _backupPlan->setYieldPolicy(policy); + } + } + + const std::string& CachedPlanRunner::ns() { + return _canonicalQuery->getParsed().ns(); + } + + void CachedPlanRunner::kill() { + _killed = true; + _collection = NULL; + _exec->kill(); + if (NULL != _backupPlan.get()) { + _backupPlan->kill(); + } + } + + Status CachedPlanRunner::getInfo(TypeExplain** explain, + PlanInfo** planInfo) const { + if (NULL != explain) { + if (NULL == _exec.get()) { + return Status(ErrorCodes::InternalError, "No plan available to provide stats"); + } + + // + // Explain for the winner plan + // + + scoped_ptr stats(_exec->getStats()); + if (NULL == stats.get()) { + return Status(ErrorCodes::InternalError, "no stats available to explain plan"); + } + + // Alternate plans not needed for explainMultiPlain. + // We don't bother showing the alternative plans because their bounds + // may not match the bounds of the query that we're running. If a user + // is explaining a query that could have been executed in >1 way it + // won't use a cached plan runner for that reason + // getInfo is used to generate explain info for system.profile and + // slow query logging only. + // User visible explain output is generated by multi plan runner's getInfo(). + std::vector emptyStats; + return explainMultiPlan(*stats, emptyStats, _solution.get(), explain); + } + else if (NULL != planInfo) { + if (NULL == _solution.get()) { + return Status(ErrorCodes::InternalError, + "no best solution available for plan info"); + } + getPlanInfo(*_solution, planInfo); + } + + return Status::OK(); + } + + void CachedPlanRunner::updateCache() { + _updatedCache = true; + + if (_killed) { + return; + } + + Database* db = cc().database(); + + // We need to check db and collection for NULL because updateCache() is called upon destruction of + // the CachedPlanRunner. In some cases, the db or collection could be dropped without kill() + // being called on the runner (for example, timeout of a ClientCursor holding the runner). + if (NULL == db) { return; } + Collection* collection = db->getCollection(_canonicalQuery->ns()); + if (NULL == collection) { return; } + PlanCache* cache = collection->infoCache()->getPlanCache(); + + std::auto_ptr feedback(new PlanCacheEntryFeedback()); + feedback->stats.reset(_exec->getStats()); + feedback->score = PlanRanker::scoreTree(feedback->stats.get()); + + Status fbs = cache->feedback(*_canonicalQuery, feedback.release()); + + if (!fbs.isOK()) { + QLOG() << _canonicalQuery->ns() << ": Failed to update cache with feedback: " + << fbs.toString() << " - " + << "(query: " << _canonicalQuery->getQueryObj() + << "; sort: " << _canonicalQuery->getParsed().getSort() + << "; projection: " << _canonicalQuery->getParsed().getProj() + << ") is no longer in plan cache."; + } + } + + void CachedPlanRunner::setBackupPlan(QuerySolution* qs, PlanStage* root, WorkingSet* ws) { + _backupSolution.reset(qs); + _backupPlan.reset(new PlanExecutor(ws, root)); + } + +} // namespace mongo diff --git a/src/mongo/db/query/cached_plan_runner.h b/src/mongo/db/query/cached_plan_runner.h new file mode 100644 index 00000000000..49d7cf8572f --- /dev/null +++ b/src/mongo/db/query/cached_plan_runner.h @@ -0,0 +1,126 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/runner.h" + +namespace mongo { + + class BSONObj; + class CachedSolution; + class CanonicalQuery; + class DiskLoc; + class PlanExecutor; + class PlanStage; + class TypeExplain; + struct PlanInfo; + class WorkingSet; + + /** + * CachedPlanRunner runs a plan retrieved from the cache. + * + * If we run a plan from the cache and behavior wildly deviates from expected behavior, we may + * remove the plan from the cache. See plan_cache.h. + */ + class CachedPlanRunner : public Runner { + public: + /** + * Takes ownership of all arguments. + */ + CachedPlanRunner(const Collection* collection, + CanonicalQuery* canonicalQuery, + QuerySolution* solution, + PlanStage* root, + WorkingSet* ws); + + virtual ~CachedPlanRunner(); + + Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + virtual bool isEOF(); + + virtual void saveState(); + + virtual bool restoreState(); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual void setYieldPolicy(Runner::YieldPolicy policy); + + virtual const std::string& ns(); + + virtual void kill(); + + virtual const Collection* collection() { return _collection; } + /** + * Returns OK, allocating and filling in '*explain' and '*planInfo' with details of + * the cached plan. Caller takes ownership of '*explain' and '*planInfo'. Otherwise, + * return a status describing the error. + */ + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const; + + /** + * Takes ownership of all arguments. + */ + void setBackupPlan(QuerySolution* qs, PlanStage* root, WorkingSet* ws); + + private: + void updateCache(); + + const Collection* _collection; + + boost::scoped_ptr _canonicalQuery; + boost::scoped_ptr _solution; + boost::scoped_ptr _exec; + + // Owned here. If non-NULL, then this plan executor is capable + // of executing a backup plan in the case of a blocking sort. + std::auto_ptr _backupPlan; + + // Owned here. If non-NULL, contains the query solution corresponding + // to the backup plan. + boost::scoped_ptr _backupSolution; + + // Whether the executor for the winning plan has produced results yet. + bool _alreadyProduced; + + // Have we updated the cache with our plan stats yet? + bool _updatedCache; + + // Has the runner been killed? + bool _killed; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp new file mode 100644 index 00000000000..72f194fad28 --- /dev/null +++ b/src/mongo/db/query/canonical_query.cpp @@ -0,0 +1,771 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/canonical_query.h" + +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/query_planner_common.h" + +namespace { + + using std::auto_ptr; + using std::string; + using namespace mongo; + + // Delimiters for cache key encoding. + const char kEncodeChildrenBegin = '['; + const char kEncodeChildrenEnd = ']'; + const char kEncodeChildrenSeparator = ','; + const char kEncodeSortSection = '~'; + const char kEncodeProjectionSection = '|'; + + /** + * Encode user-provided string. Cache key delimiters seen in the + * user string are escaped with a backslash. + */ + void encodeUserString(const StringData& s, mongoutils::str::stream* os) { + for (size_t i = 0; i < s.size(); ++i) { + char c = s[i]; + switch (c) { + case kEncodeChildrenBegin: + case kEncodeChildrenEnd: + case kEncodeChildrenSeparator: + case kEncodeSortSection: + case kEncodeProjectionSection: + case '\\': + *os << '\\'; + // Fall through to default case. + default: + *os << c; + } + } + } + + void encodePlanCacheKeyTree(const MatchExpression* tree, mongoutils::str::stream* os); + + /** + * Comparator for MatchExpression nodes. nodes by: + * 1) operator type (MatchExpression::MatchType) + * 2) path name (MatchExpression::path()) + * 3) cache key of the subtree + * + * The third item is needed to break ties, thus ensuring that + * match expression trees which should have the same cache key + * always sort the same way. If you're wondering when the tuple + * (operator type, path name) could ever be equal, consider this + * query: + * + * {$and:[{$or:[{a:1},{a:2}]},{$or:[{b:1},{b:2}]}]} + * + * The two OR nodes would compare as equal in this case were it + * not for tuple item #3 (cache key of the subtree). + */ + bool OperatorAndFieldNameComparison(const MatchExpression* lhs, const MatchExpression* rhs) { + // First compare by MatchType + MatchExpression::MatchType lhsMatchType = lhs->matchType(); + MatchExpression::MatchType rhsMatchType = rhs->matchType(); + if (lhsMatchType != rhsMatchType) { + return lhsMatchType < rhsMatchType; + } + // Second, path. + StringData lhsPath = lhs->path(); + StringData rhsPath = rhs->path(); + if (lhsPath != rhsPath) { + return lhsPath < rhsPath; + } + // Third, cache key. + mongoutils::str::stream ssLeft, ssRight; + encodePlanCacheKeyTree(lhs, &ssLeft); + encodePlanCacheKeyTree(rhs, &ssRight); + string strLeft(ssLeft); + string strRight(ssRight); + return strLeft < strRight; + } + + /** + * 2-character encoding of MatchExpression::MatchType. + */ + const char* encodeMatchType(MatchExpression::MatchType mt) { + switch(mt) { + case MatchExpression::AND: return "an"; break; + case MatchExpression::OR: return "or"; break; + case MatchExpression::NOR: return "nr"; break; + case MatchExpression::NOT: return "nt"; break; + case MatchExpression::ELEM_MATCH_OBJECT: return "eo"; break; + case MatchExpression::ELEM_MATCH_VALUE: return "ev"; break; + case MatchExpression::SIZE: return "sz"; break; + case MatchExpression::LTE: return "le"; break; + case MatchExpression::LT: return "lt"; break; + case MatchExpression::EQ: return "eq"; break; + case MatchExpression::GT: return "gt"; break; + case MatchExpression::GTE: return "ge"; break; + case MatchExpression::REGEX: return "re"; break; + case MatchExpression::MOD: return "mo"; break; + case MatchExpression::EXISTS: return "ex"; break; + case MatchExpression::MATCH_IN: return "in"; break; + case MatchExpression::NIN: return "ni"; break; + case MatchExpression::TYPE_OPERATOR: return "ty"; break; + case MatchExpression::GEO: return "go"; break; + case MatchExpression::WHERE: return "wh"; break; + case MatchExpression::ATOMIC: return "at"; break; + case MatchExpression::ALWAYS_FALSE: return "af"; break; + case MatchExpression::GEO_NEAR: return "gn"; break; + case MatchExpression::TEXT: return "te"; break; + default: verify(0); return ""; + } + } + + /** + * Encodes GEO match expression. + * Encoding includes: + * - type of geo query (within/intersect/near) + * - geometry type + * - CRS (flat or spherical) + */ + void encodeGeoMatchExpression(const GeoMatchExpression* tree, mongoutils::str::stream* os) { + const GeoQuery& geoQuery = tree->getGeoQuery(); + + // Type of geo query. + switch (geoQuery.getPred()) { + case GeoQuery::WITHIN: *os << "wi"; break; + case GeoQuery::INTERSECT: *os << "in"; break; + case GeoQuery::INVALID: *os << "id"; break; + } + + // Geometry type. + // Only one of the shared_ptrs in GeoContainer may be non-NULL. + const GeometryContainer& geoContainer = geoQuery.getGeometry(); + if (NULL != geoContainer._point) { *os << "pt"; } + else if (NULL != geoContainer._line) { *os << "ln"; } + else if (NULL != geoContainer._polygon) { *os << "pl"; } + else if (NULL != geoContainer._cap ) { *os << "cc"; } + else if (NULL != geoContainer._multiPoint) { *os << "mp"; } + else if (NULL != geoContainer._multiLine) { *os << "ml"; } + else if (NULL != geoContainer._multiPolygon) { *os << "my"; } + else if (NULL != geoContainer._geometryCollection) { *os << "gc"; } + else { invariant(NULL != geoContainer._box); *os << "bx"; } + + // CRS (flat or spherical) + if (geoContainer.hasFlatRegion()) { *os << "fl"; } + else { invariant(geoContainer.hasS2Region()); *os << "sp"; } + } + + /** + * Encodes GEO_NEAR match expression. + * Encode: + * - isNearSphere + * - CRS (flat or spherical) + */ + void encodeGeoNearMatchExpression(const GeoNearMatchExpression* tree, + mongoutils::str::stream* os) { + const NearQuery& nearQuery = tree->getData(); + + // isNearSphere + *os << (nearQuery.isNearSphere ? "ns" : "nr"); + + // CRS (flat or spherical) + switch (nearQuery.centroid.crs) { + case FLAT: *os << "fl"; break; + case SPHERE: *os << "sp"; break; + } + } + + /** + * Traverses expression tree pre-order. + * Appends an encoding of each node's match type and path name + * to the output stream. + */ + void encodePlanCacheKeyTree(const MatchExpression* tree, mongoutils::str::stream* os) { + // Encode match type and path. + *os << encodeMatchType(tree->matchType()); + + encodeUserString(tree->path(), os); + + // GEO and GEO_NEAR require additional encoding. + if (MatchExpression::GEO == tree->matchType()) { + encodeGeoMatchExpression(static_cast(tree), os); + } + else if (MatchExpression::GEO_NEAR == tree->matchType()) { + encodeGeoNearMatchExpression(static_cast(tree), os); + } + + // Traverse child nodes. + // Enclose children in []. + if (tree->numChildren() > 0) { + *os << kEncodeChildrenBegin; + } + // Use comma to separate children encoding. + for (size_t i = 0; i < tree->numChildren(); ++i) { + if (i > 0) { + *os << kEncodeChildrenSeparator; + } + encodePlanCacheKeyTree(tree->getChild(i), os); + } + if (tree->numChildren() > 0) { + *os << kEncodeChildrenEnd; + } + } + + /** + * Encodes sort order into cache key. + * Sort order is normalized because it provided by + * LiteParsedQuery. + */ + void encodePlanCacheKeySort(const BSONObj& sortObj, mongoutils::str::stream* os) { + if (sortObj.isEmpty()) { + return; + } + + *os << kEncodeSortSection; + + BSONObjIterator it(sortObj); + while (it.more()) { + BSONElement elt = it.next(); + // $meta text score + if (LiteParsedQuery::isTextScoreMeta(elt)) { + *os << "t"; + } + // Ascending + else if (elt.numberInt() == 1) { + *os << "a"; + } + // Descending + else { + *os << "d"; + } + encodeUserString(elt.fieldName(), os); + } + } + + /** + * Encodes parsed projection into cache key. + * Does a simple toString() on each projected field + * in the BSON object. + * Orders the encoded elements in the projection by field name. + * This handles all the special projection types ($meta, $elemMatch, etc.) + */ + void encodePlanCacheKeyProj(const BSONObj& projObj, mongoutils::str::stream* os) { + if (projObj.isEmpty()) { + return; + } + + *os << kEncodeProjectionSection; + + // Sorts the BSON elements by field name using a map. + std::map elements; + + BSONObjIterator it(projObj); + while (it.more()) { + BSONElement elt = it.next(); + StringData fieldName = elt.fieldNameStringData(); + elements[fieldName] = elt; + } + + // Read elements in order of field name + for (std::map::const_iterator i = elements.begin(); + i != elements.end(); ++i) { + const BSONElement& elt = (*i).second; + // BSONElement::toString() arguments + // includeFieldName - skip field name (appending after toString() result). false. + // full: choose less verbose representation of child/data values. false. + encodeUserString(elt.toString(false, false), os); + encodeUserString(elt.fieldName(), os); + } + } + +} // namespace + +namespace mongo { + + // + // These all punt to the many-argumented canonicalize below. + // + + // static + Status CanonicalQuery::canonicalize(const string& ns, const BSONObj& query, + CanonicalQuery** out) { + BSONObj emptyObj; + return CanonicalQuery::canonicalize(ns, query, emptyObj, emptyObj, 0, 0, out); + } + + // static + Status CanonicalQuery::canonicalize(const string& ns, const BSONObj& query, + long long skip, long long limit, + CanonicalQuery** out) { + BSONObj emptyObj; + return CanonicalQuery::canonicalize(ns, query, emptyObj, emptyObj, skip, limit, out); + } + + // static + Status CanonicalQuery::canonicalize(const string& ns, const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + CanonicalQuery** out) { + return CanonicalQuery::canonicalize(ns, query, sort, proj, 0, 0, out); + } + + // static + Status CanonicalQuery::canonicalize(const string& ns, const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + CanonicalQuery** out) { + BSONObj emptyObj; + return CanonicalQuery::canonicalize(ns, query, sort, proj, skip, limit, emptyObj, out); + } + + // static + Status CanonicalQuery::canonicalize(const string& ns, const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + CanonicalQuery** out) { + BSONObj emptyObj; + return CanonicalQuery::canonicalize(ns, query, sort, proj, skip, limit, hint, + emptyObj, emptyObj, + false, // snapshot + false, // explain + out); + } + + // + // These actually call init() on the CQ. + // + + // static + Status CanonicalQuery::canonicalize(const QueryMessage& qm, CanonicalQuery** out) { + // Make LiteParsedQuery. + LiteParsedQuery* lpq; + Status parseStatus = LiteParsedQuery::make(qm, &lpq); + if (!parseStatus.isOK()) { return parseStatus; } + + // Make MatchExpression. + StatusWithMatchExpression swme = MatchExpressionParser::parse(lpq->getFilter()); + if (!swme.isOK()) { + delete lpq; + return swme.getStatus(); + } + + // Make the CQ we'll hopefully return. + auto_ptr cq(new CanonicalQuery()); + // Takes ownership of lpq and the MatchExpression* in swme. + Status initStatus = cq->init(lpq, swme.getValue()); + + if (!initStatus.isOK()) { return initStatus; } + *out = cq.release(); + return Status::OK(); + } + + // static + Status CanonicalQuery::canonicalize(const CanonicalQuery& baseQuery, + MatchExpression* root, + CanonicalQuery** out) { + + LiteParsedQuery* lpq; + + // Pass empty sort and projection. + BSONObj emptyObj; + // 0, 0, 0 is 'ntoskip', 'ntoreturn', and 'queryoptions' + // false, false is 'snapshot' and 'explain' + Status parseStatus = LiteParsedQuery::make(baseQuery.ns(), + 0, 0, 0, + baseQuery.getParsed().getFilter(), + baseQuery.getParsed().getProj(), + baseQuery.getParsed().getSort(), + emptyObj, emptyObj, emptyObj, + false, false, &lpq); + if (!parseStatus.isOK()) { + return parseStatus; + } + + // Make the CQ we'll hopefully return. + auto_ptr cq(new CanonicalQuery()); + Status initStatus = cq->init(lpq, root->shallowClone()); + + if (!initStatus.isOK()) { return initStatus; } + *out = cq.release(); + return Status::OK(); + } + + // static + Status CanonicalQuery::canonicalize(const string& ns, const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj, + bool snapshot, + bool explain, + CanonicalQuery** out) { + LiteParsedQuery* lpqRaw; + // Pass empty sort and projection. + BSONObj emptyObj; + Status parseStatus = LiteParsedQuery::make(ns, skip, limit, 0, query, proj, sort, + hint, minObj, maxObj, snapshot, explain, + &lpqRaw); + if (!parseStatus.isOK()) { + return parseStatus; + } + auto_ptr lpq(lpqRaw); + + // Build a parse tree from the BSONObj in the parsed query. + StatusWithMatchExpression swme = MatchExpressionParser::parse(lpq->getFilter()); + if (!swme.isOK()) { + return swme.getStatus(); + } + + // Make the CQ we'll hopefully return. + auto_ptr cq(new CanonicalQuery()); + // Takes ownership of lpq and the MatchExpression* in swme. + Status initStatus = cq->init(lpq.release(), swme.getValue()); + + if (!initStatus.isOK()) { return initStatus; } + *out = cq.release(); + return Status::OK(); + } + + Status CanonicalQuery::init(LiteParsedQuery* lpq, MatchExpression* root) { + _pq.reset(lpq); + + // Normalize, sort and validate tree. + root = normalizeTree(root); + + sortTree(root); + _root.reset(root); + Status validStatus = isValid(root, *_pq); + if (!validStatus.isOK()) { + return validStatus; + } + + this->generateCacheKey(); + + // Validate the projection if there is one. + if (!_pq->getProj().isEmpty()) { + ParsedProjection* pp; + Status projStatus = ParsedProjection::make(_pq->getProj(), _root.get(), &pp); + if (!projStatus.isOK()) { + return projStatus; + } + _proj.reset(pp); + } + + return Status::OK(); + } + + + // static + bool CanonicalQuery::isSimpleIdQuery(const BSONObj& query) { + bool hasID = false; + + BSONObjIterator it(query); + while (it.more()) { + BSONElement elt = it.next(); + if (mongoutils::str::equals("_id", elt.fieldName() ) ) { + // Verify that the query on _id is a simple equality. + hasID = true; + + if (elt.type() == Object) { + // If the value is an object, it can't have a query operator + // (must be a literal object match). + if (elt.Obj().firstElementFieldName()[0] == '$') { + return false; + } + } + else if (!elt.isSimpleType() && BinData != elt.type()) { + // The _id fild cannot be something like { _id : { $gt : ... + // But it can be BinData. + return false; + } + } + else if (elt.fieldName()[0] == '$' && + (mongoutils::str::equals("$isolated", elt.fieldName())|| + mongoutils::str::equals("$atomic", elt.fieldName()))) { + // ok, passthrough + } + else { + // If the field is not _id, it must be $isolated/$atomic. + return false; + } + } + + return hasID; + } + + const PlanCacheKey& CanonicalQuery::getPlanCacheKey() const { + return _cacheKey; + } + + void CanonicalQuery::generateCacheKey(void) { + mongoutils::str::stream ss; + encodePlanCacheKeyTree(_root.get(), &ss); + encodePlanCacheKeySort(_pq->getSort(), &ss); + encodePlanCacheKeyProj(_pq->getProj(), &ss); + _cacheKey = ss; + } + + // static + MatchExpression* CanonicalQuery::normalizeTree(MatchExpression* root) { + // root->isLogical() is true now. We care about AND, OR, and NOT. NOR currently scares us. + if (MatchExpression::AND == root->matchType() || MatchExpression::OR == root->matchType()) { + // We could have AND of AND of AND. Make sure we clean up our children before merging + // them. + // UNITTEST 11738048 + for (size_t i = 0; i < root->getChildVector()->size(); ++i) { + (*root->getChildVector())[i] = normalizeTree(root->getChild(i)); + } + + // If any of our children are of the same logical operator that we are, we remove the + // child's children and append them to ourselves after we examine all children. + vector absorbedChildren; + + for (size_t i = 0; i < root->numChildren();) { + MatchExpression* child = root->getChild(i); + if (child->matchType() == root->matchType()) { + // AND of an AND or OR of an OR. Absorb child's children into ourself. + for (size_t j = 0; j < child->numChildren(); ++j) { + absorbedChildren.push_back(child->getChild(j)); + } + // TODO(opt): this is possibly n^2-ish + root->getChildVector()->erase(root->getChildVector()->begin() + i); + child->getChildVector()->clear(); + // Note that this only works because we cleared the child's children + delete child; + // Don't increment 'i' as the current child 'i' used to be child 'i+1' + } + else { + ++i; + } + } + + root->getChildVector()->insert(root->getChildVector()->end(), + absorbedChildren.begin(), + absorbedChildren.end()); + + // AND of 1 thing is the thing, OR of 1 thing is the thing. + if (1 == root->numChildren()) { + MatchExpression* ret = root->getChild(0); + root->getChildVector()->clear(); + delete root; + return ret; + } + } + else if (MatchExpression::NOT == root->matchType()) { + // Normalize the rest of the tree hanging off this NOT node. + NotMatchExpression* nme = static_cast(root); + MatchExpression* child = nme->releaseChild(); + // normalizeTree(...) takes ownership of 'child', and then + // transfers ownership of its return value to 'nme'. + nme->resetChild(normalizeTree(child)); + } + + return root; + } + + // static + void CanonicalQuery::sortTree(MatchExpression* tree) { + for (size_t i = 0; i < tree->numChildren(); ++i) { + sortTree(tree->getChild(i)); + } + std::vector* children = tree->getChildVector(); + if (NULL != children) { + std::sort(children->begin(), children->end(), OperatorAndFieldNameComparison); + } + } + + // static + size_t CanonicalQuery::countNodes(const MatchExpression* root, + MatchExpression::MatchType type) { + size_t sum = 0; + if (type == root->matchType()) { + sum = 1; + } + for (size_t i = 0; i < root->numChildren(); ++i) { + sum += countNodes(root->getChild(i), type); + } + return sum; + } + + /** + * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside? + */ + bool hasNodeInSubtree(MatchExpression* root, MatchExpression::MatchType childType, + MatchExpression::MatchType subtreeType) { + if (subtreeType == root->matchType()) { + return QueryPlannerCommon::hasNode(root, childType); + } + for (size_t i = 0; i < root->numChildren(); ++i) { + if (hasNodeInSubtree(root->getChild(i), childType, subtreeType)) { + return true; + } + } + return false; + } + + // static + Status CanonicalQuery::isValid(MatchExpression* root, const LiteParsedQuery& parsed) { + // Analysis below should be done after squashing the tree to make it clearer. + + // There can only be one TEXT. If there is a TEXT, it cannot appear inside a NOR. + // + // Note that the query grammar (as enforced by the MatchExpression parser) forbids TEXT + // inside of value-expression clauses like NOT, so we don't check those here. + size_t numText = countNodes(root, MatchExpression::TEXT); + if (numText > 1) { + return Status(ErrorCodes::BadValue, "Too many text expressions"); + } + else if (1 == numText) { + if (hasNodeInSubtree(root, MatchExpression::TEXT, MatchExpression::NOR)) { + return Status(ErrorCodes::BadValue, "text expression not allowed in nor"); + } + } + + // There can only be one NEAR. If there is a NEAR, it must be either the root or the root + // must be an AND and its child must be a NEAR. + size_t numGeoNear = countNodes(root, MatchExpression::GEO_NEAR); + if (numGeoNear > 1) { + return Status(ErrorCodes::BadValue, "Too many geoNear expressions"); + } + else if (1 == numGeoNear) { + bool topLevel = false; + if (MatchExpression::GEO_NEAR == root->matchType()) { + topLevel = true; + } + else if (MatchExpression::AND == root->matchType()) { + for (size_t i = 0; i < root->numChildren(); ++i) { + if (MatchExpression::GEO_NEAR == root->getChild(i)->matchType()) { + topLevel = true; + break; + } + } + } + if (!topLevel) { + return Status(ErrorCodes::BadValue, "geoNear must be top-level expr"); + } + } + + // TEXT and NEAR cannot both be in the query. + if (numText > 0 && numGeoNear > 0) { + return Status(ErrorCodes::BadValue, "text and geoNear not allowed in same query"); + } + + // TEXT and {$natural: ...} sort order cannot both be in the query. + if (numText > 0) { + const BSONObj& sortObj = parsed.getSort(); + BSONObjIterator it(sortObj); + while (it.more()) { + BSONElement elt = it.next(); + if (mongoutils::str::equals("$natural", elt.fieldName())) { + return Status(ErrorCodes::BadValue, + "text expression not allowed with $natural sort order"); + } + } + } + + // TEXT and hint cannot both be in the query. + if (numText > 0 && !parsed.getHint().isEmpty()) { + return Status(ErrorCodes::BadValue, "text and hint not allowed in same query"); + } + + // TEXT and snapshot cannot both be in the query. + if (numText > 0 && parsed.isSnapshot()) { + return Status(ErrorCodes::BadValue, "text and snapshot not allowed in same query"); + } + + return Status::OK(); + } + + // static + // XXX TODO: This does not belong here at all. + MatchExpression* CanonicalQuery::logicalRewrite(MatchExpression* tree) { + // Only thing we do is pull an OR up at the root. + if (MatchExpression::AND != tree->matchType()) { + return tree; + } + + // We want to bail out ASAP if we have nothing to do here. + size_t numOrs = 0; + for (size_t i = 0; i < tree->numChildren(); ++i) { + if (MatchExpression::OR == tree->getChild(i)->matchType()) { + ++numOrs; + } + } + + // Only do this for one OR right now. + if (1 != numOrs) { + return tree; + } + + // Detach the OR from the root. + invariant(NULL != tree->getChildVector()); + vector& rootChildren = *tree->getChildVector(); + MatchExpression* orChild = NULL; + for (size_t i = 0; i < rootChildren.size(); ++i) { + if (MatchExpression::OR == rootChildren[i]->matchType()) { + orChild = rootChildren[i]; + rootChildren.erase(rootChildren.begin() + i); + break; + } + } + + // AND the existing root with each or child. + invariant(NULL != orChild); + invariant(NULL != orChild->getChildVector()); + vector& orChildren = *orChild->getChildVector(); + for (size_t i = 0; i < orChildren.size(); ++i) { + AndMatchExpression* ama = new AndMatchExpression(); + ama->add(orChildren[i]); + ama->add(tree->shallowClone()); + orChildren[i] = ama; + } + delete tree; + + // Clean up any consequences from this tomfoolery. + return normalizeTree(orChild); + } + + std::string CanonicalQuery::toString() const { + mongoutils::str::stream ss; + ss << "ns=" << _pq->ns() << " limit=" << _pq->getNumToReturn() + << " skip=" << _pq->getSkip() << '\n'; + // The expression tree puts an endl on for us. + ss << "Tree: " << _root->toString(); + ss << "Sort: " << _pq->getSort().toString() << '\n'; + ss << "Proj: " << _pq->getProj().toString() << '\n'; + return ss; + } + + std::string CanonicalQuery::toStringShort() const { + mongoutils::str::stream ss; + ss << "query: " << _pq->getFilter().toString() + << " sort: " << _pq->getSort().toString() + << " projection: " << _pq->getProj().toString() + << " skip: " << _pq->getSkip() + << " limit: " << _pq->getNumToReturn(); + return ss; + } + +} // namespace mongo diff --git a/src/mongo/db/query/canonical_query.h b/src/mongo/db/query/canonical_query.h new file mode 100644 index 00000000000..a65fc5dbc05 --- /dev/null +++ b/src/mongo/db/query/canonical_query.h @@ -0,0 +1,187 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/dbmessage.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/query/lite_parsed_query.h" +#include "mongo/db/query/parsed_projection.h" + +namespace mongo { + + // TODO: Is this binary data really? + typedef std::string PlanCacheKey; + + class CanonicalQuery { + public: + /** + * Caller owns the pointer in 'out' if any call to canonicalize returns Status::OK(). + */ + static Status canonicalize(const QueryMessage& qm, CanonicalQuery** out); + + /** + * For testing or for internal clients to use. + */ + + /** + * Used for creating sub-queries from an existing CanonicalQuery. + * + * 'root' must be an expression in baseQuery.root(). + * + * Does not take ownership of 'root'. + */ + static Status canonicalize(const CanonicalQuery& baseQuery, + MatchExpression* root, + CanonicalQuery** out); + + static Status canonicalize(const string& ns, const BSONObj& query, CanonicalQuery** out); + + static Status canonicalize(const string& ns, const BSONObj& query, long long skip, + long long limit, CanonicalQuery** out); + + static Status canonicalize(const string& ns, const BSONObj& query, const BSONObj& sort, + const BSONObj& proj, CanonicalQuery** out); + + static Status canonicalize(const string& ns, const BSONObj& query, const BSONObj& sort, + const BSONObj& proj, + long long skip, long long limit, + CanonicalQuery** out); + + static Status canonicalize(const string& ns, const BSONObj& query, const BSONObj& sort, + const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + CanonicalQuery** out); + + static Status canonicalize(const string& ns, const BSONObj& query, const BSONObj& sort, + const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj, + bool snapshot, + bool explain, + CanonicalQuery** out); + + /** + * Returns true if "query" describes an exact-match query on _id, possibly with + * the $isolated/$atomic modifier. + */ + static bool isSimpleIdQuery(const BSONObj& query); + + // What namespace is this query over? + const string& ns() const { return _pq->ns(); } + + // + // Accessors for the query + // + MatchExpression* root() const { return _root.get(); } + BSONObj getQueryObj() const { return _pq->getFilter(); } + const LiteParsedQuery& getParsed() const { return *_pq; } + const ParsedProjection* getProj() const { return _proj.get(); } + + /** + * Get the cache key for this canonical query. + */ + const PlanCacheKey& getPlanCacheKey() const; + + // Debugging + std::string toString() const; + std::string toStringShort() const; + + /** + * Validates match expression, checking for certain + * combinations of operators in match expression and + * query options in LiteParsedQuery. + * Since 'root' is derived from 'filter' in LiteParsedQuery, + * 'filter' is not validated. + * + * TODO: Move this to query_validator.cpp + */ + static Status isValid(MatchExpression* root, const LiteParsedQuery& parsed); + + /** + * Returns the normalized version of the subtree rooted at 'root'. + * + * Takes ownership of 'root'. + */ + static MatchExpression* normalizeTree(MatchExpression* root); + + /** + * Traverses expression tree post-order. + * Sorts children at each non-leaf node by (MatchType, path(), cacheKey) + */ + static void sortTree(MatchExpression* tree); + + /** + * Returns a count of 'type' nodes in expression tree. + */ + static size_t countNodes(const MatchExpression* root, MatchExpression::MatchType type); + + /** + * Takes ownership of 'tree'. Performs some rewriting of the query to a logically + * equivalent but more digestible form. + * + * TODO: This doesn't entirely belong here. Really we'd do this while exploring + * solutions in an enumeration setting but given the current lack of pruning + * while exploring the enumeration space we do it here. + */ + static MatchExpression* logicalRewrite(MatchExpression* tree); + + private: + // You must go through canonicalize to create a CanonicalQuery. + CanonicalQuery() { } + + /** + * Computes and stores the cache key / query shape + * for this query. + */ + void generateCacheKey(void); + + /** + * Takes ownership of 'root' and 'lpq'. + */ + Status init(LiteParsedQuery* lpq, MatchExpression* root); + + scoped_ptr _pq; + + // _root points into _pq->getFilter() + scoped_ptr _root; + + scoped_ptr _proj; + + /** + * Cache key is a string-ified combination of the query and sort obfuscated + * for minimal user comprehension. + */ + PlanCacheKey _cacheKey; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/canonical_query_test.cpp b/src/mongo/db/query/canonical_query_test.cpp new file mode 100644 index 00000000000..30e5952b1ee --- /dev/null +++ b/src/mongo/db/query/canonical_query_test.cpp @@ -0,0 +1,586 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/canonical_query.h" + +#include "mongo/db/json.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + static const char* ns = "somebogusns"; + + /** + * Utility function to parse the given JSON as a MatchExpression and normalize the expression + * tree. Returns the resulting tree, or an error Status. + */ + StatusWithMatchExpression parseNormalize(const std::string& queryStr) { + StatusWithMatchExpression swme = MatchExpressionParser::parse(fromjson(queryStr)); + if (!swme.getStatus().isOK()) { + return swme; + } + return StatusWithMatchExpression(CanonicalQuery::normalizeTree(swme.getValue())); + } + + TEST(CanonicalQueryTest, IsValidText) { + // Passes in default values for LiteParsedQuery. + // Filter inside LiteParsedQuery is not used. + LiteParsedQuery* lpqRaw; + ASSERT_OK(LiteParsedQuery::make(ns, 0, 0, 0, fromjson("{}"), fromjson("{}"), + fromjson("{}"), fromjson("{}"), fromjson("{}"), + fromjson("{}"), + false, // snapshot + false, // explain + &lpqRaw)); + auto_ptr lpq(lpqRaw); + + auto_ptr me; + StatusWithMatchExpression swme(Status::OK()); + + // Valid: regular TEXT. + swme = parseNormalize("{$text: {$search: 's'}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Valid: TEXT inside OR. + swme = parseNormalize( + "{$or: [" + " {$text: {$search: 's'}}," + " {a: 1}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Valid: TEXT outside NOR. + swme = parseNormalize("{$text: {$search: 's'}, $nor: [{a: 1}, {b: 1}]}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: TEXT inside NOR. + swme = parseNormalize("{$nor: [{$text: {$search: 's'}}, {a: 1}]}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: TEXT inside NOR. + swme = parseNormalize( + "{$nor: [" + " {$or: [" + " {$text: {$search: 's'}}," + " {a: 1}" + " ]}," + " {a: 2}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: >1 TEXT. + swme = parseNormalize( + "{$and: [" + " {$text: {$search: 's'}}," + " {$text: {$search: 't'}}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: >1 TEXT. + swme = parseNormalize( + "{$and: [" + " {$or: [" + " {$text: {$search: 's'}}," + " {a: 1}" + " ]}," + " {$or: [" + " {$text: {$search: 't'}}," + " {b: 1}" + " ]}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + } + + TEST(CanonicalQueryTest, IsValidGeo) { + // Passes in default values for LiteParsedQuery. + // Filter inside LiteParsedQuery is not used. + LiteParsedQuery* lpqRaw; + ASSERT_OK(LiteParsedQuery::make(ns, 0, 0, 0, fromjson("{}"), fromjson("{}"), + fromjson("{}"), fromjson("{}"), fromjson("{}"), + fromjson("{}"), + false, // snapshot + false, // explain + &lpqRaw)); + auto_ptr lpq(lpqRaw); + + auto_ptr me; + StatusWithMatchExpression swme(Status::OK()); + + // Valid: regular GEO_NEAR. + swme = parseNormalize("{a: {$near: [0, 0]}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Valid: GEO_NEAR inside nested AND. + swme = parseNormalize( + "{$and: [" + " {$and: [" + " {a: {$near: [0, 0]}}," + " {b: 1}" + " ]}," + " {c: 1}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: >1 GEO_NEAR. + swme = parseNormalize( + "{$and: [" + " {a: {$near: [0, 0]}}," + " {b: {$near: [0, 0]}}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: >1 GEO_NEAR. + swme = parseNormalize( + "{$and: [" + " {a: {$geoNear: [0, 0]}}," + " {b: {$near: [0, 0]}}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: >1 GEO_NEAR. + swme = parseNormalize( + "{$and: [" + " {$and: [" + " {a: {$near: [0, 0]}}," + " {b: 1}" + " ]}," + " {$and: [" + " {c: {$near: [0, 0]}}," + " {d: 1}" + " ]}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: GEO_NEAR inside NOR. + swme = parseNormalize( + "{$nor: [" + " {a: {$near: [0, 0]}}," + " {b: 1}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: GEO_NEAR inside OR. + swme = parseNormalize( + "{$or: [" + " {a: {$near: [0, 0]}}," + " {b: 1}" + "]}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + } + + TEST(CanonicalQueryTest, IsValidTextAndGeo) { + // Passes in default values for LiteParsedQuery. + // Filter inside LiteParsedQuery is not used. + LiteParsedQuery* lpqRaw; + ASSERT_OK(LiteParsedQuery::make(ns, 0, 0, 0, fromjson("{}"), fromjson("{}"), + fromjson("{}"), fromjson("{}"), fromjson("{}"), + fromjson("{}"), + false, // snapshot + false, // explain + &lpqRaw)); + auto_ptr lpq(lpqRaw); + + auto_ptr me; + StatusWithMatchExpression swme(Status::OK()); + + // Invalid: TEXT and GEO_NEAR. + swme = parseNormalize("{$text: {$search: 's'}, a: {$near: [0, 0]}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: TEXT and GEO_NEAR. + swme = parseNormalize("{$text: {$search: 's'}, a: {$geoNear: [0, 0]}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + + // Invalid: TEXT and GEO_NEAR. + swme = parseNormalize( + "{$or: [" + " {$text: {$search: 's'}}," + " {a: 1}" + " ]," + " b: {$near: [0, 0]}}" + ); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + } + + TEST(CanonicalQueryTest, IsValidTextAndNaturalAscending) { + // Passes in default values for LiteParsedQuery except for sort order. + // Filter inside LiteParsedQuery is not used. + LiteParsedQuery* lpqRaw; + BSONObj sort = fromjson("{$natural: 1}"); + ASSERT_OK(LiteParsedQuery::make(ns, 0, 0, 0, fromjson("{}"), fromjson("{}"), + sort, fromjson("{}"), fromjson("{}"), + fromjson("{}"), + false, // snapshot + false, // explain + &lpqRaw)); + auto_ptr lpq(lpqRaw); + + auto_ptr me; + StatusWithMatchExpression swme(Status::OK()); + + // Invalid: TEXT and {$natural: 1} sort order. + swme = parseNormalize("{$text: {$search: 's'}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + } + + TEST(CanonicalQueryTest, IsValidTextAndNaturalDescending) { + // Passes in default values for LiteParsedQuery except for sort order. + // Filter inside LiteParsedQuery is not used. + LiteParsedQuery* lpqRaw; + BSONObj sort = fromjson("{$natural: -1}"); + ASSERT_OK(LiteParsedQuery::make(ns, 0, 0, 0, fromjson("{}"), fromjson("{}"), + sort, fromjson("{}"), fromjson("{}"), + fromjson("{}"), + false, // snapshot + false, // explain + &lpqRaw)); + auto_ptr lpq(lpqRaw); + + auto_ptr me; + StatusWithMatchExpression swme(Status::OK()); + + // Invalid: TEXT and {$natural: -1} sort order. + swme = parseNormalize("{$text: {$search: 's'}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + } + + TEST(CanonicalQueryTest, IsValidTextAndHint) { + // Passes in default values for LiteParsedQuery except for hint. + // Filter inside LiteParsedQuery is not used. + LiteParsedQuery* lpqRaw; + BSONObj hint = fromjson("{a: 1}"); + ASSERT_OK(LiteParsedQuery::make(ns, 0, 0, 0, fromjson("{}"), fromjson("{}"), + fromjson("{}"), hint, fromjson("{}"), + fromjson("{}"), + false, // snapshot + false, // explain + &lpqRaw)); + auto_ptr lpq(lpqRaw); + + auto_ptr me; + StatusWithMatchExpression swme(Status::OK()); + + // Invalid: TEXT and {$natural: -1} sort order. + swme = parseNormalize("{$text: {$search: 's'}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + } + + TEST(CanonicalQueryTest, IsValidTextAndSnapshot) { + // Passes in default values for LiteParsedQuery except for snapshot. + // Filter inside LiteParsedQuery is not used. + LiteParsedQuery* lpqRaw; + bool snapshot = true; + ASSERT_OK(LiteParsedQuery::make(ns, 0, 0, 0, fromjson("{}"), fromjson("{}"), + fromjson("{}"), fromjson("{}"), fromjson("{}"), + fromjson("{}"), + snapshot, + false, // explain + &lpqRaw)); + auto_ptr lpq(lpqRaw); + + auto_ptr me; + StatusWithMatchExpression swme(Status::OK()); + + // Invalid: TEXT and snapshot. + swme = parseNormalize("{$text: {$search: 's'}}"); + ASSERT_OK(swme.getStatus()); + me.reset(swme.getValue()); + ASSERT_NOT_OK(CanonicalQuery::isValid(me.get(), *lpq)); + } + + /** + * Utility function to create a CanonicalQuery + */ + CanonicalQuery* canonicalize(const char* queryStr) { + BSONObj queryObj = fromjson(queryStr); + CanonicalQuery* cq; + Status result = CanonicalQuery::canonicalize(ns, queryObj, &cq); + ASSERT_OK(result); + return cq; + } + + CanonicalQuery* canonicalize(const char* queryStr, const char* sortStr, + const char* projStr) { + BSONObj queryObj = fromjson(queryStr); + BSONObj sortObj = fromjson(sortStr); + BSONObj projObj = fromjson(projStr); + CanonicalQuery* cq; + Status result = CanonicalQuery::canonicalize(ns, queryObj, sortObj, + projObj, + &cq); + ASSERT_OK(result); + return cq; + } + + /** + * Utility function to create MatchExpression + */ + MatchExpression* parseMatchExpression(const BSONObj& obj) { + StatusWithMatchExpression status = MatchExpressionParser::parse(obj); + if (!status.isOK()) { + mongoutils::str::stream ss; + ss << "failed to parse query: " << obj.toString() + << ". Reason: " << status.toString(); + FAIL(ss); + } + MatchExpression* expr(status.getValue()); + return expr; + } + + void assertEquivalent(const char* queryStr, + const MatchExpression* expected, + const MatchExpression* actual) { + if (actual->equivalent(expected)) { + return; + } + mongoutils::str::stream ss; + ss << "Match expressions are not equivalent." + << "\nOriginal query: " << queryStr + << "\nExpected: " << expected->toString() + << "\nActual: " << actual->toString(); + FAIL(ss); + } + + // + // Tests for CanonicalQuery::logicalRewrite + // + + // Don't do anything with a double OR. + TEST(CanonicalQueryTest, RewriteNoDoubleOr) { + string queryStr = "{$or:[{a:1}, {b:1}], $or:[{c:1}, {d:1}], e:1}"; + BSONObj queryObj = fromjson(queryStr); + auto_ptr base(parseMatchExpression(queryObj)); + auto_ptr rewrite(CanonicalQuery::logicalRewrite(base->shallowClone())); + assertEquivalent(queryStr.c_str(), base.get(), rewrite.get()); + } + + // Do something with a single or. + TEST(CanonicalQueryTest, RewriteSingleOr) { + // Rewrite of this... + string queryStr = "{$or:[{a:1}, {b:1}], e:1}"; + BSONObj queryObj = fromjson(queryStr); + auto_ptr rewrite(CanonicalQuery::logicalRewrite(parseMatchExpression(queryObj))); + + // Should look like this. + string rewriteStr = "{$or:[{a:1, e:1}, {b:1, e:1}]}"; + BSONObj rewriteObj = fromjson(rewriteStr); + auto_ptr base(parseMatchExpression(rewriteObj)); + assertEquivalent(queryStr.c_str(), base.get(), rewrite.get()); + } + + /** + * Test function for CanonicalQuery::normalize. + */ + void testNormalizeQuery(const char* queryStr, const char* expectedExprStr) { + auto_ptr cq(canonicalize(queryStr)); + MatchExpression* me = cq->root(); + BSONObj expectedExprObj = fromjson(expectedExprStr); + auto_ptr expectedExpr(parseMatchExpression(expectedExprObj)); + assertEquivalent(queryStr, expectedExpr.get(), me); + } + + TEST(CanonicalQueryTest, NormalizeQuerySort) { + // Field names + testNormalizeQuery("{b: 1, a: 1}", "{a: 1, b: 1}"); + // Operator types + testNormalizeQuery("{a: {$gt: 5}, a: {$lt: 10}}}", "{a: {$lt: 10}, a: {$gt: 5}}"); + // Nested queries + testNormalizeQuery("{a: {$elemMatch: {c: 1, b:1}}}", + "{a: {$elemMatch: {b: 1, c:1}}}"); + } + + TEST(CanonicalQueryTest, NormalizeQueryTree) { + // Single-child $or elimination. + testNormalizeQuery("{$or: [{b: 1}]}", "{b: 1}"); + // Single-child $and elimination. + testNormalizeQuery("{$or: [{$and: [{a: 1}]}, {b: 1}]}", "{$or: [{a: 1}, {b: 1}]}"); + // $or absorbs $or children. + testNormalizeQuery("{$or: [{a: 1}, {$or: [{b: 1}, {$or: [{c: 1}]}]}, {d: 1}]}", + "{$or: [{a: 1}, {b: 1}, {c: 1}, {d: 1}]}"); + // $and absorbs $and children. + testNormalizeQuery("{$and: [{$and: [{a: 1}, {b: 1}]}, {c: 1}]}", + "{$and: [{a: 1}, {b: 1}, {c: 1}]}"); + } + + /** + * Test functions for getPlanCacheKey. + * Cache keys are intentionally obfuscated and are meaningful only + * within the current lifetime of the server process. Users should treat + * plan cache keys as opaque. + */ + void testGetPlanCacheKey(const char* queryStr, const char* sortStr, + const char* projStr, + const char *expectedStr) { + auto_ptr cq(canonicalize(queryStr, sortStr, projStr)); + const PlanCacheKey& key = cq->getPlanCacheKey(); + PlanCacheKey expectedKey(expectedStr); + if (key == expectedKey) { + return; + } + mongoutils::str::stream ss; + ss << "Unexpected plan cache key. Expected: " << expectedKey << ". Actual: " << key + << ". Query: " << cq->toString(); + FAIL(ss); + } + + TEST(PlanCacheTest, GetPlanCacheKey) { + // Generated cache keys should be treated as opaque to the user. + + // No sorts + testGetPlanCacheKey("{}", "{}", "{}", "an"); + testGetPlanCacheKey("{$or: [{a: 1}, {b: 2}]}", "{}", "{}", "or[eqa,eqb]"); + testGetPlanCacheKey("{$or: [{a: 1}, {b: 1}, {c: 1}], d: 1}", "{}", "{}", + "an[or[eqa,eqb,eqc],eqd]"); + testGetPlanCacheKey("{$or: [{a: 1}, {b: 1}], c: 1, d: 1}", "{}", "{}", + "an[or[eqa,eqb],eqc,eqd]"); + testGetPlanCacheKey("{a: 1, b: 1, c: 1}", "{}", "{}", "an[eqa,eqb,eqc]"); + testGetPlanCacheKey("{a: 1, beqc: 1}", "{}", "{}", "an[eqa,eqbeqc]"); + testGetPlanCacheKey("{ap1a: 1}", "{}", "{}", "eqap1a"); + testGetPlanCacheKey("{aab: 1}", "{}", "{}", "eqaab"); + + // With sort + testGetPlanCacheKey("{}", "{a: 1}", "{}", "an~aa"); + testGetPlanCacheKey("{}", "{a: -1}", "{}", "an~da"); + testGetPlanCacheKey("{}", "{a: {$meta: 'textScore'}}", "{a: {$meta: 'textScore'}}", + "an~ta|{ $meta: \"textScore\" }a"); + testGetPlanCacheKey("{a: 1}", "{b: 1}", "{}", "eqa~ab"); + + // With projection + testGetPlanCacheKey("{}", "{}", "{a: 1}", "an|1a"); + testGetPlanCacheKey("{}", "{}", "{a: 0}", "an|0a"); + testGetPlanCacheKey("{}", "{}", "{a: 99}", "an|99a"); + testGetPlanCacheKey("{}", "{}", "{a: 'foo'}", "an|\"foo\"a"); + testGetPlanCacheKey("{}", "{}", "{a: {$slice: [3, 5]}}", "an|{ $slice: \\[ 3\\, 5 \\] }a"); + testGetPlanCacheKey("{}", "{}", "{a: {$elemMatch: {x: 2}}}", + "an|{ $elemMatch: { x: 2 } }a"); + testGetPlanCacheKey("{a: 1}", "{}", "{'a.$': 1}", "eqa|1a.$"); + testGetPlanCacheKey("{a: 1}", "{}", "{a: 1}", "eqa|1a"); + + // Projection should be order-insensitive + testGetPlanCacheKey("{}", "{}", "{a: 1, b: 1}", "an|1a1b"); + testGetPlanCacheKey("{}", "{}", "{b: 1, a: 1}", "an|1a1b"); + + // With or-elimination and projection + testGetPlanCacheKey("{$or: [{a: 1}]}", "{}", "{_id: 0, a: 1}", "eqa|0_id1a"); + testGetPlanCacheKey("{$or: [{a: 1}]}", "{}", "{'a.$': 1}", "eqa|1a.$"); + } + + // Delimiters found in user field names or non-standard projection field values + // must be escaped. + TEST(PlanCacheTest, GetPlanCacheKeyEscaped) { + // Field name in query. + testGetPlanCacheKey("{'a,[]~|': 1}", "{}", "{}", "eqa\\,\\[\\]\\~\\|"); + + // Field name in sort. + testGetPlanCacheKey("{}", "{'a,[]~|': 1}", "{}", "an~aa\\,\\[\\]\\~\\|"); + + // Field name in projection. + testGetPlanCacheKey("{}", "{}", "{'a,[]~|': 1}", "an|1a\\,\\[\\]\\~\\|"); + + // Value in projection. + testGetPlanCacheKey("{}", "{}", "{a: 'foo,[]~|'}", "an|\"foo\\,\\[\\]\\~\\|\"a"); + } + + // Cache keys for $geoWithin queries with legacy and GeoJSON coordinates should + // not be the same. + TEST(PlanCacheTest, GetPlanCacheKeyGeoWithin) { + // Legacy coordinates. + auto_ptr cqLegacy(canonicalize("{a: {$geoWithin: " + "{$box: [[-180, -90], [180, 90]]}}}")); + // GeoJSON coordinates. + auto_ptr cqNew(canonicalize("{a: {$geoWithin: " + "{$geometry: {type: 'Polygon', coordinates: " + "[[[0, 0], [0, 90], [90, 0], [0, 0]]]}}}}")); + ASSERT_NOT_EQUALS(cqLegacy->getPlanCacheKey(), cqNew->getPlanCacheKey()); + } + + // GEO_NEAR cache keys should include information on geometry and CRS in addition + // to the match type and field name. + TEST(PlanCacheTest, GetPlanCacheKeyGeoNear) { + testGetPlanCacheKey("{a: {$near: [0,0], $maxDistance:0.3 }}", "{}", "{}", + "gnanrfl"); + testGetPlanCacheKey("{a: {$nearSphere: [0,0], $maxDistance: 0.31 }}", "{}", "{}", + "gnansfl"); + testGetPlanCacheKey("{a: {$geoNear: {$geometry: {type: 'Point', coordinates: [0,0]}," + "$maxDistance:100}}}", "{}", "{}", + "gnanrsp"); + } + +} diff --git a/src/mongo/db/query/eof_runner.cpp b/src/mongo/db/query/eof_runner.cpp new file mode 100644 index 00000000000..2bbf9993896 --- /dev/null +++ b/src/mongo/db/query/eof_runner.cpp @@ -0,0 +1,106 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/eof_runner.h" + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/type_explain.h" + +namespace mongo { + + EOFRunner::EOFRunner(CanonicalQuery* cq, const string& ns) : _cq(cq), _ns(ns) { + } + + EOFRunner::~EOFRunner() { + } + + Runner::RunnerState EOFRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { + return Runner::RUNNER_EOF; + } + + bool EOFRunner::isEOF() { + return true; + } + + void EOFRunner::saveState() { + } + + bool EOFRunner::restoreState() { + // TODO: Does this value matter? + return false; + } + + void EOFRunner::setYieldPolicy(Runner::YieldPolicy policy) { + } + + void EOFRunner::invalidate(const DiskLoc& dl, InvalidationType type) { + } + + const std::string& EOFRunner::ns() { + return _ns; + } + + void EOFRunner::kill() { + } + + Status EOFRunner::getInfo(TypeExplain** explain, + PlanInfo** planInfo) const { + if (NULL != explain) { + *explain = new TypeExplain; + + // Fill in mandatory fields. + (*explain)->setN(0); + (*explain)->setNScannedObjects(0); + (*explain)->setNScanned(0); + + // Fill in all the main fields that don't have a default in the explain data structure. + (*explain)->setCursor("BasicCursor"); + (*explain)->setScanAndOrder(false); + (*explain)->setIsMultiKey(false); + (*explain)->setIndexOnly(false); + (*explain)->setNYields(0); + (*explain)->setNChunkSkips(0); + + TypeExplain* allPlans = new TypeExplain; + allPlans->setCursor("BasicCursor"); + (*explain)->addToAllPlans(allPlans); // ownership xfer + + (*explain)->setNScannedObjectsAllPlans(0); + (*explain)->setNScannedAllPlans(0); + } + else if (NULL != planInfo) { + *planInfo = new PlanInfo(); + (*planInfo)->planSummary = "EOF"; + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/eof_runner.h b/src/mongo/db/query/eof_runner.h new file mode 100644 index 00000000000..acb88e2ce57 --- /dev/null +++ b/src/mongo/db/query/eof_runner.h @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/query/runner.h" + +namespace mongo { + + class BSONObj; + class CanonicalQuery; + class DiskLoc; + class TypeExplain; + struct PlanInfo; + + /** + * EOFRunner is EOF immediately and doesn't do anything except return EOF and possibly die + * during a yield. + */ + class EOFRunner : public Runner { + public: + + /* Takes ownership */ + EOFRunner(CanonicalQuery* cq, const std::string& ns); + + virtual ~EOFRunner(); + + virtual Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + virtual bool isEOF(); + + virtual void saveState(); + + virtual bool restoreState(); + + virtual void setYieldPolicy(Runner::YieldPolicy policy); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual const std::string& ns(); + + virtual void kill(); + + // this can return NULL since we never yield or anything over it + virtual const Collection* collection() { return NULL; } + + /** + * Always returns OK, allocating and filling in '*explain' with a fake ("zeroed") + * collection scan plan. Fills in '*planInfo' with information indicating an + * EOF runner. Caller owns '*explain', though. + */ + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const; + + private: + boost::scoped_ptr _cq; + std::string _ns; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/explain_plan.cpp b/src/mongo/db/query/explain_plan.cpp new file mode 100644 index 00000000000..f9c24ecdee6 --- /dev/null +++ b/src/mongo/db/query/explain_plan.cpp @@ -0,0 +1,670 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/explain_plan.h" + +#include "mongo/db/exec/2dcommon.h" +#include "mongo/db/query/stage_types.h" +#include "mongo/db/query/type_explain.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + using mongoutils::str::stream; + + namespace { + + bool isOrStage(StageType stageType) { + return stageType == STAGE_OR || stageType == STAGE_SORT_MERGE; + } + + bool isIntersectPlan(const PlanStageStats& stats) { + if (stats.stageType == STAGE_AND_HASH || stats.stageType == STAGE_AND_SORTED) { + return true; + } + for (size_t i = 0; i < stats.children.size(); ++i) { + if (isIntersectPlan(*stats.children[i])) { + return true; + } + } + return false; + } + + void getLeafNodes(const PlanStageStats& stats, vector* leafNodesOut) { + if (0 == stats.children.size()) { + leafNodesOut->push_back(&stats); + } + for (size_t i = 0; i < stats.children.size(); ++i) { + getLeafNodes(*stats.children[i], leafNodesOut); + } + } + + const PlanStageStats* findNode(const PlanStageStats* root, StageType type) { + if (root->stageType == type) { + return root; + } + for (size_t i = 0; i < root->children.size(); ++i) { + const PlanStageStats* ret = findNode(root->children[i], type); + if (NULL != ret) { + return ret; + } + } + return NULL; + } + + } // namespace + + Status explainIntersectPlan(const PlanStageStats& stats, TypeExplain** explainOut, bool fullDetails) { + auto_ptr res(new TypeExplain); + res->setCursor("Complex Plan"); + res->setN(stats.common.advanced); + + // Sum the various counters at the leaves. + vector leaves; + getLeafNodes(stats, &leaves); + + long long nScanned = 0; + long long nScannedObjects = 0; + for (size_t i = 0; i < leaves.size(); ++i) { + TypeExplain* leafExplain; + explainPlan(*leaves[i], &leafExplain, false); + nScanned += leafExplain->getNScanned(); + nScannedObjects += leafExplain->getNScannedObjects(); + delete leafExplain; + } + + res->setNScanned(nScanned); + // XXX: this isn't exactly "correct" -- for ixscans we have to find out if it's part of a + // subtree rooted at a fetch, etc. etc. do we want to just add the # of advances of a + // fetch node minus the number of alreadyHasObj for those nodes? + res->setNScannedObjects(nScannedObjects); + + uint64_t chunkSkips = 0; + const PlanStageStats* shardFilter = findNode(&stats, STAGE_SHARDING_FILTER); + if (NULL != shardFilter) { + const ShardingFilterStats* sfs + = static_cast(shardFilter->specific.get()); + chunkSkips = sfs->chunkSkips; + } + + res->setNChunkSkips(chunkSkips); + + if (fullDetails) { + res->setNYields(stats.common.yields); + BSONObjBuilder bob; + statsToBSON(stats, &bob); + res->stats = bob.obj(); + } + + *explainOut = res.release(); + return Status::OK(); + } + + namespace { + + Status explainPlan(const PlanStageStats& stats, TypeExplain** explainOut, + bool fullDetails, bool covered) { + // + // Temporary explain for index intersection + // + + if (isIntersectPlan(stats)) { + return explainIntersectPlan(stats, explainOut, fullDetails); + } + + // + // Legacy explain implementation + // + + // Descend the plan looking for structural properties: + // + Are there any OR clauses? If so, explain each branch. + // + What type(s) are the leaf nodes and what are their properties? + // + Did we need a sort? + + bool sortPresent = false; + size_t chunkSkips = 0; + + const PlanStageStats* orStage = NULL; + const PlanStageStats* root = &stats; + const PlanStageStats* leaf = root; + + while (leaf->children.size() > 0) { + // We shouldn't be here if there are any ANDs + if (leaf->children.size() > 1) { + verify(isOrStage(leaf->stageType)); + } + + if (isOrStage(leaf->stageType)) { + orStage = leaf; + break; + } + + if (leaf->stageType == STAGE_FETCH) { + covered = false; + } + + if (leaf->stageType == STAGE_SORT) { + sortPresent = true; + } + + if (STAGE_SHARDING_FILTER == leaf->stageType) { + const ShardingFilterStats* sfs + = static_cast(leaf->specific.get()); + chunkSkips = sfs->chunkSkips; + } + + leaf = leaf->children[0]; + } + + auto_ptr res(new TypeExplain); + + // Accounting for 'nscanned' and 'nscannedObjects' is specific to the kind of leaf: + // + // + on collection scan, both are the same; all the documents retrieved were + // fetched in practice. To get how many documents were retrieved, one simply + // looks at the number of 'advanced' in the stats. + // + // + on an index scan, we'd neeed to look into the index scan cursor to extract the + // number of keys that cursor retrieved, and into the stage's stats 'advanced' for + // nscannedObjects', which would be the number of keys that survived the IXSCAN + // filter. Those keys would have been FETCH-ed, if a fetch is present. + + if (orStage != NULL) { + size_t nScanned = 0; + size_t nScannedObjects = 0; + const std::vector& children = orStage->children; + for (std::vector::const_iterator it = children.begin(); + it != children.end(); + ++it) { + TypeExplain* childExplain = NULL; + explainPlan(**it, &childExplain, false /* no full details */, covered); + if (childExplain) { + // Override child's indexOnly value if we have a non-covered + // query (implied by a FETCH stage). + // + // As we run explain on each child, explainPlan() sets indexOnly + // based only on the information in each child. This does not + // consider the possibility of a FETCH stage above the OR/MERGE_SORT + // stage, in which case the child's indexOnly may be erroneously set + // to true. + if (!covered && childExplain->isIndexOnlySet()) { + childExplain->setIndexOnly(false); + } + + // 'res' takes ownership of 'childExplain'. + res->addToClauses(childExplain); + nScanned += childExplain->getNScanned(); + nScannedObjects += childExplain->getNScannedObjects(); + } + } + // We set the cursor name for backwards compatibility with 2.4. + res->setCursor("QueryOptimizerCursor"); + res->setNScanned(nScanned); + res->setNScannedObjects(nScannedObjects); + } + else if (leaf->stageType == STAGE_COLLSCAN) { + CollectionScanStats* csStats = static_cast(leaf->specific.get()); + res->setCursor("BasicCursor"); + res->setNScanned(csStats->docsTested); + res->setNScannedObjects(csStats->docsTested); + res->setIndexOnly(false); + res->setIsMultiKey(false); + } + else if (leaf->stageType == STAGE_GEO_2D) { + // Cursor name depends on type of GeoBrowse. + // TODO: We could omit the shape from the cursor name. + TwoDStats* nStats = static_cast(leaf->specific.get()); + res->setCursor("GeoBrowse-" + nStats->type); + res->setNScanned(leaf->common.works); + res->setNScannedObjects(leaf->common.works); + + // Generate index bounds from prefixes. + GeoHashConverter converter(nStats->converterParams); + BSONObjBuilder bob; + BSONArrayBuilder arrayBob(bob.subarrayStart(nStats->field)); + for (size_t i = 0; i < nStats->expPrefixes.size(); ++i) { + const GeoHash& prefix = nStats->expPrefixes[i]; + Box box = converter.unhashToBox(prefix); + arrayBob.append(box.toBSON()); + } + arrayBob.doneFast(); + res->setIndexBounds(bob.obj()); + + // TODO: Could be multikey. + res->setIsMultiKey(false); + res->setIndexOnly(false); + } + else if (leaf->stageType == STAGE_GEO_NEAR_2DSPHERE) { + S2NearStats* s2stats = static_cast(leaf->specific.get()); + res->setCursor("S2NearCursor"); + + res->setNScanned(s2stats->nscanned); + res->setNScannedObjects(s2stats->nscannedObjects); + + // TODO: only adding empty index bounds for backwards compatibility. + res->setIndexBounds(BSONObj()); + res->setIsMultiKey(s2stats->isMultiKey); + res->setIndexOnly(false); + } + else if (leaf->stageType == STAGE_GEO_NEAR_2D) { + TwoDNearStats* nStats = static_cast(leaf->specific.get()); + res->setCursor("GeoSearchCursor"); + // The first work() is an init. Every subsequent work examines a document. + res->setNScanned(nStats->nscanned); + res->setNScannedObjects(nStats->objectsLoaded); + // TODO: only adding empty index bounds for backwards compatibility. + res->setIndexBounds(BSONObj()); + // TODO: Could be multikey. + res->setIsMultiKey(false); + res->setIndexOnly(false); + } + else if (leaf->stageType == STAGE_TEXT) { + TextStats* tStats = static_cast(leaf->specific.get()); + res->setCursor("TextCursor"); + res->setNScanned(tStats->keysExamined); + res->setNScannedObjects(tStats->fetches); + } + else if (leaf->stageType == STAGE_IXSCAN) { + IndexScanStats* indexStats = static_cast(leaf->specific.get()); + verify(indexStats); + string direction = indexStats->direction > 0 ? "" : " reverse"; + res->setCursor(indexStats->indexType + " " + indexStats->indexName + direction); + res->setNScanned(indexStats->keysExamined); + + // If we're covered, that is, no FETCH is present, then, by definition, + // nScannedObject would be zero because no full document would have been fetched + // from disk. + res->setNScannedObjects(covered ? 0 : leaf->common.advanced); + + res->setIndexBounds(indexStats->indexBounds); + res->setIsMultiKey(indexStats->isMultiKey); + res->setIndexOnly(covered); + } + else if (leaf->stageType == STAGE_DISTINCT) { + DistinctScanStats* dss = static_cast(leaf->specific.get()); + verify(dss); + res->setCursor("DistinctCursor"); + res->setN(dss->keysExamined); + res->setNScanned(dss->keysExamined); + // Distinct hack stage is fully covered. + res->setNScannedObjects(0); + } + else { + return Status(ErrorCodes::InternalError, "cannot interpret execution plan"); + } + + // How many documents did the query return? + res->setN(root->common.advanced); + res->setScanAndOrder(sortPresent); + res->setNChunkSkips(chunkSkips); + + // Statistics for the plan (appear only in a detailed mode) + // TODO: if we can get this from the runner, we can kill "detailed mode" + if (fullDetails) { + res->setNYields(root->common.yields); + BSONObjBuilder bob; + statsToBSON(*root, &bob); + res->stats = bob.obj(); + } + + *explainOut = res.release(); + return Status::OK(); + } + + } // namespace + + Status explainPlan(const PlanStageStats& stats, TypeExplain** explainOut, bool fullDetails) { + // This function merely calls a recursive helper of the same name. The boolean "covered" is + // used to determine the value of nscannedObjects for subtrees along the way. Recursive + // calls will pass false for "covered" if a fetch stage has been seen at that point in the + // traversal. + const bool covered = true; + return explainPlan(stats, explainOut, fullDetails, covered); + } + + Status explainMultiPlan(const PlanStageStats& stats, + const std::vector& candidateStats, + QuerySolution* solution, + TypeExplain** explain) { + invariant(explain); + + Status status = explainPlan(stats, explain, true /* full details */); + if (!status.isOK()) { + return status; + } + + // TODO Hook the cached plan if there was one. + // (*explain)->setOldPlan(???); + + // + // Alternative plans' explains + // + // We get information about all the plans considered and hook them up the the main + // explain structure. If we fail to get any of them, we still return the main explain. + // Make sure we initialize the "*AllPlans" fields with the plan that was chose. + // + + TypeExplain* chosenPlan = NULL; + status = explainPlan(stats, &chosenPlan, false /* no full details */); + if (!status.isOK()) { + return status; + } + + (*explain)->addToAllPlans(chosenPlan); // ownership xfer + + size_t nScannedObjectsAllPlans = chosenPlan->getNScannedObjects(); + size_t nScannedAllPlans = chosenPlan->getNScanned(); + for (std::vector::const_iterator it = candidateStats.begin(); + it != candidateStats.end(); + ++it) { + + TypeExplain* candidateExplain = NULL; + status = explainPlan(**it, &candidateExplain, false /* no full details */); + if (status != Status::OK()) { + continue; + } + + (*explain)->addToAllPlans(candidateExplain); // ownership xfer + + nScannedObjectsAllPlans += candidateExplain->getNScannedObjects(); + nScannedAllPlans += candidateExplain->getNScanned(); + } + + (*explain)->setNScannedObjectsAllPlans(nScannedObjectsAllPlans); + (*explain)->setNScannedAllPlans(nScannedAllPlans); + + if (NULL != solution) { + (*explain)->setIndexFilterApplied(solution->indexFilterApplied); + } + + return Status::OK(); + } + + // TODO: where does this really live? stage_types.h? + string stageTypeString(StageType type) { + switch (type) { + case STAGE_AND_HASH: + return "AND_HASH"; + case STAGE_AND_SORTED: + return "AND_SORTED"; + case STAGE_COLLSCAN: + return "COLLSCAN"; + case STAGE_COUNT: + return "COUNT"; + case STAGE_DISTINCT: + return "DISTINCT"; + case STAGE_FETCH: + return "FETCH"; + case STAGE_GEO_2D: + return "GEO_2D"; + case STAGE_GEO_NEAR_2D: + return "GEO_NEAR_2D"; + case STAGE_GEO_NEAR_2DSPHERE: + return "GEO_NEAR_2DSPHERE"; + case STAGE_IXSCAN: + return "IXSCAN"; + case STAGE_KEEP_MUTATIONS: + return "KEEP_MUTATIONS"; + case STAGE_LIMIT: + return "LIMIT"; + case STAGE_OR: + return "OR"; + case STAGE_PROJECTION: + return "PROJECTION"; + case STAGE_SHARDING_FILTER: + return "SHARDING_FILTER"; + case STAGE_SKIP: + return "SKIP"; + case STAGE_SORT: + return "SORT"; + case STAGE_SORT_MERGE: + return "SORT_MERGE"; + case STAGE_TEXT: + return "TEXT"; + default: + invariant(0); + } + } + + void statsToBSON(const PlanStageStats& stats, + BSONObjBuilder* bob, + BSONObjBuilder* topLevelBob) { + invariant(bob); + invariant(topLevelBob); + + // Stop as soon as the BSON object we're building exceeds 10 MB. + static const int kMaxStatsBSONSize = 10 * 1024 * 1024; + if (topLevelBob->len() > kMaxStatsBSONSize) { + bob->append("warning", "stats tree exceeded 10 MB"); + return; + } + + // Common details. + bob->append("type", stageTypeString(stats.stageType)); + bob->appendNumber("works", stats.common.works); + bob->appendNumber("yields", stats.common.yields); + bob->appendNumber("unyields", stats.common.unyields); + bob->appendNumber("invalidates", stats.common.invalidates); + bob->appendNumber("advanced", stats.common.advanced); + bob->appendNumber("needTime", stats.common.needTime); + bob->appendNumber("needFetch", stats.common.needFetch); + bob->appendNumber("isEOF", stats.common.isEOF); + + // Stage-specific stats + if (STAGE_AND_HASH == stats.stageType) { + AndHashStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("flaggedButPassed", spec->flaggedButPassed); + bob->appendNumber("flaggedInProgress", spec->flaggedInProgress); + bob->appendNumber("memUsage", spec->memUsage); + bob->appendNumber("memLimit", spec->memLimit); + for (size_t i = 0; i < spec->mapAfterChild.size(); ++i) { + bob->appendNumber(string(stream() << "mapAfterChild_" << i), spec->mapAfterChild[i]); + } + } + else if (STAGE_AND_SORTED == stats.stageType) { + AndSortedStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("flagged", spec->flagged); + bob->appendNumber("matchTested", spec->matchTested); + for (size_t i = 0; i < spec->failedAnd.size(); ++i) { + bob->appendNumber(string(stream() << "failedAnd_" << i), spec->failedAnd[i]); + } + } + else if (STAGE_COLLSCAN == stats.stageType) { + CollectionScanStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("docsTested", spec->docsTested); + } + else if (STAGE_FETCH == stats.stageType) { + FetchStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("alreadyHasObj", spec->alreadyHasObj); + bob->appendNumber("forcedFetches", spec->forcedFetches); + bob->appendNumber("matchTested", spec->matchTested); + } + else if (STAGE_GEO_2D == stats.stageType) { + TwoDStats* spec = static_cast(stats.specific.get()); + bob->append("geometryType", spec->type); + bob->append("field", spec->field); + + // Generate verbose index bounds from prefixes + GeoHashConverter converter(spec->converterParams); + BSONArrayBuilder arrayBob(bob->subarrayStart("boundsVerbose")); + for (size_t i = 0; i < spec->expPrefixes.size(); ++i) { + const GeoHash& prefix = spec->expPrefixes[i]; + Box box = converter.unhashToBox(prefix); + arrayBob.append(box.toString()); + } + } + else if (STAGE_GEO_NEAR_2D == stats.stageType) { + TwoDNearStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("objectsLoaded", spec->objectsLoaded); + bob->appendNumber("nscanned", spec->nscanned); + } + else if (STAGE_IXSCAN == stats.stageType) { + IndexScanStats* spec = static_cast(stats.specific.get()); + // TODO: how much do we really want here? we should separate runtime stats vs. tree + // structure (soln tostring). + bob->append("keyPattern", spec->keyPattern.toString()); + bob->appendNumber("isMultiKey", spec->isMultiKey); + + // The verbose bounds can get large. Truncate to 1 MB. + static const int kMaxVerboseBoundsSize = 1024 * 1024; + bob->append("boundsVerbose", spec->indexBoundsVerbose.substr(0, kMaxVerboseBoundsSize)); + + bob->appendNumber("yieldMovedCursor", spec->yieldMovedCursor); + bob->appendNumber("dupsTested", spec->dupsTested); + bob->appendNumber("dupsDropped", spec->dupsDropped); + bob->appendNumber("seenInvalidated", spec->seenInvalidated); + bob->appendNumber("matchTested", spec->matchTested); + bob->appendNumber("keysExamined", spec->keysExamined); + } + else if (STAGE_OR == stats.stageType) { + OrStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("dupsTested", spec->dupsTested); + bob->appendNumber("dupsDropped", spec->dupsDropped); + bob->appendNumber("locsForgotten", spec->locsForgotten); + for (size_t i = 0 ; i < spec->matchTested.size(); ++i) { + bob->appendNumber(string(stream() << "matchTested_" << i), spec->matchTested[i]); + } + } + else if (STAGE_SHARDING_FILTER == stats.stageType) { + ShardingFilterStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("chunkSkips", spec->chunkSkips); + } + else if (STAGE_SORT == stats.stageType) { + SortStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("forcedFetches", spec->forcedFetches); + bob->appendNumber("memUsage", spec->memUsage); + bob->appendNumber("memLimit", spec->memLimit); + } + else if (STAGE_SORT_MERGE == stats.stageType) { + MergeSortStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("dupsTested", spec->dupsTested); + bob->appendNumber("dupsDropped", spec->dupsDropped); + bob->appendNumber("forcedFetches", spec->forcedFetches); + } + else if (STAGE_TEXT == stats.stageType) { + TextStats* spec = static_cast(stats.specific.get()); + bob->appendNumber("keysExamined", spec->keysExamined); + bob->appendNumber("fetches", spec->fetches); + bob->append("parsedTextQuery", spec->parsedTextQuery); + } + + BSONArrayBuilder childrenBob(bob->subarrayStart("children")); + for (size_t i = 0; i < stats.children.size(); ++i) { + BSONObjBuilder childBob(childrenBob.subobjStart()); + statsToBSON(*stats.children[i], &childBob, topLevelBob); + } + childrenBob.doneFast(); + } + + void statsToBSON(const PlanStageStats& stats, BSONObjBuilder* bob) { + statsToBSON(stats, bob, bob); + } + + BSONObj statsToBSON(const PlanStageStats& stats) { + BSONObjBuilder bob; + statsToBSON(stats, &bob); + return bob.obj(); + } + + namespace { + + /** + * Given a tree of stats, traverses the tree to the leaves. Appends a + * debug string for each leaf in the plan to the out-parameter 'leaves'. + */ + void getLeafStrings(const QuerySolutionNode* node, std::vector& leaves) { + if (node->children.empty()) { + // This is a leaf, append a string describing it. + mongoutils::str::stream leafInfo; + leafInfo << stageTypeString(node->getType()); + + // If the leaf is an index scan, also add the key pattern. + if (STAGE_COUNT == node->getType()) { + const CountNode* countNode = static_cast(node); + leafInfo << " " << countNode->indexKeyPattern; + } + else if (STAGE_DISTINCT == node->getType()) { + const DistinctNode* dn = static_cast(node); + leafInfo << " " << dn->indexKeyPattern; + } + else if (STAGE_GEO_2D == node->getType()) { + const Geo2DNode* g2d = static_cast(node); + leafInfo << " " << g2d->indexKeyPattern; + } + else if (STAGE_GEO_NEAR_2D == node->getType()) { + const GeoNear2DNode* g2dnear = static_cast(node); + leafInfo << " " << g2dnear->indexKeyPattern; + } + else if (STAGE_GEO_NEAR_2DSPHERE == node->getType()) { + const GeoNear2DSphereNode* g2dsphere = + static_cast(node); + leafInfo << " " << g2dsphere->indexKeyPattern; + } + else if (STAGE_IXSCAN == node->getType()) { + const IndexScanNode* ixn = static_cast(node); + leafInfo << " " << ixn->indexKeyPattern; + } + else if (STAGE_TEXT == node->getType()) { + const TextNode* textNode = static_cast(node); + leafInfo << " " << textNode->indexKeyPattern; + } + + leaves.push_back(leafInfo); + } + + for (size_t i = 0; i < node->children.size(); ++i) { + getLeafStrings(node->children[i], leaves); + } + } + + } // namespace + + std::string getPlanSummary(const QuerySolution& soln) { + std::vector leaves; + getLeafStrings(soln.root.get(), leaves); + + mongoutils::str::stream ss; + for (size_t i = 0; i < leaves.size(); i++) { + ss << leaves[i]; + if ((leaves.size() - 1) != i) { + ss << ", "; + } + } + + return ss; + } + + void getPlanInfo(const QuerySolution& soln, PlanInfo** infoOut) { + if (NULL == infoOut) { return; } + + *infoOut = new PlanInfo(); + + (*infoOut)->planSummary = getPlanSummary(soln); + } + +} // namespace mongo diff --git a/src/mongo/db/query/explain_plan.h b/src/mongo/db/query/explain_plan.h new file mode 100644 index 00000000000..37cf23c6b68 --- /dev/null +++ b/src/mongo/db/query/explain_plan.h @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/exec/plan_stats.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + class TypeExplain; + struct PlanInfo; + + /** + * Returns OK, allocating and filling in '*explainOut' describing the access paths used in + * the 'stats' tree of a given query solution. The caller has the ownership of + * '*explainOut', on success. Otherwise return an error status describing the problem. + * + * If 'fullDetails' was requested, the explain will return all available information about + * the plan, otherwise, just a summary. The fields in the summary are: 'cursor', 'n', + * 'nscannedObjects', 'nscanned', and 'indexBounds'. The remaining fields are: 'isMultKey', + * 'nscannedObjectsAllPlans', 'nscannedAllPlans', 'scanAndOrder', 'indexOnly', 'nYields', + * 'nChunkSkips', 'millis', 'allPlans', and 'oldPlan'. + * + * All these fields are documented in type_explain.h + * + * TODO: Currently, only working for single-leaf plans. + */ + Status explainPlan(const PlanStageStats& stats, TypeExplain** explainOut, bool fullDetails); + + /** + * Returns OK, allocating and filling in '*explain' with details of + * the "winner" plan. Caller takes ownership of '*explain'. Otherwise, + * return a status describing the error. + * + * 'bestStats', 'candidateStats' and 'solution' are used to fill in '*explain'. + * Used by both MultiPlanRunner and CachedPlanRunner. + */ + Status explainMultiPlan(const PlanStageStats& stats, + const std::vector& candidateStats, + QuerySolution* solution, + TypeExplain** explain); + + /** + * Returns a short plan summary string describing the leaves of the query solution. + * + * Used for logging. + */ + std::string getPlanSummary(const QuerySolution& soln); + + /** + * If the out-parameter 'info' is non-null, fills in '*infoOut' with information + * from the query solution tree 'soln' that can be determined before the query is done + * running. Whereas 'explainPlan(...)' above is for collecting runtime debug information, + * this function is for collecting static debug information that is known prior + * to query runtime. + * + * The caller is responsible for deleting '*infoOut'. + */ + void getPlanInfo(const QuerySolution& soln, PlanInfo** infoOut); + + void statsToBSON(const PlanStageStats& stats, + BSONObjBuilder* bob, + BSONObjBuilder* topLevelBob); + + void statsToBSON(const PlanStageStats& stats, BSONObjBuilder* bob); + + BSONObj statsToBSON(const PlanStageStats& stats); + +} // namespace mongo diff --git a/src/mongo/db/query/find_constants.h b/src/mongo/db/query/find_constants.h new file mode 100644 index 00000000000..e6333798682 --- /dev/null +++ b/src/mongo/db/query/find_constants.h @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +namespace mongo { + + // We cut off further objects once we cross this threshold; thus, you might get + // a little bit more than this, it is a threshold rather than a limit. + extern const int32_t MaxBytesToReturnToClientAtOnce; + +} // namespace mongo diff --git a/src/mongo/db/query/get_runner.cpp b/src/mongo/db/query/get_runner.cpp new file mode 100644 index 00000000000..8808fc8b7ae --- /dev/null +++ b/src/mongo/db/query/get_runner.cpp @@ -0,0 +1,877 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/get_runner.h" + +#include + +#include "mongo/base/parse_number.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/query/cached_plan_runner.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/eof_runner.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/query_settings.h" +#include "mongo/db/query/idhack_runner.h" +#include "mongo/db/query/index_bounds_builder.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/query/multi_plan_runner.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/planner_analysis.h" +#include "mongo/db/query/planner_access.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/single_solution_runner.h" +#include "mongo/db/query/stage_builder.h" +#include "mongo/db/query/subplan_runner.h" +#include "mongo/db/index_names.h" +#include "mongo/db/server_options.h" +#include "mongo/db/server_parameters.h" +#include "mongo/s/d_logic.h" + +namespace mongo { + + // static + void filterAllowedIndexEntries(const AllowedIndices& allowedIndices, + std::vector* indexEntries) { + invariant(indexEntries); + + // Filter index entries + // Check BSON objects in AllowedIndices::_indexKeyPatterns against IndexEntry::keyPattern. + // Removes IndexEntrys that do not match _indexKeyPatterns. + std::vector temp; + for (std::vector::const_iterator i = indexEntries->begin(); + i != indexEntries->end(); ++i) { + const IndexEntry& indexEntry = *i; + for (std::vector::const_iterator j = allowedIndices.indexKeyPatterns.begin(); + j != allowedIndices.indexKeyPatterns.end(); ++j) { + const BSONObj& index = *j; + // Copy index entry to temp vector if found in query settings. + if (0 == indexEntry.keyPattern.woCompare(index)) { + temp.push_back(indexEntry); + break; + } + } + } + + // Update results. + temp.swap(*indexEntries); + } + + /** + * For a given query, get a runner. The runner could be a SingleSolutionRunner, a + * CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc. + */ + Status getRunner(CanonicalQuery* rawCanonicalQuery, + Runner** out, size_t plannerOptions) { + verify(rawCanonicalQuery); + Database* db = cc().database(); + verify(db); + return getRunner(db->getCollection(rawCanonicalQuery->ns()), + rawCanonicalQuery, + out, + plannerOptions); + } + + Status getRunner(Collection* collection, + const std::string& ns, + const BSONObj& unparsedQuery, + Runner** outRunner, + CanonicalQuery** outCanonicalQuery, + size_t plannerOptions) { + + if (!collection) { + LOG(2) << "Collection " << ns << " does not exist." + << " Using EOF runner: " << unparsedQuery.toString(); + *outCanonicalQuery = NULL; + *outRunner = new EOFRunner(NULL, ns); + return Status::OK(); + } + if (!CanonicalQuery::isSimpleIdQuery(unparsedQuery) || + !collection->getIndexCatalog()->findIdIndex()) { + + Status status = CanonicalQuery::canonicalize( + collection->ns(), + unparsedQuery, + outCanonicalQuery); + if (!status.isOK()) + return status; + return getRunner(collection, *outCanonicalQuery, outRunner, plannerOptions); + } + + LOG(2) << "Using idhack: " << unparsedQuery.toString(); + + *outCanonicalQuery = NULL; + *outRunner = new IDHackRunner(collection, unparsedQuery["_id"].wrap()); + return Status::OK(); + } + + namespace { + // The body is below in the "count hack" section but getRunner calls it. + bool turnIxscanIntoCount(QuerySolution* soln); + } // namespace + + + void fillOutPlannerParams(Collection* collection, + CanonicalQuery* canonicalQuery, + QueryPlannerParams* plannerParams) { + // If it's not NULL, we may have indices. Access the catalog and fill out IndexEntry(s) + IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false); + while (ii.more()) { + const IndexDescriptor* desc = ii.next(); + plannerParams->indices.push_back(IndexEntry(desc->keyPattern(), + desc->getAccessMethodName(), + desc->isMultikey(), + desc->isSparse(), + desc->unique(), + desc->indexName(), + desc->infoObj())); + } + + // If query supports index filters, filter params.indices by indices in query settings. + QuerySettings* querySettings = collection->infoCache()->getQuerySettings(); + AllowedIndices* allowedIndicesRaw; + + // Filter index catalog if index filters are specified for query. + // Also, signal to planner that application hint should be ignored. + if (querySettings->getAllowedIndices(*canonicalQuery, &allowedIndicesRaw)) { + boost::scoped_ptr allowedIndices(allowedIndicesRaw); + filterAllowedIndexEntries(*allowedIndices, &plannerParams->indices); + plannerParams->indexFiltersApplied = true; + } + + // We will not output collection scans unless there are no indexed solutions. NO_TABLE_SCAN + // overrides this behavior by not outputting a collscan even if there are no indexed + // solutions. + if (storageGlobalParams.noTableScan) { + const string& ns = canonicalQuery->ns(); + // There are certain cases where we ignore this restriction: + bool ignore = canonicalQuery->getQueryObj().isEmpty() + || (string::npos != ns.find(".system.")) + || (0 == ns.find("local.")); + if (!ignore) { + plannerParams->options |= QueryPlannerParams::NO_TABLE_SCAN; + } + } + + // If the caller wants a shard filter, make sure we're actually sharded. + if (plannerParams->options & QueryPlannerParams::INCLUDE_SHARD_FILTER) { + CollectionMetadataPtr collMetadata = + shardingState.getCollectionMetadata(canonicalQuery->ns()); + + if (collMetadata) { + plannerParams->shardKey = collMetadata->getKeyPattern(); + } + else { + // If there's no metadata don't bother w/the shard filter since we won't know what + // the key pattern is anyway... + plannerParams->options &= ~QueryPlannerParams::INCLUDE_SHARD_FILTER; + } + } + + if (internalQueryPlannerEnableIndexIntersection) { + plannerParams->options |= QueryPlannerParams::INDEX_INTERSECTION; + } + + plannerParams->options |= QueryPlannerParams::KEEP_MUTATIONS; + plannerParams->options |= QueryPlannerParams::SPLIT_LIMITED_SORT; + } + + Status getRunnerFromCache(CanonicalQuery* canonicalQuery, + Collection* collection, + const QueryPlannerParams& plannerParams, + Runner** out) { + // Skip cache look up for non-cacheable queries. + if (!PlanCache::shouldCacheQuery(*canonicalQuery)) { + return Status(ErrorCodes::BadValue, "query is not cacheable"); + } + + CachedSolution* rawCS; + Status cacheLookupStatus = collection->infoCache()->getPlanCache()->get(*canonicalQuery, + &rawCS); + if (!cacheLookupStatus.isOK()) { + return cacheLookupStatus; + } + + // We have a CachedSolution. Have the planner turn it into a QuerySolution. + boost::scoped_ptr cs(rawCS); + QuerySolution *qs, *backupQs; + Status status = QueryPlanner::planFromCache(*canonicalQuery, + plannerParams, + *cs, + &qs, + &backupQs); + if (!status.isOK()) { + return status; + } + + // If our cached solution is a hit for a count query, try to turn it into a fast count + // thing. + if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) { + if (turnIxscanIntoCount(qs)) { + LOG(2) << "Using fast count: " << canonicalQuery->toStringShort() + << ", planSummary: " << getPlanSummary(*qs); + + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*qs, &root, &ws)); + *out = new SingleSolutionRunner(collection, + canonicalQuery, qs, root, ws); + if (NULL != backupQs) { + delete backupQs; + } + return Status::OK(); + } + } + + // If we're here, we're going to used the cached plan and things are normal. + LOG(2) << "Using cached query plan: " << canonicalQuery->toStringShort() + << ", planSummary: " << getPlanSummary(*qs); + + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*qs, &root, &ws)); + CachedPlanRunner* cpr = new CachedPlanRunner(collection, + canonicalQuery, + qs, + root, + ws); + + // If there's a backup solution, let the CachedPlanRunner know about it. + if (NULL != backupQs) { + WorkingSet* backupWs; + PlanStage* backupRoot; + verify(StageBuilder::build(*backupQs, &backupRoot, &backupWs)); + cpr->setBackupPlan(backupQs, backupRoot, backupWs); + } + + *out = cpr; + return Status::OK(); + } + + /** + * For a given query, get a runner. The runner could be a SingleSolutionRunner, a + * CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc. + */ + Status getRunner(Collection* collection, + CanonicalQuery* rawCanonicalQuery, + Runner** out, + size_t plannerOptions) { + + verify(rawCanonicalQuery); + auto_ptr canonicalQuery(rawCanonicalQuery); + + // This can happen as we're called by internal clients as well. + if (NULL == collection) { + const string& ns = canonicalQuery->ns(); + LOG(2) << "Collection " << ns << " does not exist." + << " Using EOF runner: " << canonicalQuery->toStringShort(); + *out = new EOFRunner(canonicalQuery.release(), ns); + return Status::OK(); + } + + // If we have an _id index we can use the idhack runner. + if (IDHackRunner::supportsQuery(*canonicalQuery) && + collection->getIndexCatalog()->findIdIndex()) { + LOG(2) << "Using idhack: " << canonicalQuery->toStringShort(); + *out = new IDHackRunner(collection, canonicalQuery.release()); + return Status::OK(); + } + + // Tailable: If the query requests tailable the collection must be capped. + if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) { + if (!collection->isCapped()) { + return Status(ErrorCodes::BadValue, + "error processing query: " + canonicalQuery->toString() + + " tailable cursor requested on non capped collection"); + } + + // If a sort is specified it must be equal to expectedSort. + const BSONObj expectedSort = BSON("$natural" << 1); + const BSONObj& actualSort = canonicalQuery->getParsed().getSort(); + if (!actualSort.isEmpty() && !(actualSort == expectedSort)) { + return Status(ErrorCodes::BadValue, + "error processing query: " + canonicalQuery->toString() + + " invalid sort specified for tailable cursor: " + + actualSort.toString()); + } + } + + // Fill out the planning params. We use these for both cached solutions and non-cached. + QueryPlannerParams plannerParams; + plannerParams.options = plannerOptions; + fillOutPlannerParams(collection, rawCanonicalQuery, &plannerParams); + + // See if the cache has what we're looking for. + Status cacheStatus = getRunnerFromCache(canonicalQuery.get(), + collection, + plannerParams, + out); + + // This can be not-OK and we can carry on. It just means the query wasn't cached. + if (cacheStatus.isOK()) { + // We got a cached runner. + canonicalQuery.release(); + return cacheStatus; + } + + if (internalQueryPlanOrChildrenIndependently + && SubplanRunner::canUseSubplanRunner(*canonicalQuery)) { + + QLOG() << "Running query as sub-queries: " << canonicalQuery->toStringShort(); + LOG(2) << "Running query as sub-queries: " << canonicalQuery->toStringShort(); + + SubplanRunner* runner; + Status runnerStatus = SubplanRunner::make(collection, plannerParams, + canonicalQuery.release(), &runner); + if (!runnerStatus.isOK()) { + return runnerStatus; + } + + *out = runner; + return Status::OK(); + } + + return getRunnerAlwaysPlan(collection, canonicalQuery.release(), plannerParams, out); + } + + Status getRunnerAlwaysPlan(Collection* collection, + CanonicalQuery* rawCanonicalQuery, + const QueryPlannerParams& plannerParams, + Runner** out) { + + invariant(collection); + invariant(rawCanonicalQuery); + auto_ptr canonicalQuery(rawCanonicalQuery); + + vector solutions; + Status status = QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions); + if (!status.isOK()) { + return Status(ErrorCodes::BadValue, + "error processing query: " + canonicalQuery->toString() + + " planner returned error: " + status.reason()); + } + + // We cannot figure out how to answer the query. Perhaps it requires an index + // we do not have? + if (0 == solutions.size()) { + return Status(ErrorCodes::BadValue, + str::stream() + << "error processing query: " + << canonicalQuery->toString() + << " No query solutions"); + } + + // See if one of our solutions is a fast count hack in disguise. + if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) { + for (size_t i = 0; i < solutions.size(); ++i) { + if (turnIxscanIntoCount(solutions[i])) { + // Great, we can use solutions[i]. Clean up the other QuerySolution(s). + for (size_t j = 0; j < solutions.size(); ++j) { + if (j != i) { + delete solutions[j]; + } + } + + LOG(2) << "Using fast count: " << canonicalQuery->toStringShort() + << ", planSummary: " << getPlanSummary(*solutions[i]); + + // We're not going to cache anything that's fast count. + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*solutions[i], &root, &ws)); + *out = new SingleSolutionRunner(collection, + canonicalQuery.release(), + solutions[i], + root, + ws); + return Status::OK(); + } + } + } + + if (1 == solutions.size()) { + LOG(2) << "Only one plan is available; it will be run but will not be cached. " + << canonicalQuery->toStringShort() + << ", planSummary: " << getPlanSummary(*solutions[0]); + + // Only one possible plan. Run it. Build the stages from the solution. + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*solutions[0], &root, &ws)); + + // And, run the plan. + *out = new SingleSolutionRunner(collection, + canonicalQuery.release(), + solutions[0], + root, + ws); + return Status::OK(); + } + else { + // Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on. + auto_ptr mpr(new MultiPlanRunner(collection,canonicalQuery.release())); + + for (size_t i = 0; i < solutions.size(); ++i) { + WorkingSet* ws; + PlanStage* root; + if (solutions[i]->cacheData.get()) { + solutions[i]->cacheData->indexFilterApplied = plannerParams.indexFiltersApplied; + } + verify(StageBuilder::build(*solutions[i], &root, &ws)); + // Takes ownership of all arguments. + mpr->addPlan(solutions[i], root, ws); + } + *out = mpr.release(); + return Status::OK(); + } + } + + // + // Count hack + // + + namespace { + + /** + * Returns 'true' if the provided solution 'soln' can be rewritten to use + * a fast counting stage. Mutates the tree in 'soln->root'. + * + * Otherwise, returns 'false'. + */ + bool turnIxscanIntoCount(QuerySolution* soln) { + QuerySolutionNode* root = soln->root.get(); + + // Root should be a fetch w/o any filters. + if (STAGE_FETCH != root->getType()) { + return false; + } + + if (NULL != root->filter.get()) { + return false; + } + + // Child should be an ixscan. + if (STAGE_IXSCAN != root->children[0]->getType()) { + return false; + } + + IndexScanNode* isn = static_cast(root->children[0]); + + // No filters allowed and side-stepping isSimpleRange for now. TODO: do we ever see + // isSimpleRange here? because we could well use it. I just don't think we ever do see + // it. + + if (NULL != isn->filter.get() || isn->bounds.isSimpleRange) { + return false; + } + + // Make sure the bounds are OK. + BSONObj startKey; + bool startKeyInclusive; + BSONObj endKey; + bool endKeyInclusive; + + if (!IndexBoundsBuilder::isSingleInterval( isn->bounds, + &startKey, + &startKeyInclusive, + &endKey, + &endKeyInclusive )) { + return false; + } + + // Make the count node that we replace the fetch + ixscan with. + CountNode* cn = new CountNode(); + cn->indexKeyPattern = isn->indexKeyPattern; + cn->startKey = startKey; + cn->startKeyInclusive = startKeyInclusive; + cn->endKey = endKey; + cn->endKeyInclusive = endKeyInclusive; + // Takes ownership of 'cn' and deletes the old root. + soln->root.reset(cn); + return true; + } + + /** + * Returns true if indices contains an index that can be + * used with DistinctNode. Sets indexOut to the array index + * of PlannerParams::indices. + * Look for the index for the fewest fields. + * Criteria for suitable index is that the index cannot be special + * (geo, hashed, text, ...). + * + * Multikey indices are not suitable for DistinctNode when the projection + * is on an array element. Arrays are flattened in a multikey index which + * makes it impossible for the distinct scan stage (plan stage generated from + * DistinctNode) to select the requested element by array index. + * + * Multikey indices cannot be used for the fast distinct hack if the field is dotted. + * Currently the solution generated for the distinct hack includes a projection stage and + * the projection stage cannot be covered with a dotted field. + */ + bool getDistinctNodeIndex(const std::vector& indices, + const std::string& field, size_t* indexOut) { + invariant(indexOut); + bool isDottedField = str::contains(field, '.'); + int minFields = std::numeric_limits::max(); + for (size_t i = 0; i < indices.size(); ++i) { + // Skip special indices. + if (!IndexNames::findPluginName(indices[i].keyPattern).empty()) { + continue; + } + // Skip multikey indices if we are projecting on a dotted field. + if (indices[i].multikey && isDottedField) { + continue; + } + int nFields = indices[i].keyPattern.nFields(); + // Pick the index with the lowest number of fields. + if (nFields < minFields) { + minFields = nFields; + *indexOut = i; + } + } + return minFields != std::numeric_limits::max(); + } + + /** + * Checks dotted field for a projection and truncates the + * field name if we could be projecting on an array element. + * Sets 'isIDOut' to true if the projection is on a sub document of _id. + * For example, _id.a.2, _id.b.c. + */ + std::string getProjectedDottedField(const std::string& field, bool* isIDOut) { + // Check if field contains an array index. + std::vector res; + mongo::splitStringDelim(field, &res, '.'); + + // Since we could exit early from the loop, + // we should check _id here and set '*isIDOut' accordingly. + *isIDOut = ("_id" == res[0]); + + // Skip the first dotted component. If the field starts + // with a number, the number cannot be an array index. + int arrayIndex = 0; + for (size_t i = 1; i < res.size(); ++i) { + if (mongo::parseNumberFromStringWithBase(res[i], 10, &arrayIndex).isOK()) { + // Array indices cannot be negative numbers (this is not $slice). + // Negative numbers are allowed as field names. + if (arrayIndex >= 0) { + // Generate prefix of field up to (but not including) array index. + std::vector prefixStrings(res); + prefixStrings.resize(i); + // Reset projectedField. Instead of overwriting, joinStringDelim() appends joined string + // to the end of projectedField. + std::string projectedField; + mongo::joinStringDelim(prefixStrings, &projectedField, '.'); + return projectedField; + } + } + } + + return field; + } + + /** + * Creates a projection spec for a distinct command from the requested field. + * In most cases, the projection spec will be {_id: 0, key: 1}. + * The exceptions are: + * 1) When the requested field is '_id', the projection spec will {_id: 1}. + * 2) When the requested field could be an array element (eg. a.0), + * the projected field will be the prefix of the field up to the array element. + * For example, a.b.2 => {_id: 0, 'a.b': 1} + * Note that we can't use a $slice projection because the distinct command filters + * the results from the runner using the dotted field name. Using $slice will + * re-order the documents in the array in the results. + */ + BSONObj getDistinctProjection(const std::string& field) { + std::string projectedField(field); + + bool isID = false; + if ("_id" == field) { + isID = true; + } + else if (str::contains(field, '.')) { + projectedField = getProjectedDottedField(field, &isID); + } + BSONObjBuilder bob; + if (!isID) { + bob.append("_id", 0); + } + bob.append(projectedField, 1); + return bob.obj(); + } + + } // namespace + + Status getRunnerCount(Collection* collection, + const BSONObj& query, + const BSONObj& hintObj, + Runner** out) { + verify(collection); + + CanonicalQuery* cq; + uassertStatusOK(CanonicalQuery::canonicalize(collection->ns().ns(), + query, + BSONObj(), + BSONObj(), + 0, + 0, + hintObj, + &cq)); + + return getRunner(collection, cq, out, QueryPlannerParams::PRIVATE_IS_COUNT); + } + + // + // Distinct hack + // + + /** + * If possible, turn the provided QuerySolution into a QuerySolution that uses a DistinctNode + * to provide results for the distinct command. + * + * If the provided solution could be mutated successfully, returns true, otherwise returns + * false. + */ + bool turnIxscanIntoDistinctIxscan(QuerySolution* soln, const string& field) { + QuerySolutionNode* root = soln->root.get(); + + // We're looking for a project on top of an ixscan. + if (STAGE_PROJECTION == root->getType() && (STAGE_IXSCAN == root->children[0]->getType())) { + IndexScanNode* isn = static_cast(root->children[0]); + + // An additional filter must be applied to the data in the key, so we can't just skip + // all the keys with a given value; we must examine every one to find the one that (may) + // pass the filter. + if (NULL != isn->filter.get()) { + return false; + } + + // We only set this when we have special query modifiers (.max() or .min()) or other + // special cases. Don't want to handle the interactions between those and distinct. + // Don't think this will ever really be true but if it somehow is, just ignore this + // soln. + if (isn->bounds.isSimpleRange) { + return false; + } + + // Make a new DistinctNode. We swap this for the ixscan in the provided solution. + DistinctNode* dn = new DistinctNode(); + dn->indexKeyPattern = isn->indexKeyPattern; + dn->direction = isn->direction; + dn->bounds = isn->bounds; + + // Figure out which field we're skipping to the next value of. TODO: We currently only + // try to distinct-hack when there is an index prefixed by the field we're distinct-ing + // over. Consider removing this code if we stick with that policy. + dn->fieldNo = 0; + BSONObjIterator it(isn->indexKeyPattern); + while (it.more()) { + if (field == it.next().fieldName()) { + break; + } + dn->fieldNo++; + } + + // Delete the old index scan, set the child of project to the fast distinct scan. + delete root->children[0]; + root->children[0] = dn; + return true; + } + + return false; + } + + Status getRunnerDistinct(Collection* collection, + const BSONObj& query, + const string& field, + Runner** out) { + // This should'a been checked by the distinct command. + verify(collection); + + // TODO: check for idhack here? + + // When can we do a fast distinct hack? + // 1. There is a plan with just one leaf and that leaf is an ixscan. + // 2. The ixscan indexes the field we're interested in. + // 2a: We are correct if the index contains the field but for now we look for prefix. + // 3. The query is covered/no fetch. + // + // We go through normal planning (with limited parameters) to see if we can produce + // a soln with the above properties. + + QueryPlannerParams plannerParams; + plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN; + + IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false); + while (ii.more()) { + const IndexDescriptor* desc = ii.next(); + // The distinct hack can work if any field is in the index but it's not always clear + // if it's a win unless it's the first field. + if (desc->keyPattern().firstElement().fieldName() == field) { + plannerParams.indices.push_back(IndexEntry(desc->keyPattern(), + desc->getAccessMethodName(), + desc->isMultikey(), + desc->isSparse(), + desc->unique(), + desc->indexName(), + desc->infoObj())); + } + } + + // If there are no suitable indices for the distinct hack bail out now into regular planning + // with no projection. + if (plannerParams.indices.empty()) { + CanonicalQuery* cq; + Status status = CanonicalQuery::canonicalize(collection->ns().ns(), + query, + BSONObj(), + BSONObj(), + &cq); + if (!status.isOK()) { + return status; + } + + // Takes ownership of cq. + return getRunner(cq, out); + } + + // + // If we're here, we have an index prefixed by the field we're distinct-ing over. + // + + // Applying a projection allows the planner to try to give us covered plans that we can turn + // into the projection hack. getDistinctProjection deals with .find() projection semantics + // (ie _id:1 being implied by default). + BSONObj projection = getDistinctProjection(field); + + // Apply a projection of the key. Empty BSONObj() is for the sort. + CanonicalQuery* cq; + Status status = CanonicalQuery::canonicalize(collection->ns().ns(), + query, + BSONObj(), + projection, + &cq); + if (!status.isOK()) { + return status; + } + + // If there's no query, we can just distinct-scan one of the indices. + // Not every index in plannerParams.indices may be suitable. Refer to + // getDistinctNodeIndex(). + size_t distinctNodeIndex = 0; + if (query.isEmpty() && + getDistinctNodeIndex(plannerParams.indices, field, &distinctNodeIndex)) { + DistinctNode* dn = new DistinctNode(); + dn->indexKeyPattern = plannerParams.indices[distinctNodeIndex].keyPattern; + dn->direction = 1; + IndexBoundsBuilder::allValuesBounds(dn->indexKeyPattern, &dn->bounds); + dn->fieldNo = 0; + + QueryPlannerParams params; + + // Takes ownership of 'dn'. + QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*cq, params, dn); + verify(soln); + + LOG(2) << "Using fast distinct: " << cq->toStringShort() + << ", planSummary: " << getPlanSummary(*soln); + + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*soln, &root, &ws)); + *out = new SingleSolutionRunner(collection, cq, soln, root, ws); + return Status::OK(); + } + + // See if we can answer the query in a fast-distinct compatible fashion. + vector solutions; + status = QueryPlanner::plan(*cq, plannerParams, &solutions); + if (!status.isOK()) { + return getRunner(cq, out); + } + + // We look for a solution that has an ixscan we can turn into a distinctixscan + for (size_t i = 0; i < solutions.size(); ++i) { + if (turnIxscanIntoDistinctIxscan(solutions[i], field)) { + // Great, we can use solutions[i]. Clean up the other QuerySolution(s). + for (size_t j = 0; j < solutions.size(); ++j) { + if (j != i) { + delete solutions[j]; + } + } + + LOG(2) << "Using fast distinct: " << cq->toStringShort() + << ", planSummary: " << getPlanSummary(*solutions[i]); + + // Build and return the SSR over solutions[i]. + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*solutions[i], &root, &ws)); + *out = new SingleSolutionRunner(collection, cq, solutions[i], root, ws); + return Status::OK(); + } + } + + // If we're here, the planner made a soln with the restricted index set but we couldn't + // translate any of them into a distinct-compatible soln. So, delete the solutions and just + // go through normal planning. + for (size_t i = 0; i < solutions.size(); ++i) { + delete solutions[i]; + } + + // We drop the projection from the 'cq'. Unfortunately this is not trivial. + delete cq; + status = CanonicalQuery::canonicalize(collection->ns().ns(), + query, + BSONObj(), + BSONObj(), + &cq); + if (!status.isOK()) { + return status; + } + + // Takes ownership of cq. + return getRunner(cq, out); + } + + ScopedRunnerRegistration::ScopedRunnerRegistration(Runner* runner) + : _runner(runner) { + // Collection can be null for EOFRunner, or other places where registration is not needed + if ( _runner->collection() ) + _runner->collection()->cursorCache()->registerRunner( runner ); + } + + ScopedRunnerRegistration::~ScopedRunnerRegistration() { + if ( _runner->collection() ) + _runner->collection()->cursorCache()->deregisterRunner( _runner ); + } + +} // namespace mongo diff --git a/src/mongo/db/query/get_runner.h b/src/mongo/db/query/get_runner.h new file mode 100644 index 00000000000..18f7c7235f1 --- /dev/null +++ b/src/mongo/db/query/get_runner.h @@ -0,0 +1,147 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/db/query/query_settings.h" +#include "mongo/db/query/runner.h" + +namespace mongo { + + class Collection; + + /** + * Filter indexes retrieved from index catalog by + * allowed indices in query settings. + * Used by getRunner(). + * This function is public to facilitate testing. + */ + void filterAllowedIndexEntries(const AllowedIndices& allowedIndices, + std::vector* indexEntries); + + /** + * Fill out the provided 'plannerParams' for the 'canonicalQuery' operating on the collection + * 'collection'. Exposed for testing. + */ + void fillOutPlannerParams(Collection* collection, + CanonicalQuery* canonicalQuery, + QueryPlannerParams* plannerParams); + + /** + * Get a runner for a query. Takes ownership of rawCanonicalQuery. + * + * If the query is valid and a runner could be created, returns Status::OK() + * and populates *out with the Runner. + * + * If the query cannot be executed, returns a Status indicating why. Deletes + * rawCanonicalQuery. + */ + Status getRunner(CanonicalQuery* rawCanonicalQuery, + Runner** out, + size_t plannerOptions = 0); + + /** + * Get a runner for a query. Takes ownership of rawCanonicalQuery. + * + * As 'getRunner' above, but takes a Collection* as the first argument, for cases where the + * work to obtain the Collection has already been done by the caller. The 'collection' + * argument may be NULL. + */ + Status getRunner(Collection* collection, + CanonicalQuery* rawCanonicalQuery, + Runner** out, + size_t plannerOptions = 0); + + /** + * Gets a runner for a query described as an unparsed BSON object over the named and optionally + * supplied collection. + * + * If necessary, parses a CanonicalQuery out of 'unparsedQuery'. + * + * Returns Status::OK() on success, in which case '*outRunner' points to a runner now owned by + * the caller, and '*outCanonicalQuery' is either NULL or points to a canonical query owned by + * the returned runner. On failure, returns other status values, and '*outRunner' and + * '*outCanonicalQuery' have unspecified values. + */ + Status getRunner(Collection* collection, + const std::string& ns, + const BSONObj& unparsedQuery, + Runner** outRunner, + CanonicalQuery** outCanonicalQuery, + size_t plannerOptions = 0); + + /* + * Get a runner for a query executing as part of a distinct command. + * + * Distinct is unique in that it doesn't care about getting all the results; it just wants all + * possible values of a certain field. As such, we can skip lots of data in certain cases (see + * body of method for detail). + */ + Status getRunnerDistinct(Collection* collection, + const BSONObj& query, + const std::string& field, + Runner** out); + /* + * Get a runner for a query executing as part of a count command. + * + * Count doesn't care about actually examining its results; it just wants to walk through them. + * As such, with certain covered queries, we can skip the overhead of fetching etc. when + * executing a count. + */ + Status getRunnerCount(Collection* collection, + const BSONObj& query, + const BSONObj& hintObj, + Runner** out); + + /** + * Get a runner for a query. Ignores the cache and always plans the full query. + */ + Status getRunnerAlwaysPlan(Collection* collection, + CanonicalQuery* rawCanonicalQuery, + const QueryPlannerParams& plannerParams, + Runner** out); + + /** + * RAII approach to ensuring that runners are deregistered in newRunQuery. + * + * While retrieving the first batch of results, newRunQuery manually registers the runner with + * ClientCursor. Certain query execution paths, namely $where, can throw an exception. If we + * fail to deregister the runner, we will call invalidate/kill on the + * still-registered-yet-deleted runner. + * + * For any subsequent calls to getMore, the runner is already registered with ClientCursor + * by virtue of being cached, so this exception-proofing is not required. + */ + struct ScopedRunnerRegistration { + ScopedRunnerRegistration(Runner* runner); + ~ScopedRunnerRegistration(); + + Runner* const _runner; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/get_runner_test.cpp b/src/mongo/db/query/get_runner_test.cpp new file mode 100644 index 00000000000..ce646770623 --- /dev/null +++ b/src/mongo/db/query/get_runner_test.cpp @@ -0,0 +1,144 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/get_runner.h + */ + +#include "mongo/db/query/get_runner.h" + +#include "mongo/db/json.h" +#include "mongo/db/query/query_settings.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/mongoutils/str.h" + +using namespace mongo; + +namespace { + + using std::auto_ptr; + + static const char* ns = "somebogusns"; + + /** + * Utility functions to create a CanonicalQuery + */ + CanonicalQuery* canonicalize(const char* queryStr, const char* sortStr, + const char* projStr) { + BSONObj queryObj = fromjson(queryStr); + BSONObj sortObj = fromjson(sortStr); + BSONObj projObj = fromjson(projStr); + CanonicalQuery* cq; + Status result = CanonicalQuery::canonicalize(ns, queryObj, sortObj, + projObj, + &cq); + ASSERT_OK(result); + return cq; + } + + // + // get_runner tests + // + + // + // filterAllowedIndexEntries + // + + /** + * Test function to check filterAllowedIndexEntries + */ + void testAllowedIndices(const char* hintKeyPatterns[], + const char* indexCatalogKeyPatterns[], + const char* expectedFilteredKeyPatterns[]) { + QuerySettings querySettings; + AllowedIndices *allowedIndicesRaw; + + // getAllowedIndices should return false when query shape is not yet in query settings. + auto_ptr cq(canonicalize("{a: 1}", "{}", "{}")); + ASSERT_FALSE(querySettings.getAllowedIndices(*cq, &allowedIndicesRaw)); + + // Add entry to query settings. + const PlanCacheKey& key = cq->getPlanCacheKey(); + std::vector indexKeyPatterns; + for (int i=0; hintKeyPatterns[i] != NULL; ++i) { + indexKeyPatterns.push_back(fromjson(hintKeyPatterns[i])); + } + querySettings.setAllowedIndices(*cq, indexKeyPatterns); + + // Index entry vector should contain 1 entry after filtering. + ASSERT_TRUE(querySettings.getAllowedIndices(*cq, &allowedIndicesRaw)); + ASSERT_FALSE(key.empty()); + ASSERT(NULL != allowedIndicesRaw); + auto_ptr allowedIndices(allowedIndicesRaw); + + // Indexes from index catalog. + std::vector indexEntries; + for (int i=0; indexCatalogKeyPatterns[i] != NULL; ++i) { + indexEntries.push_back(IndexEntry(fromjson(indexCatalogKeyPatterns[i]))); + } + + // Apply filter in allowed indices. + filterAllowedIndexEntries(*allowedIndices, &indexEntries); + size_t numExpected = 0; + while (expectedFilteredKeyPatterns[numExpected] != NULL) { + ASSERT_LESS_THAN(numExpected, indexEntries.size()); + ASSERT_EQUALS(indexEntries[numExpected].keyPattern, + fromjson(expectedFilteredKeyPatterns[numExpected])); + numExpected++; + } + ASSERT_EQUALS(indexEntries.size(), numExpected); + } + + // Use of index filters to select compound index over single key index. + TEST(GetRunnerTest, GetAllowedIndices) { + const char* hintKeyPatterns[] = {"{a: 1, b: 1}", NULL}; + const char* indexCatalogKeyPatterns[] = {"{a: 1}", "{a: 1, b: 1}", "{a: 1, c: 1}", NULL}; + const char* expectedFilteredKeyPatterns[] = {"{a: 1, b: 1}", NULL}; + testAllowedIndices(hintKeyPatterns, indexCatalogKeyPatterns, expectedFilteredKeyPatterns); + } + + // Setting index filter referring to non-existent indexes + // will effectively disregard the index catalog and + // result in the planner generating a collection scan. + TEST(GetRunnerTest, GetAllowedIndicesNonExistentIndexKeyPatterns) { + const char* hintKeyPatterns[] = {"{nosuchfield: 1}", NULL}; + const char* indexCatalogKeyPatterns[] = {"{a: 1}", "{a: 1, b: 1}", "{a: 1, c: 1}", NULL}; + const char* expectedFilteredKeyPatterns[] = {NULL}; + testAllowedIndices(hintKeyPatterns, indexCatalogKeyPatterns, expectedFilteredKeyPatterns); + } + + // This test case shows how to force query execution to use + // an index that orders items in descending order. + TEST(GetRunnerTest, GetAllowedIndicesDescendingOrder) { + const char* hintKeyPatterns[] = {"{a: -1}", NULL}; + const char* indexCatalogKeyPatterns[] = {"{a: 1}", "{a: -1}", NULL}; + const char* expectedFilteredKeyPatterns[] = {"{a: -1}", NULL}; + testAllowedIndices(hintKeyPatterns, indexCatalogKeyPatterns, expectedFilteredKeyPatterns); + } + +} // namespace diff --git a/src/mongo/db/query/idhack_runner.cpp b/src/mongo/db/query/idhack_runner.cpp new file mode 100644 index 00000000000..d587323d5f0 --- /dev/null +++ b/src/mongo/db/query/idhack_runner.cpp @@ -0,0 +1,315 @@ +/** + * Copyright 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/idhack_runner.h" + +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/structure/btree/btree.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/projection.h" +#include "mongo/db/index/btree_access_method.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/type_explain.h" +#include "mongo/db/query/plan_executor.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + +namespace { + + CollectionMetadataPtr getMetadata(const std::string& ns) { + if (shardingState.needCollectionMetadata(ns)) { + return shardingState.getCollectionMetadata(ns); + } + else { + return CollectionMetadataPtr(); + } + } + +} // namespace + + IDHackRunner::IDHackRunner(const Collection* collection, CanonicalQuery* query) + : _collection(collection), + _key(query->getQueryObj()["_id"].wrap()), + _query(query), + _killed(false), + _done(false), + _nscanned(0), + _nscannedObjects(0), + _metadata(getMetadata(collection->ns())) { + } + + IDHackRunner::IDHackRunner(Collection* collection, const BSONObj& key) + : _collection(collection), + _key(key), + _query(NULL), + _killed(false), + _done(false), + _nscanned(0), + _nscannedObjects(0), + _metadata(getMetadata(collection->ns())) { + } + + IDHackRunner::~IDHackRunner() { } + + Runner::RunnerState IDHackRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { + if (_killed) { return Runner::RUNNER_DEAD; } + if (_done) { return Runner::RUNNER_EOF; } + + // Use the index catalog to get the id index. + const IndexCatalog* catalog = _collection->getIndexCatalog(); + + // Find the index we use. + IndexDescriptor* idDesc = catalog->findIdIndex(); + if (NULL == idDesc) { + _done = true; + return Runner::RUNNER_EOF; + } + + // This may not be valid always. See SERVER-12397. + const BtreeBasedAccessMethod* accessMethod = + static_cast(catalog->getIndex(idDesc)); + + // Look up the key by going directly to the Btree. + DiskLoc loc = accessMethod->findSingle( _key ); + + // Key not found. + if (loc.isNull()) { + _done = true; + return Runner::RUNNER_EOF; + } + + _nscanned++; + + // Set out parameters and note that we're done w/lookup. + if (NULL == objOut) { + // No object requested - nothing to do. + } + else { + // If we're not sharded, consider a covered projection (we can't if we're sharded, since + // we require a fetch in order to apply the sharding filter). + if (!_metadata && hasCoveredProjection()) { + // Covered query on _id field only. Set object to search key. Search key is + // retrieved from the canonical query at construction and always contains the _id + // field name. It is possible to construct the ID hack runner with just the + // collection and the key object (which could be {"": my_obj_id}) but _query would + // be null in that case and the query would never be seen as covered. + *objOut = _key.getOwned(); + } + // Otherwise, fetch the document. + else { + Record* record = loc.rec(); + + _nscannedObjects++; + + // If the record isn't in memory... + if (!Record::likelyInPhysicalMemory(record->dataNoThrowing())) { + // And we're allowed to yield ourselves... + if (Runner::YIELD_AUTO == _policy) { + // Note what we're yielding to fetch so that we don't crash if the loc is + // deleted during a yield. + _locFetching = loc; + // Yield. TODO: Do we want to bother yielding if micros < 0? + int micros = ClientCursor::suggestYieldMicros(); + ClientCursor::staticYield(micros, "", record); + // This can happen when we're yielded for various reasons (e.g. db/idx dropped). + if (_killed) { + _done = true; + return Runner::RUNNER_DEAD; + } + } + } + + // If we're here, either the data was in memory or we paged it in. + *objOut = loc.obj(); + + // If we're sharded, make sure the key belongs to us. + if (_metadata) { + KeyPattern kp(_metadata->getKeyPattern()); + if (!_metadata->keyBelongsToMe(kp.extractSingleKey(*objOut))) { + // We have something with a matching _id but it doesn't belong to me. + _done = true; + return Runner::RUNNER_EOF; + } + } + + // Apply the projection if one was requested. + if (_query && _query->getProj()) { + *objOut = applyProjection(*objOut); + } + } + } + + // Return the DiskLoc if the caller wants it. + if (NULL != dlOut) { + *dlOut = loc; + } + + _done = true; + return Runner::RUNNER_ADVANCED; + } + + BSONObj IDHackRunner::applyProjection(const BSONObj& docObj) const { + invariant(_query && _query->getProj()); + + // We have a non-covered projection (covered projections should be handled earlier, + // in getNext(..). For simple inclusion projections we use a fast path similar to that + // implemented in the ProjectionStage. For non-simple inclusion projections we fallback + // to ProjectionExec. + const BSONObj& projObj = _query->getParsed().getProj(); + + if (_query->getProj()->wantIndexKey()) { + // $returnKey is specified. This overrides everything else. + BSONObjBuilder bob; + const BSONObj& queryObj = _query->getParsed().getFilter(); + bob.append(queryObj["_id"]); + return bob.obj(); + } + else if (_query->getProj()->requiresDocument() || _query->getProj()->wantIndexKey()) { + // Not a simple projection, so fallback on the regular projection path. + BSONObj projectedObj; + ProjectionExec projExec(projObj, _query->root()); + projExec.transform(docObj, &projectedObj); + return projectedObj; + } + else { + // This is a simple inclusion projection. Start by getting the set + // of fields to include. + unordered_set includedFields; + ProjectionStage::getSimpleInclusionFields(projObj, &includedFields); + + // Apply the simple inclusion projection. + BSONObjBuilder bob; + ProjectionStage::transformSimpleInclusion(docObj, includedFields, bob); + + return bob.obj(); + } + } + + bool IDHackRunner::isEOF() { + return _killed || _done; + } + + void IDHackRunner::saveState() { } + + bool IDHackRunner::restoreState() { return true; } + + void IDHackRunner::setYieldPolicy(Runner::YieldPolicy policy) { + if (_done || _killed) { return; } + _policy = policy; + } + + // Nothing to do here, holding no state. + void IDHackRunner::invalidate(const DiskLoc& dl, InvalidationType type) { + if (_done || _killed) { return; } + if (_locFetching == dl && (type == INVALIDATION_DELETION)) { + _locFetching = DiskLoc(); + _killed = true; + } + } + + const std::string& IDHackRunner::ns() { + return _collection->ns().ns(); + } + + void IDHackRunner::kill() { + _killed = true; + _collection = NULL; + } + + Status IDHackRunner::getInfo(TypeExplain** explain, + PlanInfo** planInfo) const { + // The explain plan simply indicates that the plan is idhack. + if (NULL != explain) { + *explain = new TypeExplain(); + // Explain format does not match 2.4 and is intended + // to indicate clearly that the ID hack has been applied. + (*explain)->setCursor("IDCursor"); + (*explain)->setIDHack(true); + (*explain)->setN(_nscanned); + (*explain)->setNScanned(_nscanned); + (*explain)->setNScannedObjects(_nscannedObjects); + BSONElement keyElt = _key.firstElement(); + BSONObj indexBounds = BSON("_id" << BSON_ARRAY( BSON_ARRAY( keyElt << keyElt ) ) ); + (*explain)->setIndexBounds(indexBounds); + // ID hack queries are only considered covered if they have the projection {_id: 1}. + (*explain)->setIndexOnly(hasCoveredProjection()); + } + else if (NULL != planInfo) { + *planInfo = new PlanInfo(); + (*planInfo)->planSummary = "IDHACK"; + } + + return Status::OK(); + } + + // static + bool IDHackRunner::supportsQuery(const CanonicalQuery& query) { + return !query.getParsed().showDiskLoc() + && query.getParsed().getHint().isEmpty() + && 0 == query.getParsed().getSkip() + && CanonicalQuery::isSimpleIdQuery(query.getParsed().getFilter()) + && !query.getParsed().hasOption(QueryOption_CursorTailable); + } + + // static + bool IDHackRunner::hasCoveredProjection() const { + // Some update operations use the IDHackRunner without creating a + // canonical query. In this case, _query will be NULL. Just return + // false, as we won't have to do any projection handling for updates. + if (NULL == _query.get()) { + return false; + } + + const ParsedProjection* proj = _query->getProj(); + if (!proj) { + return false; + } + + // If there is a projection, it has to be a covered projection on + // the _id field only. + if (proj->requiresDocument()) { + return false; + } + const std::vector& requiredFields = proj->getRequiredFields(); + if (1U != requiredFields.size()) { + return false; + } + if ("_id" != requiredFields[0]) { + return false; + } + + // Can use this projection with ID Hack. + return true; + } + +} // namespace mongo diff --git a/src/mongo/db/query/idhack_runner.h b/src/mongo/db/query/idhack_runner.h new file mode 100644 index 00000000000..bc61ea92730 --- /dev/null +++ b/src/mongo/db/query/idhack_runner.h @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/query/runner.h" +#include "mongo/s/d_logic.h" + +namespace mongo { + + class BSONObj; + class CanonicalQuery; + class Collection; + class DiskLoc; + class PlanStage; + class TypeExplain; + struct PlanInfo; + + /** + */ + class IDHackRunner : public Runner { + public: + + /** Takes ownership of all the arguments -collection. */ + IDHackRunner(const Collection* collection, CanonicalQuery* query); + + IDHackRunner(Collection* collection, const BSONObj& key); + + virtual ~IDHackRunner(); + + Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + virtual bool isEOF(); + + virtual void saveState(); + + virtual bool restoreState(); + + virtual void setYieldPolicy(Runner::YieldPolicy policy); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual const std::string& ns(); + + virtual void kill(); + + virtual const Collection* collection() { return _collection; } + + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const; + + /** + * ID Hack has a very strict criteria for the queries it supports. + */ + static bool supportsQuery(const CanonicalQuery& query); + + private: + /** + * ID Hack queries are only covered with the projection {_id: 1}. + */ + bool hasCoveredProjection() const; + + /** + * Apply the projection from '_query' to the given object and return the result. + * '_query->getProj()' must be non-NULL. + */ + BSONObj applyProjection(const BSONObj& docObj) const; + + // Not owned here. + const Collection* _collection; + + // The value to match against the _id field. + BSONObj _key; + + // TODO: When we combine the canonicalize and getRunner steps into one we can get rid of + // this. + boost::scoped_ptr _query; + + // Are we allowed to release the lock? + Runner::YieldPolicy _policy; + + // Did someone call kill() on us? + bool _killed; + + // Have we returned our one document? + bool _done; + + // If we're yielding to fetch a document, what is it's diskloc? It may be invalidated + // while we're yielded. + DiskLoc _locFetching; + + // Number of index keys scanned: should be either 0 or 1. + int _nscanned; + + // Number of objects scanned: should be either 0 or 1. + int _nscannedObjects; + + // Used to drop documents that don't belong to this shard. Must be initialized at the time + // of construction of the IDHackRunner. Since it is possible for IDHackRunner::getNext() to + // be called inside a getMore (e.g. aggregation will do this), we need to establish + // metadata information up front, not at the time that the IDHackRunner is actually used. + const CollectionMetadataPtr _metadata; + }; + +} // namespace mongo + diff --git a/src/mongo/db/query/index_bounds.cpp b/src/mongo/db/query/index_bounds.cpp new file mode 100644 index 00000000000..f90fa9b6ca0 --- /dev/null +++ b/src/mongo/db/query/index_bounds.cpp @@ -0,0 +1,577 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/index_bounds.h" + +#include +#include + +namespace mongo { + + using std::vector; + + namespace { + + // Return a value in the set {-1, 0, 1} to represent the sign of parameter i. + int sgn(int i) { + if (i == 0) + return 0; + return i > 0 ? 1 : -1; + } + + /** + * Returns BEHIND if the key is behind the interval. + * Returns WITHIN if the key is within the interval. + * Returns AHEAD if the key is ahead the interval. + * + * All directions are oriented along 'direction'. + */ + IndexBoundsChecker::Location intervalCmp(const Interval& interval, const BSONElement& key, + const int expectedDirection) { + int cmp = sgn(key.woCompare(interval.start, false)); + bool startOK = (cmp == expectedDirection) || (cmp == 0 && interval.startInclusive); + if (!startOK) { return IndexBoundsChecker::BEHIND; } + + cmp = sgn(key.woCompare(interval.end, false)); + bool endOK = (cmp == -expectedDirection) || (cmp == 0 && interval.endInclusive); + if (!endOK) { return IndexBoundsChecker::AHEAD; } + + return IndexBoundsChecker::WITHIN; + } + + } // namespace + + // For debugging. + size_t IndexBounds::size() const { + return fields.size(); + } + + string IndexBounds::getFieldName(size_t i) const { + return i < size() ? fields[i].name : ""; + } + + size_t IndexBounds::getNumIntervals(size_t i) const { + return i < size() ? fields[i].intervals.size() : 0; + } + + Interval IndexBounds::getInterval(size_t i, size_t j) const { + if (i < size() && j < fields[i].intervals.size()) { + return fields[i].intervals[j]; + } + else { + return Interval(); + } + } + + string OrderedIntervalList::toString() const { + mongoutils::str::stream ss; + ss << "['" << name << "']: "; + for (size_t j = 0; j < intervals.size(); ++j) { + ss << intervals[j].toString(); + if (j < intervals.size() - 1) { + ss << ", "; + } + } + return ss; + } + + // static + void OrderedIntervalList::complement() { + BSONObjBuilder minBob; + minBob.appendMinKey(""); + BSONObj minObj = minBob.obj(); + + // We complement by scanning the entire range of BSON values + // from MinKey to MaxKey. The value from which we must begin + // the next complemented interval is kept in 'curBoundary'. + BSONElement curBoundary = minObj.firstElement(); + + // If 'curInclusive' is true, then 'curBoundary' is + // included in one of the original intervals, and hence + // should not be included in the complement (and vice-versa + // if 'curInclusive' is false). + bool curInclusive = false; + + // We will build up a list of intervals that represents + // the inversion of those in the OIL. + vector newIntervals; + for (size_t j = 0; j < intervals.size(); ++j) { + Interval curInt = intervals[j]; + if (0 != curInt.start.woCompare(curBoundary) || + (!curInclusive && !curInt.startInclusive)) { + // Make a new interval from 'curBoundary' to + // the start of 'curInterval'. + BSONObjBuilder intBob; + intBob.append(curBoundary); + intBob.append(curInt.start); + Interval newInt(intBob.obj(), !curInclusive, !curInt.startInclusive); + newIntervals.push_back(newInt); + } + + // Reset the boundary for the next iteration. + curBoundary = curInt.end; + curInclusive = curInt.endInclusive; + } + + // We may have to add a final interval which ends in MaxKey. + BSONObjBuilder maxBob; + maxBob.appendMaxKey(""); + BSONObj maxObj = maxBob.obj(); + BSONElement maxKey = maxObj.firstElement(); + if (0 != maxKey.woCompare(curBoundary) || !curInclusive) { + BSONObjBuilder intBob; + intBob.append(curBoundary); + intBob.append(maxKey); + Interval newInt(intBob.obj(), !curInclusive, true); + newIntervals.push_back(newInt); + } + + // Replace the old list of intervals with the new one. + intervals.clear(); + intervals.insert(intervals.end(), newIntervals.begin(), newIntervals.end()); + } + + string IndexBounds::toString() const { + mongoutils::str::stream ss; + if (isSimpleRange) { + ss << "[" << startKey.toString() << ", "; + if (endKey.isEmpty()) { + ss << "]"; + } + else { + ss << endKey.toString(); + if (endKeyInclusive) { + ss << "]"; + } + else { + ss << ")"; + } + } + return ss; + } + for (size_t i = 0; i < fields.size(); ++i) { + if (i > 0) { + ss << ", "; + } + ss << "field #" << i << fields[i].toString(); + } + + return ss; + } + + BSONObj IndexBounds::toBSON() const { + BSONObjBuilder builder; + if (isSimpleRange) { + // TODO + } + else { + for (vector::const_iterator itField = fields.begin(); + itField != fields.end(); + ++itField) { + BSONArrayBuilder fieldBuilder(builder.subarrayStart(itField->name)); + for (vector::const_iterator itInterval = itField->intervals.begin(); + itInterval != itField->intervals.end(); + ++itInterval) { + BSONArrayBuilder intervalBuilder; + + // Careful to output $minElement/$maxElement if we don't have bounds. + if (itInterval->start.eoo()) { + BSONObjBuilder minBuilder; + minBuilder.appendMinKey(""); + BSONObj minKeyObj = minBuilder.obj(); + intervalBuilder.append(minKeyObj.firstElement()); + } + else { + intervalBuilder.append(itInterval->start); + } + + if (itInterval->end.eoo()) { + BSONObjBuilder maxBuilder; + maxBuilder.appendMaxKey(""); + BSONObj maxKeyObj = maxBuilder.obj(); + intervalBuilder.append(maxKeyObj.firstElement()); + } + else { + intervalBuilder.append(itInterval->end); + } + + fieldBuilder.append( + static_cast(intervalBuilder.arr().clientReadable())); + + // If the bounds object gets too large, truncate it. + static const int kMaxBoundsSize = 1024 * 1024; + if (builder.len() > kMaxBoundsSize) { + intervalBuilder.doneFast(); + fieldBuilder.append(BSON("warning" << "bounds obj exceeds 1 MB")); + fieldBuilder.doneFast(); + return builder.obj(); + } + } + } + } + + return builder.obj(); + } + + // + // Validity checking for bounds + // + + bool OrderedIntervalList::isValidFor(int expectedOrientation) const { + // Make sure each interval's start is oriented correctly with respect to its end. + for (size_t j = 0; j < intervals.size(); ++j) { + // false means don't consider field name. + int cmp = sgn(intervals[j].end.woCompare(intervals[j].start, false)); + + if (cmp == 0 && intervals[j].startInclusive + && intervals[j].endInclusive) { continue; } + + if (cmp != expectedOrientation) { + return false; + } + } + + // Make sure each interval is oriented correctly with respect to its neighbors. + for (size_t j = 1; j < intervals.size(); ++j) { + int cmp = sgn(intervals[j].start.woCompare(intervals[j - 1].end, false)); + + // TODO: We could care if the end of one interval is the start of another. The bounds + // are still valid but they're a bit sloppy; they could have been combined to form one + // interval if either of them is inclusive. + if (0 == cmp) { continue; } + + if (cmp != expectedOrientation) { + return false; + } + } + return true; + } + + bool IndexBounds::isValidFor(const BSONObj& keyPattern, int direction) { + if (isSimpleRange) { + return direction == sgn(endKey.woCompare(startKey, keyPattern, false)); + } + + BSONObjIterator it(keyPattern); + + for (size_t i = 0; i < fields.size(); ++i) { + // We expect a bound for each field in the index. + if (!it.more()) { return false; } + BSONElement elt = it.next(); + + const OrderedIntervalList& field = fields[i]; + + // Make sure the names match up. + if (field.name != elt.fieldName()) { return false; } + + // Special indices are all inserted increasing. elt.number() will return 0 if it's + // not a number. Special indices are strings, not numbers. + int expectedOrientation = direction * ((elt.number() >= 0) ? 1 : -1); + + if (!field.isValidFor(expectedOrientation)) { + return false; + } + } + + return !it.more(); + } + + // + // Iteration over index bounds + // + + IndexBoundsChecker::IndexBoundsChecker(const IndexBounds* bounds, const BSONObj& keyPattern, + int scanDirection) + : _bounds(bounds), _curInterval(bounds->fields.size(), 0) { + + BSONObjIterator it(keyPattern); + while (it.more()) { + int indexDirection = it.next().number() >= 0 ? 1 : -1; + _expectedDirection.push_back(indexDirection * scanDirection); + } + } + + bool IndexBoundsChecker::getStartKey(vector* valueOut, + vector* inclusiveOut) { + verify(valueOut->size() == _bounds->fields.size()); + verify(inclusiveOut->size() == _bounds->fields.size()); + + for (size_t i = 0; i < _bounds->fields.size(); ++i) { + if (0 == _bounds->fields[i].intervals.size()) { + return false; + } + (*valueOut)[i] = &_bounds->fields[i].intervals[0].start; + (*inclusiveOut)[i] = _bounds->fields[i].intervals[0].startInclusive; + } + + return true; + } + + bool IndexBoundsChecker::findLeftmostProblem(const vector& keyValues, + size_t* where, + Location* what) { + // For each field in the index key, see if it's in the interval it should be. + for (size_t i = 0; i < _curInterval.size(); ++i) { + const OrderedIntervalList& field = _bounds->fields[i]; + const Interval& currentInterval = field.intervals[_curInterval[i]]; + Location cmp = intervalCmp(currentInterval, keyValues[i], _expectedDirection[i]); + + // If it's not in the interval we think it is... + if (0 != cmp) { + *where = i; + *what = cmp; + return true; + } + } + + return false; + } + + bool IndexBoundsChecker::spaceLeftToAdvance(size_t fieldsToCheck, + const vector& keyValues) { + // Check end conditions. Since we need to move the keys before + // firstNonContainedField forward, let's make sure that those fields are not at the + // end of their bounds. + for (size_t i = 0; i < fieldsToCheck; ++i) { + // Field 'i' isn't at its last interval. There's possibly a key we could move forward + // to, either in the current interval or the next one. + if (_curInterval[i] != _bounds->fields[i].intervals.size() - 1) { + return true; + } + + // Field 'i' is at its last interval. + const Interval& ival = _bounds->fields[i].intervals[_curInterval[i]]; + + // We're OK if it's an open interval. There are an infinite number of keys between any + // key and the end point... + if (!ival.endInclusive) { + return true; + } + + // If it's a closed interval, we're fine so long as we haven't hit the end point of + // the interval. + if (-_expectedDirection[i] == sgn(keyValues[i].woCompare(ival.end, false))) { + return true; + } + } + + return false; + } + + bool IndexBoundsChecker::isValidKey(const BSONObj& key) { + BSONObjIterator it(key); + size_t curOil = 0; + while (it.more()) { + BSONElement elt = it.next(); + size_t whichInterval; + Location loc = findIntervalForField(elt, _bounds->fields[curOil], _expectedDirection[curOil], &whichInterval); + if (WITHIN != loc) { + return false; + } + ++curOil; + } + return true; + } + + IndexBoundsChecker::KeyState IndexBoundsChecker::checkKey(const BSONObj& key, + int* keyEltsToUse, + bool* movePastKeyElts, + vector* out, + vector* incOut) { + verify(_curInterval.size() > 0); + verify(out->size() == _curInterval.size()); + verify(incOut->size() == _curInterval.size()); + + // It's useful later to go from a field number to the value for that field. Store these. + // TODO: on optimization pass, populate the vector as-needed and keep the vector around as a + // member variable + vector keyValues; + BSONObjIterator keyIt(key); + while (keyIt.more()) { + keyValues.push_back(keyIt.next()); + } + verify(keyValues.size() == _curInterval.size()); + + size_t firstNonContainedField; + Location orientation; + + if (!findLeftmostProblem(keyValues, &firstNonContainedField, &orientation)) { + // All fields in the index are within the current interval. Caller can use the key. + return VALID; + } + + // Field number 'firstNonContainedField' of the index key is before its current interval. + if (BEHIND == orientation) { + // It's behind our current interval, but our current interval could be wrong. Start all + // intervals from firstNonContainedField to the right over... + for (size_t i = firstNonContainedField; i < _curInterval.size(); ++i) { + _curInterval[i] = 0; + } + + // ...and try again. This call modifies 'orientation', so we may check its value again + // in the clause below if field number 'firstNonContainedField' isn't in its first + // interval. + if (!findLeftmostProblem(keyValues, &firstNonContainedField, &orientation)) { + return VALID; + } + } + + // Field number 'firstNonContainedField' of the index key is before all current intervals. + if (BEHIND == orientation) { + // Tell the caller to move forward to the start of the current interval. + *keyEltsToUse = firstNonContainedField; + *movePastKeyElts = false; + + for (size_t j = firstNonContainedField; j < _curInterval.size(); ++j) { + const OrderedIntervalList& oil = _bounds->fields[j]; + (*out)[j] = &oil.intervals[_curInterval[j]].start; + (*incOut)[j] = oil.intervals[_curInterval[j]].startInclusive; + } + + return MUST_ADVANCE; + } + + verify(AHEAD == orientation); + + // Field number 'firstNonContainedField' of the index key is after interval we think it's + // in. Fields 0 through 'firstNonContained-1' are within their current intervals and we can + // ignore them. + while (firstNonContainedField < _curInterval.size()) { + // Find the interval that contains our field. + size_t newIntervalForField; + + Location where = findIntervalForField(keyValues[firstNonContainedField], + _bounds->fields[firstNonContainedField], + _expectedDirection[firstNonContainedField], + &newIntervalForField); + + if (WITHIN == where) { + // Found a new interval for field firstNonContainedField. Move our internal choice + // of interval to that. + _curInterval[firstNonContainedField] = newIntervalForField; + // Let's find valid intervals for fields to the right. + ++firstNonContainedField; + } + else if (BEHIND == where) { + // firstNonContained field is between the intervals (newIntervalForField-1) and + // newIntervalForField. We have to tell the caller to move forward until he at + // least hits our new current interval. + _curInterval[firstNonContainedField] = newIntervalForField; + + // All other fields to the right start at their first interval. + for (size_t i = firstNonContainedField + 1; i < _curInterval.size(); ++i) { + _curInterval[i] = 0; + } + + *keyEltsToUse = firstNonContainedField; + *movePastKeyElts = false; + + for (size_t i = firstNonContainedField; i < _curInterval.size(); ++i) { + const OrderedIntervalList& oil = _bounds->fields[i]; + (*out)[i] = &oil.intervals[_curInterval[i]].start; + (*incOut)[i] = oil.intervals[_curInterval[i]].startInclusive; + } + + return MUST_ADVANCE; + } + else { + verify (AHEAD == where); + // Field number 'firstNonContainedField' cannot possibly be placed into an interval, + // as it is already past its last possible interval. The caller must move forward + // to a key with a greater value for the previous field. + + // If all fields to the left have hit the end of their intervals, we can't ask them + // to move forward and we should stop iterating. + if (!spaceLeftToAdvance(firstNonContainedField, keyValues)) { + return DONE; + } + + *keyEltsToUse = firstNonContainedField; + *movePastKeyElts = true; + + for (size_t i = firstNonContainedField; i < _curInterval.size(); ++i) { + _curInterval[i] = 0; + } + + // If movePastKeyElts is true, we don't examine any fields after the keyEltsToUse + // fields of the key. As such we don't populate the out/incOut. + return MUST_ADVANCE; + } + } + + verify(firstNonContainedField == _curInterval.size()); + return VALID; + } + + namespace { + + /** + * Returns true if key (first member of pair) is AHEAD of interval + * along 'direction' (second member of pair). + */ + bool isKeyAheadOfInterval(const Interval& interval, + const std::pair& keyAndDirection) { + const BSONElement& elt = keyAndDirection.first; + int expectedDirection = keyAndDirection.second; + IndexBoundsChecker::Location where = intervalCmp(interval, elt, expectedDirection); + return IndexBoundsChecker::AHEAD == where; + } + + } // namespace + + // static + IndexBoundsChecker::Location IndexBoundsChecker::findIntervalForField(const BSONElement& elt, + const OrderedIntervalList& oil, const int expectedDirection, size_t* newIntervalIndex) { + // Binary search for interval. + // Intervals are ordered in the same direction as our keys. + // Key behind all intervals: [BEHIND, ..., BEHIND] + // Key ahead of all intervals: [AHEAD, ..., AHEAD] + // Key within one interval: [AHEAD, ..., WITHIN, BEHIND, ...] + // Key not in any inteval: [AHEAD, ..., AHEAD, BEHIND, ...] + + // Find left-most BEHIND/WITHIN interval. + vector::const_iterator i = + std::lower_bound(oil.intervals.begin(), oil.intervals.end(), + std::make_pair(elt, expectedDirection), isKeyAheadOfInterval); + + // Key ahead of all intervals. + if (i == oil.intervals.end()) { + return AHEAD; + } + + // Found either interval containing key or left-most BEHIND interval. + *newIntervalIndex = std::distance(oil.intervals.begin(), i); + + // Additional check to determine if interval contains key. + Location where = intervalCmp(*i, elt, expectedDirection); + invariant(BEHIND == where || WITHIN == where); + + return where; + } + +} // namespace mongo diff --git a/src/mongo/db/query/index_bounds.h b/src/mongo/db/query/index_bounds.h new file mode 100644 index 00000000000..8bf955aadaf --- /dev/null +++ b/src/mongo/db/query/index_bounds.h @@ -0,0 +1,241 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/db/jsobj.h" +#include "mongo/db/query/interval.h" + +namespace mongo { + + /** + * An ordered list of intervals for one field. + */ + struct OrderedIntervalList { + OrderedIntervalList() { } + OrderedIntervalList(const string& n) : name(n) { } + + // Must be ordered according to the index order. + vector intervals; + + // TODO: We could drop this. Only used in IndexBounds::isValidFor. + string name; + + bool isValidFor(int expectedOrientation) const; + std::string toString() const; + + /** + * Complements the OIL. Used by the index bounds builder in order + * to create index bounds for $not predicates. + * + * Assumes the OIL is increasing, and therefore must be called prior to + * alignBounds(...). + * + * Example: + * The complement of [3, 6), [8, 10] is [MinKey, 3), [6, 8), (20, MaxKey], + * where this OIL has direction==1. + */ + void complement(); + }; + + /** + * Tied to an index. Permissible values for all fields in the index. Requires the index to + * interpret. Previously known as FieldRangeVector. + */ + struct IndexBounds { + IndexBounds() : isSimpleRange(false), endKeyInclusive(false) { } + + // For each indexed field, the values that the field is allowed to take on. + vector fields; + + // Debugging check. + // We must have as many fields the key pattern does. + // The fields must be oriented in the direction we'd encounter them given the indexing + // direction (the value of the field in keyPattern) and index traversal direction provided + // by 'direction'. + // + // An example: [7, 20] + // We can traverse this forward if indexed ascending + // We can traverse this backwards if indexed descending. + bool isValidFor(const BSONObj& keyPattern, int direction); + + // Methods below used for debugging purpose only. Do not use outside testing code. + size_t size() const; + std::string getFieldName(size_t i) const; + size_t getNumIntervals(size_t i) const; + Interval getInterval(size_t i, size_t j) const; + std::string toString() const; + BSONObj toBSON() const; + + // TODO: we use this for max/min scan. Consider migrating that. + bool isSimpleRange; + BSONObj startKey; + BSONObj endKey; + bool endKeyInclusive; + }; + + /** + * A helper used by IndexScan to navigate an index. + */ + class IndexBoundsChecker { + public: + /** + * keyPattern is the index that we're iterating over. + * bounds are the bounds we're allowed to iterate over. + * direction is the direction we're moving over the index, 1 or -1. + * + * Bounds not owned by us. + */ + IndexBoundsChecker(const IndexBounds* bounds, const BSONObj& keyPattern, int direction); + + /** + * Get the key that we should with. + * + * Returns true if there is a valid start key. Returns false otherwise. + */ + bool getStartKey(vector* valueOut, vector* inclusiveOut); + + /** + * The states of a key from an index scan. See checkKey below. + */ + enum KeyState { + VALID, + MUST_ADVANCE, + DONE, + }; + + /** + * Is 'key' a valid key? Note that this differs from checkKey, which assumes that it + * receives keys in sorted order. + */ + bool isValidKey(const BSONObj& key); + + /** + * This function checks if the key is within the bounds we're iterating over and updates any + * internal state required to efficiently determine if the key is within our bounds. + * + * Possible outcomes: + * + * 1. The key is in our bounds. Returns VALID. Caller can use the data associated with the + * key. + * + * 2. The key is not in our bounds but has not exceeded the maximum value in our bounds. + * Returns MUST_ADVANCE. Caller must advance to the key provided in the out parameters and + * call checkKey again. + * + * 3. The key is past our bounds. Returns DONE. No further keys will satisfy the bounds + * and the caller should stop. + * + * keyEltsToUse, movePastKeyElts, out, and incOut must all be non-NULL. + * out and incOut must already be resized to have as many elements as the key has fields. + * + * In parameters: + * key is the index key. + * + * Out parameters, only valid if we return MUST_ADVANCE: + * + * keyEltsToUse: The key that the caller should advance to is made up of the first + * 'keyEltsToUse' of the key that was provided. + * + * movePastKeyElts: If true, the caller must only use the first 'keyEltsToUse' of the + * provided key to form its key. It moves to the first key that is after + * the key formed by only using those elements. + * + * out: If keyEltsToUse is less than the number of indexed fields in the key, the remaining + * fields are taken from here. out is not filled from the start but from the position + * that the key corresponds to. An example: If keyEltsToUse is 1, movePastKeyElts is + * false, and the index we're iterating over has two fields, out[1] will have the value + * for the second field. + * + * incOut: If the i-th element is false, seek to the key *after* the i-th element of out. + * If the i-th element is true, seek to the i-th element of out. + */ + KeyState checkKey(const BSONObj& key, int* keyEltsToUse, bool* movePastKeyElts, + vector* out, vector* incOut); + + /** + * Relative position of a key to an interval. + * Exposed for testing only. + */ + enum Location { + BEHIND = -1, + WITHIN = 0, + AHEAD = 1, + }; + + /** + * If 'elt' is in any interval, return WITHIN and set 'newIntervalIndex' to the index of the + * interval in the ordered interval list. + * + * If 'elt' is not in any interval but could be advanced to be in one, return BEHIND and set + * 'newIntervalIndex' to the index of the interval that 'elt' could be advanced to. + * + * If 'elt' cannot be advanced to any interval, return AHEAD. + * + * Exposed for testing only. + * + * TODO(efficiency): Start search from a given index. + */ + static Location findIntervalForField(const BSONElement &elt, const OrderedIntervalList& oil, + const int expectedDirection, size_t* newIntervalIndex); + + private: + /** + * Find the first field in the key that isn't within the interval we think it is. Returns + * false if every field is in the interval we think it is. Returns true and populates out + * parameters if a field isn't in the interval we think it is. + * + * Out parameters set if we return true: + * 'where' is the leftmost field that isn't in the interval we think it is. + * 'what' is the orientation of the field with respect to that interval. + */ + bool findLeftmostProblem(const vector& keyValues, size_t* where, + Location* what); + + /** + * Returns true if it's possible to advance any of the first 'fieldsToCheck' fields of the + * index key and still be within valid index bounds. + * + * keyValues are the elements of the index key in order. + */ + bool spaceLeftToAdvance(size_t fieldsToCheck, const vector& keyValues); + + // The actual bounds. Must outlive this object. Not owned by us. + const IndexBounds* _bounds; + + // For each field, which interval are we currently in? + vector _curInterval; + + // Direction of scan * direction of indexing. + vector _expectedDirection; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/index_bounds_builder.cpp b/src/mongo/db/query/index_bounds_builder.cpp new file mode 100644 index 00000000000..d38ed728c3e --- /dev/null +++ b/src/mongo/db/query/index_bounds_builder.cpp @@ -0,0 +1,991 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/index_bounds_builder.h" + +#include +#include "mongo/db/geo/geoconstants.h" +#include "mongo/db/geo/s2common.h" +#include "mongo/db/index/expression_index.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/query/indexability.h" +#include "mongo/db/query/qlog.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/db/geo/s2.h" +#include "third_party/s2/s2cell.h" +#include "third_party/s2/s2regioncoverer.h" + +namespace mongo { + + string IndexBoundsBuilder::simpleRegex(const char* regex, const char* flags, + BoundsTightness* tightnessOut) { + string r = ""; + *tightnessOut = IndexBoundsBuilder::INEXACT_COVERED; + + bool multilineOK; + if ( regex[0] == '\\' && regex[1] == 'A') { + multilineOK = true; + regex += 2; + } + else if (regex[0] == '^') { + multilineOK = false; + regex += 1; + } + else { + return r; + } + + bool extended = false; + while (*flags) { + switch (*(flags++)) { + case 'm': // multiline + if (multilineOK) + continue; + else + return r; + case 'x': // extended + extended = true; + break; + default: + return r; // cant use index + } + } + + mongoutils::str::stream ss; + + while(*regex) { + char c = *(regex++); + if ( c == '*' || c == '?' ) { + // These are the only two symbols that make the last char optional + r = ss; + r = r.substr( 0 , r.size() - 1 ); + return r; //breaking here fails with /^a?/ + } + else if (c == '|') { + // whole match so far is optional. Nothing we can do here. + return string(); + } + else if (c == '\\') { + c = *(regex++); + if (c == 'Q'){ + // \Q...\E quotes everything inside + while (*regex) { + c = (*regex++); + if (c == '\\' && (*regex == 'E')){ + regex++; //skip the 'E' + break; // go back to start of outer loop + } + else { + ss << c; // character should match itself + } + } + } + else if ((c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '0') || + (c == '\0')) { + // don't know what to do with these + r = ss; + break; + } + else { + // slash followed by non-alphanumeric represents the following char + ss << c; + } + } + else if (strchr("^$.[()+{", c)) { + // list of "metacharacters" from man pcrepattern + r = ss; + break; + } + else if (extended && c == '#') { + // comment + r = ss; + break; + } + else if (extended && isspace(c)) { + continue; + } + else { + // self-matching char + ss << c; + } + } + + if ( r.empty() && *regex == 0 ) { + r = ss; + *tightnessOut = r.empty() ? IndexBoundsBuilder::INEXACT_COVERED : IndexBoundsBuilder::EXACT; + } + + return r; + } + + + // static + void IndexBoundsBuilder::allValuesForField(const BSONElement& elt, OrderedIntervalList* out) { + // ARGH, BSONValue would make this shorter. + BSONObjBuilder bob; + bob.appendMinKey(""); + bob.appendMaxKey(""); + out->name = elt.fieldName(); + out->intervals.push_back(makeRangeInterval(bob.obj(), true, true)); + } + + Interval IndexBoundsBuilder::allValues() { + BSONObjBuilder bob; + bob.appendMinKey(""); + bob.appendMaxKey(""); + return makeRangeInterval(bob.obj(), true, true); + } + + bool IntervalComparison(const Interval& lhs, const Interval& rhs) { + int wo = lhs.start.woCompare(rhs.start, false); + if (0 != wo) { + return wo < 0; + } + + // The start and end are equal. + // Strict weak requires irreflexivity which implies that equivalence returns false. + if (lhs.startInclusive == rhs.startInclusive) { return false; } + + // Put the bound that's inclusive to the left. + return lhs.startInclusive; + } + + // static + void IndexBoundsBuilder::translateAndIntersect(const MatchExpression* expr, + const BSONElement& elt, + const IndexEntry& index, + OrderedIntervalList* oilOut, + BoundsTightness* tightnessOut) { + OrderedIntervalList arg; + translate(expr, elt, index, &arg, tightnessOut); + + // translate outputs arg in sorted order. intersectize assumes that its arguments are + // sorted. + intersectize(arg, oilOut); + } + + // static + void IndexBoundsBuilder::translateAndUnion(const MatchExpression* expr, + const BSONElement& elt, + const IndexEntry& index, + OrderedIntervalList* oilOut, + BoundsTightness* tightnessOut) { + OrderedIntervalList arg; + translate(expr, elt, index, &arg, tightnessOut); + + // Append the new intervals to oilOut. + oilOut->intervals.insert(oilOut->intervals.end(), arg.intervals.begin(), + arg.intervals.end()); + + // Union the appended intervals with the existing ones. + unionize(oilOut); + } + + bool typeMatch(const BSONObj& obj) { + BSONObjIterator it(obj); + verify(it.more()); + BSONElement first = it.next(); + verify(it.more()); + BSONElement second = it.next(); + return first.canonicalType() == second.canonicalType(); + } + + // static + void IndexBoundsBuilder::translate(const MatchExpression* expr, + const BSONElement& elt, + const IndexEntry& index, + OrderedIntervalList* oilOut, + BoundsTightness* tightnessOut) { + // We expect that the OIL we are constructing starts out empty. + invariant(oilOut->intervals.empty()); + + oilOut->name = elt.fieldName(); + + bool isHashed = false; + if (mongoutils::str::equals("hashed", elt.valuestrsafe())) { + isHashed = true; + } + + if (isHashed) { + verify(MatchExpression::EQ == expr->matchType() + || MatchExpression::MATCH_IN == expr->matchType()); + } + + if (MatchExpression::ELEM_MATCH_VALUE == expr->matchType()) { + OrderedIntervalList acc; + translate(expr->getChild(0), elt, index, &acc, tightnessOut); + + for (size_t i = 1; i < expr->numChildren(); ++i) { + OrderedIntervalList next; + BoundsTightness tightness; + translate(expr->getChild(i), elt, index, &next, &tightness); + intersectize(next, &acc); + } + + for (size_t i = 0; i < acc.intervals.size(); ++i) { + oilOut->intervals.push_back(acc.intervals[i]); + } + + if (!oilOut->intervals.empty()) { + std::sort(oilOut->intervals.begin(), oilOut->intervals.end(), IntervalComparison); + } + + // $elemMatch value requires an array. + // Scalars and directly nested objects are not matched with $elemMatch. + // We can't tell if a multi-key index key is derived from an array field. + // Therefore, a fetch is required. + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + else if (MatchExpression::NOT == expr->matchType()) { + MatchExpression* child = expr->getChild(0); + + // If we have a NOT -> EXISTS, we must handle separately. + if (MatchExpression::EXISTS == child->matchType()) { + // We should never try to use a sparse index for $exists:false. + invariant(!index.sparse); + BSONObjBuilder bob; + bob.appendNull(""); + bob.appendNull(""); + BSONObj dataObj = bob.obj(); + oilOut->intervals.push_back(makeRangeInterval(dataObj, true, true)); + + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + return; + } + else if (Indexability::nodeCanUseIndexOnOwnField(child)) { + // We have a NOT of a bounds-generating expression. Get the + // bounds of the NOT's child and then complement them. + translate(expr->getChild(0), elt, index, oilOut, tightnessOut); + oilOut->complement(); + + // If the index is multikey, it doesn't matter what the tightness + // of the child is, we must return INEXACT_FETCH. Consider a multikey + // index on 'a' with document {a: [1, 2, 3]} and query {a: {$ne: 3}}. + // If we treated the bounds [MinKey, 3), (3, MaxKey] as exact, then + // we would erroneously return the document! + if (index.multikey) { + *tightnessOut = INEXACT_FETCH; + } + } + else { + // TODO: In the future we shouldn't need this. We handle this case for the time + // being because we have some deficiencies in tree normalization (see SERVER-12735). + // + // For example, we will get here if there is an index {a: 1} + // and the query is {a: {$elemMatch: {$not: {$gte: 6}}}}. + oilOut->intervals.push_back(allValues()); + *tightnessOut = INEXACT_FETCH; + } + } + else if (MatchExpression::EXISTS == expr->matchType()) { + // We only handle the {$exists:true} case, as {$exists:false} + // will have been translated to {$not:{ $exists:true }}. + // + // Documents with a missing value are stored *as if* they were + // explicitly given the value 'null'. Given: + // X = { b : 1 } + // Y = { a : null, b : 1 } + // X and Y look identical from within a standard index on { a : 1 }. + // HOWEVER a sparse index on { a : 1 } will treat X and Y differently, + // storing Y and not storing X. + // + // We can safely use an index in the following cases: + // {a:{ $exists:true }} - normal index helps, but we must still fetch + // {a:{ $exists:true }} - sparse index is exact + // {a:{ $exists:false }} - normal index requires a fetch + // {a:{ $exists:false }} - sparse indexes cannot be used at all. + // + // Noted in SERVER-12869, in case this ever changes some day. + if (index.sparse) { + oilOut->intervals.push_back(allValues()); + // A sparse, compound index on { a:1, b:1 } will include entries + // for all of the following documents: + // { a:1 }, { b:1 }, { a:1, b:1 } + // So we must use INEXACT bounds in this case. + if ( 1 < index.keyPattern.nFields() ) { + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + else { + *tightnessOut = IndexBoundsBuilder::EXACT; + } + } + else { + oilOut->intervals.push_back(allValues()); + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + } + else if (MatchExpression::EQ == expr->matchType()) { + const EqualityMatchExpression* node = static_cast(expr); + translateEquality(node->getData(), isHashed, oilOut, tightnessOut); + } + else if (MatchExpression::LTE == expr->matchType()) { + const LTEMatchExpression* node = static_cast(expr); + BSONElement dataElt = node->getData(); + + // Everything is <= MaxKey. + if (MaxKey == dataElt.type()) { + oilOut->intervals.push_back(allValues()); + *tightnessOut = IndexBoundsBuilder::EXACT; + return; + } + + BSONObjBuilder bob; + // Use -infinity for one-sided numerical bounds + if (dataElt.isNumber()) { + bob.appendNumber("", -std::numeric_limits::infinity()); + } + else { + bob.appendMinForType("", dataElt.type()); + } + bob.appendAs(dataElt, ""); + BSONObj dataObj = bob.obj(); + verify(dataObj.isOwned()); + oilOut->intervals.push_back(makeRangeInterval(dataObj, typeMatch(dataObj), true)); + + if (dataElt.isSimpleType() || dataElt.type() == BinData) { + *tightnessOut = IndexBoundsBuilder::EXACT; + } + else { + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + } + else if (MatchExpression::LT == expr->matchType()) { + const LTMatchExpression* node = static_cast(expr); + BSONElement dataElt = node->getData(); + + // Everything is <= MaxKey. + if (MaxKey == dataElt.type()) { + oilOut->intervals.push_back(allValues()); + *tightnessOut = IndexBoundsBuilder::EXACT; + return; + } + + BSONObjBuilder bob; + // Use -infinity for one-sided numerical bounds + if (dataElt.isNumber()) { + bob.appendNumber("", -std::numeric_limits::infinity()); + } + else { + bob.appendMinForType("", dataElt.type()); + } + bob.appendAs(dataElt, ""); + BSONObj dataObj = bob.obj(); + verify(dataObj.isOwned()); + Interval interval = makeRangeInterval(dataObj, typeMatch(dataObj), false); + + // If the operand to LT is equal to the lower bound X, the interval [X, X) is invalid + // and should not be added to the bounds. + if (!interval.isNull()) { + oilOut->intervals.push_back(interval); + } + + if (dataElt.isSimpleType() || dataElt.type() == BinData) { + *tightnessOut = IndexBoundsBuilder::EXACT; + } + else { + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + } + else if (MatchExpression::GT == expr->matchType()) { + const GTMatchExpression* node = static_cast(expr); + BSONElement dataElt = node->getData(); + + // Everything is > MinKey. + if (MinKey == dataElt.type()) { + oilOut->intervals.push_back(allValues()); + *tightnessOut = IndexBoundsBuilder::EXACT; + return; + } + + BSONObjBuilder bob; + bob.appendAs(node->getData(), ""); + if (dataElt.isNumber()) { + bob.appendNumber("", std::numeric_limits::infinity()); + } + else { + bob.appendMaxForType("", dataElt.type()); + } + BSONObj dataObj = bob.obj(); + verify(dataObj.isOwned()); + Interval interval = makeRangeInterval(dataObj, false, typeMatch(dataObj)); + + // If the operand to GT is equal to the upper bound X, the interval (X, X] is invalid + // and should not be added to the bounds. + if (!interval.isNull()) { + oilOut->intervals.push_back(interval); + } + + if (dataElt.isSimpleType() || dataElt.type() == BinData) { + *tightnessOut = IndexBoundsBuilder::EXACT; + } + else { + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + } + else if (MatchExpression::GTE == expr->matchType()) { + const GTEMatchExpression* node = static_cast(expr); + BSONElement dataElt = node->getData(); + + // Everything is >= MinKey. + if (MinKey == dataElt.type()) { + oilOut->intervals.push_back(allValues()); + *tightnessOut = IndexBoundsBuilder::EXACT; + return; + } + + BSONObjBuilder bob; + bob.appendAs(dataElt, ""); + if (dataElt.isNumber()) { + bob.appendNumber("", std::numeric_limits::infinity()); + } + else { + bob.appendMaxForType("", dataElt.type()); + } + BSONObj dataObj = bob.obj(); + verify(dataObj.isOwned()); + + oilOut->intervals.push_back(makeRangeInterval(dataObj, true, typeMatch(dataObj))); + if (dataElt.isSimpleType() || dataElt.type() == BinData) { + *tightnessOut = IndexBoundsBuilder::EXACT; + } + else { + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + } + else if (MatchExpression::REGEX == expr->matchType()) { + const RegexMatchExpression* rme = static_cast(expr); + translateRegex(rme, oilOut, tightnessOut); + } + else if (MatchExpression::MOD == expr->matchType()) { + BSONObjBuilder bob; + bob.appendMinForType("", NumberDouble); + bob.appendMaxForType("", NumberDouble); + BSONObj dataObj = bob.obj(); + verify(dataObj.isOwned()); + oilOut->intervals.push_back(makeRangeInterval(dataObj, true, true)); + *tightnessOut = IndexBoundsBuilder::INEXACT_COVERED; + } + else if (MatchExpression::TYPE_OPERATOR == expr->matchType()) { + const TypeMatchExpression* tme = static_cast(expr); + BSONObjBuilder bob; + bob.appendMinForType("", tme->getData()); + bob.appendMaxForType("", tme->getData()); + BSONObj dataObj = bob.obj(); + verify(dataObj.isOwned()); + oilOut->intervals.push_back(makeRangeInterval(dataObj, true, true)); + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + else if (MatchExpression::MATCH_IN == expr->matchType()) { + const InMatchExpression* ime = static_cast(expr); + const ArrayFilterEntries& afr = ime->getData(); + + *tightnessOut = IndexBoundsBuilder::EXACT; + + // Create our various intervals. + + IndexBoundsBuilder::BoundsTightness tightness; + for (BSONElementSet::iterator it = afr.equalities().begin(); + it != afr.equalities().end(); ++it) { + translateEquality(*it, isHashed, oilOut, &tightness); + if (tightness != IndexBoundsBuilder::EXACT) { + *tightnessOut = tightness; + } + } + + for (size_t i = 0; i < afr.numRegexes(); ++i) { + translateRegex(afr.regex(i), oilOut, &tightness); + if (tightness != IndexBoundsBuilder::EXACT) { + *tightnessOut = tightness; + } + } + + if (afr.hasNull()) { + // A null index key does not always match a null query value so we must fetch the + // doc and run a full comparison. See SERVER-4529. + // TODO: Do we already set the tightnessOut by calling translateEquality? + *tightnessOut = INEXACT_FETCH; + } + + if (afr.hasEmptyArray()) { + // Empty arrays are indexed as undefined. + BSONObjBuilder undefinedBob; + undefinedBob.appendUndefined(""); + oilOut->intervals.push_back(makePointInterval(undefinedBob.obj())); + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + + unionize(oilOut); + } + else if (MatchExpression::GEO == expr->matchType()) { + const GeoMatchExpression* gme = static_cast(expr); + // Can only do this for 2dsphere. + if (!mongoutils::str::equals("2dsphere", elt.valuestrsafe())) { + warning() << "Planner error, trying to build geo bounds for non-2dsphere" + << " index element: " << elt.toString() << endl; + verify(0); + } + + const S2Region& region = gme->getGeoQuery().getRegion(); + ExpressionMapping::cover2dsphere(region, index.infoObj, oilOut); + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + else { + warning() << "Planner error, trying to build bounds for expression: " + << expr->toString() << endl; + verify(0); + } + } + + // static + Interval IndexBoundsBuilder::makeRangeInterval(const BSONObj& obj, bool startInclusive, + bool endInclusive) { + Interval ret; + ret._intervalData = obj; + ret.startInclusive = startInclusive; + ret.endInclusive = endInclusive; + BSONObjIterator it(obj); + verify(it.more()); + ret.start = it.next(); + verify(it.more()); + ret.end = it.next(); + return ret; + } + + // static + void IndexBoundsBuilder::intersectize(const OrderedIntervalList& arg, + OrderedIntervalList* oilOut) { + verify(arg.name == oilOut->name); + + size_t argidx = 0; + const vector& argiv = arg.intervals; + + size_t ividx = 0; + vector& iv = oilOut->intervals; + + vector result; + + while (argidx < argiv.size() && ividx < iv.size()) { + Interval::IntervalComparison cmp = argiv[argidx].compare(iv[ividx]); + + verify(Interval::INTERVAL_UNKNOWN != cmp); + + if (cmp == Interval::INTERVAL_PRECEDES + || cmp == Interval::INTERVAL_PRECEDES_COULD_UNION) { + // argiv is before iv. move argiv forward. + ++argidx; + } + else if (cmp == Interval::INTERVAL_SUCCEEDS) { + // iv is before argiv. move iv forward. + ++ividx; + } + else { + // argiv[argidx] (cmpresults) iv[ividx] + Interval newInt = argiv[argidx]; + newInt.intersect(iv[ividx], cmp); + result.push_back(newInt); + + if (Interval::INTERVAL_EQUALS == cmp) { + ++argidx; + ++ividx; + } + else if (Interval::INTERVAL_WITHIN == cmp) { + ++argidx; + } + else if (Interval::INTERVAL_CONTAINS == cmp) { + ++ividx; + } + else if (Interval::INTERVAL_OVERLAPS_BEFORE == cmp) { + ++argidx; + } + else if (Interval::INTERVAL_OVERLAPS_AFTER == cmp) { + ++ividx; + } + else { + verify(0); + } + } + } + + oilOut->intervals.swap(result); + } + + // static + void IndexBoundsBuilder::unionize(OrderedIntervalList* oilOut) { + vector& iv = oilOut->intervals; + + // This can happen. + if (iv.empty()) { return; } + + // Step 1: sort. + std::sort(iv.begin(), iv.end(), IntervalComparison); + + // Step 2: Walk through and merge. + size_t i = 0; + while (i < iv.size() - 1) { + // Compare i with i + 1. + Interval::IntervalComparison cmp = iv[i].compare(iv[i + 1]); + + // This means our sort didn't work. + verify(Interval::INTERVAL_SUCCEEDS != cmp); + + // Intervals are correctly ordered. + if (Interval::INTERVAL_PRECEDES == cmp) { + // We can move to the next pair. + ++i; + } + else if (Interval::INTERVAL_EQUALS == cmp || Interval::INTERVAL_WITHIN == cmp) { + // Interval 'i' is equal to i+1, or is contained within i+1. + // Remove interval i and don't move to the next value of 'i'. + iv.erase(iv.begin() + i); + } + else if (Interval::INTERVAL_CONTAINS == cmp) { + // Interval 'i' contains i+1, remove i+1 and don't move to the next value of 'i'. + iv.erase(iv.begin() + i + 1); + } + else if (Interval::INTERVAL_OVERLAPS_BEFORE == cmp + || Interval::INTERVAL_PRECEDES_COULD_UNION == cmp) { + // We want to merge intervals i and i+1. + // Interval 'i' starts before interval 'i+1'. + BSONObjBuilder bob; + bob.appendAs(iv[i].start, ""); + bob.appendAs(iv[i + 1].end, ""); + BSONObj data = bob.obj(); + bool startInclusive = iv[i].startInclusive; + bool endInclusive = iv[i + 1].endInclusive; + iv.erase(iv.begin() + i); + // iv[i] is now the former iv[i + 1] + iv[i] = makeRangeInterval(data, startInclusive, endInclusive); + // Don't increment 'i'. + } + } + } + + // static + Interval IndexBoundsBuilder::makeRangeInterval(const string& start, const string& end, + bool startInclusive, bool endInclusive) { + BSONObjBuilder bob; + bob.append("", start); + bob.append("", end); + return makeRangeInterval(bob.obj(), startInclusive, endInclusive); + } + + // static + Interval IndexBoundsBuilder::makePointInterval(const BSONObj& obj) { + Interval ret; + ret._intervalData = obj; + ret.startInclusive = ret.endInclusive = true; + ret.start = ret.end = obj.firstElement(); + return ret; + } + + // static + Interval IndexBoundsBuilder::makePointInterval(const string& str) { + BSONObjBuilder bob; + bob.append("", str); + return makePointInterval(bob.obj()); + } + + // static + BSONObj IndexBoundsBuilder::objFromElement(const BSONElement& elt) { + BSONObjBuilder bob; + bob.appendAs(elt, ""); + return bob.obj(); + } + + // static + void IndexBoundsBuilder::reverseInterval(Interval* ival) { + BSONElement tmp = ival->start; + ival->start = ival->end; + ival->end = tmp; + + bool tmpInc = ival->startInclusive; + ival->startInclusive = ival->endInclusive; + ival->endInclusive = tmpInc; + } + + // static + void IndexBoundsBuilder::translateRegex(const RegexMatchExpression* rme, + OrderedIntervalList* oilOut, BoundsTightness* tightnessOut) { + + const string start = simpleRegex(rme->getString().c_str(), rme->getFlags().c_str(), tightnessOut); + + // Note that 'tightnessOut' is set by simpleRegex above. + if (!start.empty()) { + string end = start; + end[end.size() - 1]++; + oilOut->intervals.push_back(makeRangeInterval(start, end, true, false)); + } + else { + BSONObjBuilder bob; + bob.appendMinForType("", String); + bob.appendMaxForType("", String); + BSONObj dataObj = bob.obj(); + verify(dataObj.isOwned()); + oilOut->intervals.push_back(makeRangeInterval(dataObj, true, false)); + } + + // Regexes are after strings. + BSONObjBuilder bob; + bob.appendRegex("", rme->getString(), rme->getFlags()); + oilOut->intervals.push_back(makePointInterval(bob.obj())); + } + + // static + void IndexBoundsBuilder::translateEquality(const BSONElement& data, bool isHashed, + OrderedIntervalList* oil, BoundsTightness* tightnessOut) { + // We have to copy the data out of the parse tree and stuff it into the index + // bounds. BSONValue will be useful here. + if (Array != data.type()) { + BSONObj dataObj; + if (isHashed) { + dataObj = ExpressionMapping::hash(data); + } + else { + dataObj = objFromElement(data); + } + + verify(dataObj.isOwned()); + oil->intervals.push_back(makePointInterval(dataObj)); + + if (dataObj.firstElement().isNull() || isHashed) { + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + else { + *tightnessOut = IndexBoundsBuilder::EXACT; + } + return; + } + + // If we're here, Array == data.type(). + // + // Using arrays with hashed indices is currently not supported, so we don't have to worry + // about that case. + // + // Arrays are indexed by either: + // + // 1. the first element if there is one. Note that using the first is arbitrary; we could + // just as well use any array element.). If the query is {a: [1, 2, 3]}, for example, then + // using the bounds [1, 1] for the multikey index will pick up every document containing the + // array [1, 2, 3]. + // + // 2. undefined if the array is empty. + // + // Also, arrays are indexed by: + // + // 3. the full array if it's inside of another array. We check for this so that the query + // {a: [1, 2, 3]} will match documents like {a: [[1, 2, 3], 4, 5]}. + + // Case 3. + oil->intervals.push_back(makePointInterval(objFromElement(data))); + + if (data.Obj().isEmpty()) { + // Case 2. + BSONObjBuilder undefinedBob; + undefinedBob.appendUndefined(""); + oil->intervals.push_back(makePointInterval(undefinedBob.obj())); + } + else { + // Case 1. + BSONElement firstEl = data.Obj().firstElement(); + oil->intervals.push_back(makePointInterval(objFromElement(firstEl))); + } + + std::sort(oil->intervals.begin(), oil->intervals.end(), IntervalComparison); + *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH; + } + + // static + void IndexBoundsBuilder::allValuesBounds(const BSONObj& keyPattern, IndexBounds* bounds) { + bounds->fields.resize(keyPattern.nFields()); + + BSONObjIterator it(keyPattern); + int field = 0; + while (it.more()) { + IndexBoundsBuilder::allValuesForField(it.next(), &bounds->fields[field]); + ++field; + } + + alignBounds(bounds, keyPattern); + } + + // static + void IndexBoundsBuilder::alignBounds(IndexBounds* bounds, const BSONObj& kp, int scanDir) { + BSONObjIterator it(kp); + size_t oilIdx = 0; + while (it.more()) { + BSONElement elt = it.next(); + int direction = (elt.numberInt() >= 0) ? 1 : -1; + direction *= scanDir; + if (-1 == direction) { + vector& iv = bounds->fields[oilIdx].intervals; + // Step 1: reverse the list. + std::reverse(iv.begin(), iv.end()); + // Step 2: reverse each interval. + for (size_t i = 0; i < iv.size(); ++i) { + iv[i].reverse(); + } + } + ++oilIdx; + } + + if (!bounds->isValidFor(kp, scanDir)) { + QLOG() << "INVALID BOUNDS: " << bounds->toString() << endl + << "kp = " << kp.toString() << endl + << "scanDir = " << scanDir << endl; + verify(0); + } + } + + // static + bool IndexBoundsBuilder::isSingleInterval(const IndexBounds& bounds, + BSONObj* startKey, + bool* startKeyInclusive, + BSONObj* endKey, + bool* endKeyInclusive) { + // We build our start/end keys as we go. + BSONObjBuilder startBob; + BSONObjBuilder endBob; + + // The start and end keys are inclusive unless we have a non-point interval, in which case + // we take the inclusivity from there. + *startKeyInclusive = true; + *endKeyInclusive = true; + + size_t fieldNo = 0; + + // First, we skip over point intervals. + for (; fieldNo < bounds.fields.size(); ++fieldNo) { + const OrderedIntervalList& oil = bounds.fields[fieldNo]; + // A point interval requires just one interval... + if (1 != oil.intervals.size()) { + break; + } + if (!oil.intervals[0].isPoint()) { + break; + } + // Since it's a point, start == end. + startBob.append(oil.intervals[0].start); + endBob.append(oil.intervals[0].end); + } + + if (fieldNo >= bounds.fields.size()) { + // All our intervals are points. We count for all values of one field. + *startKey = startBob.obj(); + *endKey = endBob.obj(); + return true; + } + + // After point intervals we can have exactly one non-point interval. + const OrderedIntervalList& nonPoint = bounds.fields[fieldNo]; + if (1 != nonPoint.intervals.size()) { + return false; + } + + // Add the non-point interval to our builder and set the inclusivity from it. + startBob.append(nonPoint.intervals[0].start); + *startKeyInclusive = nonPoint.intervals[0].startInclusive; + endBob.append(nonPoint.intervals[0].end); + *endKeyInclusive = nonPoint.intervals[0].endInclusive; + + ++fieldNo; + + // Get some "all values" intervals for comparison's sake. + // TODO: make static? + Interval minMax = IndexBoundsBuilder::allValues(); + Interval maxMin = minMax; + maxMin.reverse(); + + // And after the non-point interval we can have any number of "all values" intervals. + for (; fieldNo < bounds.fields.size(); ++fieldNo) { + const OrderedIntervalList& oil = bounds.fields[fieldNo]; + // "All Values" is just one point. + if (1 != oil.intervals.size()) { + break; + } + + // Must be min->max or max->min. + if (oil.intervals[0].equals(minMax)) { + // As an example for the logic below, consider the index {a:1, b:1} and a count for + // {a: {$gt: 2}}. Our start key isn't inclusive (as it's $gt: 2) and looks like + // {"":2} so far. If we move to the key greater than {"":2, "": MaxKey} we will get + // the first value of 'a' that is greater than 2. + if (!*startKeyInclusive) { + startBob.appendMaxKey(""); + } + else { + // In this case, consider the index {a:1, b:1} and a count for {a:{$gte: 2}}. + // We want to look at all values where a is 2, so our start key is {"":2, + // "":MinKey}. + startBob.appendMinKey(""); + } + + // Same deal as above. Consider the index {a:1, b:1} and a count for {a: {$lt: 2}}. + // Our end key isn't inclusive as ($lt: 2) and looks like {"":2} so far. We can't + // look at any values where a is 2 so we have to stop at {"":2, "": MinKey} as + // that's the smallest key where a is still 2. + if (!*endKeyInclusive) { + endBob.appendMinKey(""); + } + else { + endBob.appendMaxKey(""); + } + } + else if (oil.intervals[0].equals(maxMin)) { + // The reasoning here is the same as above but with the directions reversed. + if (!*startKeyInclusive) { + startBob.appendMinKey(""); + } + else { + startBob.appendMaxKey(""); + } + if (!*endKeyInclusive) { + endBob.appendMaxKey(""); + } + else { + endBob.appendMinKey(""); + } + } + else { + // No dice. + break; + } + } + + if (fieldNo >= bounds.fields.size()) { + *startKey = startBob.obj(); + *endKey = endBob.obj(); + return true; + } + else { + return false; + } + } + +} // namespace mongo diff --git a/src/mongo/db/query/index_bounds_builder.h b/src/mongo/db/query/index_bounds_builder.h new file mode 100644 index 00000000000..1666ef4aad8 --- /dev/null +++ b/src/mongo/db/query/index_bounds_builder.h @@ -0,0 +1,195 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/db/hasher.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/index_bounds.h" +#include "mongo/db/query/index_entry.h" + +namespace mongo { + + /** + * Translates expressions over fields into bounds on an index. + */ + class IndexBoundsBuilder { + public: + /** + * Describes various degrees of precision with which predicates can be evaluated based + * on the index bounds. + * + * The integer values of the enum are significant, and are assigned in order of + * increasing tightness. These values are used when we need to do comparison between two + * BoundsTightness values. Such comparisons can answer questions such as "Does predicate + * X have tighter or looser bounds than predicate Y?". + */ + enum BoundsTightness { + // Index bounds are inexact, and a fetch is required. + INEXACT_FETCH = 0, + + // Index bounds are inexact, but no fetch is required + INEXACT_COVERED = 1, + + // Index bounds are exact. + EXACT = 2 + }; + + /** + * Populate the provided O.I.L. with one interval goes from MinKey to MaxKey (or vice-versa + * depending on the index direction). + */ + static void allValuesForField(const BSONElement& elt, OrderedIntervalList* out); + + /** + * Turn the MatchExpression in 'expr' into a set of index bounds. The field that 'expr' is + * concerned with is indexed according to the keypattern element 'elt' from index 'index'. + * + * If 'expr' is elemMatch, the index tag is affixed to a child. + * + * The expression must be a predicate over one field. That is, expr->isLeaf() or + * expr->isArray() must be true, and expr->isLogical() must be false. + */ + static void translate(const MatchExpression* expr, + const BSONElement& elt, + const IndexEntry& index, + OrderedIntervalList* oilOut, + BoundsTightness* tightnessOut); + + /** + * Creates bounds for 'expr' (indexed according to 'elt'). Intersects those bounds + * with the bounds in oilOut, which is an in/out parameter. + */ + static void translateAndIntersect(const MatchExpression* expr, + const BSONElement& elt, + const IndexEntry& index, + OrderedIntervalList* oilOut, + BoundsTightness* tightnessOut); + + /** + * Creates bounds for 'expr' (indexed according to 'elt'). Unions those bounds + * with the bounds in oilOut, which is an in/out parameter. + */ + static void translateAndUnion(const MatchExpression* expr, + const BSONElement& elt, + const IndexEntry& index, + OrderedIntervalList* oilOut, + BoundsTightness* tightnessOut); + + /** + * Make a range interval from the provided object. + * The object must have exactly two fields. The first field is the start, the second the + * end. + * The two inclusive flags indicate whether or not the start/end fields are included in the + * interval (closed interval if included, open if not). + */ + static Interval makeRangeInterval(const BSONObj& obj, + bool startInclusive, + bool endInclusive); + + static Interval makeRangeInterval(const string& start, + const string& end, + bool startInclusive, + bool endInclusive); + + /** + * Make a point interval from the provided object. + * The object must have exactly one field which is the value of the point interval. + */ + static Interval makePointInterval(const BSONObj& obj); + static Interval makePointInterval(const string& str); + + /** + * Since we have no BSONValue we must make an object that's a copy of a piece of another + * object. + */ + static BSONObj objFromElement(const BSONElement& elt); + + /** + * Swap start/end in the provided interval. + */ + static void reverseInterval(Interval* ival); + + /** + * Copied almost verbatim from db/queryutil.cpp. + * + * returns a string that when used as a matcher, would match a super set of regex() + * + * returns "" for complex regular expressions + * + * used to optimize queries in some simple regex cases that start with '^' + */ + static string simpleRegex(const char* regex, + const char* flags, + BoundsTightness* tightnessOut); + + /** + * Returns an Interval from minKey to maxKey + */ + static Interval allValues(); + + static void translateRegex(const RegexMatchExpression* rme, + OrderedIntervalList* oil, + BoundsTightness* tightnessOut); + + static void translateEquality(const BSONElement& data, + bool isHashed, + OrderedIntervalList* oil, + BoundsTightness* tightnessOut); + + static void unionize(OrderedIntervalList* oilOut); + static void intersectize(const OrderedIntervalList& arg, + OrderedIntervalList* oilOut); + + /** + * Fills out 'bounds' with the bounds for an index scan over all values of the + * index described by 'keyPattern' in the default forward direction. + */ + static void allValuesBounds(const BSONObj& keyPattern, IndexBounds* bounds); + + /** + * Assumes each OIL in 'bounds' is increasing. + * + * Aligns OILs (and bounds) according to the 'kp' direction * the scanDir. + */ + static void alignBounds(IndexBounds* bounds, const BSONObj& kp, int scanDir = 1); + + /** + * Returns 'true' if the bounds 'bounds' can be represented as one interval between + * 'startKey' and 'endKey'. Inclusivity of each bound is set through the relevant + * (name)KeyInclusive parameter. Returns 'false' if otherwise. + */ + static bool isSingleInterval(const IndexBounds& bounds, + BSONObj* startKey, + bool* startKeyInclusive, + BSONObj* endKey, + bool* endKeyInclusive); + }; + +} // namespace mongo diff --git a/src/mongo/db/query/index_bounds_builder_test.cpp b/src/mongo/db/query/index_bounds_builder_test.cpp new file mode 100644 index 00000000000..66fd241c618 --- /dev/null +++ b/src/mongo/db/query/index_bounds_builder_test.cpp @@ -0,0 +1,1195 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/index_bounds_builder.cpp + */ + +#include "mongo/db/query/index_bounds_builder.h" + +#include +#include +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + using std::auto_ptr; + + double numberMin = -numeric_limits::max(); + double numberMax = numeric_limits::max(); + double negativeInfinity = -numeric_limits::infinity(); + double positiveInfinity = numeric_limits::infinity(); + + /** + * Utility function to create MatchExpression + */ + MatchExpression* parseMatchExpression(const BSONObj& obj) { + StatusWithMatchExpression status = MatchExpressionParser::parse(obj); + ASSERT_TRUE(status.isOK()); + MatchExpression* expr(status.getValue()); + return expr; + } + + /** + * Given a list of queries in 'toUnion', translate into index bounds and return + * the union of these bounds in the out-parameter 'oilOut'. + */ + void testTranslateAndUnion(const vector& toUnion, OrderedIntervalList* oilOut, + IndexBoundsBuilder::BoundsTightness* tightnessOut) { + + IndexEntry testIndex = IndexEntry(BSONObj()); + + for (vector::const_iterator it = toUnion.begin(); + it != toUnion.end(); + ++it) { + auto_ptr expr(parseMatchExpression(*it)); + BSONElement elt = it->firstElement(); + if (toUnion.begin() == it) { + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, oilOut, tightnessOut); + } + else { + IndexBoundsBuilder::translateAndUnion(expr.get(), elt, testIndex, oilOut, tightnessOut); + } + } + } + + /** + * Given a list of queries in 'toUnion', translate into index bounds and return + * the intersection of these bounds in the out-parameter 'oilOut'. + */ + void testTranslateAndIntersect(const vector& toIntersect, OrderedIntervalList* oilOut, + IndexBoundsBuilder::BoundsTightness* tightnessOut) { + + IndexEntry testIndex = IndexEntry(BSONObj()); + + for (vector::const_iterator it = toIntersect.begin(); + it != toIntersect.end(); + ++it) { + auto_ptr expr(parseMatchExpression(*it)); + BSONElement elt = it->firstElement(); + if (toIntersect.begin() == it) { + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, oilOut, tightnessOut); + } + else { + IndexBoundsBuilder::translateAndIntersect(expr.get(), elt, testIndex, oilOut, tightnessOut); + } + } + } + + /** + * 'constraints' is a vector of BSONObj's representing match expressions, where + * each filter is paired with a boolean. If the boolean is true, then the filter's + * index bounds should be intersected with the other constraints; if false, then + * they should be unioned. The resulting bounds are returned in the + * out-parameter 'oilOut'. + */ + void testTranslate(const vector< std::pair >& constraints, + OrderedIntervalList* oilOut, + IndexBoundsBuilder::BoundsTightness* tightnessOut) { + + IndexEntry testIndex = IndexEntry(BSONObj()); + + for (vector< std::pair >::const_iterator it = constraints.begin(); + it != constraints.end(); + ++it) { + BSONObj obj = it->first; + bool isIntersect = it->second; + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + if (constraints.begin() == it) { + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, oilOut, tightnessOut); + } + else if (isIntersect) { + IndexBoundsBuilder::translateAndIntersect(expr.get(), elt, testIndex, oilOut, tightnessOut); + } + else { + IndexBoundsBuilder::translateAndUnion(expr.get(), elt, testIndex, oilOut, tightnessOut); + } + } + } + + /** + * run isSingleInterval and return the result to calling test. + */ + bool testSingleInterval(IndexBounds bounds) { + BSONObj startKey; + bool startKeyIn; + BSONObj endKey; + bool endKeyIn; + return IndexBoundsBuilder::isSingleInterval( bounds, + &startKey, + &startKeyIn, + &endKey, + &endKeyIn ); + } + + // + // $elemMatch value + // Example: {a: {$elemMatch: {$gt: 2}}} + // + + TEST(IndexBoundsBuilderTest, TranslateElemMatchValue) { + IndexEntry testIndex = IndexEntry(BSONObj()); + // Bounds generated should be the same as the embedded expression + // except for the tightness. + BSONObj obj = fromjson("{a: {$elemMatch: {$gt: 2}}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 2, '': Infinity}"), false, true))); + ASSERT(tightness == IndexBoundsBuilder::INEXACT_FETCH); + } + + // + // Comparison operators ($lte, $lt, $gt, $gte, $eq) + // + + TEST(IndexBoundsBuilderTest, TranslateLteNumber) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$lte: 1}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': -Infinity, '': 1}"), true, true))); + ASSERT(tightness == IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateLteNumberMin) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = BSON("a" << BSON("$lte" << numberMin)); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(BSON("" << negativeInfinity << "" << numberMin), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateLteNegativeInfinity) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$lte: -Infinity}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': -Infinity, '': -Infinity}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateLtNumber) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$lt: 1}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': -Infinity, '': 1}"), true, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateLtNumberMin) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = BSON("a" << BSON("$lt" << numberMin)); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(BSON("" << negativeInfinity << "" << numberMin), true, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateLtNegativeInfinity) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$lt: -Infinity}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 0U); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateLtDate) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = BSON("a" << LT << Date_t(5000)); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': true, '': new Date(5000)}"), false, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGtNumber) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$gt: 1}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 1, '': Infinity}"), false, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGtNumberMax) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = BSON("a" << BSON("$gt" << numberMax)); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(BSON("" << numberMax << "" << positiveInfinity), false, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGtPositiveInfinity) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$gt: Infinity}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 0U); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGteNumber) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$gte: 1}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 1, '': Infinity}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGteNumberMax) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = BSON("a" << BSON("$gte" << numberMax)); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(BSON("" << numberMax << "" << positiveInfinity), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGtePositiveInfinity) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$gte: Infinity}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': Infinity, '': Infinity}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGtString) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$gt: 'abc'}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 'abc', '': {}}"), false, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateEqual) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = BSON("a" << 4); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 4, '': 4}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateArrayEqualBasic) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: [1, 2, 3]}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 1, '': 1}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(fromjson("{'': [1, 2, 3], '': [1, 2, 3]}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH); + } + + TEST(IndexBoundsBuilderTest, TranslateIn) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$in: [8, 44, -1, -3]}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 4U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': -3, '': -3}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(fromjson("{'': -1, '': -1}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[2].compare( + Interval(fromjson("{'': 8, '': 8}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[3].compare( + Interval(fromjson("{'': 44, '': 44}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateInArray) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$in: [[1], 2]}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 3U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 1, '': 1}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(fromjson("{'': 2, '': 2}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[2].compare( + Interval(fromjson("{'': [1], '': [1]}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH); + } + + TEST(IndexBoundsBuilderTest, TranslateLteBinData) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$lte: {$binary: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAA'," + "$type: '00'}}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': {$binary: '', $type: '00'}," + "'': {$binary: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAA', $type: '00'}}"), + true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateLtBinData) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$lt: {$binary: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAA'," + "$type: '00'}}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': {$binary: '', $type: '00'}," + "'': {$binary: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAA', $type: '00'}}"), + true, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGtBinData) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$gt: {$binary: '////////////////////////////'," + "$type: '00'}}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': {$binary: '////////////////////////////', $type: '00'}," + "'': ObjectId('000000000000000000000000')}"), + false, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, TranslateGteBinData) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$gte: {$binary: '////////////////////////////'," + "$type: '00'}}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': {$binary: '////////////////////////////', $type: '00'}," + "'': ObjectId('000000000000000000000000')}"), + true, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + // + // $exists tests + // + + TEST(IndexBoundsBuilderTest, ExistsTrue) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$exists: true}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, + oil.intervals[0].compare(IndexBoundsBuilder::allValues())); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH); + } + + TEST(IndexBoundsBuilderTest, ExistsFalse) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$exists: false}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': null, '': null}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH); + } + + TEST(IndexBoundsBuilderTest, ExistsTrueSparse) { + IndexEntry testIndex = IndexEntry(BSONObj(), + false, // multikey + true, // sparse + false, // unique + "exists_true_sparse", + BSONObj()); + BSONObj obj = fromjson("{a: {$exists: true}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, + oil.intervals[0].compare(IndexBoundsBuilder::allValues())); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + // + // Union tests + // + + TEST(IndexBoundsBuilderTest, UnionTwoLt) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toUnion; + toUnion.push_back(fromjson("{a: {$lt: 1}}")); + toUnion.push_back(fromjson("{a: {$lt: 5}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndUnion(toUnion, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': -Infinity, '': 5}"), true, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, UnionDupEq) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toUnion; + toUnion.push_back(fromjson("{a: 1}")); + toUnion.push_back(fromjson("{a: 5}")); + toUnion.push_back(fromjson("{a: 1}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndUnion(toUnion, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 1, '': 1}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(fromjson("{'': 5, '': 5}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, UnionGtLt) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toUnion; + toUnion.push_back(fromjson("{a: {$gt: 1}}")); + toUnion.push_back(fromjson("{a: {$lt: 3}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndUnion(toUnion, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': -Infinity, '': Infinity}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, UnionTwoEmptyRanges) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector< std::pair > constraints; + constraints.push_back(std::make_pair(fromjson("{a: {$gt: 1}}"), true)); + constraints.push_back(std::make_pair(fromjson("{a: {$lte: 0}}"), true)); + constraints.push_back(std::make_pair(fromjson("{a: {$in:[]}}"), false)); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslate(constraints, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 0U); + } + + // + // Intersection tests + // + + TEST(IndexBoundsBuilderTest, IntersectTwoLt) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: {$lt: 1}}")); + toIntersect.push_back(fromjson("{a: {$lt: 5}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': -Infinity, '': 1}"), true, false))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, IntersectEqGte) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: 1}}")); + toIntersect.push_back(fromjson("{a: {$gte: 1}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 1, '': 1}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, IntersectGtLte) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: {$gt: 0}}")); + toIntersect.push_back(fromjson("{a: {$lte: 10}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 0, '': 10}"), false, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, IntersectGtIn) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: {$gt: 4}}")); + toIntersect.push_back(fromjson("{a: {$in: [1,2,3,4,5,6]}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 5, '': 5}"), true, true))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(fromjson("{'': 6, '': 6}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, IntersectionIsPointInterval) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: {$gte: 1}}")); + toIntersect.push_back(fromjson("{a: {$lte: 1}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 1, '': 1}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, IntersectFullyContained) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: {$gt: 5}}")); + toIntersect.push_back(fromjson("{a: {$lt: 15}}")); + toIntersect.push_back(fromjson("{a: {$gte: 6}}")); + toIntersect.push_back(fromjson("{a: {$lte: 13}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 6, '': 13}"), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, EmptyIntersection) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: 1}}")); + toIntersect.push_back(fromjson("{a: {$gte: 2}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 0U); + } + + // + // $mod + // + + TEST(IndexBoundsBuilderTest, TranslateMod) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$mod: [2, 0]}}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(BSON("" << numberMin << "" << numberMax), true, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_COVERED); + } + + // + // Test simpleRegex + // + + TEST(SimpleRegexTest, RootedLine) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex("^foo", "", &tightness); + ASSERT_EQUALS(prefix, "foo"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedString) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex("\\Afoo", "", &tightness); + ASSERT_EQUALS(prefix, "foo"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedOptionalFirstChar) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex("^f?oo", "", &tightness); + ASSERT_EQUALS(prefix, ""); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_COVERED); + } + + TEST(SimpleRegexTest, RootedOptionalSecondChar) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex("^fz?oo", "", &tightness); + ASSERT_EQUALS(prefix, "f"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_COVERED); + } + + TEST(SimpleRegexTest, RootedMultiline) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex("^foo", "m", &tightness); + ASSERT_EQUALS(prefix, ""); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_COVERED); + } + + TEST(SimpleRegexTest, RootedStringMultiline) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex("\\Afoo", "m", &tightness); + ASSERT_EQUALS(prefix, "foo"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedCaseInsensitiveMulti) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex("\\Afoo", "mi", &tightness); + ASSERT_EQUALS(prefix, ""); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_COVERED); + } + + TEST(SimpleRegexTest, RootedComplex) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "\\Af \t\vo\n\ro \\ \\# #comment", "mx", &tightness); + ASSERT_EQUALS(prefix, "foo #"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_COVERED); + } + + TEST(SimpleRegexTest, RootedLiteral) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "^\\Qasdf\\E", "", &tightness); + ASSERT_EQUALS(prefix, "asdf"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedLiteralWithExtra) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "^\\Qasdf\\E.*", "", &tightness); + ASSERT_EQUALS(prefix, "asdf"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_COVERED); + } + + TEST(SimpleRegexTest, RootedLiteralNoEnd) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "^\\Qasdf", "", &tightness); + ASSERT_EQUALS(prefix, "asdf"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedLiteralBackslash) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "^\\Qasdf\\\\E", "", &tightness); + ASSERT_EQUALS(prefix, "asdf\\"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedLiteralDotStar) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "^\\Qas.*df\\E", "", &tightness); + ASSERT_EQUALS(prefix, "as.*df"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedLiteralNestedEscape) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "^\\Qas\\Q[df\\E", "", &tightness); + ASSERT_EQUALS(prefix, "as\\Q[df"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(SimpleRegexTest, RootedLiteralNestedEscapeEnd) { + IndexBoundsBuilder::BoundsTightness tightness; + string prefix = IndexBoundsBuilder::simpleRegex( + "^\\Qas\\E\\\\E\\Q$df\\E", "", &tightness); + ASSERT_EQUALS(prefix, "as\\E$df"); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + // + // Regex bounds + // + + TEST(IndexBoundsBuilderTest, SimpleNonPrefixRegex) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: /foo/}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': '', '': {}}"), true, false))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(fromjson("{'': /foo/, '': /foo/}"), true, true))); + ASSERT(tightness == IndexBoundsBuilder::INEXACT_COVERED); + } + + TEST(IndexBoundsBuilderTest, SimplePrefixRegex) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: /^foo/}"); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(fromjson("{'': 'foo', '': 'fop'}"), true, false))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(fromjson("{'': /^foo/, '': /^foo/}"), true, true))); + ASSERT(tightness == IndexBoundsBuilder::EXACT); + } + + // + // isSingleInterval + // + + TEST(IndexBoundsBuilderTest, SingleFieldEqualityInterval) { + // Equality on a single field is a single interval. + OrderedIntervalList oil("a"); + IndexBounds bounds; + oil.intervals.push_back(Interval(BSON("" << 5 << "" << 5), true, true)); + bounds.fields.push_back(oil); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, SingleIntervalSingleFieldInterval) { + // Single interval on a single field is a single interval. + OrderedIntervalList oil("a"); + IndexBounds bounds; + oil.intervals.push_back(Interval(fromjson("{ '':5, '':Infinity }"), true, true)); + bounds.fields.push_back(oil); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, MultipleIntervalsSingleFieldInterval) { + // Multiple intervals on a single field is not a single interval. + OrderedIntervalList oil("a"); + IndexBounds bounds; + oil.intervals.push_back(Interval(fromjson( "{ '':4, '':5 }" ), true, true)); + oil.intervals.push_back(Interval(fromjson( "{ '':7, '':Infinity }" ), true, true)); + bounds.fields.push_back(oil); + ASSERT(!testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, EqualityTwoFieldsInterval) { + // Equality on two fields is a compound single interval. + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON("" << 5 << "" << 5), true, true)); + oil_b.intervals.push_back(Interval(BSON("" << 6 << "" << 6), true, true)); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, EqualityFirstFieldSingleIntervalSecondFieldInterval) { + // Equality on first field and single interval on second field + // is a compound single interval. + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON("" << 5 << "" << 5), true, true)); + oil_b.intervals.push_back(Interval(fromjson( "{ '':6, '':Infinity }" ), true, true)); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, SingleIntervalFirstAndSecondFieldsInterval) { + // Single interval on first field and single interval on second field is + // not a compound single interval. + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(fromjson( "{ '':-Infinity, '':5 }" ), true, true)); + oil_b.intervals.push_back(Interval(fromjson( "{ '':6, '':Infinity }" ), true, true)); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + ASSERT(!testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, MultipleIntervalsTwoFieldsInterval) { + // Multiple intervals on two fields is not a compound single interval. + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON( "" << 4 << "" << 4 ), true, true)); + oil_a.intervals.push_back(Interval(BSON( "" << 5 << "" << 5 ), true, true)); + oil_b.intervals.push_back(Interval(BSON( "" << 7 << "" << 7 ), true, true)); + oil_b.intervals.push_back(Interval(BSON( "" << 8 << "" << 8 ), true, true)); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + ASSERT(!testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, MissingSecondFieldInterval) { + // when second field is not specified, still a compound single interval + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON( "" << 5 << "" << 5 ), true, true)); + oil_b.intervals.push_back(IndexBoundsBuilder::allValues()); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, EqualityTwoFieldsIntervalThirdInterval) { + // Equality on first two fields and single interval on third is a + // compound single interval. + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + OrderedIntervalList oil_c("c"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON( "" << 5 << "" << 5 ), true, true)); + oil_b.intervals.push_back(Interval(BSON( "" << 6 << "" << 6 ), true, true)); + oil_c.intervals.push_back(Interval(fromjson( "{ '':7, '':Infinity }" ), true, true)); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + bounds.fields.push_back(oil_c); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, EqualitySingleIntervalMissingInterval) { + // Equality, then Single Interval, then missing is a compound single interval + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + OrderedIntervalList oil_c("c"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON( "" << 5 << "" << 5 ), true, true)); + oil_b.intervals.push_back(Interval(fromjson( "{ '':7, '':Infinity }" ), true, true)); + oil_c.intervals.push_back(IndexBoundsBuilder::allValues()); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + bounds.fields.push_back(oil_c); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, EqualitySingleMissingMissingInterval) { + // Equality, then single interval, then missing, then missing, + // is a compound single interval + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + OrderedIntervalList oil_c("c"); + OrderedIntervalList oil_d("d"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON( "" << 5 << "" << 5 ), true, true)); + oil_b.intervals.push_back(Interval(fromjson( "{ '':7, '':Infinity }" ), true, true)); + oil_c.intervals.push_back(IndexBoundsBuilder::allValues()); + oil_d.intervals.push_back(IndexBoundsBuilder::allValues()); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + bounds.fields.push_back(oil_c); + bounds.fields.push_back(oil_d); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, EqualitySingleMissingMissingMixedInterval) { + // Equality, then single interval, then missing, then missing, with mixed order + // fields is a compound single interval. + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + OrderedIntervalList oil_c("c"); + OrderedIntervalList oil_d("d"); + IndexBounds bounds; + Interval allValues = IndexBoundsBuilder::allValues(); + oil_a.intervals.push_back(Interval(BSON( "" << 5 << "" << 5 ), true, true)); + oil_b.intervals.push_back(Interval(fromjson( "{ '':7, '':Infinity }" ), true, true)); + oil_c.intervals.push_back(allValues); + IndexBoundsBuilder::reverseInterval(&allValues); + oil_d.intervals.push_back(allValues); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + bounds.fields.push_back(oil_c); + bounds.fields.push_back(oil_d); + ASSERT(testSingleInterval(bounds)); + } + + TEST(IndexBoundsBuilderTest, EqualitySingleMissingSingleInterval) { + // Equality, then single interval, then missing, then single interval is not + // a compound single interval. + OrderedIntervalList oil_a("a"); + OrderedIntervalList oil_b("b"); + OrderedIntervalList oil_c("c"); + OrderedIntervalList oil_d("d"); + IndexBounds bounds; + oil_a.intervals.push_back(Interval(BSON( "" << 5 << "" << 5 ), true, true)); + oil_b.intervals.push_back(Interval(fromjson( "{ '':7, '':Infinity }" ), true, true)); + oil_c.intervals.push_back(IndexBoundsBuilder::allValues()); + oil_d.intervals.push_back(Interval(fromjson( "{ '':1, '':Infinity }" ), true, true)); + bounds.fields.push_back(oil_a); + bounds.fields.push_back(oil_b); + bounds.fields.push_back(oil_c); + bounds.fields.push_back(oil_d); + ASSERT(!testSingleInterval(bounds)); + } + + // + // Complementing bounds for negations + // + + /** + * Get a BSONObj which represents the interval from + * MinKey to 'end'. + */ + BSONObj minKeyIntObj(int end) { + BSONObjBuilder bob; + bob.appendMinKey(""); + bob.appendNumber("", end); + return bob.obj(); + } + + /** + * Get a BSONObj which represents the interval from + * 'start' to MaxKey. + */ + BSONObj maxKeyIntObj(int start) { + BSONObjBuilder bob; + bob.appendNumber("", start); + bob.appendMaxKey(""); + return bob.obj(); + } + + // Expected oil: [MinKey, 3), (3, MaxKey] + TEST(IndexBoundsBuilderTest, SimpleNE) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = BSON("a" << BSON("$ne" << 3)); + auto_ptr expr(parseMatchExpression(obj)); + BSONElement elt = obj.firstElement(); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + IndexBoundsBuilder::translate(expr.get(), elt, testIndex, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(minKeyIntObj(3), true, false))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(maxKeyIntObj(3), false, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, IntersectWithNE) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toIntersect; + toIntersect.push_back(fromjson("{a: {$gt: 1}}")); + toIntersect.push_back(fromjson("{a: {$ne: 2}}}")); + toIntersect.push_back(fromjson("{a: {$lte: 6}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndIntersect(toIntersect, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(BSON("" << 1 << "" << 2), false, false))); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(BSON("" << 2 << "" << 6), false, true))); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + + TEST(IndexBoundsBuilderTest, UnionizeWithNE) { + IndexEntry testIndex = IndexEntry(BSONObj()); + vector toUnionize; + toUnionize.push_back(fromjson("{a: {$ne: 3}}")); + toUnionize.push_back(fromjson("{a: {$ne: 4}}}")); + OrderedIntervalList oil; + IndexBoundsBuilder::BoundsTightness tightness; + testTranslateAndUnion(toUnionize, &oil, &tightness); + ASSERT_EQUALS(oil.name, "a"); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + IndexBoundsBuilder::allValues())); + ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT); + } + +} // namespace diff --git a/src/mongo/db/query/index_bounds_test.cpp b/src/mongo/db/query/index_bounds_test.cpp new file mode 100644 index 00000000000..2e285f227ce --- /dev/null +++ b/src/mongo/db/query/index_bounds_test.cpp @@ -0,0 +1,850 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/index_bounds.cpp + */ + +#include "mongo/db/query/index_bounds.h" +#include "mongo/db/json.h" +#include "mongo/db/jsobj.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/text.h" +#include "mongo/util/mongoutils/str.h" + +using namespace mongo; + +namespace { + + using std::string; + using std::vector; + + // + // Validation + // + + TEST(IndexBoundsTest, ValidBasic) { + OrderedIntervalList list("foo"); + list.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + IndexBounds bounds; + bounds.fields.push_back(list); + + // Go forwards with data indexed forwards. + ASSERT(bounds.isValidFor(BSON("foo" << 1), 1)); + // Go backwards with data indexed backwards. + ASSERT(bounds.isValidFor(BSON("foo" << -1), -1)); + // Bounds are not oriented along the direction of traversal. + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << 1), -1)); + + // Bounds must match the index exactly. + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << 1 << "bar" << 1), 1)); + ASSERT_FALSE(bounds.isValidFor(BSON("bar" << 1), 1)); + } + + TEST(IndexBoundsTest, ValidTwoFields) { + OrderedIntervalList list("foo"); + list.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + IndexBounds bounds; + bounds.fields.push_back(list); + + // Let's add another field + OrderedIntervalList otherList("bar"); + otherList.intervals.push_back(Interval(BSON("" << 0 << "" << 3), true, true)); + bounds.fields.push_back(otherList); + + // These are OK. + ASSERT(bounds.isValidFor(BSON("foo" << 1 << "bar" << 1), 1)); + ASSERT(bounds.isValidFor(BSON("foo" << -1 << "bar" << -1), -1)); + + // Direction(s) don't match. + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << -1 << "bar" << 1), -1)); + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << 1 << "bar" << -1), -1)); + + // Index doesn't match. + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << 1), 1)); + ASSERT_FALSE(bounds.isValidFor(BSON("bar" << 1 << "foo" << 1), 1)); + } + + TEST(IndexBoundsTest, ValidIntervalsInOrder) { + OrderedIntervalList list("foo"); + // Whether navigated forward or backward, there's no valid ordering for these two intervals. + list.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + list.intervals.push_back(Interval(BSON("" << 0 << "" << 5), true, true)); + IndexBounds bounds; + bounds.fields.push_back(list); + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << 1), 1)); + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << -1), 1)); + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << 1), -1)); + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << -1), -1)); + } + + TEST(IndexBoundsTest, ValidNoOverlappingIntervals) { + OrderedIntervalList list("foo"); + // overlapping intervals not allowed. + list.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + list.intervals.push_back(Interval(BSON("" << 19 << "" << 25), true, true)); + IndexBounds bounds; + bounds.fields.push_back(list); + ASSERT_FALSE(bounds.isValidFor(BSON("foo" << 1), 1)); + } + + TEST(IndexBoundsTest, ValidOverlapOnlyWhenBothOpen) { + OrderedIntervalList list("foo"); + list.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, false)); + list.intervals.push_back(Interval(BSON("" << 20 << "" << 25), false, true)); + IndexBounds bounds; + bounds.fields.push_back(list); + ASSERT(bounds.isValidFor(BSON("foo" << 1), 1)); + } + + // + // Tests for OrderedIntervalList::complement() + // + + /** + * Get a BSONObj which represents the interval from + * MinKey to 'end'. + */ + BSONObj minKeyIntObj(int end) { + BSONObjBuilder bob; + bob.appendMinKey(""); + bob.appendNumber("", end); + return bob.obj(); + } + + /** + * Get a BSONObj which represents the interval from + * 'start' to MaxKey. + */ + BSONObj maxKeyIntObj(int start) { + BSONObjBuilder bob; + bob.appendNumber("", start); + bob.appendMaxKey(""); + return bob.obj(); + } + + /** + * Get a BSONObj which represents the interval + * [MinKey, MaxKey]. + */ + BSONObj allValues() { + BSONObjBuilder bob; + bob.appendMinKey(""); + bob.appendMaxKey(""); + return bob.obj(); + } + + /** + * Test that if we complement the OIL twice, + * we get back the original OIL. + */ + void testDoubleComplement(const OrderedIntervalList* oil) { + OrderedIntervalList clone; + for (size_t i = 0; i < oil->intervals.size(); ++i) { + clone.intervals.push_back(oil->intervals[i]); + } + + clone.complement(); + clone.complement(); + + ASSERT_EQUALS(oil->intervals.size(), clone.intervals.size()); + for (size_t i = 0; i < oil->intervals.size(); ++i) { + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, + oil->intervals[i].compare(clone.intervals[i])); + } + } + + // Complement of empty is [MinKey, MaxKey] + TEST(IndexBoundsTest, ComplementEmptyOil) { + OrderedIntervalList oil; + testDoubleComplement(&oil); + oil.complement(); + ASSERT_EQUALS(oil.intervals.size(), 1U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(allValues(), true, true))); + } + + // Complement of [MinKey, MaxKey] is empty + TEST(IndexBoundsTest, ComplementAllValues) { + OrderedIntervalList oil; + oil.intervals.push_back(Interval(allValues(), true, true)); + testDoubleComplement(&oil); + oil.complement(); + ASSERT_EQUALS(oil.intervals.size(), 0U); + } + + // Complement of [MinKey, 3), [5, MaxKey) is + // [3, 5), [MaxKey, MaxKey]. + TEST(IndexBoundsTest, ComplementRanges) { + OrderedIntervalList oil; + oil.intervals.push_back(Interval(minKeyIntObj(3), true, false)); + oil.intervals.push_back(Interval(maxKeyIntObj(5), true, false)); + testDoubleComplement(&oil); + oil.complement(); + ASSERT_EQUALS(oil.intervals.size(), 2U); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(BSON("" << 3 << "" << 5), true, false))); + + // Make the interval [MaxKey, MaxKey]. + BSONObjBuilder bob; + bob.appendMaxKey(""); + bob.appendMaxKey(""); + BSONObj maxKeyInt = bob.obj(); + + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(maxKeyInt, true, true))); + } + + // Complement of (MinKey, 3), (3, MaxKey) is + // [MinKey, MinKey], [3, 3], [MaxKey, MaxKey]. + TEST(IndexBoundsTest, ComplementRanges2) { + OrderedIntervalList oil; + oil.intervals.push_back(Interval(minKeyIntObj(3), false, false)); + oil.intervals.push_back(Interval(maxKeyIntObj(3), false, false)); + testDoubleComplement(&oil); + oil.complement(); + ASSERT_EQUALS(oil.intervals.size(), 3U); + + // First interval is [MinKey, MinKey] + BSONObjBuilder minBob; + minBob.appendMinKey(""); + minBob.appendMinKey(""); + BSONObj minObj = minBob.obj(); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[0].compare( + Interval(minObj, true, true))); + + // Second is [3, 3] + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[1].compare( + Interval(BSON("" << 3 << "" << 3), true, true))); + + // Third is [MaxKey, MaxKey] + BSONObjBuilder maxBob; + maxBob.appendMaxKey(""); + maxBob.appendMaxKey(""); + BSONObj maxObj = maxBob.obj(); + ASSERT_EQUALS(Interval::INTERVAL_EQUALS, oil.intervals[2].compare( + Interval(maxObj, true, true))); + } + + // + // Iteration over + // + + TEST(IndexBoundsCheckerTest, StartKey) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + + OrderedIntervalList barList("bar"); + barList.intervals.push_back(Interval(BSON("" << 0 << "" << 5), false, false)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + bounds.fields.push_back(barList); + IndexBoundsChecker it(&bounds, BSON("foo" << 1 << "bar" << 1), 1); + + vector elt(2); + vector inc(2); + + it.getStartKey(&elt, &inc); + + ASSERT_EQUALS(elt[0]->numberInt(), 7); + ASSERT_EQUALS(inc[0], true); + ASSERT_EQUALS(elt[1]->numberInt(), 0); + ASSERT_EQUALS(inc[1], false); + } + + TEST(IndexBoundsCheckerTest, CheckEnd) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + fooList.intervals.push_back(Interval(BSON("" << 21 << "" << 30), true, false)); + + OrderedIntervalList barList("bar"); + barList.intervals.push_back(Interval(BSON("" << 0 << "" << 5), false, false)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + bounds.fields.push_back(barList); + IndexBoundsChecker it(&bounds, BSON("foo" << 1 << "bar" << 1), 1); + + int keyEltsToUse; + bool movePastKeyElts; + vector elt(2); + vector inc(2); + + IndexBoundsChecker::KeyState state; + + // Start at something in our range. + state = it.checkKey(BSON("" << 7 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // Second field moves past the end, but we're not done, since there's still an interval in + // the previous field that the key hasn't advanced to. + state = it.checkKey(BSON("" << 20 << "" << 5), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT(movePastKeyElts); + + // The next index key is in the second interval for 'foo' and there is a valid interval for + // 'bar'. + state = it.checkKey(BSON("" << 22 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // The next index key is very close to the end of the open interval for foo, and it's past + // the interval for 'bar'. Since the interval for foo is open, we are asked to move + // forward, since we possibly could. + state = it.checkKey(BSON("" << 29.9 << "" << 5), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT(movePastKeyElts); + } + + TEST(IndexBoundsCheckerTest, MoveIntervalForwardToNextInterval) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + fooList.intervals.push_back(Interval(BSON("" << 21 << "" << 30), true, false)); + + OrderedIntervalList barList("bar"); + barList.intervals.push_back(Interval(BSON("" << 0 << "" << 5), false, false)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + bounds.fields.push_back(barList); + IndexBoundsChecker it(&bounds, BSON("foo" << 1 << "bar" << 1), 1); + + int keyEltsToUse; + bool movePastKeyElts; + vector elt(2); + vector inc(2); + + IndexBoundsChecker::KeyState state; + + // Start at something in our range. + state = it.checkKey(BSON("" << 7 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // "foo" moves between two intervals. + state = it.checkKey(BSON("" << 20.5 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 0); + // Should be told to move exactly to the next interval's beginning. + ASSERT_EQUALS(movePastKeyElts, false); + ASSERT_EQUALS(elt[0]->numberInt(), 21); + ASSERT_EQUALS(inc[0], true); + ASSERT_EQUALS(elt[1]->numberInt(), 0); + ASSERT_EQUALS(inc[1], false); + } + + TEST(IndexBoundsCheckerTest, MoveIntervalForwardManyIntervals) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + fooList.intervals.push_back(Interval(BSON("" << 21 << "" << 30), true, false)); + fooList.intervals.push_back(Interval(BSON("" << 31 << "" << 40), true, false)); + fooList.intervals.push_back(Interval(BSON("" << 41 << "" << 50), true, false)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + IndexBoundsChecker it(&bounds, BSON("foo" << 1), 1); + + int keyEltsToUse; + bool movePastKeyElts; + vector elt(1); + vector inc(1); + + IndexBoundsChecker::KeyState state; + + // Start at something in our range. + state = it.checkKey(BSON("" << 7), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // "foo" moves forward a few intervals. + state = it.checkKey(BSON("" << 42), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + } + + TEST(IndexBoundsCheckerTest, SimpleCheckKey) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, true)); + + OrderedIntervalList barList("bar"); + barList.intervals.push_back(Interval(BSON("" << 0 << "" << 5), false, true)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + bounds.fields.push_back(barList); + IndexBoundsChecker it(&bounds, BSON("foo" << 1 << "bar" << 1), 1); + + int keyEltsToUse; + bool movePastKeyElts; + vector elt(2); + vector inc(2); + + IndexBoundsChecker::KeyState state; + + // Start at something in our range. + state = it.checkKey(BSON("" << 7 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // The rightmost key is past the range. We should be told to move past the key before the + // one whose interval we exhausted. + state = it.checkKey(BSON("" << 7 << "" << 5.00001), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT_EQUALS(movePastKeyElts, true); + + // Move a little forward, but note that the rightmost key isn't in the interval yet. + state = it.checkKey(BSON("" << 7.2 << "" << 0), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT_EQUALS(movePastKeyElts, false); + ASSERT_EQUALS(elt[1]->numberInt(), 0); + ASSERT_EQUALS(inc[1], false); + + // Move to the edge of both intervals, 20,5 + state = it.checkKey(BSON("" << 20 << "" << 5), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // And a little beyond. + state = it.checkKey(BSON("" << 20 << "" << 5.1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::DONE); + } + + TEST(IndexBoundsCheckerTest, FirstKeyMovedIsOKSecondKeyMustMove) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 0 << "" << 9), true, true)); + fooList.intervals.push_back(Interval(BSON("" << 10 << "" << 20), true, true)); + + OrderedIntervalList barList("bar"); + barList.intervals.push_back(Interval(BSON("" << 0 << "" << 5), false, true)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + bounds.fields.push_back(barList); + IndexBoundsChecker it(&bounds, BSON("foo" << 1 << "bar" << 1), 1); + + int keyEltsToUse; + bool movePastKeyElts; + vector elt(2); + vector inc(2); + + IndexBoundsChecker::KeyState state; + + // Start at something in our range. + state = it.checkKey(BSON("" << 0 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // First key moves to next interval, second key needs to be advanced. + state = it.checkKey(BSON("" << 10 << "" << -1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT_EQUALS(movePastKeyElts, false); + ASSERT_EQUALS(elt[1]->numberInt(), 0); + ASSERT_EQUALS(inc[1], false); + } + + TEST(IndexBoundsCheckerTest, SecondIntervalMustRewind) { + OrderedIntervalList first("first"); + first.intervals.push_back(Interval(BSON("" << 25 << "" << 30), true, true)); + + OrderedIntervalList second("second"); + second.intervals.push_back(Interval(BSON("" << 0 << "" << 0), true, true)); + second.intervals.push_back(Interval(BSON("" << 9 << "" << 9), true, true)); + + IndexBounds bounds; + bounds.fields.push_back(first); + bounds.fields.push_back(second); + + BSONObj idx = BSON("first" << 1 << "second" << 1); + ASSERT(bounds.isValidFor(idx, 1)); + IndexBoundsChecker it(&bounds, idx, 1); + + int keyEltsToUse; + bool movePastKeyElts; + + vector elt(2); + vector inc(2); + + IndexBoundsChecker::KeyState state; + + state = it.checkKey(BSON("" << 25 << "" << 0), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + state = it.checkKey(BSON("" << 25 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT_EQUALS(movePastKeyElts, false); + ASSERT_EQUALS(elt[1]->numberInt(), 9); + ASSERT_EQUALS(inc[1], true); + + state = it.checkKey(BSON("" << 25 << "" << 9), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // First key moved forward. The second key moved back to a valid state but it's behind + // the interval that the checker thought it was in. + state = it.checkKey(BSON("" << 26 << "" << 0), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + } + + TEST(IndexBoundsCheckerTest, SimpleCheckKeyBackwards) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 20 << "" << 7), true, true)); + + OrderedIntervalList barList("bar"); + barList.intervals.push_back(Interval(BSON("" << 5 << "" << 0), true, false)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + bounds.fields.push_back(barList); + + BSONObj idx = BSON("foo" << -1 << "bar" << -1); + ASSERT(bounds.isValidFor(idx, 1)); + IndexBoundsChecker it(&bounds, idx, 1); + + int keyEltsToUse; + bool movePastKeyElts; + vector elt(2); + vector inc(2); + + IndexBoundsChecker::KeyState state; + + // Start at something in our range. + state = it.checkKey(BSON("" << 20 << "" << 5), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // The rightmost key is past the range. We should be told to move past the key before the + // one whose interval we exhausted. + state = it.checkKey(BSON("" << 20 << "" << 0), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT_EQUALS(movePastKeyElts, true); + + // Move a little forward, but note that the rightmost key isn't in the interval yet. + state = it.checkKey(BSON("" << 19 << "" << 6), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT_EQUALS(movePastKeyElts, false); + ASSERT_EQUALS(elt[1]->numberInt(), 5); + ASSERT_EQUALS(inc[1], true); + + // Move to the edge of both intervals + state = it.checkKey(BSON("" << 7 << "" << 0.01), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // And a little beyond. + state = it.checkKey(BSON("" << 7 << "" << 0), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::DONE); + } + + TEST(IndexBoundsCheckerTest, CheckEndBackwards) { + OrderedIntervalList fooList("foo"); + fooList.intervals.push_back(Interval(BSON("" << 30 << "" << 21), true, true)); + fooList.intervals.push_back(Interval(BSON("" << 20 << "" << 7), true, false)); + + OrderedIntervalList barList("bar"); + barList.intervals.push_back(Interval(BSON("" << 0 << "" << 5), false, false)); + + IndexBounds bounds; + bounds.fields.push_back(fooList); + bounds.fields.push_back(barList); + + BSONObj idx = BSON("foo" << 1 << "bar" << -1); + ASSERT(bounds.isValidFor(idx, -1)); + IndexBoundsChecker it(&bounds, idx, -1); + + int keyEltsToUse; + bool movePastKeyElts; + vector elt(2); + vector inc(2); + + IndexBoundsChecker::KeyState state; + + // Start at something in our range. + state = it.checkKey(BSON("" << 30 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // Second field moves past the end, but we're not done, since there's still an interval in + // the previous field that the key hasn't advanced to. + state = it.checkKey(BSON("" << 30 << "" << 5), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT(movePastKeyElts); + + // The next index key is in the second interval for 'foo' and there is a valid interval for + // 'bar'. + state = it.checkKey(BSON("" << 20 << "" << 1), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::VALID); + + // The next index key is very close to the end of the open interval for foo, and it's past + // the interval for 'bar'. Since the interval for foo is open, we are asked to move + // forward, since we possibly could. + state = it.checkKey(BSON("" << 7.001 << "" << 5), + &keyEltsToUse, + &movePastKeyElts, + &elt, + &inc); + ASSERT_EQUALS(state, IndexBoundsChecker::MUST_ADVANCE); + ASSERT_EQUALS(keyEltsToUse, 1); + ASSERT(movePastKeyElts); + } + + // + // IndexBoundsChecker::findIntervalForField + // + + /** + * Returns string representation of IndexBoundsChecker::Location. + */ + string toString(IndexBoundsChecker::Location location) { + switch(location) { + case IndexBoundsChecker::BEHIND: return "BEHIND"; + case IndexBoundsChecker::WITHIN: return "WITHIN"; + case IndexBoundsChecker::AHEAD: return "AHEAD"; + } + invariant(0); + } + + /** + * Test function for findIntervalForField. + * Constructs a list of point intervals from 'points' and searches for 'key' + * using findIntervalForField(). Verifies expected location and index (if expectedLocation + * is BEHIND or WITHIN). + * 'points' is provided in BSON format: {points: [pt1, pt2, pt4, ...] + */ + void testFindIntervalForField(int key, const BSONObj& pointsObj, const int expectedDirection, + IndexBoundsChecker::Location expectedLocation, + size_t expectedIntervalIndex) { + // Create key BSONElement. + BSONObj keyObj = BSON("" << key); + BSONElement keyElt = keyObj.firstElement(); + + // Construct point intervals. + OrderedIntervalList oil("foo"); + BSONObjIterator i(pointsObj.getObjectField("points")); + while (i.more()) { + BSONElement e = i.next(); + int j = e.numberInt(); + oil.intervals.push_back(Interval(BSON("" << j << "" << j), true, true)); + } + size_t intervalIndex = 0; + IndexBoundsChecker::Location location = + IndexBoundsChecker::findIntervalForField(keyElt, oil, expectedDirection, &intervalIndex); + if (expectedLocation != location) { + mongoutils::str::stream ss; + ss << "Unexpected location from findIntervalForField: key=" << keyElt + << "; intervals=" << oil.toString() << "; direction=" << expectedDirection + << ". Expected: " << toString(expectedLocation) + << ". Actual: " << toString(location); + FAIL(ss); + } + // Check interval index if location is BEHIND or WITHIN. + if ((IndexBoundsChecker::BEHIND == expectedLocation || + IndexBoundsChecker::WITHIN == expectedLocation) && + expectedIntervalIndex != intervalIndex) { + mongoutils::str::stream ss; + ss << "Unexpected interval index from findIntervalForField: key=" << keyElt + << "; intervals=" << oil.toString() << "; direction=" << expectedDirection + << "; location= " << toString(location) + << ". Expected: " << expectedIntervalIndex + << ". Actual: " << intervalIndex; + FAIL(ss); + } + } + + TEST(IndexBoundsCheckerTest, FindIntervalForField) { + // No intervals + BSONObj pointsObj = fromjson("{points: []}"); + testFindIntervalForField(5, pointsObj, 1, IndexBoundsChecker::AHEAD, 0U); + testFindIntervalForField(5, pointsObj, -1, IndexBoundsChecker::AHEAD, 0U); + + // One interval + pointsObj = fromjson("{points: [5]}"); + testFindIntervalForField(4, pointsObj, 1, IndexBoundsChecker::BEHIND, 0U); + testFindIntervalForField(5, pointsObj, 1, IndexBoundsChecker::WITHIN, 0U); + testFindIntervalForField(6, pointsObj, 1, IndexBoundsChecker::AHEAD, 0U); + + // One interval - reverse direction + pointsObj = fromjson("{points: [5]}"); + testFindIntervalForField(6, pointsObj, -1, IndexBoundsChecker::BEHIND, 0U); + testFindIntervalForField(5, pointsObj, -1, IndexBoundsChecker::WITHIN, 0U); + testFindIntervalForField(4, pointsObj, -1, IndexBoundsChecker::AHEAD, 0U); + + // Two intervals + // Verifies off-by-one handling in upper bound of binary search. + pointsObj = fromjson("{points: [5, 7]}"); + testFindIntervalForField(4, pointsObj, 1, IndexBoundsChecker::BEHIND, 0U); + testFindIntervalForField(5, pointsObj, 1, IndexBoundsChecker::WITHIN, 0U); + testFindIntervalForField(6, pointsObj, 1, IndexBoundsChecker::BEHIND, 1U); + testFindIntervalForField(7, pointsObj, 1, IndexBoundsChecker::WITHIN, 1U); + testFindIntervalForField(8, pointsObj, 1, IndexBoundsChecker::AHEAD, 0U); + + // Two intervals - reverse direction + // Verifies off-by-one handling in upper bound of binary search. + pointsObj = fromjson("{points: [7, 5]}"); + testFindIntervalForField(8, pointsObj, -1, IndexBoundsChecker::BEHIND, 0U); + testFindIntervalForField(7, pointsObj, -1, IndexBoundsChecker::WITHIN, 0U); + testFindIntervalForField(6, pointsObj, -1, IndexBoundsChecker::BEHIND, 1U); + testFindIntervalForField(5, pointsObj, -1, IndexBoundsChecker::WITHIN, 1U); + testFindIntervalForField(4, pointsObj, -1, IndexBoundsChecker::AHEAD, 0U); + + // Multiple intervals - odd number of intervals. + pointsObj = fromjson("{points: [1, 3, 5, 7, 9]}"); + testFindIntervalForField(0, pointsObj, 1, IndexBoundsChecker::BEHIND, 0U); + testFindIntervalForField(1, pointsObj, 1, IndexBoundsChecker::WITHIN, 0U); + testFindIntervalForField(2, pointsObj, 1, IndexBoundsChecker::BEHIND, 1U); + testFindIntervalForField(3, pointsObj, 1, IndexBoundsChecker::WITHIN, 1U); + testFindIntervalForField(4, pointsObj, 1, IndexBoundsChecker::BEHIND, 2U); + testFindIntervalForField(5, pointsObj, 1, IndexBoundsChecker::WITHIN, 2U); + testFindIntervalForField(6, pointsObj, 1, IndexBoundsChecker::BEHIND, 3U); + testFindIntervalForField(7, pointsObj, 1, IndexBoundsChecker::WITHIN, 3U); + testFindIntervalForField(8, pointsObj, 1, IndexBoundsChecker::BEHIND, 4U); + testFindIntervalForField(9, pointsObj, 1, IndexBoundsChecker::WITHIN, 4U); + testFindIntervalForField(10, pointsObj, 1, IndexBoundsChecker::AHEAD, 0U); + + // Multiple intervals - even number of intervals, reverse direction + // Interval order has to match direction. + pointsObj = fromjson("{points: [7, 5, 3, 1]}"); + testFindIntervalForField(8, pointsObj, -1, IndexBoundsChecker::BEHIND, 0U); + testFindIntervalForField(7, pointsObj, -1, IndexBoundsChecker::WITHIN, 0U); + testFindIntervalForField(6, pointsObj, -1, IndexBoundsChecker::BEHIND, 1U); + testFindIntervalForField(5, pointsObj, -1, IndexBoundsChecker::WITHIN, 1U); + testFindIntervalForField(4, pointsObj, -1, IndexBoundsChecker::BEHIND, 2U); + testFindIntervalForField(3, pointsObj, -1, IndexBoundsChecker::WITHIN, 2U); + testFindIntervalForField(2, pointsObj, -1, IndexBoundsChecker::BEHIND, 3U); + testFindIntervalForField(1, pointsObj, -1, IndexBoundsChecker::WITHIN, 3U); + testFindIntervalForField(0, pointsObj, -1, IndexBoundsChecker::AHEAD, 0U); + } + +} // namespace diff --git a/src/mongo/db/query/index_entry.h b/src/mongo/db/query/index_entry.h new file mode 100644 index 00000000000..0391dcb7d95 --- /dev/null +++ b/src/mongo/db/query/index_entry.h @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/index_names.h" +#include "mongo/db/jsobj.h" + +namespace mongo { + + /** + * This name sucks, but every name involving 'index' is used somewhere. + */ + struct IndexEntry { + /** + * Use this constructor if you're making an IndexEntry from the catalog. + */ + IndexEntry(const BSONObj& kp, + const string& accessMethod, + bool mk, + bool sp, + bool unq, + const string& n, + const BSONObj& io) + : keyPattern(kp), + multikey(mk), + sparse(sp), + unique(unq), + name(n), + infoObj(io) { + + type = IndexNames::nameToType(accessMethod); + } + + /** + * For testing purposes only. + */ + IndexEntry(const BSONObj& kp, + bool mk, + bool sp, + bool unq, + const string& n, + const BSONObj& io) + : keyPattern(kp), + multikey(mk), + sparse(sp), + unique(unq), + name(n), + infoObj(io) { + + type = IndexNames::nameToType(IndexNames::findPluginName(keyPattern)); + } + + /** + * For testing purposes only. + */ + IndexEntry(const BSONObj& kp) + : keyPattern(kp), + multikey(false), + sparse(false), + unique(false), + name("test_foo"), + infoObj(BSONObj()) { + + type = IndexNames::nameToType(IndexNames::findPluginName(keyPattern)); + } + + BSONObj keyPattern; + + bool multikey; + + bool sparse; + + bool unique; + + string name; + + // Geo indices have extra parameters. We need those available to plan correctly. + BSONObj infoObj; + + // What type of index is this? (What access method can we use on the index described + // by the keyPattern?) + IndexType type; + + std::string toString() const { + mongoutils::str::stream ss; + ss << "kp: " << keyPattern.toString(); + + if (multikey) { + ss << " multikey"; + } + + if (sparse) { + ss << " sparse"; + } + + if (!infoObj.isEmpty()) { + ss << " io: " << infoObj.toString(); + } + + return ss; + } + }; + +} // namespace mongo diff --git a/src/mongo/db/query/index_tag.cpp b/src/mongo/db/query/index_tag.cpp new file mode 100644 index 00000000000..7b81a23f556 --- /dev/null +++ b/src/mongo/db/query/index_tag.cpp @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/query/index_tag.h" + +#include "mongo/db/query/indexability.h" + +#include +#include + +namespace mongo { + + // TODO: Move out of the enumerator and into the planner. + + const size_t IndexTag::kNoIndex = std::numeric_limits::max(); + + void tagForSort(MatchExpression* tree) { + if (!Indexability::nodeCanUseIndexOnOwnField(tree)) { + size_t myTagValue = IndexTag::kNoIndex; + for (size_t i = 0; i < tree->numChildren(); ++i) { + MatchExpression* child = tree->getChild(i); + tagForSort(child); + IndexTag* childTag = static_cast(child->getTag()); + if (NULL != childTag) { + myTagValue = std::min(myTagValue, childTag->index); + } + } + if (myTagValue != IndexTag::kNoIndex) { + tree->setTag(new IndexTag(myTagValue)); + } + } + } + + bool TagComparison(const MatchExpression* lhs, const MatchExpression* rhs) { + IndexTag* lhsTag = static_cast(lhs->getTag()); + size_t lhsValue = (NULL == lhsTag) ? IndexTag::kNoIndex : lhsTag->index; + size_t lhsPos = (NULL == lhsTag) ? IndexTag::kNoIndex : lhsTag->pos; + + IndexTag* rhsTag = static_cast(rhs->getTag()); + size_t rhsValue = (NULL == rhsTag) ? IndexTag::kNoIndex : rhsTag->index; + size_t rhsPos = (NULL == rhsTag) ? IndexTag::kNoIndex : rhsTag->pos; + + // First, order on indices. + if (lhsValue != rhsValue) { + // This relies on kNoIndex being larger than every other possible index. + return lhsValue < rhsValue; + } + + // Next, order so that if there's a GEO_NEAR it's first. + if (MatchExpression::GEO_NEAR == lhs->matchType()) { + return true; + } + else if (MatchExpression::GEO_NEAR == rhs->matchType()) { + return false; + } + + // Ditto text. + if (MatchExpression::TEXT == lhs->matchType()) { + return true; + } + else if (MatchExpression::TEXT == rhs->matchType()) { + return false; + } + + // Next, order so that the first field of a compound index appears first. + if (lhsPos != rhsPos) { + return lhsPos < rhsPos; + } + + // Next, order on fields. + int cmp = lhs->path().compare(rhs->path()); + if (0 != cmp) { + return 0; + } + + // Finally, order on expression type. + return lhs->matchType() < rhs->matchType(); + } + + void sortUsingTags(MatchExpression* tree) { + for (size_t i = 0; i < tree->numChildren(); ++i) { + sortUsingTags(tree->getChild(i)); + } + std::vector* children = tree->getChildVector(); + if (NULL != children) { + std::sort(children->begin(), children->end(), TagComparison); + } + } + +} // namespace mongo diff --git a/src/mongo/db/query/index_tag.h b/src/mongo/db/query/index_tag.h new file mode 100644 index 00000000000..060bf7212cc --- /dev/null +++ b/src/mongo/db/query/index_tag.h @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include + +#include "mongo/bson/util/builder.h" +#include "mongo/db/matcher/expression.h" + +namespace mongo { + + // output from enumerator to query planner + class IndexTag : public MatchExpression::TagData { + public: + static const size_t kNoIndex; + + IndexTag() : index(kNoIndex), pos(0) {} + IndexTag(size_t i) : index(i), pos(0) { } + IndexTag(size_t i, size_t p) : index(i), pos(p) { } + + virtual ~IndexTag() { } + + virtual void debugString(StringBuilder* builder) const { + *builder << " || Selected Index #" << index << " pos " << pos; + } + + virtual MatchExpression::TagData* clone() const { + return new IndexTag(index, pos); + } + + // What index should we try to use for this leaf? + size_t index; + + // What position are we in the index? (Compound.) + size_t pos; + }; + + // used internally + class RelevantTag : public MatchExpression::TagData { + public: + RelevantTag() : elemMatchExpr(NULL), pathPrefix("") { } + + std::vector first; + std::vector notFirst; + + // We don't know the full path from a node unless we keep notes as we traverse from the + // root. We do this once and store it. + // TODO: Do a FieldRef / StringData pass. + // TODO: We might want this inside of the MatchExpression. + string path; + + // Points to the innermost containing $elemMatch. If this tag is + // attached to an expression not contained in an $elemMatch, then + // 'elemMatchExpr' is NULL. Not owned here. + MatchExpression* elemMatchExpr; + + // If not contained inside an elemMatch, 'pathPrefix' contains the + // part of 'path' prior to the first dot. For example, if 'path' is + // "a.b.c", then 'pathPrefix' is "a". If 'path' is just "a", then + // 'pathPrefix' is also "a". + // + // If tagging a predicate contained in an $elemMatch, 'pathPrefix' + // holds the prefix of the path *inside* the $elemMatch. If this + // tags predicate {a: {$elemMatch: {"b.c": {$gt: 1}}}}, then + // 'pathPrefix' is "b". + // + // Used by the plan enumerator to make sure that we never + // compound two predicates sharing a path prefix. + std::string pathPrefix; + + virtual void debugString(StringBuilder* builder) const { + *builder << " || First: "; + for (size_t i = 0; i < first.size(); ++i) { + *builder << first[i] << " "; + } + *builder << "notFirst: "; + for (size_t i = 0; i < notFirst.size(); ++i) { + *builder << notFirst[i] << " "; + } + *builder << "full path: " << path; + } + + virtual MatchExpression::TagData* clone() const { + RelevantTag* ret = new RelevantTag(); + ret->first = first; + ret->notFirst = notFirst; + return ret; + } + }; + + /** + * Tags each node of the tree with the lowest numbered index that the sub-tree rooted at that + * node uses. + * + * Nodes that satisfy Indexability::nodeCanUseIndexOnOwnField are already tagged if there + * exists an index that that node can use. + */ + void tagForSort(MatchExpression* tree); + + /** + * Sorts the tree using its IndexTag(s). Nodes that use the same index are adjacent to one + * another. + */ + void sortUsingTags(MatchExpression* tree); + +} // namespace mongo diff --git a/src/mongo/db/query/indexability.h b/src/mongo/db/query/indexability.h new file mode 100644 index 00000000000..03820679436 --- /dev/null +++ b/src/mongo/db/query/indexability.h @@ -0,0 +1,133 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/matcher/expression.h" + +#pragma once + +namespace mongo { + + /** + * Logic for how indices can be used with an expression. + */ + class Indexability { + public: + /** + * Is an index over me->path() useful? + * This is the same thing as being sargable, if you have a RDBMS background. + */ + static bool nodeCanUseIndexOnOwnField(const MatchExpression* me) { + if (me->path().empty()) { + return false; + } + + if (arrayUsesIndexOnOwnField(me)) { + return true; + } + + return isIndexOnOwnFieldTypeNode(me); + } + + /** + * This array operator doesn't have any children with fields and can use an index. + * + * Example: a: {$elemMatch: {$gte: 1, $lte: 1}}. + */ + static bool arrayUsesIndexOnOwnField(const MatchExpression* me) { + if (!me->isArray()) { + return false; + } + + if (MatchExpression::ELEM_MATCH_VALUE != me->matchType()) { + return false; + } + + // We have an ELEM_MATCH_VALUE expression. In order to be + // considered "indexable" all children of the ELEM_MATCH_VALUE + // must be "indexable" type expressions as well. + for (size_t i = 0; i < me->numChildren(); i++) { + if (!isIndexOnOwnFieldTypeNode(me->getChild(i))) { + return false; + } + } + + return true; + } + + /** + * Certain array operators require that the field for that operator is prepended + * to all fields in that operator's children. + * + * Example: a: {$elemMatch: {b:1, c:1}}. + */ + static bool arrayUsesIndexOnChildren(const MatchExpression* me) { + return me->isArray() && MatchExpression::ELEM_MATCH_OBJECT == me->matchType(); + } + + /** + * Returns true if 'me' is a NOT, and the child of the NOT can use + * an index on its own field. + */ + static bool isBoundsGeneratingNot(const MatchExpression* me) { + return MatchExpression::NOT == me->matchType() && + nodeCanUseIndexOnOwnField(me->getChild(0)); + } + + /** + * Returns true if either 'me' is a bounds generating NOT, + * or 'me' can use an index on its own field. + */ + static bool isBoundsGenerating(const MatchExpression* me) { + return isBoundsGeneratingNot(me) || nodeCanUseIndexOnOwnField(me); + } + + private: + /** + * Returns true if 'me' is "sargable" but is not a negation and + * is not an array node such as ELEM_MATCH_VALUE. + * + * Used as a helper for nodeCanUseIndexOnOwnField(). + */ + static bool isIndexOnOwnFieldTypeNode(const MatchExpression* me) { + return me->matchType() == MatchExpression::LTE + || me->matchType() == MatchExpression::LT + || me->matchType() == MatchExpression::EQ + || me->matchType() == MatchExpression::GT + || me->matchType() == MatchExpression::GTE + || me->matchType() == MatchExpression::REGEX + || me->matchType() == MatchExpression::MOD + || me->matchType() == MatchExpression::MATCH_IN + || me->matchType() == MatchExpression::TYPE_OPERATOR + || me->matchType() == MatchExpression::GEO + || me->matchType() == MatchExpression::GEO_NEAR + || me->matchType() == MatchExpression::EXISTS + || me->matchType() == MatchExpression::TEXT; + } + }; + +} // namespace mongo diff --git a/src/mongo/db/query/internal_plans.h b/src/mongo/db/query/internal_plans.h new file mode 100644 index 00000000000..fa5597dfbc1 --- /dev/null +++ b/src/mongo/db/query/internal_plans.h @@ -0,0 +1,118 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/catalog/database.h" +#include "mongo/db/client.h" +#include "mongo/db/exec/collection_scan.h" +#include "mongo/db/exec/fetch.h" +#include "mongo/db/exec/index_scan.h" +#include "mongo/db/query/eof_runner.h" +#include "mongo/db/query/internal_runner.h" + +namespace mongo { + + /** + * The internal planner is a one-stop shop for "off-the-shelf" plans. Most internal procedures + * that do not require advanced queries could be served by plans already in here. + */ + class InternalPlanner { + public: + enum Direction { + FORWARD = 1, + BACKWARD = -1, + }; + + enum IndexScanOptions { + // The client is interested in the default outputs of an index scan: BSONObj of the key, + // DiskLoc of the record that's indexed. The client does its own fetching if required. + IXSCAN_DEFAULT = 0, + + // The client wants the fetched object and the DiskLoc that refers to it. Delegating + // the fetch to the runner allows fetching outside of a lock. + IXSCAN_FETCH = 1, + }; + + /** + * Return a collection scan. Caller owns pointer. + */ + static Runner* collectionScan(const StringData& ns, // TODO: make this a Collection* + const Direction direction = FORWARD, + const DiskLoc startLoc = DiskLoc()) { + Collection* collection = cc().database()->getCollection(ns); + if (NULL == collection) { return new EOFRunner(NULL, ns.toString()); } + + CollectionScanParams params; + params.ns = ns.toString(); + params.start = startLoc; + + if (FORWARD == direction) { + params.direction = CollectionScanParams::FORWARD; + } + else { + params.direction = CollectionScanParams::BACKWARD; + } + + WorkingSet* ws = new WorkingSet(); + CollectionScan* cs = new CollectionScan(params, ws, NULL); + return new InternalRunner(collection, cs, ws); + } + + /** + * Return an index scan. Caller owns returned pointer. + */ + static Runner* indexScan(const Collection* collection, + const IndexDescriptor* descriptor, + const BSONObj& startKey, const BSONObj& endKey, + bool endKeyInclusive, Direction direction = FORWARD, + int options = 0) { + invariant(collection); + invariant(descriptor); + + IndexScanParams params; + params.descriptor = descriptor; + params.direction = direction; + params.bounds.isSimpleRange = true; + params.bounds.startKey = startKey; + params.bounds.endKey = endKey; + params.bounds.endKeyInclusive = endKeyInclusive; + + WorkingSet* ws = new WorkingSet(); + IndexScan* ix = new IndexScan(params, ws, NULL); + + if (IXSCAN_FETCH & options) { + return new InternalRunner(collection, new FetchStage(ws, ix, NULL), ws); + } + else { + return new InternalRunner(collection, ix, ws); + } + } + }; + +} // namespace mongo diff --git a/src/mongo/db/query/internal_runner.cpp b/src/mongo/db/query/internal_runner.cpp new file mode 100644 index 00000000000..eea40aa6c7c --- /dev/null +++ b/src/mongo/db/query/internal_runner.cpp @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/internal_runner.h" + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/plan_executor.h" +#include "mongo/db/query/type_explain.h" + +namespace mongo { + + InternalRunner::InternalRunner(const Collection* collection, PlanStage* root, WorkingSet* ws) + : _collection(collection), + _exec(new PlanExecutor(ws, root)), + _policy(Runner::YIELD_MANUAL) { + invariant( collection ); + } + + InternalRunner::~InternalRunner() { + if (Runner::YIELD_AUTO == _policy && _collection) { + _collection->cursorCache()->deregisterRunner(this); + } + } + + Runner::RunnerState InternalRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { + return _exec->getNext(objOut, dlOut); + } + + bool InternalRunner::isEOF() { + return _exec->isEOF(); + } + + void InternalRunner::saveState() { + _exec->saveState(); + } + + bool InternalRunner::restoreState() { + return _exec->restoreState(); + } + + const std::string& InternalRunner::ns() { + return _collection->ns().ns(); + } + + void InternalRunner::invalidate(const DiskLoc& dl, InvalidationType type) { + _exec->invalidate(dl, type); + } + + void InternalRunner::setYieldPolicy(Runner::YieldPolicy policy) { + // No-op. + if (_policy == policy) { return; } + + invariant( _collection ); + + if (Runner::YIELD_AUTO == policy) { + // Going from manual to auto. + _collection->cursorCache()->registerRunner(this); + } + else { + // Going from auto to manual. + _collection->cursorCache()->deregisterRunner(this); + } + + _policy = policy; + _exec->setYieldPolicy(policy); + } + + void InternalRunner::kill() { + _exec->kill(); + _collection = NULL; + } + + Status InternalRunner::getInfo(TypeExplain** explain, + PlanInfo** planInfo) const { + if (NULL != explain) { + verify(_exec.get()); + + scoped_ptr stats(_exec->getStats()); + if (NULL == stats.get()) { + return Status(ErrorCodes::InternalError, "no stats available to explain plan"); + } + + Status status = explainPlan(*stats, explain, true /* full details */); + if (!status.isOK()) { + return status; + } + + // Fill in explain fields that are accounted by on the runner level. + TypeExplain* chosenPlan = NULL; + explainPlan(*stats, &chosenPlan, false /* no full details */); + if (chosenPlan) { + (*explain)->addToAllPlans(chosenPlan); + } + (*explain)->setNScannedObjectsAllPlans((*explain)->getNScannedObjects()); + (*explain)->setNScannedAllPlans((*explain)->getNScanned()); + } + else if (NULL != planInfo) { + *planInfo = new PlanInfo(); + (*planInfo)->planSummary = "INTERNAL"; + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/internal_runner.h b/src/mongo/db/query/internal_runner.h new file mode 100644 index 00000000000..b2147bbc529 --- /dev/null +++ b/src/mongo/db/query/internal_runner.h @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/query/runner.h" + +namespace mongo { + + class BSONObj; + class CanonicalQuery; + class DiskLoc; + class PlanExecutor; + class PlanStage; + struct QuerySolution; + class TypeExplain; + struct PlanInfo; + class WorkingSet; + + /** + * This is a runner that was requested by an internal client of the query system, as opposed to + * runners that are built in response to a query entering the system. It is only used by + * internal clients of the query systems (e.g., chunk migration, index building, commands that + * traverse data such as md5, ... ) + * + * The salient feature of this Runner is that it does not interact with the cache at all. + */ + class InternalRunner : public Runner { + public: + + /** Takes ownership of root and ws. */ + InternalRunner(const Collection* collection, PlanStage* root, WorkingSet* ws); + + virtual ~InternalRunner(); + + Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + virtual bool isEOF(); + + virtual void saveState(); + + virtual bool restoreState(); + + virtual const std::string& ns(); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual void setYieldPolicy(Runner::YieldPolicy policy); + + virtual void kill(); + + virtual const Collection* collection() { return _collection; } + + /** + * Returns OK, allocating and filling in '*explain' with details of the plan used by + * this runner. Caller takes ownership of '*explain'. Similarly fills in '*planInfo', + * which the caller takes ownership of. Otherwise, return a status describing the + * error. + * + * Strictly speaking, an InternalRunner's explain is never exposed, simply because an + * InternalRunner itself is not exposed. But we implement the explain here anyway so + * to help in debugging situations. + */ + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const; + + private: + const Collection* _collection; + + boost::scoped_ptr _exec; + Runner::YieldPolicy _policy; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/interval.cpp b/src/mongo/db/query/interval.cpp new file mode 100644 index 00000000000..c42feefa2c2 --- /dev/null +++ b/src/mongo/db/query/interval.cpp @@ -0,0 +1,316 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/query/interval.h" + +namespace mongo { + + Interval::Interval() + : _intervalData(BSONObj()), start(BSONElement()), startInclusive(false), end(BSONElement()), + endInclusive(false) { } + + Interval::Interval(BSONObj base, bool si, bool ei) { + init(base, si, ei); + } + + void Interval::init(BSONObj base, bool si, bool ei) { + verify(base.nFields() >= 2); + + _intervalData = base.getOwned(); + BSONObjIterator it(_intervalData); + start = it.next(); + end = it.next(); + startInclusive = si; + endInclusive = ei; + } + + bool Interval::isEmpty() const { + return _intervalData.nFields() == 0; + } + + bool Interval::isPoint() const { + return startInclusive && endInclusive && 0 == start.woCompare(end, false); + } + + bool Interval::isNull() const { + return (!startInclusive || !endInclusive) && 0 == start.woCompare(end, false); + } + + // + // Comparison + // + + bool Interval::equals(const Interval& other) const { + if (this->startInclusive != other.startInclusive) { + return false; + } + + if (this->endInclusive != other.endInclusive) { + return false; + } + + int res = this->start.woCompare(other.start, false); + if (res != 0) { + return false; + } + + res = this->end.woCompare(other.end, false); + if (res != 0) { + return false; + } + + return true; + } + + bool Interval::intersects(const Interval& other) const { + int res = this->start.woCompare(other.end, false); + if (res > 0) { + return false; + } + else if (res == 0 && (!this->startInclusive || !other.endInclusive)) { + return false; + } + + res = other.start.woCompare(this->end, false); + if (res > 0) { + return false; + } + else if (res == 0 && (!other.startInclusive || !this->endInclusive)) { + return false; + } + + return true; + } + + bool Interval::within(const Interval& other) const { + int res = this->start.woCompare(other.start, false); + if (res < 0) { + return false; + } + else if (res == 0 && this->startInclusive && !other.startInclusive) { + return false; + } + + res = this->end.woCompare(other.end, false); + if (res > 0) { + return false; + } + else if (res == 0 && this->endInclusive && !other.endInclusive) { + return false; + } + + return true; + } + + /** Returns true if the start of comes before the start of other */ + bool Interval::precedes(const Interval& other) const { + int res = this->start.woCompare(other.start, false); + if (res < 0) { + return true; + } + else if (res == 0 && this->startInclusive && !other.startInclusive) { + return true; + } + return false; + } + + + Interval::IntervalComparison Interval::compare(const Interval& other) const { + // + // Intersect cases + // + + if (this->intersects(other)) { + if (this->equals(other)) { + return INTERVAL_EQUALS; + } + if (this->within(other)) { + return INTERVAL_WITHIN; + } + if (other.within(*this)) { + return INTERVAL_CONTAINS; + } + if (this->precedes(other)) { + return INTERVAL_OVERLAPS_BEFORE; + } + return INTERVAL_OVERLAPS_AFTER; + } + + // + // Non-intersect cases + // + + if (this->precedes(other)) { + // It's not possible for both endInclusive and other.startInclusive to be true because + // the bounds would intersect. Refer to section on "Intersect cases" above. + if ((endInclusive || other.startInclusive) && 0 == end.woCompare(other.start, false)) { + return INTERVAL_PRECEDES_COULD_UNION; + } + return INTERVAL_PRECEDES; + } + + return INTERVAL_SUCCEEDS; + } + + // + // Mutation: Union and Intersection + // + + void Interval::intersect(const Interval& other, IntervalComparison cmp) { + if (cmp == INTERVAL_UNKNOWN) { + cmp = this->compare(other); + } + + BSONObjBuilder builder; + switch (cmp) { + + case INTERVAL_EQUALS: + case INTERVAL_WITHIN: + break; + + case INTERVAL_CONTAINS: + builder.append(other.start); + builder.append(other.end); + init(builder.obj(), other.startInclusive, other.endInclusive); + break; + + case INTERVAL_OVERLAPS_AFTER: + builder.append(start); + builder.append(other.end); + init(builder.obj(), startInclusive, other.endInclusive); + break; + + case INTERVAL_OVERLAPS_BEFORE: + builder.append(other.start); + builder.append(end); + init(builder.obj(), other.startInclusive, endInclusive); + break; + + case INTERVAL_PRECEDES: + case INTERVAL_SUCCEEDS: + *this = Interval(); + break; + + default: + verify(false); + } + } + + void Interval::combine(const Interval& other, IntervalComparison cmp) { + if (cmp == INTERVAL_UNKNOWN) { + cmp = this->compare(other); + } + + BSONObjBuilder builder; + switch (cmp) { + + case INTERVAL_EQUALS: + case INTERVAL_CONTAINS: + break; + + case INTERVAL_WITHIN: + builder.append(other.start); + builder.append(other.end); + init(builder.obj(), other.startInclusive, other.endInclusive); + break; + + case INTERVAL_OVERLAPS_AFTER: + case INTERVAL_SUCCEEDS: + builder.append(other.start); + builder.append(end); + init(builder.obj(), other.startInclusive, endInclusive); + break; + + case INTERVAL_OVERLAPS_BEFORE: + case INTERVAL_PRECEDES: + builder.append(start); + builder.append(other.end); + init(builder.obj(), startInclusive, other.endInclusive); + break; + + default: + verify(false); + } + } + + void Interval::reverse() { + std::swap(start, end); + std::swap(startInclusive, endInclusive); + } + + // + // Debug info + // + + // static + string Interval::cmpstr(IntervalComparison c) { + if (c == INTERVAL_EQUALS) { + return "INTERVAL_EQUALS"; + } + + // 'this' contains the other interval. + if (c == INTERVAL_CONTAINS) { + return "INTERVAL_CONTAINS"; + } + + // 'this' is contained by the other interval. + if (c == INTERVAL_WITHIN) { + return "INTERVAL_WITHIN"; + } + + // The two intervals intersect and 'this' is before the other interval. + if (c == INTERVAL_OVERLAPS_BEFORE) { + return "INTERVAL_OVERLAPS_BEFORE"; + } + + // The two intervals intersect and 'this is after the other interval. + if (c == INTERVAL_OVERLAPS_AFTER) { + return "INTERVAL_OVERLAPS_AFTER"; + } + + // There is no intersection. + if (c == INTERVAL_PRECEDES) { + return "INTERVAL_PRECEDES"; + } + + if (c == INTERVAL_PRECEDES_COULD_UNION) { + return "INTERVAL_PRECEDES_COULD_UNION"; + } + + if (c == INTERVAL_SUCCEEDS) { + return "INTERVAL_SUCCEEDS"; + } + + if (c == INTERVAL_UNKNOWN) { + return "INTERVAL_UNKNOWN"; + } + + return "NO IDEA DUDE"; + } + +} // namespace mongo diff --git a/src/mongo/db/query/interval.h b/src/mongo/db/query/interval.h new file mode 100644 index 00000000000..1d3e156af4a --- /dev/null +++ b/src/mongo/db/query/interval.h @@ -0,0 +1,196 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + /** A range of values for one field. */ + struct Interval { + + // No BSONValue means we have to keep a BSONObj and pointers (BSONElement) into it. + // 'start' may not point at the first field in _intervalData. + // 'end' may not point at the last field in _intervalData. + // 'start' and 'end' may point at the same field. + BSONObj _intervalData; + + // Start and End must be ordered according to the index order. + BSONElement start; + bool startInclusive; + + BSONElement end; + bool endInclusive; + + /** Creates an empty interval */ + Interval(); + + string toString() const { + mongoutils::str::stream ss; + if (startInclusive) { + ss << "["; + } + else { + ss << "("; + } + // false means omit the field name + ss << start.toString(false); + ss << ", "; + ss << end.toString(false); + if (endInclusive) { + ss << "]"; + } + else { + ss << ")"; + } + return ss; + } + + /** + * Creates an interval that starts at the first field of 'base' and ends at the second + * field of 'base'. (In other words, 'base' is a bsonobj with at least two elements, of + * which we don't care about field names.) + * + * The interval's extremities are closed or not depending on whether + * 'start'/'endIncluded' are true or not. + */ + Interval(BSONObj base, bool startIncluded, bool endIncluded); + + /** Sets the current interval to the given values (see constructor) */ + void init(BSONObj base, bool startIncluded, bool endIncluded); + + /** + * Returns true if an empty-constructed interval hasn't been init()-ialized yet + */ + bool isEmpty() const; + + /** + * Does this interval represent exactly one point? + */ + bool isPoint() const; + + /** + * Returns true if start is same as end and interval is open at either end + */ + bool isNull() const; + + // + // Comparison with other intervals + // + + /** + * Returns true if 'this' is the same interval as 'other' + */ + bool equals(const Interval& other) const; + + /** + * Returns true if 'this' overlaps with 'other', false otherwise. + */ + bool intersects(const Interval& rhs) const; + + /** + * Returns true if 'this' is within 'other', false otherwise. + */ + bool within(const Interval& other) const; + + /** + * Returns true if 'this' is located before 'other', false otherwise. + */ + bool precedes(const Interval& other) const; + + /** Returns how 'this' compares to 'other' */ + enum IntervalComparison { + // + // There is some intersection. + // + + // The two intervals are *exactly* equal. + INTERVAL_EQUALS, + + // 'this' contains the other interval. + INTERVAL_CONTAINS, + + // 'this' is contained by the other interval. + INTERVAL_WITHIN, + + // The two intervals intersect and 'this' is before the other interval. + INTERVAL_OVERLAPS_BEFORE, + + // The two intervals intersect and 'this is after the other interval. + INTERVAL_OVERLAPS_AFTER, + + // + // There is no intersection. + // + + INTERVAL_PRECEDES, + + // This happens if we have [a,b) [b,c] + INTERVAL_PRECEDES_COULD_UNION, + + INTERVAL_SUCCEEDS, + + INTERVAL_UNKNOWN + }; + + IntervalComparison compare(const Interval& other) const; + + /** + * toString for IntervalComparison + */ + static string cmpstr(IntervalComparison c); + + // + // Mutation of intervals + // + + /** + * Swap start and end points of interval. + */ + void reverse(); + + /** + * Updates 'this' with the intersection of 'this' and 'other'. If 'this' and 'other' + * have been compare()d before, that result can be optionally passed in 'cmp' + */ + void intersect(const Interval& other, IntervalComparison cmp = INTERVAL_UNKNOWN); + + /** + * Updates 'this" with the union of 'this' and 'other'. If 'this' and 'other' have + * been compare()d before, that result can be optionaly passed in 'cmp'. + */ + void combine(const Interval& other, IntervalComparison cmp = INTERVAL_UNKNOWN); + }; + + inline bool operator==(const Interval& lhs, const Interval& rhs) { + return lhs.compare(rhs) == Interval::INTERVAL_EQUALS; + } + +} // namespace mongo diff --git a/src/mongo/db/query/interval_test.cpp b/src/mongo/db/query/interval_test.cpp new file mode 100644 index 00000000000..62c4b815e53 --- /dev/null +++ b/src/mongo/db/query/interval_test.cpp @@ -0,0 +1,296 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/query/interval.h" + +#include "mongo/db/jsobj.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::BSONObj; + using mongo::Interval; + + // + // Comparison + // + + TEST(Comparison, Equality) { + Interval a(BSON("" << 0 << "" << 10), true, true); + ASSERT_EQUALS(a.compare(a), Interval::INTERVAL_EQUALS); + + Interval b(BSON("" << 0 << "" << 10), true, false); + ASSERT_NOT_EQUALS(a.compare(b), Interval::INTERVAL_EQUALS); + + Interval c(BSON("" << 0 << "" << 10), false, true); + ASSERT_NOT_EQUALS(a.compare(c), Interval::INTERVAL_EQUALS); + + Interval d(BSON("" << 0 << "" << 11), true, true); + ASSERT_NOT_EQUALS(a.compare(d), Interval::INTERVAL_EQUALS); + + Interval e(BSON("" << 1 << "" << 10), true, true); + ASSERT_NOT_EQUALS(a.compare(e), Interval::INTERVAL_EQUALS); + } + + TEST(Comparison, Contains) { + Interval a(BSON("" << 0 << "" << 10), true, true); + Interval b(BSON("" << 1 << "" << 9), true, true); + ASSERT_EQUALS(a.compare(b), Interval::INTERVAL_CONTAINS); + + Interval c(BSON("" << 0 << "" << 10), true, false); + ASSERT_EQUALS(a.compare(c), Interval::INTERVAL_CONTAINS); + + Interval d(BSON("" << 0 << "" << 10), false, true); + ASSERT_EQUALS(a.compare(d), Interval::INTERVAL_CONTAINS); + + Interval e(BSON("" << 0 << "" << 11), false, true); + ASSERT_NOT_EQUALS(a.compare(e), Interval::INTERVAL_CONTAINS); + } + + TEST(Comparison, Within) { + Interval a(BSON("" << 0 << "" << 10), true, true); + ASSERT_NOT_EQUALS(a.compare(a), Interval::INTERVAL_WITHIN); + + Interval b(BSON("" << 1 << "" << 9), true, true); + ASSERT_EQUALS(b.compare(a), Interval::INTERVAL_WITHIN); + + Interval c(BSON("" << 0 << "" << 10), true, false); + ASSERT_EQUALS(c.compare(a), Interval::INTERVAL_WITHIN); + + Interval d(BSON("" << 0 << "" << 10), false, true); + ASSERT_EQUALS(d.compare(a), Interval::INTERVAL_WITHIN); + + Interval e(BSON("" << 0 << "" << 11), false, true); + ASSERT_NOT_EQUALS(e.compare(a), Interval::INTERVAL_CONTAINS); + } + + TEST(Comparison, OverlapsBefore) { + Interval a(BSON("" << 1 << "" << 9), true, false); + ASSERT_NOT_EQUALS(a.compare(a), Interval::INTERVAL_OVERLAPS_BEFORE); + + Interval b(BSON("" << 1 << "" << 9), false, true); + ASSERT_EQUALS(a.compare(b), Interval::INTERVAL_OVERLAPS_BEFORE); + + Interval c(BSON("" << 1 << "" << 9), false, false); + ASSERT_NOT_EQUALS(a.compare(c), Interval::INTERVAL_OVERLAPS_BEFORE); + + Interval d(BSON("" << 2 << "" << 10), true, true); + ASSERT_EQUALS(a.compare(d), Interval::INTERVAL_OVERLAPS_BEFORE); + + Interval e(BSON("" << 0 << "" << 9), true, false); + ASSERT_NOT_EQUALS(a.compare(e), Interval::INTERVAL_OVERLAPS_BEFORE); + + Interval f(BSON("" << 0 << "" << 8), true, false); + ASSERT_NOT_EQUALS(a.compare(f), Interval::INTERVAL_OVERLAPS_BEFORE); + } + + TEST(Comparison, OverlapsAfter) { + Interval a(BSON("" << 1 << "" << 9), false, true); + ASSERT_NOT_EQUALS(a.compare(a), Interval::INTERVAL_OVERLAPS_AFTER); + + Interval b(BSON("" << 1 << "" << 9), true, false); + ASSERT_EQUALS(a.compare(b), Interval::INTERVAL_OVERLAPS_AFTER); + + Interval c(BSON("" << 1 << "" << 9), true, true); + ASSERT_NOT_EQUALS(a.compare(c), Interval::INTERVAL_OVERLAPS_AFTER); + + Interval d(BSON("" << 2 << "" << 10), true, true); + ASSERT_NOT_EQUALS(a.compare(d), Interval::INTERVAL_OVERLAPS_AFTER); + + Interval e(BSON("" << 0 << "" << 9), true, false); + ASSERT_EQUALS(a.compare(e), Interval::INTERVAL_OVERLAPS_AFTER); + } + + TEST(Comparison, Precedes) { + Interval a(BSON("" << 10 << "" << 20), true, true); + ASSERT_NOT_EQUALS(a.compare(a), Interval::INTERVAL_PRECEDES); + + Interval b(BSON("" << 0 << "" << 10), true, true); + ASSERT_NOT_EQUALS(b.compare(a), Interval::INTERVAL_PRECEDES); + + Interval c(BSON("" << 0 << "" << 10), true, false); + ASSERT_EQUALS(c.compare(a), Interval::INTERVAL_PRECEDES_COULD_UNION); + + Interval d(BSON("" << 0 << "" << 9), true, true); + ASSERT_EQUALS(d.compare(a), Interval::INTERVAL_PRECEDES); + + Interval e(BSON("" << 5 << "" << 15), true, true); + ASSERT_NOT_EQUALS(e.compare(a), Interval::INTERVAL_PRECEDES); + + Interval f(BSON("" << 5 << "" << 20), true, false); + ASSERT_NOT_EQUALS(f.compare(a), Interval::INTERVAL_PRECEDES); + } + + TEST(Comparison, PrecedesCouldUnion) { + Interval a(BSON("" << 10 << "" << 20), false, true); + ASSERT_NOT_EQUALS(a.compare(a), Interval::INTERVAL_PRECEDES); + + Interval b(BSON("" << 0 << "" << 10), true, false); + ASSERT_EQUALS(b.compare(a), Interval::INTERVAL_PRECEDES); + + Interval c(BSON("" << 0 << "" << 10), true, true); + ASSERT_EQUALS(c.compare(a), Interval::INTERVAL_PRECEDES_COULD_UNION); + } + + TEST(Comparison, Succeds) { + Interval a(BSON("" << 10 << "" << 20), true, true); + ASSERT_NOT_EQUALS(a.compare(a), Interval::INTERVAL_SUCCEEDS); + + Interval b(BSON("" << 20 << "" << 30), true, true); + ASSERT_NOT_EQUALS(b.compare(a), Interval::INTERVAL_SUCCEEDS); + + Interval c(BSON("" << 20 << "" << 30), false, true); + ASSERT_EQUALS(c.compare(a), Interval::INTERVAL_SUCCEEDS); + + Interval d(BSON("" << 21 << "" << 30), true, true); + ASSERT_EQUALS(d.compare(a), Interval::INTERVAL_SUCCEEDS); + + Interval e(BSON("" << 15 << "" << 30), true, true); + ASSERT_NOT_EQUALS(e.compare(a), Interval::INTERVAL_SUCCEEDS); + } + + // + // intersection + // + + TEST(Intersection, Equals) { + BSONObj itv = BSON("" << 10 << "" << 20); + Interval a(itv, true, true); + a.intersect(a); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Intersection, Contains) { + Interval a(BSON("" << 10 << "" << 20), true, true); + BSONObj itv = BSON("" << 11 << "" << 19); + Interval b(itv, true, true); + a.intersect(b); + ASSERT_EQUALS(a.compare(b), Interval::INTERVAL_EQUALS); + } + + TEST(Intersection, Within) { + BSONObj itv = BSON("" << 10 << "" << 20); + Interval a(itv, true, true); + Interval b(BSON("" << 9 << "" << 21), true, true); + a.intersect(b); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Intersection, OverlapsBefore) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 15 << "" << 25), true, true); + a.intersect(b); + + BSONObj itv = BSON("" << 15 << "" << 20); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Intersection, OverlapsAfter) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 5 << "" << 15), true, true); + a.intersect(b); + + BSONObj itv = BSON("" << 10 << "" << 15); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Intersection, Procedes) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 0 << "" << 5), true, true); + a.intersect(b); + + ASSERT_TRUE(a.isEmpty()); + } + + TEST(Intersection, Succeds) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 25 << "" << 30), true, true); + a.intersect(b); + + ASSERT_TRUE(a.isEmpty()); + } + + // + // combine (union) + // + + TEST(Union, Equals) { + BSONObj itv = BSON("" << 10 << "" << 20); + Interval a(itv, true, true); + a.combine(a); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Union, Contains) { + BSONObj itv = BSON("" << 10 << "" << 20); + Interval a(itv, true, true); + Interval b(BSON("" << 11 << "" << 19), true, true); + a.combine(b); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Union, Within) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 9 << "" << 21), true, true); + a.combine(b); + ASSERT_EQUALS(a.compare(b), Interval::INTERVAL_EQUALS); + } + + TEST(Union, OverlapsBefore) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 15 << "" << 25), true, true); + a.combine(b); + BSONObj itv = BSON("" << 10 << "" << 25); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Union, OverlapsAfter) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 5 << "" << 15), true, true); + a.combine(b); + BSONObj itv = BSON("" << 5 << "" << 20); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Union, Precedes) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 20 << "" << 30), true, true); + a.combine(b); + BSONObj itv = BSON("" << 10 << "" << 30); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + + TEST(Union, Succeds) { + Interval a(BSON("" << 10 << "" << 20), true, true); + Interval b(BSON("" << 0 << "" << 5), true, true); + a.combine(b); + BSONObj itv = BSON("" << 0 << "" << 20); + ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); + } + +} // unnamed namespace diff --git a/src/mongo/db/query/lite_parsed_query.cpp b/src/mongo/db/query/lite_parsed_query.cpp new file mode 100644 index 00000000000..c81efba9dcb --- /dev/null +++ b/src/mongo/db/query/lite_parsed_query.cpp @@ -0,0 +1,414 @@ +/** + * Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/db/query/lite_parsed_query.h" + +#include + +#include "mongo/db/dbmessage.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + const string LiteParsedQuery::cmdOptionMaxTimeMS("maxTimeMS"); + const string LiteParsedQuery::queryOptionMaxTimeMS("$maxTimeMS"); + + const string LiteParsedQuery::metaTextScore("textScore"); + const string LiteParsedQuery::metaGeoNearDistance("geoNearDistance"); + const string LiteParsedQuery::metaGeoNearPoint("geoNearPoint"); + const string LiteParsedQuery::metaDiskLoc("diskloc"); + const string LiteParsedQuery::metaIndexKey("indexKey"); + + // static + Status LiteParsedQuery::make(const QueryMessage& qm, LiteParsedQuery** out) { + auto_ptr pq(new LiteParsedQuery()); + + Status status = pq->init(qm.ns, qm.ntoskip, qm.ntoreturn, qm.queryOptions, qm.query, + qm.fields, true); + if (status.isOK()) { *out = pq.release(); } + return status; + } + + // static + Status LiteParsedQuery::make(const string& ns, int ntoskip, int ntoreturn, int queryOptions, + const BSONObj& query, const BSONObj& proj, const BSONObj& sort, + const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj, + bool snapshot, + bool explain, + LiteParsedQuery** out) { + auto_ptr pq(new LiteParsedQuery()); + pq->_sort = sort.getOwned(); + pq->_hint = hint.getOwned(); + pq->_min = minObj.getOwned(); + pq->_max = maxObj.getOwned(); + pq->_snapshot = snapshot; + pq->_explain = explain; + + Status status = pq->init(ns, ntoskip, ntoreturn, queryOptions, query, proj, false); + if (status.isOK()) { *out = pq.release(); } + return status; + } + + // static + StatusWith LiteParsedQuery::parseMaxTimeMSCommand(const BSONObj& cmdObj) { + return parseMaxTimeMS(cmdObj[cmdOptionMaxTimeMS]); + } + + // static + StatusWith LiteParsedQuery::parseMaxTimeMSQuery(const BSONObj& queryObj) { + return parseMaxTimeMS(queryObj[queryOptionMaxTimeMS]); + } + + // static + StatusWith LiteParsedQuery::parseMaxTimeMS(const BSONElement& maxTimeMSElt) { + if (!maxTimeMSElt.eoo() && !maxTimeMSElt.isNumber()) { + return StatusWith(ErrorCodes::BadValue, + (StringBuilder() + << maxTimeMSElt.fieldNameStringData() + << " must be a number").str()); + } + long long maxTimeMSLongLong = maxTimeMSElt.safeNumberLong(); // returns 0 on EOO + if (maxTimeMSLongLong < 0 || maxTimeMSLongLong > INT_MAX) { + return StatusWith(ErrorCodes::BadValue, + (StringBuilder() + << maxTimeMSElt.fieldNameStringData() + << " is out of range").str()); + } + double maxTimeMSDouble = maxTimeMSElt.numberDouble(); + if (maxTimeMSElt.type() == mongo::NumberDouble + && floor(maxTimeMSDouble) != maxTimeMSDouble) { + return StatusWith(ErrorCodes::BadValue, + (StringBuilder() + << maxTimeMSElt.fieldNameStringData() + << " has non-integral value").str()); + } + return StatusWith(static_cast(maxTimeMSLongLong)); + } + + // static + bool LiteParsedQuery::isTextScoreMeta(BSONElement elt) { + // elt must be foo: {$meta: "textScore"} + if (mongo::Object != elt.type()) { + return false; + } + BSONObj metaObj = elt.Obj(); + BSONObjIterator metaIt(metaObj); + // must have exactly 1 element + if (!metaIt.more()) { + return false; + } + BSONElement metaElt = metaIt.next(); + if (!mongoutils::str::equals("$meta", metaElt.fieldName())) { + return false; + } + if (mongo::String != metaElt.type()) { + return false; + } + if (LiteParsedQuery::metaTextScore != metaElt.valuestr()) { + return false; + } + // must have exactly 1 element + if (metaIt.more()) { + return false; + } + return true; + } + + // static + bool LiteParsedQuery::isDiskLocMeta(BSONElement elt) { + // elt must be foo: {$meta: "diskloc"} + if (mongo::Object != elt.type()) { + return false; + } + BSONObj metaObj = elt.Obj(); + BSONObjIterator metaIt(metaObj); + // must have exactly 1 element + if (!metaIt.more()) { + return false; + } + BSONElement metaElt = metaIt.next(); + if (!mongoutils::str::equals("$meta", metaElt.fieldName())) { + return false; + } + if (mongo::String != metaElt.type()) { + return false; + } + if (LiteParsedQuery::metaDiskLoc != metaElt.valuestr()) { + return false; + } + // must have exactly 1 element + if (metaIt.more()) { + return false; + } + return true; + } + + // static + bool LiteParsedQuery::isValidSortOrder(const BSONObj& sortObj) { + BSONObjIterator i(sortObj); + while (i.more()) { + BSONElement e = i.next(); + // fieldNameSize() includes NULL terminator. For empty field name, + // we should be checking for 1 instead of 0. + if (1 == e.fieldNameSize()) { + return false; + } + if (isTextScoreMeta(e)) { + continue; + } + long long n = e.safeNumberLong(); + if (!(e.isNumber() && (n == -1LL || n == 1LL))) { + return false; + } + } + return true; + } + + // static + bool LiteParsedQuery::isQueryIsolated(const BSONObj& query) { + BSONObjIterator iter(query); + while (iter.more()) { + BSONElement elt = iter.next(); + if (str::equals(elt.fieldName(), "$isolated") && elt.trueValue()) + return true; + if (str::equals(elt.fieldName(), "$atomic") && elt.trueValue()) + return true; + } + return false; + } + + LiteParsedQuery::LiteParsedQuery() : _wantMore(true), _explain(false), _snapshot(false), + _returnKey(false), _showDiskLoc(false), _maxScan(0), + _maxTimeMS(0) { } + + Status LiteParsedQuery::init(const string& ns, int ntoskip, int ntoreturn, int queryOptions, + const BSONObj& queryObj, const BSONObj& proj, + bool fromQueryMessage) { + _ns = ns; + _ntoskip = ntoskip; + _ntoreturn = ntoreturn; + _options = queryOptions; + _proj = proj.getOwned(); + + if (_ntoskip < 0) { + return Status(ErrorCodes::BadValue, "bad skip value in query"); + } + + if (_ntoreturn == std::numeric_limits::min()) { + // _ntoreturn is negative but can't be negated. + return Status(ErrorCodes::BadValue, "bad limit value in query"); + } + + if (_ntoreturn < 0) { + // _ntoreturn greater than zero is simply a hint on how many objects to send back per + // "cursor batch". A negative number indicates a hard limit. + _wantMore = false; + _ntoreturn = -_ntoreturn; + } + + if (fromQueryMessage) { + BSONElement queryField = queryObj["query"]; + if (!queryField.isABSONObj()) { queryField = queryObj["$query"]; } + if (queryField.isABSONObj()) { + _filter = queryField.embeddedObject().getOwned(); + Status status = initFullQuery(queryObj); + if (!status.isOK()) { return status; } + } + else { + // TODO: Does this ever happen? + _filter = queryObj.getOwned(); + } + } + else { + // This is the debugging code path. + _filter = queryObj.getOwned(); + } + + _hasReadPref = queryObj.hasField("$readPreference"); + + if (!isValidSortOrder(_sort)) { + return Status(ErrorCodes::BadValue, "bad sort specification"); + } + + // Min and Max objects must have the same fields. + if (!_min.isEmpty() && !_max.isEmpty()) { + if (!_min.isFieldNamePrefixOf(_max) || (_min.nFields() != _max.nFields())) { + return Status(ErrorCodes::BadValue, "min and max must have the same field names"); + } + } + + // Can't combine a normal sort and a $meta projection on the same field. + BSONObjIterator projIt(_proj); + while (projIt.more()) { + BSONElement projElt = projIt.next(); + if (isTextScoreMeta(projElt)) { + BSONElement sortElt = _sort[projElt.fieldName()]; + if (!sortElt.eoo() && !isTextScoreMeta(sortElt)) { + return Status(ErrorCodes::BadValue, + "can't have a non-$meta sort on a $meta projection"); + } + } + } + + // All fields with a $meta sort must have a corresponding $meta projection. + BSONObjIterator sortIt(_sort); + while (sortIt.more()) { + BSONElement sortElt = sortIt.next(); + if (isTextScoreMeta(sortElt)) { + BSONElement projElt = _proj[sortElt.fieldName()]; + if (projElt.eoo() || !isTextScoreMeta(projElt)) { + return Status(ErrorCodes::BadValue, + "must have $meta projection for all $meta sort keys"); + } + } + } + + return Status::OK(); + } + + Status LiteParsedQuery::initFullQuery(const BSONObj& top) { + BSONObjIterator i(top); + + while (i.more()) { + BSONElement e = i.next(); + const char* name = e.fieldName(); + + if (0 == strcmp("$orderby", name) || 0 == strcmp("orderby", name)) { + if (Object == e.type()) { + _sort = e.embeddedObject().getOwned(); + } + else if (Array == e.type()) { + _sort = e.embeddedObject(); + + // TODO: Is this ever used? I don't think so. + // Quote: + // This is for languages whose "objects" are not well ordered (JSON is well + // ordered). + // [ { a : ... } , { b : ... } ] -> { a : ..., b : ... } + // note: this is slow, but that is ok as order will have very few pieces + BSONObjBuilder b; + char p[2] = "0"; + + while (1) { + BSONObj j = _sort.getObjectField(p); + if (j.isEmpty()) { break; } + BSONElement e = j.firstElement(); + if (e.eoo()) { + return Status(ErrorCodes::BadValue, "bad order array"); + } + if (!e.isNumber()) { + return Status(ErrorCodes::BadValue, "bad order array [2]"); + } + b.append(e); + (*p)++; + if (!(*p <= '9')) { + return Status(ErrorCodes::BadValue, "too many ordering elements"); + } + } + + _sort = b.obj(); + } + else { + return Status(ErrorCodes::BadValue, "sort must be object or array"); + } + } + else if ('$' == *name) { + name++; + if (str::equals("explain", name)) { + // Won't throw. + _explain = e.trueValue(); + } + else if (str::equals("snapshot", name)) { + // Won't throw. + _snapshot = e.trueValue(); + } + else if (str::equals("min", name)) { + if (!e.isABSONObj()) { + return Status(ErrorCodes::BadValue, "$min must be a BSONObj"); + } + _min = e.embeddedObject().getOwned(); + } + else if (str::equals("max", name)) { + if (!e.isABSONObj()) { + return Status(ErrorCodes::BadValue, "$max must be a BSONObj"); + } + _max = e.embeddedObject().getOwned(); + } + else if (str::equals("hint", name)) { + if (e.isABSONObj()) { + _hint = e.embeddedObject().getOwned(); + } + else { + // Hint can be specified as an object or as a string. Wrap takes care of + // it. + _hint = e.wrap(); + } + } + else if (str::equals("returnKey", name)) { + // Won't throw. + if (e.trueValue()) { + _returnKey = true; + BSONObjBuilder projBob; + projBob.appendElements(_proj); + // We use $$ because it's never going to show up in a user's projection. + // The exact text doesn't matter. + BSONObj indexKey = BSON("$$" << + BSON("$meta" << LiteParsedQuery::metaIndexKey)); + projBob.append(indexKey.firstElement()); + _proj = projBob.obj(); + } + } + else if (str::equals("maxScan", name)) { + // Won't throw. + _maxScan = e.numberInt(); + } + else if (str::equals("showDiskLoc", name)) { + // Won't throw. + if (e.trueValue()) { + _showDiskLoc = true; + + BSONObjBuilder projBob; + projBob.appendElements(_proj); + BSONObj metaDiskLoc = BSON("$diskLoc" << + BSON("$meta" << LiteParsedQuery::metaDiskLoc)); + projBob.append(metaDiskLoc.firstElement()); + _proj = projBob.obj(); + } + } + else if (str::equals("maxTimeMS", name)) { + StatusWith maxTimeMS = parseMaxTimeMS(e); + if (!maxTimeMS.isOK()) { + return maxTimeMS.getStatus(); + } + _maxTimeMS = maxTimeMS.getValue(); + } + } + } + + if (_snapshot) { + if (!_sort.isEmpty()) { + return Status(ErrorCodes::BadValue, "E12001 can't use sort with $snapshot"); + } + if (!_hint.isEmpty()) { + return Status(ErrorCodes::BadValue, "E12002 can't use hint with $snapshot"); + } + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/lite_parsed_query.h b/src/mongo/db/query/lite_parsed_query.h new file mode 100644 index 00000000000..05b516fe91b --- /dev/null +++ b/src/mongo/db/query/lite_parsed_query.h @@ -0,0 +1,162 @@ +/** + * Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongo/db/jsobj.h" + +namespace mongo { + + class QueryMessage; + + /** + * Parses the QueryMessage received from the user and makes the various fields more easily + * accessible. + */ + class LiteParsedQuery { + public: + /** + * Parse the provided QueryMessage and set *out to point to the output. + * + * Return Status::OK() if parsing succeeded. Caller owns *out. + * Otherwise, *out is invalid and the returned Status indicates why parsing failed. + */ + static Status make(const QueryMessage& qm, LiteParsedQuery** out); + + /** + * Fills out a LiteParsedQuery. Used for debugging and testing, when we don't have a + * QueryMessage. + */ + static Status make(const string& ns, + int ntoskip, + int ntoreturn, + int queryoptions, + const BSONObj& query, + const BSONObj& proj, + const BSONObj& sort, + const BSONObj& hint, + const BSONObj& minObj, + const BSONObj& maxObj, + bool snapshot, + bool explain, + LiteParsedQuery** out); + + /** + * Helper functions to parse maxTimeMS from a command object. Returns the contained value, + * or an error on parsing fail. When passed an EOO-type element, returns 0 (special value + * for "allow to run indefinitely"). + */ + static StatusWith parseMaxTimeMSCommand(const BSONObj& cmdObj); + + /** + * Same as parseMaxTimeMSCommand, but for a query object. + */ + static StatusWith parseMaxTimeMSQuery(const BSONObj& queryObj); + + /** + * Helper function to identify text search sort key + * Example: {a: {$meta: "textScore"}} + */ + static bool isTextScoreMeta(BSONElement elt); + + /** + * Helper function to identify diskLoc projection + * Example: {a: {$meta: "diskloc"}}. + */ + static bool isDiskLocMeta(BSONElement elt); + + /** + * Helper function to validate a sort object. + * Returns true if each element satisfies one of: + * 1. a number with value 1 + * 2. a number with value -1 + * 3. isTextScoreMeta + */ + static bool isValidSortOrder(const BSONObj& sortObj); + + /** + * Returns true if the query described by "query" should execute + * at an elevated level of isolation (i.e., $isolated was specified). + */ + static bool isQueryIsolated(const BSONObj& query); + + // Names of the maxTimeMS command and query option. + static const string cmdOptionMaxTimeMS; + static const string queryOptionMaxTimeMS; + + // Names of the $meta projection values. + static const string metaTextScore; + static const string metaGeoNearDistance; + static const string metaGeoNearPoint; + static const string metaDiskLoc; + static const string metaIndexKey; + + const string& ns() const { return _ns; } + bool isLocalDB() const { return _ns.compare(0, 6, "local.") == 0; } + + const BSONObj& getFilter() const { return _filter; } + const BSONObj& getProj() const { return _proj; } + const BSONObj& getSort() const { return _sort; } + const BSONObj& getHint() const { return _hint; } + + int getSkip() const { return _ntoskip; } + int getNumToReturn() const { return _ntoreturn; } + bool wantMore() const { return _wantMore; } + int getOptions() const { return _options; } + bool hasOption(int x) const { return ( x & _options ) != 0; } + bool hasReadPref() const { return _hasReadPref; } + + bool isExplain() const { return _explain; } + bool isSnapshot() const { return _snapshot; } + bool returnKey() const { return _returnKey; } + bool showDiskLoc() const { return _showDiskLoc; } + + const BSONObj& getMin() const { return _min; } + const BSONObj& getMax() const { return _max; } + int getMaxScan() const { return _maxScan; } + int getMaxTimeMS() const { return _maxTimeMS; } + + private: + LiteParsedQuery(); + + Status init(const string& ns, int ntoskip, int ntoreturn, int queryOptions, + const BSONObj& queryObj, const BSONObj& proj, bool fromQueryMessage); + + Status initFullQuery(const BSONObj& top); + + static StatusWith parseMaxTimeMS(const BSONElement& maxTimeMSElt); + + string _ns; + int _ntoskip; + int _ntoreturn; + BSONObj _filter; + BSONObj _sort; + BSONObj _proj; + int _options; + bool _wantMore; + bool _explain; + bool _snapshot; + bool _returnKey; + bool _showDiskLoc; + bool _hasReadPref; + BSONObj _min; + BSONObj _max; + BSONObj _hint; + int _maxScan; + int _maxTimeMS; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/lite_parsed_query_test.cpp b/src/mongo/db/query/lite_parsed_query_test.cpp new file mode 100644 index 00000000000..6884459c779 --- /dev/null +++ b/src/mongo/db/query/lite_parsed_query_test.cpp @@ -0,0 +1,258 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/list_parsed_query.h + */ + +#include "mongo/db/query/lite_parsed_query.h" + +#include "mongo/db/json.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + TEST(LiteParsedQueryTest, InitSortOrder) { + LiteParsedQuery* lpq = NULL; + Status result = LiteParsedQuery::make("testns", 0, 1, 0, BSONObj(), BSONObj(), + fromjson("{a: 1}"), BSONObj(), + BSONObj(), BSONObj(), + false, // snapshot + false, // explain + &lpq); + ASSERT_OK(result); + } + + TEST(LiteParsedQueryTest, InitSortOrderString) { + LiteParsedQuery* lpq = NULL; + Status result = LiteParsedQuery::make("testns", 0, 1, 0, BSONObj(), BSONObj(), + fromjson("{a: \"\"}"), BSONObj(), + BSONObj(), BSONObj(), + false, // snapshot + false, // explain + &lpq); + ASSERT_NOT_OK(result); + } + + TEST(LiteParsedQueryTest, GetFilter) { + LiteParsedQuery* lpq = NULL; + Status result = LiteParsedQuery::make("testns", 5, 6, 9, BSON( "x" << 5 ), BSONObj(), + BSONObj(), BSONObj(), + BSONObj(), BSONObj(), + false, // snapshot + false, // explain + &lpq); + ASSERT_OK(result); + ASSERT_EQUALS(BSON("x" << 5 ), lpq->getFilter()); + } + + TEST(LiteParsedQueryTest, NumToReturn) { + LiteParsedQuery* lpq = NULL; + Status result = LiteParsedQuery::make("testns", 5, 6, 9, BSON( "x" << 5 ), BSONObj(), + BSONObj(), BSONObj(), + BSONObj(), BSONObj(), + false, // snapshot + false, // explain + &lpq); + ASSERT_OK(result); + ASSERT_EQUALS(6, lpq->getNumToReturn()); + ASSERT(lpq->wantMore()); + + lpq = NULL; + result = LiteParsedQuery::make("testns", 5, -6, 9, BSON( "x" << 5 ), BSONObj(), + BSONObj(), BSONObj(), + BSONObj(), BSONObj(), + false, // snapshot + false, // explain + &lpq); + ASSERT_OK(result); + ASSERT_EQUALS(6, lpq->getNumToReturn()); + ASSERT(!lpq->wantMore()); + } + + TEST(LiteParsedQueryTest, MinFieldsNotPrefixOfMax) { + LiteParsedQuery* lpq = NULL; + Status result = LiteParsedQuery::make("testns", 0, 0, 0, BSONObj(), BSONObj(), + BSONObj(), BSONObj(), + fromjson("{a: 1}"), fromjson("{b: 1}"), + false, // snapshot + false, // explain + &lpq); + ASSERT_NOT_OK(result); + } + + TEST(LiteParsedQueryTest, MinFieldsMoreThanMax) { + LiteParsedQuery* lpq = NULL; + Status result = LiteParsedQuery::make("testns", 0, 0, 0, BSONObj(), BSONObj(), + BSONObj(), BSONObj(), + fromjson("{a: 1, b: 1}"), fromjson("{a: 1}"), + false, // snapshot + false, // explain + &lpq); + ASSERT_NOT_OK(result); + } + + TEST(LiteParsedQueryTest, MinFieldsLessThanMax) { + LiteParsedQuery* lpq = NULL; + Status result = LiteParsedQuery::make("testns", 0, 0, 0, BSONObj(), BSONObj(), + BSONObj(), BSONObj(), + fromjson("{a: 1}"), fromjson("{a: 1, b: 1}"), + false, // snapshot + false, // explain + &lpq); + ASSERT_NOT_OK(result); + } + + // Helper function which returns the Status of creating a LiteParsedQuery object with the given + // parameters. + Status makeLiteParsedQuery(const BSONObj& query, const BSONObj& proj, const BSONObj& sort) { + LiteParsedQuery* lpqRaw; + Status result = LiteParsedQuery::make("testns", 0, 0, 0, query, proj, sort, BSONObj(), + BSONObj(), BSONObj(), + false, // snapshot + false, // explain + &lpqRaw); + if (result.isOK()) { + boost::scoped_ptr lpq(lpqRaw); + } + + return result; + } + + // + // Test compatibility of various projection and sort objects. + // + + TEST(LiteParsedQueryTest, ValidSortProj) { + Status result = Status::OK(); + + result = makeLiteParsedQuery(BSONObj(), + fromjson("{a: 1}"), + fromjson("{a: 1}")); + ASSERT_OK(result); + + result = makeLiteParsedQuery(BSONObj(), + fromjson("{a: {$meta: \"textScore\"}}"), + fromjson("{a: {$meta: \"textScore\"}}")); + ASSERT_OK(result); + + } + + TEST(LiteParsedQueryTest, ForbidNonMetaSortOnFieldWithMetaProject) { + Status result = Status::OK(); + + result = makeLiteParsedQuery(BSONObj(), + fromjson("{a: {$meta: \"textScore\"}}"), + fromjson("{a: 1}")); + ASSERT_NOT_OK(result); + + result = makeLiteParsedQuery(BSONObj(), + fromjson("{a: {$meta: \"textScore\"}}"), + fromjson("{b: 1}")); + ASSERT_OK(result); + } + + TEST(LiteParsedQueryTest, ForbidMetaSortOnFieldWithoutMetaProject) { + Status result = Status::OK(); + + result = makeLiteParsedQuery(BSONObj(), + fromjson("{a: 1}"), + fromjson("{a: {$meta: \"textScore\"}}")); + ASSERT_NOT_OK(result); + + result = makeLiteParsedQuery(BSONObj(), + fromjson("{b: 1}"), + fromjson("{a: {$meta: \"textScore\"}}")); + ASSERT_NOT_OK(result); + } + + // + // Text meta BSON element validation + // + + bool isFirstElementTextScoreMeta(const char* sortStr) { + BSONObj sortObj = fromjson(sortStr); + BSONElement elt = sortObj.firstElement(); + bool result = LiteParsedQuery::isTextScoreMeta(elt); + return result; + } + + // Check validation of $meta expressions + TEST(LiteParsedQueryTest, IsTextScoreMeta) { + // Valid textScore meta sort + ASSERT(isFirstElementTextScoreMeta("{a: {$meta: \"textScore\"}}")); + + // Invalid textScore meta sorts + ASSERT_FALSE(isFirstElementTextScoreMeta("{a: {$meta: 1}}")); + ASSERT_FALSE(isFirstElementTextScoreMeta("{a: {$meta: \"image\"}}")); + ASSERT_FALSE(isFirstElementTextScoreMeta("{a: {$world: \"textScore\"}}")); + ASSERT_FALSE(isFirstElementTextScoreMeta("{a: {$meta: \"textScore\", b: 1}}")); + } + + // + // Sort order validation + // In a valid sort order, each element satisfies one of: + // 1. a number with value 1 + // 2. a number with value -1 + // 3. isTextScoreMeta + // + + TEST(LiteParsedQueryTest, ValidateSortOrder) { + // Valid sorts + ASSERT(LiteParsedQuery::isValidSortOrder(fromjson("{}"))); + ASSERT(LiteParsedQuery::isValidSortOrder(fromjson("{a: 1}"))); + ASSERT(LiteParsedQuery::isValidSortOrder(fromjson("{a: -1}"))); + ASSERT(LiteParsedQuery::isValidSortOrder(fromjson("{a: {$meta: \"textScore\"}}"))); + + // Invalid sorts + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: 100}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: 0}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: -100}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: Infinity}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: -Infinity}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: true}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: false}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: null}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: {}}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: {b: 1}}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: []}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: [1, 2, 3]}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: \"\"}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: \"bb\"}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: {$meta: 1}}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: {$meta: \"image\"}}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: {$world: \"textScore\"}}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{a: {$meta: \"textScore\"," + " b: 1}}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{'': 1}"))); + ASSERT_FALSE(LiteParsedQuery::isValidSortOrder(fromjson("{'': -1}"))); + } + +} // namespace diff --git a/src/mongo/db/query/lru_key_value.h b/src/mongo/db/query/lru_key_value.h new file mode 100644 index 00000000000..808e0584662 --- /dev/null +++ b/src/mongo/db/query/lru_key_value.h @@ -0,0 +1,222 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + /** + * A key-value store structure with a least recently used (LRU) replacement + * policy. The number of entries allowed in the kv-store is set as a constant + * upon construction. + * + * Caveat: + * This kv-store is NOT thread safe! The client to this utility is responsible + * for protecting concurrent access to the LRU store if used in a threaded + * context. + * + * Implemented as a doubly-linked list (std::list) with a hash map + * (boost::unordered_map) for quickly locating the kv-store entries. The + * add(), get(), and remove() operations are all O(1). + * + * The keys of generic type K map to values of type V*. The V* + * pointers are owned by the kv-store. + * + * TODO: We could move this into the util/ directory and do any cleanup necessary to make it + * fully general. + */ + template + class LRUKeyValue { + public: + LRUKeyValue(size_t maxSize) : _maxSize(maxSize), _currentSize(0) { }; + + ~LRUKeyValue() { + clear(); + } + + typedef std::pair KVListEntry; + + typedef std::list KVList; + typedef typename KVList::iterator KVListIt; + typedef typename KVList::const_iterator KVListConstIt; + + typedef boost::unordered_map KVMap; + typedef typename KVMap::const_iterator KVMapConstIt; + + /** + * Add an (K, V*) pair to the store, where 'key' can + * be used to retrieve value 'entry' from the store. + * + * Takes ownership of 'entry'. + * + * If 'key' already exists in the kv-store, 'entry' will + * simply replace what is already there. + * + * The least recently used entry is evicted if the + * kv-store is full prior to the add() operation. + * + * If an entry is evicted, it will be returned in + * an auto_ptr for the caller to use before disposing. + */ + std::auto_ptr add(const K& key, V* entry) { + // If the key already exists, delete it first. + KVMapConstIt i = _kvMap.find(key); + if (i != _kvMap.end()) { + KVListIt found = i->second; + delete found->second; + _kvMap.erase(i); + _kvList.erase(found); + _currentSize--; + } + + _kvList.push_front(std::make_pair(key, entry)); + _kvMap[key] = _kvList.begin(); + _currentSize++; + + // If the store has grown beyond its allowed size, + // evict the least recently used entry. + if (_currentSize > _maxSize) { + V* evictedEntry = _kvList.back().second; + invariant(evictedEntry); + + _kvMap.erase(_kvList.back().first); + _kvList.pop_back(); + _currentSize--; + invariant(_currentSize == _maxSize); + + // Pass ownership of evicted entry to caller. + // If caller chooses to ignore this auto_ptr, + // the evicted entry will be deleted automatically. + return std::auto_ptr(evictedEntry); + } + return std::auto_ptr(); + } + + /** + * Retrieve the value associated with 'key' from + * the kv-store. The value is returned through the + * out-parameter 'entryOut'. + * + * The kv-store retains ownership of 'entryOut', so + * it should not be deleted by the caller. + * + * As a side effect, the retrieved entry is promoted + * to the most recently used. + */ + Status get(const K& key, V** entryOut) const { + KVMapConstIt i = _kvMap.find(key); + if (i == _kvMap.end()) { + return Status(ErrorCodes::NoSuchKey, "no such key in LRU key-value store"); + } + KVListIt found = i->second; + V* foundEntry = found->second; + + // Promote the kv-store entry to the front of the list. + // It is now the most recently used. + _kvMap.erase(i); + _kvList.erase(found); + _kvList.push_front(std::make_pair(key, foundEntry)); + _kvMap[key] = _kvList.begin(); + + *entryOut = foundEntry; + return Status::OK(); + } + + /** + * Remove the kv-store entry keyed by 'key'. + */ + Status remove(const K& key) { + KVMapConstIt i = _kvMap.find(key); + if (i == _kvMap.end()) { + return Status(ErrorCodes::NoSuchKey, "no such key in LRU key-value store"); + } + KVListIt found = i->second; + delete found->second; + _kvMap.erase(i); + _kvList.erase(found); + _currentSize--; + return Status::OK(); + } + + /** + * Deletes all entries in the kv-store. + */ + void clear() { + for (KVListIt i = _kvList.begin(); i != _kvList.end(); i++) { + delete i->second; + } + _kvList.clear(); + _kvMap.clear(); + _currentSize = 0; + } + + /** + * Returns true if entry is found in the kv-store. + */ + bool hasKey(const K& key) const { + return _kvMap.find(key) != _kvMap.end(); + } + + /** + * Returns the number of entries currently in the kv-store. + */ + size_t size() const { return _currentSize; } + + /** + * TODO: The kv-store should implement its own iterator. Calling through to the underlying + * iterator exposes the internals, and forces the caller to make a horrible type + * declaration. + */ + KVListConstIt begin() const { return _kvList.begin(); } + + KVListConstIt end() const { return _kvList.end(); } + + private: + // The maximum allowable number of entries in the kv-store. + const size_t _maxSize; + + // The number of entries currently in the kv-store. + size_t _currentSize; + + // (K, V*) pairs are stored in this std::list. They are sorted in order + // of use, where the front is the most recently used and the back is the + // least recently used. + mutable KVList _kvList; + + // Maps from a key to the corresponding std::list entry. + mutable KVMap _kvMap; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/lru_key_value_test.cpp b/src/mongo/db/query/lru_key_value_test.cpp new file mode 100644 index 00000000000..28cc2147dda --- /dev/null +++ b/src/mongo/db/query/lru_key_value_test.cpp @@ -0,0 +1,197 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/lru_key_value.h" + +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +using namespace mongo; + +namespace { + + // + // Convenience functions + // + + void assertInKVStore(LRUKeyValue& cache, int key, int value) { + int* cachedValue = NULL; + ASSERT_TRUE(cache.hasKey(key)); + Status s = cache.get(key, &cachedValue); + ASSERT_OK(s); + ASSERT_EQUALS(*cachedValue, value); + } + + void assertNotInKVStore(LRUKeyValue& cache, int key) { + int* cachedValue = NULL; + ASSERT_FALSE(cache.hasKey(key)); + Status s = cache.get(key, &cachedValue); + ASSERT_NOT_OK(s); + } + + /** + * Test that we can add an entry and get it back out. + */ + TEST(LRUKeyValueTest, BasicAddGet) { + LRUKeyValue cache(100); + cache.add(1, new int(2)); + assertInKVStore(cache, 1, 2); + } + + /** + * A kv-store with a max size of 0 isn't too useful, but test + * that at the very least we don't blow up. + */ + TEST(LRUKeyValueTest, SizeZeroCache) { + LRUKeyValue cache(0); + cache.add(1, new int(2)); + assertNotInKVStore(cache, 1); + } + + /** + * Make sure eviction and promotion work properly with + * a kv-store of size 1. + */ + TEST(LRUKeyValueTest, SizeOneCache) { + LRUKeyValue cache(1); + cache.add(0, new int(0)); + assertInKVStore(cache, 0, 0); + + // Second entry should immediately evict the first. + cache.add(1, new int(1)); + assertNotInKVStore(cache, 0); + assertInKVStore(cache, 1, 1); + } + + /** + * Fill up a size 10 kv-store with 10 entries. Call get() + * on every entry except for one. Then call add() and + * make sure that the proper entry got evicted. + */ + TEST(LRUKeyValueTest, EvictionTest) { + int maxSize = 10; + LRUKeyValue cache(maxSize); + for (int i = 0; i < maxSize; ++i) { + std::auto_ptr evicted = cache.add(i, new int(i)); + ASSERT(NULL == evicted.get()); + } + ASSERT_EQUALS(cache.size(), (size_t)maxSize); + + // Call get() on all but one key. + int evictKey = 5; + for (int i = 0; i < maxSize; ++i) { + if (i == evictKey) { continue; } + assertInKVStore(cache, i, i); + } + + // Adding another entry causes an eviction. + std::auto_ptr evicted = cache.add(maxSize + 1, new int(maxSize + 1)); + ASSERT_EQUALS(cache.size(), (size_t)maxSize); + ASSERT(NULL != evicted.get()); + ASSERT_EQUALS(*evicted, evictKey); + + // Check that the least recently accessed has been evicted. + for (int i = 0; i < maxSize; ++i) { + if (i == evictKey) { + assertNotInKVStore(cache, evictKey); + } + else { + assertInKVStore(cache, i, i); + } + } + } + + /** + * Fill up a size 10 kv-store with 10 entries. Call get() + * on a single entry to promote it to most recently + * accessed. Then cause 9 evictions and make sure only + * the entry on which we called get() remains. + */ + TEST(LRUKeyValueTest, PromotionTest) { + int maxSize = 10; + LRUKeyValue cache(maxSize); + for (int i = 0; i < maxSize; ++i) { + std::auto_ptr evicted = cache.add(i, new int(i)); + ASSERT(NULL == evicted.get()); + } + ASSERT_EQUALS(cache.size(), (size_t)maxSize); + + // Call get() on a particular key. + int promoteKey = 5; + assertInKVStore(cache, promoteKey, promoteKey); + + // Evict all but one of the original entries. + for (int i = maxSize; i < (maxSize + maxSize - 1); ++i) { + std::auto_ptr evicted = cache.add(i, new int(i)); + ASSERT(NULL != evicted.get()); + } + ASSERT_EQUALS(cache.size(), (size_t)maxSize); + + // Check that the promoteKey has not been evicted. + for (int i = 0; i < maxSize; ++i) { + if (i == promoteKey) { + assertInKVStore(cache, promoteKey, promoteKey); + } + else { + assertNotInKVStore(cache, i); + } + } + } + + /** + * Test that calling add() with a key that already exists + * in the kv-store deletes the existing entry. + */ + TEST(LRUKeyValueTest, ReplaceKeyTest) { + LRUKeyValue cache(10); + cache.add(4, new int(4)); + assertInKVStore(cache, 4, 4); + cache.add(4, new int(5)); + assertInKVStore(cache, 4, 5); + } + + /** + * Test iteration over the kv-store. + */ + TEST(LRUKeyValueTest, IterationTest) { + LRUKeyValue cache(2); + cache.add(1, new int(1)); + cache.add(2, new int(2)); + + typedef std::list< std::pair >::const_iterator CacheIterator; + CacheIterator i = cache.begin(); + ASSERT_EQUALS(i->first, 2); + ASSERT_EQUALS(*i->second, 2); + ++i; + ASSERT_EQUALS(i->first, 1); + ASSERT_EQUALS(*i->second, 1); + ++i; + ASSERT(i == cache.end()); + } + +} // namespace diff --git a/src/mongo/db/query/multi_plan_runner.cpp b/src/mongo/db/query/multi_plan_runner.cpp new file mode 100644 index 00000000000..002b3ebac62 --- /dev/null +++ b/src/mongo/db/query/multi_plan_runner.cpp @@ -0,0 +1,666 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/multi_plan_runner.h" + +#include +#include +#include + +#include "mongo/db/client.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/pdfile.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/plan_executor.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/db/query/type_explain.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + MultiPlanRunner::MultiPlanRunner(const Collection* collection, CanonicalQuery* query) + : _collection(collection), + _killed(false), + _failure(false), + _failureCount(0), + _policy(Runner::YIELD_MANUAL), + _query(query), + _bestChild(numeric_limits::max()), + _backupSolution(NULL), + _backupPlan(NULL) { } + + MultiPlanRunner::~MultiPlanRunner() { + for (size_t i = 0; i < _candidates.size(); ++i) { + delete _candidates[i].solution; + delete _candidates[i].root; + // ws must die after the root. + delete _candidates[i].ws; + } + + if (NULL != _backupSolution) { + delete _backupSolution; + } + + if (NULL != _backupPlan) { + delete _backupPlan; + } + + for (vector::iterator it = _candidateStats.begin(); + it != _candidateStats.end(); + ++it) { + delete *it; + } + } + + void MultiPlanRunner::addPlan(QuerySolution* solution, PlanStage* root, WorkingSet* ws) { + _candidates.push_back(CandidatePlan(solution, root, ws)); + } + + void MultiPlanRunner::setYieldPolicy(Runner::YieldPolicy policy) { + if (_failure || _killed) { return; } + + _policy = policy; + + if (NULL != _bestPlan) { + _bestPlan->setYieldPolicy(policy); + if (NULL != _backupPlan) { + _backupPlan->setYieldPolicy(policy); + } + } else { + // Still running our candidates and doing our own yielding. + if (Runner::YIELD_MANUAL == policy) { + _yieldPolicy.reset(); + } + else { + _yieldPolicy.reset(new RunnerYieldPolicy()); + } + } + } + + void MultiPlanRunner::saveState() { + if (_failure || _killed) { return; } + + if (NULL != _bestPlan) { + _bestPlan->saveState(); + if (NULL != _backupPlan) { + _backupPlan->saveState(); + } + } + else { + allPlansSaveState(); + } + } + + bool MultiPlanRunner::restoreState() { + if (_failure || _killed) { return false; } + + if (NULL != _bestPlan) { + bool best = _bestPlan->restoreState(); + // backup plan is OK by default. Only can be set to not-OK if it exists and fails. + bool backup = true; + if (NULL != _backupPlan) { + backup = _backupPlan->restoreState(); + } + // We're OK to continue if the best plan and the backup plan are OK. + return best && backup; + } + else { + allPlansRestoreState(); + return true; + } + } + + void MultiPlanRunner::invalidate(const DiskLoc& dl, InvalidationType type) { + if (_failure || _killed) { return; } + + if (NULL != _bestPlan) { + _bestPlan->invalidate(dl, type); + for (list::iterator it = _alreadyProduced.begin(); + it != _alreadyProduced.end();) { + WorkingSetMember* member = _bestPlan->getWorkingSet()->get(*it); + if (member->hasLoc() && member->loc == dl) { + list::iterator next = it; + next++; + WorkingSetCommon::fetchAndInvalidateLoc(member); + _bestPlan->getWorkingSet()->flagForReview(*it); + _alreadyProduced.erase(it); + it = next; + } + else { + it++; + } + } + if (NULL != _backupPlan) { + _backupPlan->invalidate(dl, type); + for (list::iterator it = _backupAlreadyProduced.begin(); + it != _backupAlreadyProduced.end();) { + WorkingSetMember* member = _backupPlan->getWorkingSet()->get(*it); + if (member->hasLoc() && member->loc == dl) { + list::iterator next = it; + next++; + WorkingSetCommon::fetchAndInvalidateLoc(member); + _backupPlan->getWorkingSet()->flagForReview(*it); + _backupAlreadyProduced.erase(it); + it = next; + } + else { + it++; + } + } + } + } + else { + for (size_t i = 0; i < _candidates.size(); ++i) { + _candidates[i].root->invalidate(dl, type); + for (list::iterator it = _candidates[i].results.begin(); + it != _candidates[i].results.end();) { + WorkingSetMember* member = _candidates[i].ws->get(*it); + if (member->hasLoc() && member->loc == dl) { + list::iterator next = it; + next++; + WorkingSetCommon::fetchAndInvalidateLoc(member); + _candidates[i].ws->flagForReview(*it); + _candidates[i].results.erase(it); + it = next; + } + else { + it++; + } + } + } + } + } + + bool MultiPlanRunner::isEOF() { + if (_failure || _killed) { return true; } + // If _bestPlan is not NULL, you haven't picked the best plan yet, so you're not EOF. + if (NULL == _bestPlan) { return false; } + // We must return all our cached results and there must be no results from the best plan. + return _alreadyProduced.empty() && _bestPlan->isEOF(); + } + + const std::string& MultiPlanRunner::ns() { + return _query->getParsed().ns(); + } + + void MultiPlanRunner::kill() { + _killed = true; + _collection = NULL; + if (NULL != _bestPlan) { _bestPlan->kill(); } + if (NULL != _backupPlan) { _backupPlan->kill(); } + } + + Runner::RunnerState MultiPlanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { + if (_killed) { return Runner::RUNNER_DEAD; } + if (_failure) { return Runner::RUNNER_ERROR; } + + // If we haven't picked the best plan yet... + if (NULL == _bestPlan) { + if (!pickBestPlan(NULL, objOut)) { + verify(_failure || _killed); + if (_killed) { return Runner::RUNNER_DEAD; } + if (_failure) { return Runner::RUNNER_ERROR; } + } + cacheBestPlan(); + } + + // Look for an already produced result that provides the data the caller wants. + while (!_alreadyProduced.empty()) { + WorkingSetID id = _alreadyProduced.front(); + _alreadyProduced.pop_front(); + + WorkingSetMember* member = _bestPlan->getWorkingSet()->get(id); + + // Note that this copies code from PlanExecutor. + if (NULL != objOut) { + if (WorkingSetMember::LOC_AND_IDX == member->state) { + if (1 != member->keyData.size()) { + _bestPlan->getWorkingSet()->free(id); + // If the caller needs the key data and the WSM doesn't have it, drop the + // result and carry on. + continue; + } + *objOut = member->keyData[0].keyData; + } + else if (member->hasObj()) { + *objOut = member->obj; + } + else { + // If the caller needs an object and the WSM doesn't have it, drop and + // try the next result. + _bestPlan->getWorkingSet()->free(id); + continue; + } + } + + if (NULL != dlOut) { + if (member->hasLoc()) { + *dlOut = member->loc; + } + else { + // If the caller needs a DiskLoc and the WSM doesn't have it, drop and carry on. + _bestPlan->getWorkingSet()->free(id); + continue; + } + } + + // If we're here, the caller has all the data needed and we've set the out + // parameters. Remove the result from the WorkingSet. + _bestPlan->getWorkingSet()->free(id); + return Runner::RUNNER_ADVANCED; + } + + RunnerState state = _bestPlan->getNext(objOut, dlOut); + + if (Runner::RUNNER_ERROR == state && (NULL != _backupSolution)) { + QLOG() << "Best plan errored out; switching to backup.\n"; + // Uncache the bad solution if we fall back + // on the backup solution. + // + // XXX: Instead of uncaching we should find a way for the + // cached plan runner to fall back on a different solution + // if the best solution fails. Alternatively we could try to + // defer cache insertion to be after the first produced result. + Database* db = cc().database(); + verify(NULL != db); + Collection* collection = db->getCollection(_query->ns()); + verify(NULL != collection); + PlanCache* cache = collection->infoCache()->getPlanCache(); + cache->remove(*_query); + + // Move the backup info into the bestPlan info and clear the backup + // info. + _bestPlan.reset(_backupPlan); + _backupPlan = NULL; + _bestSolution.reset(_backupSolution); + _backupSolution = NULL; + _alreadyProduced = _backupAlreadyProduced; + _backupAlreadyProduced.clear(); + return getNext(objOut, dlOut); + } + + if (NULL != _backupSolution && Runner::RUNNER_ADVANCED == state) { + QLOG() << "Best plan had a blocking sort, became unblocked; deleting backup plan.\n"; + delete _backupSolution; + delete _backupPlan; + _backupSolution = NULL; + _backupPlan = NULL; + // TODO: free from WS? + _backupAlreadyProduced.clear(); + } + + return state; + } + + bool MultiPlanRunner::pickBestPlan(size_t* out, BSONObj* objOut) { + // Run each plan some number of times. This number is at least as great as + // 'internalQueryPlanEvaluationWorks', but may be larger for big collections. + size_t numWorks = internalQueryPlanEvaluationWorks; + if (NULL != _collection) { + // For large collections, the number of works is set to be this + // fraction of the collection size. + double fraction = internalQueryPlanEvaluationCollFraction; + + numWorks = std::max(size_t(internalQueryPlanEvaluationWorks), + size_t(fraction * _collection->numRecords())); + } + + // We treat ntoreturn as though it is a limit during plan ranking. + // This means that ranking might not be great for sort + batchSize. + // But it also means that we don't buffer too much data for sort + limit. + // See SERVER-14174 for details. + size_t numToReturn = _query->getParsed().getNumToReturn(); + + // Determine the number of results which we will produce during the plan + // ranking phase before stopping. + size_t numResults = (size_t)internalQueryPlanEvaluationMaxResults; + if (numToReturn > 0) { + numResults = std::min(numToReturn, numResults); + } + + // Work the plans, stopping when a plan hits EOF or returns some + // fixed number of results. + for (size_t i = 0; i < numWorks; ++i) { + bool moreToDo = workAllPlans(objOut, numResults); + if (!moreToDo) { break; } + } + + if (_failure || _killed) { return false; } + + // After picking best plan, ranking will own plan stats from + // candidate solutions (winner and losers). + _ranking.reset(new PlanRankingDecision()); + _bestChild = PlanRanker::pickBestPlan(_candidates, _ranking.get()); + if (NULL != out) { *out = _bestChild; } + return true; + } + + void MultiPlanRunner::cacheBestPlan() { + // Must call pickBestPlan before. + verify(_bestChild != numeric_limits::max()); + + // Copy candidate order. We will need this to sort candidate stats for explain + // after transferring ownership of 'ranking' to plan cache. + std::vector candidateOrder = _ranking->candidateOrder; + + // Run the best plan. Store it. + _bestPlan.reset(new PlanExecutor(_candidates[_bestChild].ws, + _candidates[_bestChild].root)); + _bestPlan->setYieldPolicy(_policy); + _alreadyProduced = _candidates[_bestChild].results; + _bestSolution.reset(_candidates[_bestChild].solution); + + QLOG() << "Winning solution:\n" << _bestSolution->toString() << endl; + LOG(2) << "Winning plan: " << getPlanSummary(*_bestSolution); + + size_t backupChild = _bestChild; + if (_bestSolution->hasBlockingStage && (0 == _alreadyProduced.size())) { + QLOG() << "Winner has blocking stage, looking for backup plan.\n"; + for (size_t i = 0; i < _candidates.size(); ++i) { + if (!_candidates[i].solution->hasBlockingStage) { + QLOG() << "Candidate " << i << " is backup child.\n"; + backupChild = i; + _backupSolution = _candidates[i].solution; + _backupAlreadyProduced = _candidates[i].results; + _backupPlan = new PlanExecutor(_candidates[i].ws, _candidates[i].root); + _backupPlan->setYieldPolicy(_policy); + break; + } + } + } + + // Logging for tied plans. + if (_ranking->tieForBest && NULL != _collection) { + // These arrays having two or more entries is implied by 'tieForBest'. + invariant(_ranking->scores.size() > 1); + invariant(_ranking->candidateOrder.size() > 1); + + size_t winnerIdx = _ranking->candidateOrder[0]; + size_t runnerUpIdx = _ranking->candidateOrder[1]; + + LOG(1) << "Winning plan tied with runner-up." + << " ns: " << _collection->ns() + << " " << _query->toStringShort() + << " winner score: " << _ranking->scores[0] + << " winner summary: " + << getPlanSummary(*_candidates[winnerIdx].solution) + << " runner-up score: " << _ranking->scores[1] + << " runner-up summary: " + << getPlanSummary(*_candidates[runnerUpIdx].solution); + + // There could be more than a 2-way tie, so log the stats for the remaining plans + // involved in the tie. + static const double epsilon = 1e-10; + for (size_t i = 2; i < _ranking->scores.size(); i++) { + if (fabs(_ranking->scores[i] - _ranking->scores[0]) >= epsilon) { + break; + } + + size_t planIdx = _ranking->candidateOrder[i]; + + LOG(1) << "Plan " << i << " involved in multi-way tie." + << " ns: " << _collection->ns() + << " " << _query->toStringShort() + << " score: " << _ranking->scores[i] + << " summary: " << getPlanSummary(*_candidates[planIdx].solution); + } + } + + // If the winning plan produces no results during the ranking period (and, therefore, no + // plan produced results during the ranking period), then we will not create a plan cache + // entry. + if (_alreadyProduced.empty()) { + size_t winnerIdx = _ranking->candidateOrder[0]; + LOG(1) << "Winning plan had zero results. Not caching." + << " ns: " << _collection->ns() + << " " << _query->toStringShort() + << " winner score: " << _ranking->scores[0] + << " winner summary: " + << getPlanSummary(*_candidates[winnerIdx].solution); + } + + // Store the choice we just made in the cache. In order to do so, + // 1) the query must be of a type that is safe to cache, + // 2) two or more plans cannot have tied for the win. Caching in the case of ties can cause + // successive queries of the same shape to use a bad index. + // 3) Furthermore, the winning plan must have returned returned at least one result. Plans + // which return zero results cannot be reliably ranked. Such query shapes are generally + // existence type queries, and a winning plan should get cached once the query finds a + // result. + if (PlanCache::shouldCacheQuery(*_query) + && !_ranking->tieForBest + && !_alreadyProduced.empty()) { + // If we're here, then the winning plan qualifies for caching. + Database* db = cc().database(); + verify(NULL != db); + Collection* collection = db->getCollection(_query->ns()); + verify(NULL != collection); + PlanCache* cache = collection->infoCache()->getPlanCache(); + // Create list of candidate solutions for the cache with + // the best solution at the front. + std::vector solutions; + + // Generate solutions and ranking decisions sorted by score. + for (size_t orderingIndex = 0; + orderingIndex < candidateOrder.size(); ++orderingIndex) { + // index into candidates/ranking + size_t i = candidateOrder[orderingIndex]; + solutions.push_back(_candidates[i].solution); + } + + // Check solution cache data. Do not add to cache if + // we have any invalid SolutionCacheData data. + // XXX: One known example is 2D queries + bool validSolutions = true; + for (size_t i = 0; i < solutions.size(); ++i) { + if (NULL == solutions[i]->cacheData.get()) { + QLOG() << "Not caching query because this solution has no cache data: " + << solutions[i]->toString(); + validSolutions = false; + break; + } + } + + if (validSolutions) { + cache->add(*_query, solutions, _ranking.release()); + } + } + + // Clear out the candidate plans, leaving only stats as we're all done w/them. + // Traverse candidate plans in order or score + for (size_t orderingIndex = 0; + orderingIndex < candidateOrder.size(); ++orderingIndex) { + // index into candidates/ranking + size_t i = candidateOrder[orderingIndex]; + + if (i == _bestChild) { continue; } + if (i == backupChild) { continue; } + + delete _candidates[i].solution; + + // Remember the stats for the candidate plan because we always show it on an + // explain. (The {verbose:false} in explain() is client-side trick; we always + // generate a "verbose" explain.) + PlanStageStats* stats = _candidates[i].root->getStats(); + if (stats) { + _candidateStats.push_back(stats); + } + delete _candidates[i].root; + + // ws must die after the root. + delete _candidates[i].ws; + } + + _candidates.clear(); + } + + bool MultiPlanRunner::hasBackupPlan() const { + return NULL != _backupPlan; + } + + bool MultiPlanRunner::workAllPlans(BSONObj* objOut, size_t numResults) { + bool doneWorking = false; + + for (size_t i = 0; i < _candidates.size(); ++i) { + CandidatePlan& candidate = _candidates[i]; + if (candidate.failed) { continue; } + + // Yield, if we can yield ourselves. + if (NULL != _yieldPolicy.get() && _yieldPolicy->shouldYield()) { + saveState(); + _yieldPolicy->yield(); + if (_failure || _killed) { return false; } + restoreState(); + } + + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState state = candidate.root->work(&id); + + if (PlanStage::ADVANCED == state) { + // Save result for later. + candidate.results.push_back(id); + + // Once a plan returns enough results, stop working. + if (candidate.results.size() >= numResults) { + doneWorking = true; + } + } + else if (PlanStage::NEED_TIME == state) { + // Fall through to yield check at end of large conditional. + } + else if (PlanStage::NEED_FETCH == state) { + // id has a loc and refers to an obj we need to fetch. + WorkingSetMember* member = candidate.ws->get(id); + + // This must be true for somebody to request a fetch and can only change when an + // invalidation happens, which is when we give up a lock. Don't give up the + // lock between receiving the NEED_FETCH and actually fetching(?). + verify(member->hasLoc()); + + // Actually bring record into memory. + Record* record = member->loc.rec(); + + // If we're allowed to, go to disk outside of the lock. + if (NULL != _yieldPolicy.get()) { + saveState(); + _yieldPolicy->yield(record); + if (_failure || _killed) { return false; } + restoreState(); + } + else { + // We're set to manually yield. We go to disk in the lock. + record->touch(); + } + + // Record should be in memory now. Log if it's not. + if (!Record::likelyInPhysicalMemory(record->dataNoThrowing())) { + OCCASIONALLY { + warning() << "Record wasn't in memory immediately after fetch: " + << member->loc.toString() << endl; + } + } + + // Note that we're not freeing id. Fetch semantics say that we shouldn't. + } + else if (PlanStage::IS_EOF == state) { + // First plan to hit EOF wins automatically. Stop evaluating other plans. + // Assumes that the ranking will pick this plan. + doneWorking = true; + } + else { + // FAILURE or DEAD. Do we want to just tank that plan and try the rest? We + // probably want to fail globally as this shouldn't happen anyway. + + candidate.failed = true; + ++_failureCount; + + // Propage most recent seen failure to parent. + if (PlanStage::FAILURE == state && (NULL != objOut)) { + WorkingSetCommon::getStatusMemberObject(*candidate.ws, id, objOut); + } + + if (_failureCount == _candidates.size()) { + _failure = true; + return false; + } + } + } + + return !doneWorking; + } + + void MultiPlanRunner::allPlansSaveState() { + for (size_t i = 0; i < _candidates.size(); ++i) { + _candidates[i].root->prepareToYield(); + } + } + + void MultiPlanRunner::allPlansRestoreState() { + for (size_t i = 0; i < _candidates.size(); ++i) { + _candidates[i].root->recoverFromYield(); + } + } + + Status MultiPlanRunner::getInfo(TypeExplain** explain, + PlanInfo** planInfo) const { + if (NULL != explain) { + if (NULL == _bestPlan.get()) { + return Status(ErrorCodes::InternalError, "No plan available to provide stats"); + } + + // + // Explain for the winner plan + // + + scoped_ptr stats(_bestPlan->getStats()); + if (NULL == stats.get()) { + return Status(ErrorCodes::InternalError, "no stats available to explain plan"); + } + + return explainMultiPlan(*stats, _candidateStats, _bestSolution.get(), explain); + } + else if (NULL != planInfo) { + if (NULL == _bestSolution.get()) { + return Status(ErrorCodes::InternalError, + "no best solution available for plan info"); + } + getPlanInfo(*_bestSolution, planInfo); + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/multi_plan_runner.h b/src/mongo/db/query/multi_plan_runner.h new file mode 100644 index 00000000000..9f8eb22e477 --- /dev/null +++ b/src/mongo/db/query/multi_plan_runner.h @@ -0,0 +1,198 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/plan_ranker.h" // for CandidatePlan +#include "mongo/db/query/runner.h" +#include "mongo/db/query/runner_yield_policy.h" + +namespace mongo { + + class BSONObj; + class CanonicalQuery; + class DiskLoc; + class PlanExecutor; + class PlanStage; + struct QuerySolution; + class TypeExplain; + struct PlanInfo; + class WorkingSet; + + /** + * Runs several plans in parallel and picks the best one. Caches the selection for future use. + */ + class MultiPlanRunner : public Runner { + public: + /** + * Takes ownership of query. + */ + MultiPlanRunner(const Collection* collection, CanonicalQuery* query); + virtual ~MultiPlanRunner(); + + /** + * Takes ownership of all arguments + */ + void addPlan(QuerySolution* solution, PlanStage* root, WorkingSet* ws); + + /** + * Get the next result. Yielding is handled internally. If a best plan is not picked when + * this is called, we call pickBestPlan() internally. + */ + Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + virtual bool isEOF(); + + /** + * Runs all plans added by addPlan, ranks them, and picks a best. Deletes all loser plans. + * All further calls to getNext(...) will return results from the best plan. + * + * Returns true if a best plan was picked, false if there was an error. + * If there was a failure in the underlying plan, *objOut may hold error details. + * + * If out is not-NULL, set *out to the index of the picked plan. + */ + bool pickBestPlan(size_t* out, BSONObj* objOut); + + /** + * Returns true if a backup plan was picked. + * This is the case when the best plan has a blocking stage. + * Exposed for testing. + */ + bool hasBackupPlan() const; + + /** + * Caching the best plan is (currently implemented as) a destructive act so we separate it + * from ranking so that inspection of the winning solution is possible. Also sets a backup + * plan if a backup plan is needed. Exposed for testing. + */ + void cacheBestPlan(); + + virtual void saveState(); + virtual bool restoreState(); + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual void setYieldPolicy(Runner::YieldPolicy policy); + + virtual const std::string& ns(); + + virtual void kill(); + + virtual const Collection* collection() { return _collection; } + + /** + * Returns OK, allocating and filling in '*explain' and '*planInfo' with details of + * the "winner" plan. Caller takes ownership of '*explain' and '*planInfo'. Otherwise, + * return a status describing the error. + */ + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const; + + private: + // + // Have all our candidate plans do something. + // If all our candidate plans fail, *objOut will contain + // information on the failure. + // + + /** + * Calls work on each child plan in a round-robin fashion. We stop when any plan + * hits EOF or returns 'numResults' results. + * + * Returns true if we need to keep working the plans and false otherwise. + */ + bool workAllPlans(BSONObj* objOut, size_t numResults); + + void allPlansSaveState(); + + void allPlansRestoreState(); + + const Collection* _collection; + + // Were we killed by an invalidate? + bool _killed; + + // Did all plans fail while we were running them? Note that one plan can fail + // during normal execution of the plan competition. Here is an example: + // + // Plan 1: collection scan with sort. Sort runs out of memory. + // Plan 2: ixscan that provides sort. Won't run out of memory. + // + // We want to choose plan 2 even if plan 1 fails. + bool _failure; + + // If everything fails during the plan competition, we can't pick one. + size_t _failureCount; + + // We need to cache this so that when we switch from running our candidates to using a + // PlanExecutor, we can set the right yielding policy on it. + Runner::YieldPolicy _policy; + + // The winner of the plan competition... + boost::scoped_ptr _bestPlan; + + // ...and any results it produced while working toward winning. + std::list _alreadyProduced; + + // ...and the solution, for caching. + boost::scoped_ptr _bestSolution; + + // Candidate plans. + std::vector _candidates; + + // Candidate plans' stats. Owned here. + std::vector _candidateStats; + + // Yielding policy we use when we're running candidates. + boost::scoped_ptr _yieldPolicy; + + // The query that we're trying to figure out the best solution to. + boost::scoped_ptr _query; + + // What's the ranking? Produced by pickBestPlan, consumed by cacheBestPlan. + auto_ptr _ranking; + + // What's the best child? Filled out by pickBestPlan, consumed by cacheBestPlan. + size_t _bestChild; + + // + // Backup plan for sort + // + + QuerySolution* _backupSolution; + PlanExecutor* _backupPlan; + std::list _backupAlreadyProduced; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/new_find.cpp b/src/mongo/db/query/new_find.cpp new file mode 100644 index 00000000000..b2214008a7a --- /dev/null +++ b/src/mongo/db/query/new_find.cpp @@ -0,0 +1,857 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/new_find.h" + +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/commands.h" +#include "mongo/db/exec/filter.h" +#include "mongo/db/exec/oplogstart.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/keypattern.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/query/find_constants.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/internal_plans.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/db/query/single_solution_runner.h" +#include "mongo/db/query/type_explain.h" +#include "mongo/db/repl/repl_reads_ok.h" +#include "mongo/db/server_options.h" +#include "mongo/db/server_parameters.h" +#include "mongo/db/storage_options.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/s/chunk_version.h" +#include "mongo/s/d_logic.h" +#include "mongo/s/stale_exception.h" +#include "mongo/util/fail_point_service.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + // The .h for this in find_constants.h. + const int32_t MaxBytesToReturnToClientAtOnce = 4 * 1024 * 1024; +} // namespace mongo + +namespace { + + // TODO: Remove this or use it. + bool hasIndexSpecifier(const mongo::LiteParsedQuery& pq) { + return !pq.getHint().isEmpty() || !pq.getMin().isEmpty() || !pq.getMax().isEmpty(); + } + + /** + * Quote: + * if ntoreturn is zero, we return up to 101 objects. on the subsequent getmore, there + * is only a size limit. The idea is that on a find() where one doesn't use much results, + * we don't return much, but once getmore kicks in, we start pushing significant quantities. + * + * The n limit (vs. size) is important when someone fetches only one small field from big + * objects, which causes massive scanning server-side. + */ + bool enoughForFirstBatch(const mongo::LiteParsedQuery& pq, int n, int len) { + if (0 == pq.getNumToReturn()) { + return (len > 1024 * 1024) || n >= 101; + } + return n >= pq.getNumToReturn() || len > mongo::MaxBytesToReturnToClientAtOnce; + } + + bool enough(const mongo::LiteParsedQuery& pq, int n) { + if (0 == pq.getNumToReturn()) { return false; } + return n >= pq.getNumToReturn(); + } + + bool enoughForExplain(const mongo::LiteParsedQuery& pq, long long n) { + if (pq.wantMore() || 0 == pq.getNumToReturn()) { return false; } + return n >= pq.getNumToReturn(); + } + + /** + * Returns true if 'me' is a GTE or GE predicate over the "ts" field. + * Such predicates can be used for the oplog start hack. + */ + bool isOplogTsPred(const mongo::MatchExpression* me) { + if (mongo::MatchExpression::GT != me->matchType() + && mongo::MatchExpression::GTE != me->matchType()) { + return false; + } + + return mongoutils::str::equals(me->path().rawData(), "ts"); + } + +} // namespace + +namespace mongo { + + // Failpoint for checking whether we've received a getmore. + MONGO_FP_DECLARE(failReceivedGetmore); + + // TODO: Move this and the other command stuff in newRunQuery outta here and up a level. + static bool runCommands(const char *ns, + BSONObj& jsobj, + CurOp& curop, + BufBuilder &b, + BSONObjBuilder& anObjBuilder, + bool fromRepl, + int queryOptions) { + try { + return _runCommands(ns, jsobj, b, anObjBuilder, fromRepl, queryOptions); + } + catch( SendStaleConfigException& ){ + throw; + } + catch ( AssertionException& e ) { + verify( e.getCode() != SendStaleConfigCode && e.getCode() != RecvStaleConfigCode ); + + Command::appendCommandStatus(anObjBuilder, e.toStatus()); + curop.debug().exceptionInfo = e.getInfo(); + } + BSONObj x = anObjBuilder.done(); + b.appendBuf((void*) x.objdata(), x.objsize()); + return true; + } + + /** + * Also called by db/ops/query.cpp. This is the new getMore entry point. + */ + QueryResult* newGetMore(const char* ns, int ntoreturn, long long cursorid, CurOp& curop, + int pass, bool& exhaust, bool* isCursorAuthorized) { + // For testing, we may want to fail if we receive a getmore. + if (MONGO_FAIL_POINT(failReceivedGetmore)) { + invariant(0); + } + + exhaust = false; + int bufSize = 512 + sizeof(QueryResult) + MaxBytesToReturnToClientAtOnce; + + BufBuilder bb(bufSize); + bb.skip(sizeof(QueryResult)); + + // This is a read lock. + scoped_ptr ctx(new Client::ReadContext(ns)); + Collection* collection = ctx->ctx().db()->getCollection(ns); + uassert( 17356, "collection dropped between getMore calls", collection ); + + QLOG() << "Running getMore, cursorid: " << cursorid << endl; + + // This checks to make sure the operation is allowed on a replicated node. Since we are not + // passing in a query object (necessary to check SlaveOK query option), the only state where + // reads are allowed is PRIMARY (or master in master/slave). This function uasserts if + // reads are not okay. + replVerifyReadsOk(); + + // A pin performs a CC lookup and if there is a CC, increments the CC's pin value so it + // doesn't time out. Also informs ClientCursor that there is somebody actively holding the + // CC, so don't delete it. + ClientCursorPin ccPin(collection, cursorid); + ClientCursor* cc = ccPin.c(); + + // These are set in the QueryResult msg we return. + int resultFlags = ResultFlag_AwaitCapable; + + int numResults = 0; + int startingResult = 0; + + if (NULL == cc) { + cursorid = 0; + resultFlags = ResultFlag_CursorNotFound; + } + else { + // Quote: check for spoofing of the ns such that it does not match the one originally + // there for the cursor + uassert(17011, "auth error", str::equals(ns, cc->ns().c_str())); + *isCursorAuthorized = true; + + // Reset timeout timer on the cursor since the cursor is still in use. + cc->setIdleTime(0); + + // TODO: fail point? + + // If the operation that spawned this cursor had a time limit set, apply leftover + // time to this getmore. + curop.setMaxTimeMicros(cc->getLeftoverMaxTimeMicros()); + killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. + + // Ensure that the original query or command object is available in the slow query log, + // profiler, and currentOp. + curop.debug().query = cc->getQuery(); + curop.setQuery(cc->getQuery()); + + // TODO: What is pass? + if (0 == pass) { cc->updateSlaveLocation(curop); } + + if (cc->isAggCursor) { + // Agg cursors handle their own locking internally. + ctx.reset(); // unlocks + } + + CollectionMetadataPtr collMetadata = cc->getCollMetadata(); + + // If we're replaying the oplog, we save the last time that we read. + OpTime slaveReadTill; + + // What number result are we starting at? Used to fill out the reply. + startingResult = cc->pos(); + + // What gives us results. + Runner* runner = cc->getRunner(); + const int queryOptions = cc->queryOptions(); + + // Get results out of the runner. + runner->restoreState(); + + BSONObj obj; + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { + // Add result to output buffer. + bb.appendBuf((void*)obj.objdata(), obj.objsize()); + + // Count the result. + ++numResults; + + // Possibly note slave's position in the oplog. + if (queryOptions & QueryOption_OplogReplay) { + BSONElement e = obj["ts"]; + if (Date == e.type() || Timestamp == e.type()) { + slaveReadTill = e._opTime(); + } + } + + if ((ntoreturn && numResults >= ntoreturn) + || bb.len() > MaxBytesToReturnToClientAtOnce) { + break; + } + } + + if (Runner::RUNNER_EOF == state && 0 == numResults + && (queryOptions & QueryOption_CursorTailable) + && (queryOptions & QueryOption_AwaitData) && (pass < 1000)) { + // If the cursor is tailable we don't kill it if it's eof. We let it try to get + // data some # of times first. + return 0; + } + + // We save the client cursor when there might be more results, and hence we may receive + // another getmore. If we receive a EOF or an error, or the runner is dead, then we know + // that we will not be producing more results. We indicate that the cursor is closed by + // sending a cursorId of 0 back to the client. + // + // On the other hand, if we retrieve all results necessary for this batch, then + // 'saveClientCursor' is true and we send a valid cursorId back to the client. In + // this case, there may or may not actually be more results (for example, the next call + // to getNext(...) might just return EOF). + bool saveClientCursor = false; + + if (Runner::RUNNER_DEAD == state || Runner::RUNNER_ERROR == state) { + // Propagate this error to caller. + if (Runner::RUNNER_ERROR == state) { + // Stats are helpful when errors occur. + TypeExplain* bareExplain; + Status res = runner->getInfo(&bareExplain, NULL); + if (res.isOK()) { + boost::scoped_ptr errorExplain(bareExplain); + error() << "Runner error, stats:\n" + << errorExplain->stats.jsonString(Strict, true); + } + + uasserted(17406, "getMore runner error: " + + WorkingSetCommon::toStatusString(obj)); + } + + // If we're dead there's no way to get more results. + saveClientCursor = false; + + // In the old system tailable capped cursors would be killed off at the + // cursorid level. If a tailable capped cursor is nuked the cursorid + // would vanish. + // + // In the new system they die and are cleaned up later (or time out). + // So this is where we get to remove the cursorid. + if (0 == numResults) { + resultFlags = ResultFlag_CursorNotFound; + } + } + else if (Runner::RUNNER_EOF == state) { + // EOF is also end of the line unless it's tailable. + saveClientCursor = queryOptions & QueryOption_CursorTailable; + } + else { + verify(Runner::RUNNER_ADVANCED == state); + saveClientCursor = true; + } + + if (!saveClientCursor) { + ccPin.deleteUnderlying(); + // cc is now invalid, as is the runner + cursorid = 0; + cc = NULL; + QLOG() << "getMore NOT saving client cursor, ended with state " + << Runner::statestr(state) + << endl; + } + else { + // Continue caching the ClientCursor. + cc->incPos(numResults); + runner->saveState(); + QLOG() << "getMore saving client cursor ended with state " + << Runner::statestr(state) + << endl; + + // Possibly note slave's position in the oplog. + if ((queryOptions & QueryOption_OplogReplay) && !slaveReadTill.isNull()) { + cc->slaveReadTill(slaveReadTill); + } + + exhaust = (queryOptions & QueryOption_Exhaust); + + // If the getmore had a time limit, remaining time is "rolled over" back to the + // cursor (for use by future getmore ops). + cc->setLeftoverMaxTimeMicros( curop.getRemainingMaxTimeMicros() ); + } + } + + QueryResult* qr = reinterpret_cast(bb.buf()); + qr->len = bb.len(); + qr->setOperation(opReply); + qr->_resultFlags() = resultFlags; + qr->cursorId = cursorid; + qr->startingFrom = startingResult; + qr->nReturned = numResults; + bb.decouple(); + QLOG() << "getMore returned " << numResults << " results\n"; + return qr; + } + + Status getOplogStartHack(Collection* collection, CanonicalQuery* cq, Runner** runnerOut) { + if ( collection == NULL ) + return Status(ErrorCodes::InternalError, + "getOplogStartHack called with a NULL collection" ); + + // A query can only do oplog start finding if it has a top-level $gt or $gte predicate over + // the "ts" field (the operation's timestamp). Find that predicate and pass it to + // the OplogStart stage. + MatchExpression* tsExpr = NULL; + if (MatchExpression::AND == cq->root()->matchType()) { + // The query has an AND at the top-level. See if any of the children + // of the AND are $gt or $gte predicates over 'ts'. + for (size_t i = 0; i < cq->root()->numChildren(); ++i) { + MatchExpression* me = cq->root()->getChild(i); + if (isOplogTsPred(me)) { + tsExpr = me; + break; + } + } + } + else if (isOplogTsPred(cq->root())) { + // The root of the tree is a $gt or $gte predicate over 'ts'. + tsExpr = cq->root(); + } + + if (NULL == tsExpr) { + return Status(ErrorCodes::OplogOperationUnsupported, + "OplogReplay query does not contain top-level " + "$gt or $gte over the 'ts' field."); + } + + // Make an oplog start finding stage. + WorkingSet* oplogws = new WorkingSet(); + OplogStart* stage = new OplogStart(cq->ns(), tsExpr, oplogws); + + // Takes ownership of ws and stage. + auto_ptr runner(new InternalRunner(collection, stage, oplogws)); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + // The stage returns a DiskLoc of where to start. + DiskLoc startLoc; + Runner::RunnerState state = runner->getNext(NULL, &startLoc); + + // This is normal. The start of the oplog is the beginning of the collection. + if (Runner::RUNNER_EOF == state) { return getRunner(cq, runnerOut); } + + // This is not normal. An error was encountered. + if (Runner::RUNNER_ADVANCED != state) { + return Status(ErrorCodes::InternalError, + "quick oplog start location had error...?"); + } + + // cout << "diskloc is " << startLoc.toString() << endl; + + // Build our collection scan... + CollectionScanParams params; + params.ns = cq->ns(); + params.start = startLoc; + params.direction = CollectionScanParams::FORWARD; + params.tailable = cq->getParsed().hasOption(QueryOption_CursorTailable); + + WorkingSet* ws = new WorkingSet(); + CollectionScan* cs = new CollectionScan(params, ws, cq->root()); + // Takes ownership of cq, cs, ws. + *runnerOut = new SingleSolutionRunner(collection, cq, NULL, cs, ws); + return Status::OK(); + } + + std::string newRunQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) { + // Validate the namespace. + const char *ns = q.ns; + uassert(16332, "can't have an empty ns", ns[0]); + + const NamespaceString nsString(ns); + uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid()); + + // Set curop information. + curop.debug().ns = ns; + curop.debug().ntoreturn = q.ntoreturn; + curop.debug().query = q.query; + curop.setQuery(q.query); + + // If the query is really a command, run it. + if (nsString.isCommand()) { + int nToReturn = q.ntoreturn; + uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn + << ") for $cmd type ns - can only be 1 or -1", + nToReturn == 1 || nToReturn == -1); + + curop.markCommand(); + + BufBuilder bb; + bb.skip(sizeof(QueryResult)); + + BSONObjBuilder cmdResBuf; + if (!runCommands(ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) { + uasserted(13530, "bad or malformed command request?"); + } + + curop.debug().iscommand = true; + // TODO: Does this get overwritten/do we really need to set this twice? + curop.debug().query = q.query; + + QueryResult* qr = reinterpret_cast(bb.buf()); + bb.decouple(); + qr->setResultFlagsToOk(); + qr->len = bb.len(); + curop.debug().responseLength = bb.len(); + qr->setOperation(opReply); + qr->cursorId = 0; + qr->startingFrom = 0; + qr->nReturned = 1; + result.setData(qr, true); + return ""; + } + + // This is a read lock. We require this because if we're parsing a $where, the + // where-specific parsing code assumes we have a lock and creates execution machinery that + // requires it. + Client::ReadContext ctx(q.ns); + Collection* collection = ctx.ctx().db()->getCollection( ns ); + + // Parse the qm into a CanonicalQuery. + CanonicalQuery* cq; + Status canonStatus = CanonicalQuery::canonicalize(q, &cq); + if (!canonStatus.isOK()) { + uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString()); + } + verify(cq); + + QLOG() << "Running query:\n" << cq->toString(); + LOG(2) << "Running query: " << cq->toStringShort(); + + // Parse, canonicalize, plan, transcribe, and get a runner. + Runner* rawRunner = NULL; + + // We use this a lot below. + const LiteParsedQuery& pq = cq->getParsed(); + + // We'll now try to get the query runner that will execute this query for us. There + // are a few cases in which we know upfront which runner we should get and, therefore, + // we shortcut the selection process here. + // + // (a) If the query is over a collection that doesn't exist, we get a special runner + // that's is so (a runner) which doesn't return results, the EOFRunner. + // + // (b) if the query is a replication's initial sync one, we get a SingleSolutinRunner + // that uses a specifically designed stage that skips extents faster (see details in + // exec/oplogstart.h) + // + // Otherwise we go through the selection of which runner is most suited to the + // query + run-time context at hand. + Status status = Status::OK(); + if (collection == NULL) { + rawRunner = new EOFRunner(cq, cq->ns()); + } + else if (pq.hasOption(QueryOption_OplogReplay)) { + status = getOplogStartHack(collection, cq, &rawRunner); + } + else { + // Takes ownership of cq. + size_t options = QueryPlannerParams::DEFAULT; + if (shardingState.needCollectionMetadata(pq.ns())) { + options |= QueryPlannerParams::INCLUDE_SHARD_FILTER; + } + status = getRunner(cq, &rawRunner, options); + } + + if (!status.isOK()) { + // NOTE: Do not access cq as getRunner has deleted it. + uasserted(17007, "Unable to execute query: " + status.reason()); + } + + verify(NULL != rawRunner); + auto_ptr runner(rawRunner); + + // We freak out later if this changes before we're done with the query. + const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns()); + + // Handle query option $maxTimeMS (not used with commands). + curop.setMaxTimeMicros(static_cast(pq.getMaxTimeMS()) * 1000); + killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. + + // uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set. + replVerifyReadsOk(&pq); + + // If this exists, the collection is sharded. + // If it doesn't exist, we can assume we're not sharded. + // If we're sharded, we might encounter data that is not consistent with our sharding state. + // We must ignore this data. + CollectionMetadataPtr collMetadata; + if (!shardingState.needCollectionMetadata(pq.ns())) { + collMetadata = CollectionMetadataPtr(); + } + else { + collMetadata = shardingState.getCollectionMetadata(pq.ns()); + } + + // Run the query. + // bb is used to hold query results + // this buffer should contain either requested documents per query or + // explain information, but not both + BufBuilder bb(32768); + bb.skip(sizeof(QueryResult)); + + // How many results have we obtained from the runner? + int numResults = 0; + + // If we're replaying the oplog, we save the last time that we read. + OpTime slaveReadTill; + + // Do we save the Runner in a ClientCursor for getMore calls later? + bool saveClientCursor = false; + + // We turn on auto-yielding for the runner here. The runner registers itself with the + // active runners list in ClientCursor. + auto_ptr safety(new ScopedRunnerRegistration(runner.get())); + runner->setYieldPolicy(Runner::YIELD_AUTO); + + BSONObj obj; + Runner::RunnerState state; + // uint64_t numMisplacedDocs = 0; + + // set this outside loop. we will need to use this both within loop and when deciding + // to fill in explain information + const bool isExplain = pq.isExplain(); + + // Have we retrieved info about which plan the runner will + // use to execute the query yet? + bool gotPlanInfo = false; + PlanInfo* rawInfo; + boost::scoped_ptr planInfo; + + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) { + // Add result to output buffer. This is unnecessary if explain info is requested + if (!isExplain) { + bb.appendBuf((void*)obj.objdata(), obj.objsize()); + } + + // Count the result. + ++numResults; + + // In the case of the multi plan runner, we may not be able to + // successfully retrieve plan info until after the query starts + // to run. This is because the multi plan runner doesn't know what + // plan it will end up using until it runs candidates and selects + // the best. + // + // TODO: Do we ever want to output what the MPR is comparing? + if (!gotPlanInfo) { + Status infoStatus = runner->getInfo(NULL, &rawInfo); + if (infoStatus.isOK()) { + gotPlanInfo = true; + planInfo.reset(rawInfo); + // planSummary is really a ThreadSafeString which copies the data from + // the provided pointer. + curop.debug().planSummary = planInfo->planSummary.c_str(); + } + } + + // Possibly note slave's position in the oplog. + if (pq.hasOption(QueryOption_OplogReplay)) { + BSONElement e = obj["ts"]; + if (Date == e.type() || Timestamp == e.type()) { + slaveReadTill = e._opTime(); + } + } + + // TODO: only one type of 2d search doesn't support this. We need a way to pull it out + // of CanonicalQuery. :( + const bool supportsGetMore = true; + if (isExplain) { + if (enoughForExplain(pq, numResults)) { + break; + } + } + else if (!supportsGetMore && (enough(pq, numResults) + || bb.len() >= MaxBytesToReturnToClientAtOnce)) { + break; + } + else if (enoughForFirstBatch(pq, numResults, bb.len())) { + QLOG() << "Enough for first batch, wantMore=" << pq.wantMore() + << " numToReturn=" << pq.getNumToReturn() + << " numResults=" << numResults + << endl; + // If only one result requested assume it's a findOne() and don't save the cursor. + if (pq.wantMore() && 1 != pq.getNumToReturn()) { + QLOG() << " runner EOF=" << runner->isEOF() << endl; + saveClientCursor = !runner->isEOF(); + } + break; + } + } + + // Try to get information about the plan which the runner + // will use to execute the query, it we don't have it already. + if (!gotPlanInfo) { + Status infoStatus = runner->getInfo(NULL, &rawInfo); + if (infoStatus.isOK()) { + gotPlanInfo = true; + planInfo.reset(rawInfo); + // planSummary is really a ThreadSafeString which copies the data from + // the provided pointer. + curop.debug().planSummary = planInfo->planSummary.c_str(); + } + } + + // If we cache the runner later, we want to deregister it as it receives notifications + // anyway by virtue of being cached. + // + // If we don't cache the runner later, we are deleting it, so it must be deregistered. + // + // So, no matter what, deregister the runner. + safety.reset(); + + // Caller expects exceptions thrown in certain cases. + if (Runner::RUNNER_ERROR == state) { + TypeExplain* bareExplain; + Status res = runner->getInfo(&bareExplain, NULL); + if (res.isOK()) { + boost::scoped_ptr errorExplain(bareExplain); + error() << "Runner error, stats:\n" + << errorExplain->stats.jsonString(Strict, true); + } + uasserted(17144, "Runner error: " + WorkingSetCommon::toStatusString(obj)); + } + + // Why save a dead runner? + if (Runner::RUNNER_DEAD == state) { + saveClientCursor = false; + } + else if (pq.hasOption(QueryOption_CursorTailable)) { + // If we're tailing a capped collection, we don't bother saving the cursor if the + // collection is empty. Otherwise, the semantics of the tailable cursor is that the + // client will keep trying to read from it. So we'll keep it around. + Collection* collection = ctx.ctx().db()->getCollection(cq->ns()); + if (collection && collection->numRecords() != 0 && pq.getNumToReturn() != 1) { + saveClientCursor = true; + } + } + + // TODO(greg): This will go away soon. + if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) { + // If the version changed during the query we might be missing some data, and it's safe + // to send this as mongos can resend at this point. + // + // This check is particularly important for the IDHackRunner, which can yield between + // getting the config metadata and checking ownership of the document to be returned. + // If the version changes during that yield, this branch will be taken to force a retry + // of the query. + throw SendStaleConfigException(pq.ns(), "version changed during initial query", + shardingVersionAtStart, + shardingState.getVersion(pq.ns())); + } + + // Used to fill in explain and to determine if the query is slow enough to be logged. + int elapsedMillis = curop.elapsedMillis(); + + // Get explain information if: + // 1) it is needed by an explain query; + // 2) profiling is enabled; + // 3) profiling is disabled but we still need explain details to log a "slow" query; or + // 4) the log level is set to at least logLevel 1. + // Producing explain information is expensive and should be done only if we are certain + // the information will be used. + boost::scoped_ptr explain(NULL); + if (isExplain || + ctx.ctx().db()->getProfilingLevel() > 0 || + elapsedMillis > serverGlobalParams.slowMS || + logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + // Ask the runner to produce explain information. + TypeExplain* bareExplain; + Status res = runner->getInfo(&bareExplain, NULL); + if (res.isOK()) { + explain.reset(bareExplain); + } + else if (isExplain) { + error() << "could not produce explain of query '" << pq.getFilter() + << "', error: " << res.reason(); + // If numResults and the data in bb don't correspond, we'll crash later when rooting + // through the reply msg. + BSONObj emptyObj; + bb.appendBuf((void*)emptyObj.objdata(), emptyObj.objsize()); + // The explain output is actually a result. + numResults = 1; + // TODO: we can fill out millis etc. here just fine even if the plan screwed up. + } + } + + // Fill in the missing run-time fields in explain, starting with propeties of + // the process running the query. + if (isExplain && NULL != explain.get()) { + std::string server = mongoutils::str::stream() + << getHostNameCached() << ":" << serverGlobalParams.port; + explain->setServer(server); + + // We might have skipped some results due to chunk migration etc. so our count is + // correct. + explain->setN(numResults); + + // Clock the whole operation. + explain->setMillis(elapsedMillis); + + BSONObj explainObj = explain->toBSON(); + bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize()); + + // The explain output is actually a result. + numResults = 1; + } + + long long ccId = 0; + if (saveClientCursor) { + // We won't use the runner until it's getMore'd. + runner->saveState(); + + // Allocate a new ClientCursor. We don't have to worry about leaking it as it's + // inserted into a global map by its ctor. + ClientCursor* cc = new ClientCursor(collection, runner.get(), + cq->getParsed().getOptions(), + cq->getParsed().getFilter()); + ccId = cc->cursorid(); + + QLOG() << "caching runner with cursorid " << ccId + << " after returning " << numResults << " results" << endl; + + // ClientCursor takes ownership of runner. Release to make sure it's not deleted. + runner.release(); + + // TODO document + if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) { + cc->slaveReadTill(slaveReadTill); + } + + // TODO document + if (pq.hasOption(QueryOption_Exhaust)) { + curop.debug().exhaust = true; + } + + // Set attributes for getMore. + cc->setCollMetadata(collMetadata); + cc->setPos(numResults); + + // If the query had a time limit, remaining time is "rolled over" to the cursor (for + // use by future getmore ops). + cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros()); + } + else { + QLOG() << "Not caching runner but returning " << numResults << " results.\n"; + } + + // Add the results from the query into the output buffer. + result.appendData(bb.buf(), bb.len()); + bb.decouple(); + + // Fill out the output buffer's header. + QueryResult* qr = static_cast(result.header()); + qr->cursorId = ccId; + curop.debug().cursorid = (0 == ccId ? -1 : ccId); + qr->setResultFlagsToOk(); + qr->setOperation(opReply); + qr->startingFrom = 0; + qr->nReturned = numResults; + + // Set debug information for consumption by the profiler. + curop.debug().ntoskip = pq.getSkip(); + curop.debug().nreturned = numResults; + if (NULL != explain.get()) { + if (explain->isScanAndOrderSet()) { + curop.debug().scanAndOrder = explain->getScanAndOrder(); + } + else { + curop.debug().scanAndOrder = false; + } + + if (explain->isNScannedSet()) { + curop.debug().nscanned = explain->getNScanned(); + } + + if (explain->isNScannedObjectsSet()) { + curop.debug().nscannedObjects = explain->getNScannedObjects(); + } + + if (explain->isIDHackSet()) { + curop.debug().idhack = explain->getIDHack(); + } + + if (!explain->stats.isEmpty()) { + // execStats is a CachedBSONObj because it lives in the race-prone + // curop. + curop.debug().execStats.set(explain->stats); + + // Replace exec stats with plan summary if stats cannot fit into CachedBSONObj. + if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) { + BSONObjBuilder bob; + bob.append("summary", curop.debug().planSummary.toString()); + curop.debug().execStats.set(bob.done()); + } + + } + } + + // curop.debug().exhaust is set above. + return curop.debug().exhaust ? pq.ns() : ""; + } + +} // namespace mongo diff --git a/src/mongo/db/query/new_find.h b/src/mongo/db/query/new_find.h new file mode 100644 index 00000000000..2977a83ab51 --- /dev/null +++ b/src/mongo/db/query/new_find.h @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/clientcursor.h" +#include "mongo/db/curop.h" +#include "mongo/db/dbmessage.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/runner.h" +#include "mongo/util/net/message.h" + +namespace mongo { + + /** + * Called from the getMore entry point in ops/query.cpp. + */ + QueryResult* newGetMore(const char* ns, int ntoreturn, long long cursorid, CurOp& curop, + int pass, bool& exhaust, bool* isCursorAuthorized); + + /** + * Run the query 'q' and place the result in 'result'. + */ + std::string newRunQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result); + +} // namespace mongo diff --git a/src/mongo/db/query/parsed_projection.cpp b/src/mongo/db/query/parsed_projection.cpp new file mode 100644 index 00000000000..a30623e36e6 --- /dev/null +++ b/src/mongo/db/query/parsed_projection.cpp @@ -0,0 +1,317 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/parsed_projection.h" + +#include "mongo/db/query/lite_parsed_query.h" + +namespace mongo { + + /** + * Parses the projection 'spec' and checks its validity with respect to the query 'query'. + * Puts covering information into 'out'. + * + * Does not take ownership of 'query'. + * + * Returns Status::OK() if it's a valid spec. + * Returns a Status indicating how it's invalid otherwise. + */ + // static + Status ParsedProjection::make(const BSONObj& spec, const MatchExpression* const query, + ParsedProjection** out) { + // Are we including or excluding fields? Values: + // -1 when we haven't initialized it. + // 1 when we're including + // 0 when we're excluding. + int include_exclude = -1; + + // If any of these are 'true' the projection isn't covered. + bool include = true; + bool hasNonSimple = false; + bool hasDottedField = false; + + bool includeID = true; + + bool hasIndexKeyProjection = false; + + bool wantGeoNearPoint = false; + bool wantGeoNearDistance = false; + + // Until we see a positional or elemMatch operator we're normal. + ArrayOpType arrayOpType = ARRAY_OP_NORMAL; + + BSONObjIterator it(spec); + while (it.more()) { + BSONElement e = it.next(); + + if (!e.isNumber() && !e.isBoolean()) { + hasNonSimple = true; + } + + if (Object == e.type()) { + BSONObj obj = e.embeddedObject(); + if (1 != obj.nFields()) { + return Status(ErrorCodes::BadValue, ">1 field in obj: " + obj.toString()); + } + + BSONElement e2 = obj.firstElement(); + if (mongoutils::str::equals(e2.fieldName(), "$slice")) { + if (e2.isNumber()) { + // This is A-OK. + } + else if (e2.type() == Array) { + BSONObj arr = e2.embeddedObject(); + if (2 != arr.nFields()) { + return Status(ErrorCodes::BadValue, "$slice array wrong size"); + } + + BSONObjIterator it(arr); + // Skip over 'skip'. + it.next(); + int limit = it.next().numberInt(); + if (limit <= 0) { + return Status(ErrorCodes::BadValue, "$slice limit must be positive"); + } + } + else { + return Status(ErrorCodes::BadValue, + "$slice only supports numbers and [skip, limit] arrays"); + } + } + else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) { + // Validate $elemMatch arguments and dependencies. + if (Object != e2.type()) { + return Status(ErrorCodes::BadValue, + "elemMatch: Invalid argument, object required."); + } + + if (ARRAY_OP_POSITIONAL == arrayOpType) { + return Status(ErrorCodes::BadValue, + "Cannot specify positional operator and $elemMatch."); + } + + if (mongoutils::str::contains(e.fieldName(), '.')) { + return Status(ErrorCodes::BadValue, + "Cannot use $elemMatch projection on a nested field."); + } + + arrayOpType = ARRAY_OP_ELEM_MATCH; + + // Create a MatchExpression for the elemMatch. + BSONObj elemMatchObj = e.wrap(); + verify(elemMatchObj.isOwned()); + + // TODO: Is there a faster way of validating the elemMatchObj? + StatusWithMatchExpression swme = MatchExpressionParser::parse(elemMatchObj); + if (!swme.isOK()) { + return swme.getStatus(); + } + delete swme.getValue(); + } + else if (mongoutils::str::equals(e2.fieldName(), "$meta")) { + // Field for meta must be top level. We can relax this at some point. + if (mongoutils::str::contains(e.fieldName(), '.')) { + return Status(ErrorCodes::BadValue, "field for $meta cannot be nested"); + } + + // Make sure the argument to $meta is something we recognize. + // e.g. {x: {$meta: "textScore"}} + if (String != e2.type()) { + return Status(ErrorCodes::BadValue, "unexpected argument to $meta in proj"); + } + + if (e2.valuestr() != LiteParsedQuery::metaTextScore + && e2.valuestr() != LiteParsedQuery::metaDiskLoc + && e2.valuestr() != LiteParsedQuery::metaIndexKey + && e2.valuestr() != LiteParsedQuery::metaGeoNearDistance + && e2.valuestr() != LiteParsedQuery::metaGeoNearPoint) { + return Status(ErrorCodes::BadValue, + "unsupported $meta operator: " + e2.str()); + } + + // This clobbers everything else. + if (e2.valuestr() == LiteParsedQuery::metaIndexKey) { + hasIndexKeyProjection = true; + } + else if (e2.valuestr() == LiteParsedQuery::metaGeoNearDistance) { + wantGeoNearDistance = true; + } + else if (e2.valuestr() == LiteParsedQuery::metaGeoNearPoint) { + wantGeoNearPoint = true; + } + } + else { + return Status(ErrorCodes::BadValue, + string("Unsupported projection option: ") + e.toString()); + } + } + else if (mongoutils::str::equals(e.fieldName(), "_id") && !e.trueValue()) { + includeID = false; + } + else { + // Projections of dotted fields aren't covered. + if (mongoutils::str::contains(e.fieldName(), '.')) { + hasDottedField = true; + } + + // Validate input. + if (include_exclude == -1) { + // If we haven't specified an include/exclude, initialize include_exclude. + // We expect further include/excludes to match it. + include_exclude = e.trueValue(); + include = !e.trueValue(); + } + else if (static_cast(include_exclude) != e.trueValue()) { + // Make sure that the incl./excl. matches the previous. + return Status(ErrorCodes::BadValue, + "Projection cannot have a mix of inclusion and exclusion."); + } + } + + + if (_isPositionalOperator(e.fieldName())) { + // Validate the positional op. + if (!e.trueValue()) { + return Status(ErrorCodes::BadValue, + "Cannot exclude array elements with the positional operator."); + } + + if (ARRAY_OP_POSITIONAL == arrayOpType) { + return Status(ErrorCodes::BadValue, + "Cannot specify more than one positional proj. per query."); + } + + if (ARRAY_OP_ELEM_MATCH == arrayOpType) { + return Status(ErrorCodes::BadValue, + "Cannot specify positional operator and $elemMatch."); + } + + std::string after = mongoutils::str::after(e.fieldName(), ".$"); + if (mongoutils::str::contains(after, ".$")) { + mongoutils::str::stream ss; + ss << "Positional projection '" << e.fieldName() << "' contains " + << "the positional operator more than once."; + return Status(ErrorCodes::BadValue, ss); + } + + std::string matchfield = mongoutils::str::before(e.fieldName(), '.'); + if (!_hasPositionalOperatorMatch(query, matchfield)) { + mongoutils::str::stream ss; + ss << "Positional projection '" << e.fieldName() << "' does not " + << "match the query document."; + return Status(ErrorCodes::BadValue, ss); + } + + arrayOpType = ARRAY_OP_POSITIONAL; + } + } + + // Fill out the returned obj. + auto_ptr pp(new ParsedProjection()); + + // Save the raw spec. It should be owned by the LiteParsedQuery. + verify(spec.isOwned()); + pp->_source = spec; + pp->_returnKey = hasIndexKeyProjection; + + // Dotted fields aren't covered, non-simple require match details, and as for include, "if + // we default to including then we can't use an index because we don't know what we're + // missing." + pp->_requiresDocument = include || hasNonSimple || hasDottedField; + + // Add geoNear projections. + pp->_wantGeoNearPoint = wantGeoNearPoint; + pp->_wantGeoNearDistance = wantGeoNearDistance; + + // If it's possible to compute the projection in a covered fashion, populate _requiredFields + // so the planner can perform projection analysis. + if (!pp->_requiresDocument) { + if (includeID) { + pp->_requiredFields.push_back("_id"); + } + + // The only way we could be here is if spec is only simple non-dotted-field projections. + // Therefore we can iterate over spec to get the fields required. + BSONObjIterator srcIt(spec); + while (srcIt.more()) { + BSONElement elt = srcIt.next(); + // We've already handled the _id field before entering this loop. + if (includeID && mongoutils::str::equals(elt.fieldName(), "_id")) { + continue; + } + if (elt.trueValue()) { + pp->_requiredFields.push_back(elt.fieldName()); + } + } + } + + // returnKey clobbers everything. + if (hasIndexKeyProjection) { + pp->_requiresDocument = false; + } + + *out = pp.release(); + return Status::OK(); + } + + // static + bool ParsedProjection::_isPositionalOperator(const char* fieldName) { + return mongoutils::str::contains(fieldName, ".$") && + !mongoutils::str::contains(fieldName, ".$ref") && + !mongoutils::str::contains(fieldName, ".$id") && + !mongoutils::str::contains(fieldName, ".$db"); + + } + + // static + bool ParsedProjection::_hasPositionalOperatorMatch(const MatchExpression* const query, + const std::string& matchfield) { + if (query->isLogical()) { + for (unsigned int i = 0; i < query->numChildren(); ++i) { + if (_hasPositionalOperatorMatch(query->getChild(i), matchfield)) { + return true; + } + } + } + else { + StringData queryPath = query->path(); + const char* pathRawData = queryPath.rawData(); + // We have to make a distinction between match expressions that are + // initialized with an empty field/path name "" and match expressions + // for which the path is not meaningful (eg. $where and the internal + // expression type ALWAYS_FALSE). + if (!pathRawData) { + return false; + } + std::string pathPrefix = mongoutils::str::before(pathRawData, '.'); + return pathPrefix == matchfield; + } + return false; + } + +} // namespace mongo diff --git a/src/mongo/db/query/parsed_projection.h b/src/mongo/db/query/parsed_projection.h new file mode 100644 index 00000000000..1d94057dff6 --- /dev/null +++ b/src/mongo/db/query/parsed_projection.h @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + class ParsedProjection { + public: + // TODO: this is duplicated in here and in the proj exec code. When we have + // ProjectionExpression we can remove dups. + enum ArrayOpType { + ARRAY_OP_NORMAL = 0, + ARRAY_OP_ELEM_MATCH, + ARRAY_OP_POSITIONAL + }; + + /** + * Parses the projection 'spec' and checks its validity with respect to the query 'query'. + * Puts covering information into 'out'. + * + * Returns Status::OK() if it's a valid spec. + * Returns a Status indicating how it's invalid otherwise. + */ + static Status make(const BSONObj& spec, const MatchExpression* const query, + ParsedProjection** out); + + /** + * Is the full document required to compute this projection? + */ + bool requiresDocument() const { return _requiresDocument; } + + /** + * If requiresDocument() == false, what fields are required to compute + * the projection? + */ + const vector& getRequiredFields() const { + return _requiredFields; + } + + /** + * Get the raw BSONObj proj spec obj + */ + const BSONObj& getProjObj() const { + return _source; + } + + /** + * Does the projection want geoNear metadata? If so any geoNear stage should include them. + */ + bool wantGeoNearDistance() const { + return _wantGeoNearDistance; + } + + bool wantGeoNearPoint() const { + return _wantGeoNearPoint; + } + + bool wantIndexKey() const { + return _returnKey; + } + + private: + /** + * Must go through ::make + */ + ParsedProjection() : _requiresDocument(true) { } + + /** + * Returns true if field name refers to a positional projection. + */ + static bool _isPositionalOperator(const char* fieldName); + + /** + * Returns true if the MatchExpression 'query' queries against + * the field named by 'matchfield'. This deeply traverses logical + * nodes in the matchfield and returns true if any of the children + * have the field (so if 'query' is {$and: [{a: 1}, {b: 1}]} and + * 'matchfield' is "b", the return value is true). + * + * Does not take ownership of 'query'. + */ + static bool _hasPositionalOperatorMatch(const MatchExpression* const query, + const std::string& matchfield); + + // TODO: stringdata? + vector _requiredFields; + + bool _requiresDocument; + + BSONObj _source; + + bool _wantGeoNearDistance; + + bool _wantGeoNearPoint; + + bool _returnKey; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/parsed_projection_test.cpp b/src/mongo/db/query/parsed_projection_test.cpp new file mode 100644 index 00000000000..20beaa70a6c --- /dev/null +++ b/src/mongo/db/query/parsed_projection_test.cpp @@ -0,0 +1,214 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/query/parsed_projection.h" + +#include +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using std::auto_ptr; + using std::string; + using std::vector; + + using namespace mongo; + + // + // creation function + // + + ParsedProjection* createParsedProjection(const BSONObj& query, const BSONObj& projObj) { + StatusWithMatchExpression swme = MatchExpressionParser::parse(query); + ASSERT(swme.isOK()); + MatchExpression* queryMatchExpr = swme.getValue(); + ParsedProjection* out = NULL; + Status status = ParsedProjection::make(projObj, queryMatchExpr, &out); + if (!status.isOK()) { + FAIL(mongoutils::str::stream() << "failed to parse projection " << projObj + << " (query: " << query << "): " << status.toString()); + } + ASSERT(out); + return out; + } + + ParsedProjection* createParsedProjection(const char* queryStr, const char* projStr) { + BSONObj query = fromjson(queryStr); + BSONObj projObj = fromjson(projStr); + return createParsedProjection(query, projObj); + } + + // + // Failure to create a parsed projection is expected + // + + void assertInvalidProjection(const char* queryStr, const char* projStr) { + BSONObj query = fromjson(queryStr); + BSONObj projObj = fromjson(projStr); + StatusWithMatchExpression swme = MatchExpressionParser::parse(query); + ASSERT(swme.isOK()); + MatchExpression* queryMatchExpr = swme.getValue(); + ParsedProjection* out = NULL; + Status status = ParsedProjection::make(projObj, queryMatchExpr, &out); + ASSERT(!status.isOK()); + } + + // canonical_query.cpp will invoke ParsedProjection::make only when + // the projection spec is non-empty. This test case is included for + // completeness and do not reflect actual usage. + TEST(ParsedProjectionTest, MakeId) { + auto_ptr parsedProj(createParsedProjection("{}", "{}")); + ASSERT(parsedProj->requiresDocument()); + } + + TEST(ParsedProjectionTest, MakeEmpty) { + auto_ptr parsedProj(createParsedProjection("{}", "{_id: 0}")); + ASSERT(parsedProj->requiresDocument()); + } + + TEST(ParsedProjectionTest, MakeSingleField) { + auto_ptr parsedProj(createParsedProjection("{}", "{a: 1}")); + ASSERT(!parsedProj->requiresDocument()); + const vector& fields = parsedProj->getRequiredFields(); + ASSERT_EQUALS(fields.size(), 2U); + ASSERT_EQUALS(fields[0], "_id"); + ASSERT_EQUALS(fields[1], "a"); + } + + TEST(ParsedProjectionTest, MakeSingleFieldCovered) { + auto_ptr parsedProj(createParsedProjection("{}", "{_id: 0, a: 1}")); + ASSERT(!parsedProj->requiresDocument()); + const vector& fields = parsedProj->getRequiredFields(); + ASSERT_EQUALS(fields.size(), 1U); + ASSERT_EQUALS(fields[0], "a"); + } + + TEST(ParsedProjectionTest, MakeSingleFieldIDCovered) { + auto_ptr parsedProj(createParsedProjection("{}", "{_id: 1}")); + ASSERT(!parsedProj->requiresDocument()); + const vector& fields = parsedProj->getRequiredFields(); + ASSERT_EQUALS(fields.size(), 1U); + ASSERT_EQUALS(fields[0], "_id"); + } + + // boolean support is undocumented + TEST(ParsedProjectionTest, MakeSingleFieldCoveredBoolean) { + auto_ptr parsedProj(createParsedProjection("{}", "{_id: 0, a: true}")); + ASSERT(!parsedProj->requiresDocument()); + const vector& fields = parsedProj->getRequiredFields(); + ASSERT_EQUALS(fields.size(), 1U); + ASSERT_EQUALS(fields[0], "a"); + } + + // boolean support is undocumented + TEST(ParsedProjectionTest, MakeSingleFieldCoveredIdBoolean) { + auto_ptr parsedProj(createParsedProjection("{}", "{_id: false, a: 1}")); + ASSERT(!parsedProj->requiresDocument()); + const vector& fields = parsedProj->getRequiredFields(); + ASSERT_EQUALS(fields.size(), 1U); + ASSERT_EQUALS(fields[0], "a"); + } + + // + // Positional operator validation + // + + TEST(ParsedProjectionTest, InvalidPositionalOperatorProjections) { + assertInvalidProjection("{}", "{'a.$': 1}"); + assertInvalidProjection("{a: 1}", "{'b.$': 1}"); + assertInvalidProjection("{a: 1}", "{'a.$': 0}"); + assertInvalidProjection("{a: 1}", "{'a.$.d.$': 1}"); + assertInvalidProjection("{a: 1}", "{'a.$.$': 1}"); + assertInvalidProjection("{a: 1}", "{'a.$.$': 1}"); + assertInvalidProjection("{a: 1, b: 1, c: 1}", "{'abc.$': 1}"); + assertInvalidProjection("{$or: [{a: 1}, {$or: [{b: 1}, {c: 1}]}]}", "{'d.$': 1}"); + assertInvalidProjection("{a: [1, 2, 3]}", "{'.$': 1}"); + } + + TEST(ParsedProjectionTest, ValidPositionalOperatorProjections) { + createParsedProjection("{a: 1}", "{'a.$': 1}"); + createParsedProjection("{a: 1}", "{'a.foo.bar.$': 1}"); + createParsedProjection("{a: 1}", "{'a.foo.bar.$.x.y': 1}"); + createParsedProjection("{'a.b.c': 1}", "{'a.b.c.$': 1}"); + createParsedProjection("{'a.b.c': 1}", "{'a.e.f.$': 1}"); + createParsedProjection("{a: {b: 1}}", "{'a.$': 1}"); + createParsedProjection("{a: 1, b: 1}}", "{'a.$': 1}"); + createParsedProjection("{a: 1, b: 1}}", "{'b.$': 1}"); + createParsedProjection("{$and: [{a: 1}, {b: 1}]}", "{'a.$': 1}"); + createParsedProjection("{$and: [{a: 1}, {b: 1}]}", "{'b.$': 1}"); + createParsedProjection("{$or: [{a: 1}, {b: 1}]}", "{'a.$': 1}"); + createParsedProjection("{$or: [{a: 1}, {b: 1}]}", "{'b.$': 1}"); + createParsedProjection("{$and: [{$or: [{a: 1}, {$and: [{b: 1}, {c: 1}]}]}]}", + "{'c.d.f.$': 1}"); + // Fields with empty name can be projected using the positional $ operator. + createParsedProjection("{'': [1, 2, 3]}", "{'.$': 1}"); + } + + // Some match expressions (eg. $where) do not override MatchExpression::path() + // In this test case, we use an internal match expression implementation ALWAYS_FALSE + // to achieve the same effect. + // Projection parser should handle this the same way as an empty path. + TEST(ParsedProjectionTest, InvalidPositionalProjectionDefaultPathMatchExpression) { + auto_ptr queryMatchExpr(new FalseMatchExpression()); + ASSERT(NULL == queryMatchExpr->path().rawData()); + + ParsedProjection* out = NULL; + BSONObj projObj = fromjson("{'a.$': 1}"); + Status status = ParsedProjection::make(projObj, queryMatchExpr.get(), &out); + ASSERT(!status.isOK()); + + // Projecting onto empty field should fail. + BSONObj emptyFieldProjObj = fromjson("{'.$': 1}"); + status = ParsedProjection::make(emptyFieldProjObj, queryMatchExpr.get(), &out); + ASSERT(!status.isOK()); + } + + // + // DBRef projections + // + + TEST(ParsedProjectionTest, DBRefProjections) { + // non-dotted + createParsedProjection(BSONObj(), BSON( "$ref" << 1)); + createParsedProjection(BSONObj(), BSON( "$id" << 1)); + createParsedProjection(BSONObj(), BSON( "$ref" << 1)); + // dotted before + createParsedProjection("{}", "{'a.$ref': 1}"); + createParsedProjection("{}", "{'a.$id': 1}"); + createParsedProjection("{}", "{'a.$db': 1}"); + // dotted after + createParsedProjection("{}", "{'$id.a': 1}"); + // position operator on $id + // $ref and $db hold the collection and database names respectively, + // so these fields cannot be arrays. + createParsedProjection("{'a.$id': {$elemMatch: {x: 1}}}", "{'a.$id.$': 1}"); + + } +} // unnamed namespace diff --git a/src/mongo/db/query/plan_cache.cpp b/src/mongo/db/query/plan_cache.cpp new file mode 100644 index 00000000000..3c7144b0b14 --- /dev/null +++ b/src/mongo/db/query/plan_cache.cpp @@ -0,0 +1,513 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/plan_cache.h" + +#include +#include +#include +#include "boost/thread/locks.hpp" +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/client/dbclientinterface.h" // For QueryOption_foobar +#include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + // + // Cache-related functions for CanonicalQuery + // + + bool PlanCache::shouldCacheQuery(const CanonicalQuery& query) { + const LiteParsedQuery& lpq = query.getParsed(); + const MatchExpression* expr = query.root(); + + // Collection scan + // No sort order requested + if (lpq.getSort().isEmpty() && + expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) { + return false; + } + + // Hint provided + if (!lpq.getHint().isEmpty()) { + return false; + } + + // Min provided + // Min queries are a special case of hinted queries. + if (!lpq.getMin().isEmpty()) { + return false; + } + + // Max provided + // Similar to min, max queries are a special case of hinted queries. + if (!lpq.getMax().isEmpty()) { + return false; + } + + // Explain queries are not-cacheable. This is primarily because of + // the need to generate current and accurate information in allPlans. + // If the explain report is generated by the cached plan runner using + // stale information from the cache for the losing plans, allPlans would + // simply be wrong. + if (lpq.isExplain()) { + return false; + } + + // Tailable cursors won't get cached, just turn into collscans. + if (query.getParsed().hasOption(QueryOption_CursorTailable)) { + return false; + } + + // Snapshot is really a hint. + if (query.getParsed().isSnapshot()) { + return false; + } + + return true; + } + + // + // CachedSolution + // + + CachedSolution::CachedSolution(const PlanCacheKey& key, const PlanCacheEntry& entry) + : plannerData(entry.plannerData.size()), + backupSoln(entry.backupSoln), + key(key), + query(entry.query.getOwned()), + sort(entry.sort.getOwned()), + projection(entry.projection.getOwned()) { + // CachedSolution should not having any references into + // cache entry. All relevant data should be cloned/copied. + for (size_t i = 0; i < entry.plannerData.size(); ++i) { + verify(entry.plannerData[i]); + plannerData[i] = entry.plannerData[i]->clone(); + } + } + + CachedSolution::~CachedSolution() { + for (std::vector::const_iterator i = plannerData.begin(); + i != plannerData.end(); ++i) { + SolutionCacheData* scd = *i; + delete scd; + } + } + + // + // PlanCacheEntry + // + + PlanCacheEntry::PlanCacheEntry(const std::vector& solutions, + PlanRankingDecision* why) + : plannerData(solutions.size()), + decision(why) { + invariant(why); + + // The caller of this constructor is responsible for ensuring + // that the QuerySolution 's' has valid cacheData. If there's no + // data to cache you shouldn't be trying to construct a PlanCacheEntry. + + // Copy the solution's cache data into the plan cache entry. + for (size_t i = 0; i < solutions.size(); ++i) { + invariant(solutions[i]->cacheData.get()); + plannerData[i] = solutions[i]->cacheData->clone(); + } + } + + PlanCacheEntry::~PlanCacheEntry() { + for (size_t i = 0; i < feedback.size(); ++i) { + delete feedback[i]; + } + for (size_t i = 0; i < plannerData.size(); ++i) { + delete plannerData[i]; + } + } + + PlanCacheEntry* PlanCacheEntry::clone() const { + OwnedPointerVector solutions; + for (size_t i = 0; i < plannerData.size(); ++i) { + QuerySolution* qs = new QuerySolution(); + qs->cacheData.reset(plannerData[i]->clone()); + solutions.mutableVector().push_back(qs); + } + PlanCacheEntry* entry = new PlanCacheEntry(solutions.vector(), decision->clone()); + + entry->backupSoln = backupSoln; + + // Copy query shape. + entry->query = query.getOwned(); + entry->sort = sort.getOwned(); + entry->projection = projection.getOwned(); + + // Copy performance stats. + for (size_t i = 0; i < feedback.size(); ++i) { + PlanCacheEntryFeedback* fb = new PlanCacheEntryFeedback(); + fb->stats.reset(feedback[i]->stats->clone()); + fb->score = feedback[i]->score; + entry->feedback.push_back(fb); + } + entry->averageScore = averageScore; + entry->stddevScore = stddevScore; + return entry; + } + + string PlanCacheEntry::toString() const { + mongoutils::str::stream ss; + ss << "(query: " << query.toString() + << ";sort: " << sort.toString() + << ";projection: " << projection.toString() + << ";solutions: " << plannerData.size() + << ")"; + return ss; + } + + string CachedSolution::toString() const { + mongoutils::str::stream ss; + ss << "key: " << key << '\n'; + return ss; + } + + + // static + const double PlanCacheEntry::kMinDeviation = 0.0001; + + // + // PlanCacheIndexTree + // + + void PlanCacheIndexTree::setIndexEntry(const IndexEntry& ie) { + entry.reset(new IndexEntry(ie)); + } + + PlanCacheIndexTree* PlanCacheIndexTree::clone() const { + PlanCacheIndexTree* root = new PlanCacheIndexTree(); + if (NULL != entry.get()) { + root->index_pos = index_pos; + root->setIndexEntry(*entry.get()); + } + + for (vector::const_iterator it = children.begin(); + it != children.end(); ++it) { + PlanCacheIndexTree* clonedChild = (*it)->clone(); + root->children.push_back(clonedChild); + } + return root; + } + + std::string PlanCacheIndexTree::toString(int indents) const { + mongoutils::str::stream ss; + if (!children.empty()) { + ss << string(3 * indents, '-') << "Node\n"; + int newIndent = indents + 1; + for (vector::const_iterator it = children.begin(); + it != children.end(); ++it) { + ss << (*it)->toString(newIndent); + } + return ss; + } + else { + ss << string(3 * indents, '-') << "Leaf "; + if (NULL != entry.get()) { + ss << entry->keyPattern.toString() << ", pos: " << index_pos; + } + ss << '\n'; + } + return ss; + } + + // + // SolutionCacheData + // + + SolutionCacheData* SolutionCacheData::clone() const { + SolutionCacheData* other = new SolutionCacheData(); + if (NULL != this->tree.get()) { + // 'tree' could be NULL if the cached solution + // is a collection scan. + other->tree.reset(this->tree->clone()); + } + other->solnType = this->solnType; + other->wholeIXSolnDir = this->wholeIXSolnDir; + other->indexFilterApplied = this->indexFilterApplied; + return other; + } + + std::string SolutionCacheData::toString() const { + mongoutils::str::stream ss; + switch (this->solnType) { + case WHOLE_IXSCAN_SOLN: + verify(this->tree.get()); + ss << "(whole index scan solution: " + << "dir=" << this->wholeIXSolnDir << "; " + << "tree=" << this->tree->toString() + << ")"; + break; + case COLLSCAN_SOLN: + ss << "(collection scan)"; + break; + case USE_INDEX_TAGS_SOLN: + verify(this->tree.get()); + ss << "(index-tagged expression tree: " + << "tree=" << this->tree->toString() + << ")"; + } + return ss; + } + + // + // PlanCache + // + + PlanCache::PlanCache() : _cache(internalQueryCacheSize) { } + + PlanCache::PlanCache(const std::string& ns) : _cache(internalQueryCacheSize), _ns(ns) { } + + PlanCache::~PlanCache() { } + + Status PlanCache::add(const CanonicalQuery& query, + const std::vector& solns, + PlanRankingDecision* why) { + invariant(why); + + if (solns.empty()) { + return Status(ErrorCodes::BadValue, "no solutions provided"); + } + + if (why->stats.size() != solns.size()) { + return Status(ErrorCodes::BadValue, + "number of stats in decision must match solutions"); + } + + if (why->scores.size() != solns.size()) { + return Status(ErrorCodes::BadValue, + "number of scores in decision must match solutions"); + } + + if (why->candidateOrder.size() != solns.size()) { + return Status(ErrorCodes::BadValue, + "candidate ordering entries in decision must match solutions"); + } + + PlanCacheEntry* entry = new PlanCacheEntry(solns, why); + const LiteParsedQuery& pq = query.getParsed(); + entry->query = pq.getFilter().getOwned(); + entry->sort = pq.getSort().getOwned(); + entry->projection = pq.getProj().getOwned(); + + // If the winning solution uses a blocking stage, then try and + // find a fallback solution that has no blocking stage. + if (solns[0]->hasBlockingStage) { + for (size_t i = 1; i < solns.size(); ++i) { + if (!solns[i]->hasBlockingStage) { + entry->backupSoln.reset(i); + break; + } + } + } + + boost::lock_guard cacheLock(_cacheMutex); + std::auto_ptr evictedEntry = _cache.add(query.getPlanCacheKey(), entry); + + if (NULL != evictedEntry.get()) { + LOG(1) << _ns << ": plan cache maximum size exceeded - " + << "removed least recently used entry " + << evictedEntry->toString(); + } + + return Status::OK(); + } + + Status PlanCache::get(const CanonicalQuery& query, CachedSolution** crOut) const { + const PlanCacheKey& key = query.getPlanCacheKey(); + verify(crOut); + + boost::lock_guard cacheLock(_cacheMutex); + PlanCacheEntry* entry; + Status cacheStatus = _cache.get(key, &entry); + if (!cacheStatus.isOK()) { + return cacheStatus; + } + invariant(entry); + + *crOut = new CachedSolution(key, *entry); + + return Status::OK(); + } + + // TODO: Figure out what the right policy is here for determining if the cached solution is bad. + // This is a solution but may not be the right one, if there even is a right one... + static bool hasCachedPlanPerformanceDegraded(PlanCacheEntry* entry, + PlanCacheEntryFeedback* latestFeedback) { + + if (!entry->averageScore) { + // We haven't computed baseline performance stats for this cached plan yet. + // Let's do that now. + + // Compute mean score. + double sum = 0; + for (size_t i = 0; i < entry->feedback.size(); ++i) { + sum += entry->feedback[i]->score; + } + double mean = sum / entry->feedback.size(); + + // Compute std deviation of scores. + double sum_of_squares = 0; + for (size_t i = 0; i < entry->feedback.size(); ++i) { + double iscore = entry->feedback[i]->score; + sum_of_squares += (iscore - mean) * (iscore - mean); + } + double stddev = sqrt(sum_of_squares / (entry->feedback.size() - 1)); + + entry->averageScore.reset(mean); + entry->stddevScore.reset(stddev); + } + + // If the latest use of this plan cache entry is too far from the expected + // performance, then we should uncache the entry. Only uncache if the deviation + // also exceeds a minimum value. + double deviation = *entry->averageScore - latestFeedback->score; + + if (deviation < PlanCacheEntry::kMinDeviation) { + // The plan performed better then the average or is only worse by + // epsilon. Keep the cache entry, regardless of the std dev. + return false; + } + + if (deviation > (internalQueryCacheStdDeviations * (*entry->stddevScore))) { + // This run of the plan was much worse than average. + // Kick it out of the plan cache. + return true; + } + + // If we're here, the performance deviated from the average, but + // not by enough to warrant uncaching. + + return false; + } + + Status PlanCache::feedback(const CanonicalQuery& cq, PlanCacheEntryFeedback* feedback) { + if (NULL == feedback) { + return Status(ErrorCodes::BadValue, "feedback is NULL"); + } + std::auto_ptr autoFeedback(feedback); + const PlanCacheKey& ck = cq.getPlanCacheKey(); + + boost::lock_guard cacheLock(_cacheMutex); + PlanCacheEntry* entry; + Status cacheStatus = _cache.get(ck, &entry); + if (!cacheStatus.isOK()) { + return cacheStatus; + } + invariant(entry); + + if (entry->feedback.size() >= size_t(internalQueryCacheFeedbacksStored)) { + // If we have enough feedback, then use it to determine whether + // we should get rid of the cached solution. + if (hasCachedPlanPerformanceDegraded(entry, autoFeedback.get())) { + LOG(1) << _ns << ": removing plan cache entry " << entry->toString() + << " - detected degradation in performance of cached solution."; + _cache.remove(ck); + } + } + else { + // We don't have enough feedback yet---just store it and move on. + entry->feedback.push_back(autoFeedback.release()); + } + + return Status::OK(); + } + + Status PlanCache::remove(const CanonicalQuery& canonicalQuery) { + boost::lock_guard cacheLock(_cacheMutex); + return _cache.remove(canonicalQuery.getPlanCacheKey()); + } + + void PlanCache::clear() { + boost::lock_guard cacheLock(_cacheMutex); + _cache.clear(); + _writeOperations.store(0); + } + + Status PlanCache::getEntry(const CanonicalQuery& query, PlanCacheEntry** entryOut) const { + const PlanCacheKey& key = query.getPlanCacheKey(); + verify(entryOut); + + boost::lock_guard cacheLock(_cacheMutex); + PlanCacheEntry* entry; + Status cacheStatus = _cache.get(key, &entry); + if (!cacheStatus.isOK()) { + return cacheStatus; + } + invariant(entry); + + *entryOut = entry->clone(); + + return Status::OK(); + } + + std::vector PlanCache::getAllEntries() const { + boost::lock_guard cacheLock(_cacheMutex); + std::vector entries; + typedef std::list< std::pair >::const_iterator ConstIterator; + for (ConstIterator i = _cache.begin(); i != _cache.end(); i++) { + PlanCacheEntry* entry = i->second; + entries.push_back(entry->clone()); + } + + return entries; + } + + bool PlanCache::contains(const CanonicalQuery& cq) const { + boost::lock_guard cacheLock(_cacheMutex); + return _cache.hasKey(cq.getPlanCacheKey()); + } + + size_t PlanCache::size() const { + boost::lock_guard cacheLock(_cacheMutex); + return _cache.size(); + } + + void PlanCache::notifyOfWriteOp() { + // It's fine to clear the cache multiple times if multiple threads + // increment the counter to kPlanCacheMaxWriteOperations or greater. + if (_writeOperations.addAndFetch(1) < internalQueryCacheWriteOpsBetweenFlush) { + return; + } + + LOG(1) << _ns << ": clearing collection plan cache - " + << internalQueryCacheWriteOpsBetweenFlush + << " write operations detected since last refresh."; + clear(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/plan_cache.h b/src/mongo/db/query/plan_cache.h new file mode 100644 index 00000000000..402633c8c5b --- /dev/null +++ b/src/mongo/db/query/plan_cache.h @@ -0,0 +1,417 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/db/exec/plan_stats.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/index_tag.h" +#include "mongo/db/query/lru_key_value.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/platform/atomic_word.h" + +namespace mongo { + + struct PlanRankingDecision; + struct QuerySolution; + struct QuerySolutionNode; + + /** + * When the CachedPlanRunner runs a cached query, it can provide feedback to the cache. This + * feedback is available to anyone who retrieves that query in the future. + */ + struct PlanCacheEntryFeedback { + // How well did the cached plan perform? + boost::scoped_ptr stats; + + // The "goodness" score produced by the plan ranker + // corresponding to 'stats'. + double score; + }; + + // TODO: Replace with opaque type. + typedef std::string PlanID; + + /** + * A PlanCacheIndexTree is the meaty component of the data + * stored in SolutionCacheData. It is a tree structure with + * index tags that indicates to the access planner which indices + * it should try to use. + * + * How a PlanCacheIndexTree is created: + * The query planner tags a match expression with indices. It + * then uses the tagged tree to create a PlanCacheIndexTree, + * using QueryPlanner::cacheDataFromTaggedTree. The PlanCacheIndexTree + * is isomorphic to the tagged match expression, and has matching + * index tags. + * + * How a PlanCacheIndexTree is used: + * When the query planner is planning from the cache, it uses + * the PlanCacheIndexTree retrieved from the cache in order to + * recreate index assignments. Specifically, a raw MatchExpression + * is tagged according to the index tags in the PlanCacheIndexTree. + * This is done by QueryPlanner::tagAccordingToCache. + */ + struct PlanCacheIndexTree { + PlanCacheIndexTree() : entry(NULL), index_pos(0) { } + + ~PlanCacheIndexTree() { + for (vector::const_iterator it = children.begin(); + it != children.end(); ++it) { + delete *it; + } + } + + /** + * Clone 'ie' and set 'this->entry' to be the clone. + */ + void setIndexEntry(const IndexEntry& ie); + + /** + * Make a deep copy. + */ + PlanCacheIndexTree* clone() const; + + /** + * For debugging. + */ + std::string toString(int indents = 0) const; + + // Children owned here. + std::vector children; + + // Owned here. + boost::scoped_ptr entry; + + size_t index_pos; + }; + + /** + * Data stored inside a QuerySolution which can subsequently be + * used to create a cache entry. When this data is retrieved + * from the cache, it is sufficient to reconstruct the original + * QuerySolution. + */ + struct SolutionCacheData { + SolutionCacheData() : + tree(NULL), + solnType(USE_INDEX_TAGS_SOLN), + wholeIXSolnDir(1), + indexFilterApplied(false) { + } + + // Make a deep copy. + SolutionCacheData* clone() const; + + // For debugging. + std::string toString() const; + + // Owned here. If 'wholeIXSoln' is false, then 'tree' + // can be used to tag an isomorphic match expression. If 'wholeIXSoln' + // is true, then 'tree' is used to store the relevant IndexEntry. + // If 'collscanSoln' is true, then 'tree' should be NULL. + scoped_ptr tree; + + enum SolutionType { + // Indicates that the plan should use + // the index as a proxy for a collection + // scan (e.g. using index to provide sort). + WHOLE_IXSCAN_SOLN, + + // The cached plan is a collection scan. + COLLSCAN_SOLN, + + // Build the solution by using 'tree' + // to tag the match expression. + USE_INDEX_TAGS_SOLN + } solnType; + + // The direction of the index scan used as + // a proxy for a collection scan. Used only + // for WHOLE_IXSCAN_SOLN. + int wholeIXSolnDir; + + // True if index filter was applied. + bool indexFilterApplied; + }; + + class PlanCacheEntry; + + /** + * Information returned from a get(...) query. + */ + class CachedSolution { + private: + MONGO_DISALLOW_COPYING(CachedSolution); + public: + CachedSolution(const PlanCacheKey& key, const PlanCacheEntry& entry); + ~CachedSolution(); + + // Owned here. + std::vector plannerData; + + // An index into plannerData indicating the SolutionCacheData which should be + // used to produce a backup solution in the case of a blocking sort. + boost::optional backupSoln; + + // Key used to provide feedback on the entry. + PlanCacheKey key; + + // For debugging. + std::string toString() const; + + // We are extracting just enough information from the canonical + // query. We could clone the canonical query but the following + // items are all that is displayed to the user. + BSONObj query; + BSONObj sort; + BSONObj projection; + }; + + /** + * Used by the cache to track entries and their performance over time. + * Also used by the plan cache commands to display plan cache state. + */ + class PlanCacheEntry { + private: + MONGO_DISALLOW_COPYING(PlanCacheEntry); + public: + /** + * Create a new PlanCacheEntry. + * Grabs any planner-specific data required from the solutions. + * Takes ownership of the PlanRankingDecision that placed the plan in the cache. + */ + PlanCacheEntry(const std::vector& solutions, + PlanRankingDecision* why); + + ~PlanCacheEntry(); + + /** + * Make a deep copy. + */ + PlanCacheEntry* clone() const; + + // For debugging. + std::string toString() const; + + // + // Planner data + // + + // Data provided to the planner to allow it to recreate the solutions this entry + // represents. Each SolutionCacheData is fully owned here, so in order to return + // it from the cache a deep copy is made and returned inside CachedSolution. + std::vector plannerData; + + // An index into plannerData indicating the SolutionCacheData which should be + // used to produce a backup solution in the case of a blocking sort. + boost::optional backupSoln; + + // TODO: Do we really want to just hold a copy of the CanonicalQuery? For now we just + // extract the data we need. + // + // Used by the plan cache commands to display an example query + // of the appropriate shape. + BSONObj query; + BSONObj sort; + BSONObj projection; + + // + // Performance stats + // + + // Information that went into picking the winning plan and also why + // the other plans lost. + boost::scoped_ptr decision; + + // Annotations from cached runs. The CachedSolutionRunner provides these stats about its + // runs when they complete. + std::vector feedback; + + // The average score of all stored feedback. + boost::optional averageScore; + + // The standard deviation of the scores from stored as feedback. + boost::optional stddevScore; + + // In order to justify eviction, the deviation from the mean must exceed a + // minimum threshold. + static const double kMinDeviation; + }; + + /** + * Caches the best solution to a query. Aside from the (CanonicalQuery -> QuerySolution) + * mapping, the cache contains information on why that mapping was made and statistics on the + * cache entry's actual performance on subsequent runs. + * + */ + class PlanCache { + private: + MONGO_DISALLOW_COPYING(PlanCache); + public: + /** + * We don't want to cache every possible query. This function + * encapsulates the criteria for what makes a canonical query + * suitable for lookup/inclusion in the cache. + */ + static bool shouldCacheQuery(const CanonicalQuery& query); + + /** + * If omitted, namespace set to empty string. + */ + PlanCache(); + + PlanCache(const std::string& ns); + + ~PlanCache(); + + /** + * Record solutions for query. Best plan is first element in list. + * Each query in the cache will have more than 1 plan because we only + * add queries which are considered by the multi plan runner (which happens + * only when the query planner generates multiple candidate plans). + * + * Takes ownership of 'why'. + * + * If the mapping was added successfully, returns Status::OK(). + * If the mapping already existed or some other error occurred, returns another Status. + */ + Status add(const CanonicalQuery& query, + const std::vector& solns, + PlanRankingDecision* why); + + /** + * Look up the cached data access for the provided 'query'. Used by the query planner + * to shortcut planning. + * + * If there is no entry in the cache for the 'query', returns an error Status. + * + * If there is an entry in the cache, populates 'crOut' and returns Status::OK(). Caller + * owns '*crOut'. + */ + Status get(const CanonicalQuery& query, CachedSolution** crOut) const; + + /** + * When the CachedPlanRunner runs a plan out of the cache, we want to record data about the + * plan's performance. The CachedPlanRunner calls feedback(...) at the end of query + * execution in order to do this. + * + * Cache takes ownership of 'feedback'. + * + * If the entry corresponding to 'cq' isn't in the cache anymore, the feedback is ignored + * and an error Status is returned. + * + * If the entry corresponding to 'cq' still exists, 'feedback' is added to the run + * statistics about the plan. Status::OK() is returned. + * + * May cause the cache entry to be removed if it is determined that the cached plan + * is badly performing. + */ + Status feedback(const CanonicalQuery& cq, PlanCacheEntryFeedback* feedback); + + /** + * Remove the entry corresponding to 'ck' from the cache. Returns Status::OK() if the plan + * was present and removed and an error status otherwise. + */ + Status remove(const CanonicalQuery& canonicalQuery); + + /** + * Remove *all* entries. + */ + void clear(); + + /** + * Returns a copy of a cache entry. + * Used by planCacheListPlans to display plan details. + * + * If there is no entry in the cache for the 'query', returns an error Status. + * + * If there is an entry in the cache, populates 'entryOut' and returns Status::OK(). Caller + * owns '*entryOut'. + */ + Status getEntry(const CanonicalQuery& cq, PlanCacheEntry** entryOut) const; + + /** + * Returns a vector of all cache entries. + * Caller owns the result vector and is responsible for cleaning up + * the cache entry copies. + * Used by planCacheListQueryShapes and index_filter_commands_test.cpp. + */ + std::vector getAllEntries() const; + + /** + * Returns true if there is an entry in the cache for the 'query'. + * Internally calls hasKey() on the LRU cache. + */ + bool contains(const CanonicalQuery& cq) const; + + /** + * Returns number of entries in cache. + * Used for testing. + */ + size_t size() const; + + /** + * You must notify the cache if you are doing writes, as query plan utility will change. + * Cache is flushed after every 1000 notifications. + */ + void notifyOfWriteOp(); + + private: + + /** + * Releases resources associated with each cache entry + * and clears map. + * Invoked by clear() and during destruction. + */ + void _clear(); + + LRUKeyValue _cache; + + /** + * Protects _cache. + */ + mutable boost::mutex _cacheMutex; + + /** + * Counter for write notifications since initialization or last clear() invocation. + * Starts at 0. + */ + AtomicInt32 _writeOperations; + + /** + * Full namespace of collection. + */ + std::string _ns; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/plan_cache_test.cpp b/src/mongo/db/query/plan_cache_test.cpp new file mode 100644 index 00000000000..aa0ceff0479 --- /dev/null +++ b/src/mongo/db/query/plan_cache_test.cpp @@ -0,0 +1,1036 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/plan_cache.h + */ + +#include "mongo/db/query/plan_cache.h" + +#include +#include +#include +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/plan_ranker.h" +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_planner_test_lib.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +using namespace mongo; + +namespace { + + using std::auto_ptr; + + static const char* ns = "somebogusns"; + + /** + * Utility functions to create a CanonicalQuery + */ + CanonicalQuery* canonicalize(const BSONObj& queryObj) { + CanonicalQuery* cq; + Status result = CanonicalQuery::canonicalize(ns, queryObj, &cq); + ASSERT_OK(result); + return cq; + } + + CanonicalQuery* canonicalize(const char* queryStr) { + BSONObj queryObj = fromjson(queryStr); + return canonicalize(queryObj); + } + + CanonicalQuery* canonicalize(const char* queryStr, const char* sortStr, + const char* projStr) { + BSONObj queryObj = fromjson(queryStr); + BSONObj sortObj = fromjson(sortStr); + BSONObj projObj = fromjson(projStr); + CanonicalQuery* cq; + Status result = CanonicalQuery::canonicalize(ns, queryObj, sortObj, + projObj, + &cq); + ASSERT_OK(result); + return cq; + } + + CanonicalQuery* canonicalize(const char* queryStr, const char* sortStr, + const char* projStr, + long long skip, long long limit, + const char* hintStr, + const char* minStr, const char* maxStr) { + BSONObj queryObj = fromjson(queryStr); + BSONObj sortObj = fromjson(sortStr); + BSONObj projObj = fromjson(projStr); + BSONObj hintObj = fromjson(hintStr); + BSONObj minObj = fromjson(minStr); + BSONObj maxObj = fromjson(maxStr); + CanonicalQuery* cq; + Status result = CanonicalQuery::canonicalize(ns, queryObj, sortObj, + projObj, + skip, limit, + hintObj, + minObj, maxObj, + false, // snapshot + false, // explain + &cq); + ASSERT_OK(result); + return cq; + } + + CanonicalQuery* canonicalize(const char* queryStr, const char* sortStr, + const char* projStr, + long long skip, long long limit, + const char* hintStr, + const char* minStr, const char* maxStr, + bool snapshot, + bool explain) { + BSONObj queryObj = fromjson(queryStr); + BSONObj sortObj = fromjson(sortStr); + BSONObj projObj = fromjson(projStr); + BSONObj hintObj = fromjson(hintStr); + BSONObj minObj = fromjson(minStr); + BSONObj maxObj = fromjson(maxStr); + CanonicalQuery* cq; + Status result = CanonicalQuery::canonicalize(ns, queryObj, sortObj, + projObj, + skip, limit, + hintObj, + minObj, maxObj, + snapshot, + explain, + &cq); + ASSERT_OK(result); + return cq; + } + + /** + * Utility function to create MatchExpression + */ + MatchExpression* parseMatchExpression(const BSONObj& obj) { + StatusWithMatchExpression status = MatchExpressionParser::parse(obj); + if (!status.isOK()) { + mongoutils::str::stream ss; + ss << "failed to parse query: " << obj.toString() + << ". Reason: " << status.toString(); + FAIL(ss); + } + MatchExpression* expr(status.getValue()); + return expr; + } + + void assertEquivalent(const char* queryStr, const MatchExpression* expected, const MatchExpression* actual) { + if (actual->equivalent(expected)) { + return; + } + mongoutils::str::stream ss; + ss << "Match expressions are not equivalent." + << "\nOriginal query: " << queryStr + << "\nExpected: " << expected->toString() + << "\nActual: " << actual->toString(); + FAIL(ss); + } + + // + // Tests for CachedSolution + // + + /** + * Generator for vector of QuerySolution shared pointers. + */ + struct GenerateQuerySolution { + QuerySolution* operator()() const { + auto_ptr qs(new QuerySolution()); + qs->cacheData.reset(new SolutionCacheData()); + qs->cacheData->solnType = SolutionCacheData::COLLSCAN_SOLN; + qs->cacheData->tree.reset(new PlanCacheIndexTree()); + return qs.release(); + } + }; + + /** + * Utility function to create a PlanRankingDecision + */ + PlanRankingDecision* createDecision(size_t numPlans) { + auto_ptr why(new PlanRankingDecision()); + for (size_t i = 0; i < numPlans; ++i) { + auto_ptr stats(new PlanStageStats(CommonStats(), STAGE_COLLSCAN)); + stats->specific.reset(new CollectionScanStats()); + why->stats.mutableVector().push_back(stats.release()); + why->scores.push_back(0U); + why->candidateOrder.push_back(i); + } + return why.release(); + } + + /** + * Test functions for shouldCacheQuery + * Use these functions to assert which categories + * of canonicalized queries are suitable for inclusion + * in the planner cache. + */ + void assertShouldCacheQuery(const CanonicalQuery& query) { + if (PlanCache::shouldCacheQuery(query)) { + return; + } + mongoutils::str::stream ss; + ss << "Canonical query should be cacheable: " << query.toString(); + FAIL(ss); + } + + void assertShouldNotCacheQuery(const CanonicalQuery& query) { + if (!PlanCache::shouldCacheQuery(query)) { + return; + } + mongoutils::str::stream ss; + ss << "Canonical query should not be cacheable: " << query.toString(); + FAIL(ss); + } + + void assertShouldNotCacheQuery(const BSONObj& query) { + auto_ptr cq(canonicalize(query)); + assertShouldNotCacheQuery(*cq); + } + + void assertShouldNotCacheQuery(const char* queryStr) { + auto_ptr cq(canonicalize(queryStr)); + assertShouldNotCacheQuery(*cq); + } + + /** + * Cacheable queries + * These queries will be added to the cache with run-time statistics + * and can be managed with the cache DB commands. + */ + + TEST(PlanCacheTest, ShouldCacheQueryBasic) { + auto_ptr cq(canonicalize("{a: 1}")); + assertShouldCacheQuery(*cq); + } + + TEST(PlanCacheTest, ShouldCacheQuerySort) { + auto_ptr cq(canonicalize("{}", "{a: -1}", "{_id: 0, a: 1}")); + assertShouldCacheQuery(*cq); + } + + /* + * Non-cacheable queries. + * These queries will be sent through the planning process everytime. + */ + + /** + * Collection scan + * This should normally be handled by the IDHack runner. + */ + TEST(PlanCacheTest, ShouldNotCacheQueryCollectionScan) { + auto_ptr cq(canonicalize("{}")); + assertShouldNotCacheQuery(*cq); + } + + /** + * Hint + * A hinted query implies strong user preference for a particular index. + * Therefore, not much point in caching. + */ + TEST(PlanCacheTest, ShouldNotCacheQueryWithHint) { + auto_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{a: 1, b: 1}", + "{}", "{}")); + assertShouldNotCacheQuery(*cq); + } + + /** + * Min queries are a specialized case of hinted queries + */ + TEST(PlanCacheTest, ShouldNotCacheQueryWithMin) { + auto_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", + "{a: 100}", "{}")); + assertShouldNotCacheQuery(*cq); + } + + /** + * Max queries are non-cacheable for the same reasons as min queries. + */ + TEST(PlanCacheTest, ShouldNotCacheQueryWithMax) { + auto_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", + "{}", "{a: 100}")); + assertShouldNotCacheQuery(*cq); + } + + /** + * $geoWithin queries with legacy coordinates are cacheable as long as + * the planner is able to come up with a cacheable solution. + */ + TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyCoordinates) { + auto_ptr cq(canonicalize("{a: {$geoWithin: " + "{$box: [[-180, -90], [180, 90]]}}}")); + assertShouldCacheQuery(*cq); + } + + /** + * $geoWithin queries with GeoJSON coordinates are supported by the index bounds builder. + */ + TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinJSONCoordinates) { + auto_ptr cq(canonicalize("{a: {$geoWithin: " + "{$geometry: {type: 'Polygon', coordinates: " + "[[[0, 0], [0, 90], [90, 0], [0, 0]]]}}}}")); + assertShouldCacheQuery(*cq); + } + + /** + * $geoWithin queries with both legacy and GeoJSON coordinates are cacheable. + */ + TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyAndJSONCoordinates) { + auto_ptr cq(canonicalize( + "{$or: [{a: {$geoWithin: {$geometry: {type: 'Polygon', " + "coordinates: [[[0, 0], [0, 90], " + "[90, 0], [0, 0]]]}}}}," + "{a: {$geoWithin: {$box: [[-180, -90], [180, 90]]}}}]}")); + assertShouldCacheQuery(*cq); + } + + /** + * $geoIntersects queries are always cacheable because they support GeoJSON coordinates only. + */ + TEST(PlanCacheTest, ShouldCacheQueryWithGeoIntersects) { + auto_ptr cq(canonicalize("{a: {$geoIntersects: " + "{$geometry: {type: 'Point', coordinates: " + "[10.0, 10.0]}}}}")); + assertShouldCacheQuery(*cq); + } + + /** + * $geoNear queries are cacheable because we are able to distinguish + * between flat and spherical queries. + */ + TEST(PlanCacheTest, ShouldNotCacheQueryWithGeoNear) { + auto_ptr cq(canonicalize("{a: {$geoNear: {$geometry: {type: 'Point'," + "coordinates: [0,0]}, $maxDistance:100}}}")); + assertShouldCacheQuery(*cq); + } + + /** + * Explain queries are not-cacheable because of allPlans cannot + * be accurately generated from stale cached stats in the plan cache for + * non-winning plans. + */ + TEST(PlanCacheTest, ShouldNotCacheQueryExplain) { + auto_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", + "{}", "{}", // min, max + false, // snapshot + true // explain + )); + const LiteParsedQuery& pq = cq->getParsed(); + ASSERT_TRUE(pq.isExplain()); + assertShouldNotCacheQuery(*cq); + } + + // Adding an empty vector of query solutions should fail. + TEST(PlanCacheTest, AddEmptySolutions) { + PlanCache planCache; + auto_ptr cq(canonicalize("{a: 1}")); + std::vector solns; + ASSERT_NOT_OK(planCache.add(*cq, solns, createDecision(1U))); + } + + TEST(PlanCacheTest, AddValidSolution) { + PlanCache planCache; + auto_ptr cq(canonicalize("{a: 1}")); + QuerySolution qs; + qs.cacheData.reset(new SolutionCacheData()); + qs.cacheData->tree.reset(new PlanCacheIndexTree()); + std::vector solns; + solns.push_back(&qs); + + // Check if key is in cache before and after add(). + ASSERT_FALSE(planCache.contains(*cq)); + ASSERT_OK(planCache.add(*cq, solns, createDecision(1U))); + + ASSERT_TRUE(planCache.contains(*cq)); + ASSERT_EQUALS(planCache.size(), 1U); + } + + TEST(PlanCacheTest, NotifyOfWriteOp) { + PlanCache planCache; + auto_ptr cq(canonicalize("{a: 1}")); + QuerySolution qs; + qs.cacheData.reset(new SolutionCacheData()); + qs.cacheData->tree.reset(new PlanCacheIndexTree()); + std::vector solns; + solns.push_back(&qs); + ASSERT_OK(planCache.add(*cq, solns, createDecision(1U))); + ASSERT_EQUALS(planCache.size(), 1U); + + // First (N - 1) write ops should have no effect on cache contents. + for (int i = 0; i < (internalQueryCacheWriteOpsBetweenFlush - 1); ++i) { + planCache.notifyOfWriteOp(); + } + ASSERT_EQUALS(planCache.size(), 1U); + + // N-th notification will cause cache to be cleared. + planCache.notifyOfWriteOp(); + ASSERT_EQUALS(planCache.size(), 0U); + + // Clearing the cache should reset the internal write + // operation counter. + // Repopulate cache. Write (N - 1) times. + // Clear cache. + // Add cache entry again. + // After clearing and adding a new entry, the next write operation should not + // clear the cache. + ASSERT_OK(planCache.add(*cq, solns, createDecision(1U))); + for (int i = 0; i < (internalQueryCacheWriteOpsBetweenFlush - 1); ++i) { + planCache.notifyOfWriteOp(); + } + ASSERT_EQUALS(planCache.size(), 1U); + planCache.clear(); + ASSERT_OK(planCache.add(*cq, solns, createDecision(1U))); + // Notification after clearing will not flush cache. + planCache.notifyOfWriteOp(); + ASSERT_EQUALS(planCache.size(), 1U); + } + + /** + * Each test in the CachePlanSelectionTest suite goes through + * the following flow: + * + * 1) Run QueryPlanner::plan on the query, with specified indices + * available. This simulates the case in which we failed to plan from + * the plan cache, and fell back on selecting a plan ourselves. The + * enumerator will run, and cache data will be stashed into each solution + * that it generates. + * + * 2) Use firstMatchingSolution to select one of the solutions generated + * by QueryPlanner::plan. This simulates the multi plan runner picking + * the "best solution". + * + * 3) The cache data stashed inside the "best solution" is used to + * make a CachedSolution which looks exactly like the data structure that + * would be returned from the cache. This simulates a plan cache hit. + * + * 4) Call QueryPlanner::planFromCache, passing it the CachedSolution. + * This exercises the code which is able to map from a CachedSolution to + * a full-blown QuerySolution. Finally, assert that the query solution + * recovered from the cache is identical to the original "best solution". + */ + class CachePlanSelectionTest : public mongo::unittest::Test { + protected: + void setUp() { + params.options = QueryPlannerParams::INCLUDE_COLLSCAN; + addIndex(BSON("_id" << 1)); + } + + void tearDown() { + delete cq; + + for (vector::iterator it = solns.begin(); it != solns.end(); ++it) { + delete *it; + } + } + + void addIndex(BSONObj keyPattern, bool multikey = false) { + // The first false means not multikey. + // The second false means not sparse. + // The third arg is the index name and I am egotistical. + params.indices.push_back(IndexEntry(keyPattern, + multikey, + false, + false, + "hari_king_of_the_stove", + BSONObj())); + } + + void addIndex(BSONObj keyPattern, bool multikey, bool sparse) { + params.indices.push_back(IndexEntry(keyPattern, + multikey, + sparse, + false, + "note_to_self_dont_break_build", + BSONObj())); + } + + // + // Execute planner. + // + + void runQuery(BSONObj query) { + runQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), 0, 0); + } + + void runQuerySortProj(const BSONObj& query, const BSONObj& sort, const BSONObj& proj) { + runQuerySortProjSkipLimit(query, sort, proj, 0, 0); + } + + void runQuerySkipLimit(const BSONObj& query, long long skip, long long limit) { + runQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), skip, limit); + } + + void runQueryHint(const BSONObj& query, const BSONObj& hint) { + runQuerySortProjSkipLimitHint(query, BSONObj(), BSONObj(), 0, 0, hint); + } + + void runQuerySortProjSkipLimit(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit) { + runQuerySortProjSkipLimitHint(query, sort, proj, skip, limit, BSONObj()); + } + + void runQuerySortHint(const BSONObj& query, const BSONObj& sort, const BSONObj& hint) { + runQuerySortProjSkipLimitHint(query, sort, BSONObj(), 0, 0, hint); + } + + void runQueryHintMinMax(const BSONObj& query, const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj) { + + runQueryFull(query, BSONObj(), BSONObj(), 0, 0, hint, minObj, maxObj, false); + } + + void runQuerySortProjSkipLimitHint(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint) { + runQueryFull(query, sort, proj, skip, limit, hint, BSONObj(), BSONObj(), false); + } + + void runQuerySnapshot(const BSONObj& query) { + runQueryFull(query, BSONObj(), BSONObj(), 0, 0, BSONObj(), BSONObj(), + BSONObj(), true); + } + + void runQueryFull(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + const BSONObj& minObj, + const BSONObj& maxObj, + bool snapshot) { + solns.clear(); + Status s = CanonicalQuery::canonicalize(ns, query, sort, proj, skip, limit, hint, + minObj, maxObj, snapshot, + false, // explain + &cq); + if (!s.isOK()) { cq = NULL; } + ASSERT_OK(s); + s = QueryPlanner::plan(*cq, params, &solns); + ASSERT_OK(s); + } + + // + // Solution introspection. + // + + void dumpSolutions(mongoutils::str::stream& ost) const { + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + ost << (*it)->toString() << '\n'; + } + } + + /** + * Returns number of generated solutions matching JSON. + */ + size_t numSolutionMatches(const string& solnJson) const { + BSONObj testSoln = fromjson(solnJson); + size_t matches = 0; + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + QuerySolutionNode* root = (*it)->root.get(); + if (QueryPlannerTestLib::solutionMatches(testSoln, root)) { + ++matches; + } + } + return matches; + } + + /** + * Verifies that the solution tree represented in json by 'solnJson' is + * one of the solutions generated by QueryPlanner. + * + * The number of expected matches, 'numMatches', could be greater than + * 1 if solutions differ only by the pattern of index tags on a filter. + */ + void assertSolutionExists(const string& solnJson, size_t numMatches = 1) const { + size_t matches = numSolutionMatches(solnJson); + if (numMatches == matches) { + return; + } + mongoutils::str::stream ss; + ss << "expected " << numMatches << " matches for solution " << solnJson + << " but got " << matches + << " instead. all solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + } + + /** + * Plan 'query' from the cache. A mock cache entry is created using + * the cacheData stored inside the QuerySolution 'soln'. + * + * Does not take ownership of 'soln'. + */ + QuerySolution* planQueryFromCache(const BSONObj& query, const QuerySolution& soln) const { + return planQueryFromCache(query, BSONObj(), BSONObj(), soln); + } + + /** + * Plan 'query' from the cache with sort order 'sort' and + * projection 'proj'. A mock cache entry is created using + * the cacheData stored inside the QuerySolution 'soln'. + * + * Does not take ownership of 'soln'. + */ + QuerySolution* planQueryFromCache(const BSONObj& query, + const BSONObj& sort, + const BSONObj& proj, + const QuerySolution& soln) const { + CanonicalQuery* cq; + Status s = CanonicalQuery::canonicalize(ns, query, sort, proj, &cq); + ASSERT_OK(s); + scoped_ptr scopedCq(cq); + cq = NULL; + + // Create a CachedSolution the long way.. + // QuerySolution -> PlanCacheEntry -> CachedSolution + QuerySolution qs; + qs.cacheData.reset(soln.cacheData->clone()); + std::vector solutions; + solutions.push_back(&qs); + PlanCacheEntry entry(solutions, createDecision(1U)); + CachedSolution cachedSoln(ck, entry); + + QuerySolution *out, *backupOut; + s = QueryPlanner::planFromCache(*scopedCq.get(), params, cachedSoln, + &out, &backupOut); + ASSERT_OK(s); + + return out; + } + + /** + * @param solnJson -- a json representation of a query solution. + * + * Returns the first solution matching 'solnJson', or fails if + * no match is found. + */ + QuerySolution* firstMatchingSolution(const string& solnJson) const { + BSONObj testSoln = fromjson(solnJson); + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + QuerySolutionNode* root = (*it)->root.get(); + if (QueryPlannerTestLib::solutionMatches(testSoln, root)) { + return *it; + } + } + + mongoutils::str::stream ss; + ss << "Could not find a match for solution " << solnJson + << " All solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + + return NULL; + } + + /** + * Assert that the QuerySolution 'trueSoln' matches the JSON-based representation + * of the solution in 'solnJson'. + * + * Relies on solutionMatches() -- see query_planner_test_lib.h + */ + void assertSolutionMatches(QuerySolution* trueSoln, const string& solnJson) const { + BSONObj testSoln = fromjson(solnJson); + if (!QueryPlannerTestLib::solutionMatches(testSoln, trueSoln->root.get())) { + mongoutils::str::stream ss; + ss << "Expected solution " << solnJson << " did not match true solution: " + << trueSoln->toString() << '\n'; + FAIL(ss); + } + } + + /** + * Overloaded so that it is not necessary to specificy sort and project. + */ + void assertPlanCacheRecoversSolution(const BSONObj& query, const string& solnJson) { + assertPlanCacheRecoversSolution(query, BSONObj(), BSONObj(), solnJson); + } + + /** + * First, the solution matching 'solnJson' is retrieved from the vector + * of solutions generated by QueryPlanner::plan. This solution is + * then passed into planQueryFromCache(). Asserts that the solution + * generated by QueryPlanner::planFromCache matches 'solnJson'. + * + * Must be called after calling one of the runQuery* methods. + * + * Together, 'query', 'sort', and 'proj' should specify the query which + * was previously run using one of the runQuery* methods. + */ + void assertPlanCacheRecoversSolution(const BSONObj& query, + const BSONObj& sort, + const BSONObj& proj, + const string& solnJson) { + QuerySolution* bestSoln = firstMatchingSolution(solnJson); + QuerySolution* planSoln = planQueryFromCache(query, sort, proj, *bestSoln); + assertSolutionMatches(planSoln, solnJson); + } + + /** + * Check that the solution will not be cached. The planner will store + * cache data inside non-cachable solutions, but will not do so for + * non-cachable solutions. Therefore, we just have to check that + * cache data is NULL. + */ + void assertNotCached(const string& solnJson) { + QuerySolution* bestSoln = firstMatchingSolution(solnJson); + ASSERT(NULL != bestSoln); + ASSERT(NULL == bestSoln->cacheData.get()); + } + + static const PlanCacheKey ck; + + BSONObj queryObj; + CanonicalQuery* cq; + QueryPlannerParams params; + vector solns; + }; + + const PlanCacheKey CachePlanSelectionTest::ck = "mock_cache_key"; + + // + // Equality + // + + TEST_F(CachePlanSelectionTest, EqualityIndexScan) { + addIndex(BSON("x" << 1)); + runQuery(BSON("x" << 5)); + + assertPlanCacheRecoversSolution(BSON("x" << 5), + "{fetch: {filter: null, node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(CachePlanSelectionTest, EqualityIndexScanWithTrailingFields) { + addIndex(BSON("x" << 1 << "y" << 1)); + runQuery(BSON("x" << 5)); + + assertPlanCacheRecoversSolution(BSON("x" << 5), + "{fetch: {filter: null, node: {ixscan: {pattern: {x: 1, y: 1}}}}}"); + } + + // + // Geo + // + + TEST_F(CachePlanSelectionTest, Basic2DSphereNonNear) { + addIndex(BSON("a" << "2dsphere")); + BSONObj query; + + query = fromjson("{a: {$geoIntersects: {$geometry: {type: 'Point'," + "coordinates: [10.0, 10.0]}}}}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + + query = fromjson("{a : { $geoWithin : { $centerSphere : [[ 10, 20 ], 0.01 ] } }}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + } + + TEST_F(CachePlanSelectionTest, Basic2DSphereGeoNear) { + addIndex(BSON("a" << "2dsphere")); + BSONObj query; + + query = fromjson("{a: {$nearSphere: [0,0], $maxDistance: 0.31 }}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, "{geoNear2dsphere: {a: '2dsphere'}}"); + + query = fromjson("{a: {$geoNear: {$geometry: {type: 'Point', coordinates: [0,0]}," + "$maxDistance:100}}}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, "{geoNear2dsphere: {a: '2dsphere'}}"); + } + + TEST_F(CachePlanSelectionTest, Basic2DSphereGeoNearReverseCompound) { + addIndex(BSON("x" << 1)); + addIndex(BSON("x" << 1 << "a" << "2dsphere")); + BSONObj query = fromjson("{x:1, a: {$nearSphere: [0,0], $maxDistance: 0.31 }}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, "{geoNear2dsphere: {x: 1, a: '2dsphere'}}"); + } + + TEST_F(CachePlanSelectionTest, TwoDSphereNoGeoPred) { + addIndex(BSON("x" << 1 << "a" << "2dsphere")); + runQuery(BSON("x" << 1)); + assertPlanCacheRecoversSolution(BSON("x" << 1), + "{fetch: {node: {ixscan: {pattern: {x: 1, a: '2dsphere'}}}}}"); + } + + TEST_F(CachePlanSelectionTest, Or2DSphereNonNear) { + addIndex(BSON("a" << "2dsphere")); + addIndex(BSON("b" << "2dsphere")); + BSONObj query = fromjson("{$or: [ {a: {$geoIntersects: {$geometry: {type: 'Point', coordinates: [10.0, 10.0]}}}}," + " {b: {$geoWithin: { $centerSphere: [[ 10, 20 ], 0.01 ] } }} ]}"); + + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{or: {nodes: [{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}," + "{fetch: {node: {ixscan: {pattern: {b: '2dsphere'}}}}}]}}"); + } + + // + // tree operations + // + + TEST_F(CachePlanSelectionTest, TwoPredicatesAnding) { + addIndex(BSON("x" << 1)); + BSONObj query = fromjson("{$and: [ {x: {$gt: 1}}, {x: {$lt: 3}} ] }"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{fetch: {filter: null, node: {ixscan: {filter: null, pattern: {x: 1}}}}}"); + } + + TEST_F(CachePlanSelectionTest, SimpleOr) { + addIndex(BSON("a" << 1)); + BSONObj query = fromjson("{$or: [{a: 20}, {a: 21}]}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{fetch: {filter: null, node: {ixscan: {filter: null, pattern: {a:1}}}}}"); + } + + TEST_F(CachePlanSelectionTest, OrWithAndChild) { + addIndex(BSON("a" << 1)); + BSONObj query = fromjson("{$or: [{a: 20}, {$and: [{a:1}, {b:7}]}]}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a: 1}}}, " + "{fetch: {filter: {b: 7}, node: {ixscan: " + "{filter: null, pattern: {a: 1}}}}}]}}}}"); + } + + TEST_F(CachePlanSelectionTest, AndWithUnindexedOrChild) { + addIndex(BSON("a" << 1)); + BSONObj query = fromjson("{a:20, $or: [{b:1}, {c:7}]}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{fetch: {filter: {$or: [{b: 1}, {c: 7}]}, node: " + "{ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + + TEST_F(CachePlanSelectionTest, AndWithOrWithOneIndex) { + addIndex(BSON("b" << 1)); + addIndex(BSON("a" << 1)); + BSONObj query = fromjson("{$or: [{b:1}, {c:7}], a:20}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{fetch: {filter: {$or: [{b: 1}, {c: 7}]}, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + // + // Sort orders + // + + // SERVER-1205. + TEST_F(CachePlanSelectionTest, MergeSort) { + addIndex(BSON("a" << 1 << "c" << 1)); + addIndex(BSON("b" << 1 << "c" << 1)); + + BSONObj query = fromjson("{$or: [{a:1}, {b:1}]}"); + BSONObj sort = BSON("c" << 1); + runQuerySortProj(query, sort, BSONObj()); + + assertPlanCacheRecoversSolution(query, sort, BSONObj(), + "{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a: 1, c: 1}}}, {ixscan: {pattern: {b: 1, c: 1}}}]}}}}"); + } + + // SERVER-1205 as well. + TEST_F(CachePlanSelectionTest, NoMergeSortIfNoSortWanted) { + addIndex(BSON("a" << 1 << "c" << 1)); + addIndex(BSON("b" << 1 << "c" << 1)); + + BSONObj query = fromjson("{$or: [{a:1}, {b:1}]}"); + runQuerySortProj(query, BSONObj(), BSONObj()); + + assertPlanCacheRecoversSolution(query, BSONObj(), BSONObj(), + "{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a: 1, c: 1}}}, " + "{ixscan: {filter: null, pattern: {b: 1, c: 1}}}]}}}}"); + } + + // Disabled: SERVER-10801. + /* + TEST_F(CachePlanSelectionTest, SortOnGeoQuery) { + addIndex(BSON("timestamp" << -1 << "position" << "2dsphere")); + BSONObj query = fromjson("{position: {$geoWithin: {$geometry: {type: \"Polygon\", " + "coordinates: [[[1, 1], [1, 90], [180, 90], " + "[180, 1], [1, 1]]]}}}}"); + BSONObj sort = fromjson("{timestamp: -1}"); + runQuerySortProj(query, sort, BSONObj()); + + assertPlanCacheRecoversSolution(query, sort, BSONObj(), + "{fetch: {node: {ixscan: {pattern: {timestamp: -1, position: '2dsphere'}}}}}"); + } + */ + + // SERVER-9257 + TEST_F(CachePlanSelectionTest, CompoundGeoNoGeoPredicate) { + addIndex(BSON("creationDate" << 1 << "foo.bar" << "2dsphere")); + BSONObj query = fromjson("{creationDate: {$gt: 7}}"); + BSONObj sort = fromjson("{creationDate: 1}"); + runQuerySortProj(query, sort, BSONObj()); + + assertPlanCacheRecoversSolution(query, sort, BSONObj(), + "{fetch: {node: {ixscan: {pattern: {creationDate: 1, 'foo.bar': '2dsphere'}}}}}"); + } + + TEST_F(CachePlanSelectionTest, ReverseScanForSort) { + addIndex(BSON("_id" << 1)); + runQuerySortProj(BSONObj(), fromjson("{_id: -1}"), BSONObj()); + assertPlanCacheRecoversSolution(BSONObj(), fromjson("{_id: -1}"), BSONObj(), + "{fetch: {filter: null, node: {ixscan: {filter: null, pattern: {_id: 1}}}}}"); + } + + // + // Caching collection scans. + // + + TEST_F(CachePlanSelectionTest, CollscanNoUsefulIndices) { + addIndex(BSON("a" << 1 << "b" << 1)); + addIndex(BSON("c" << 1)); + runQuery(BSON("b" << 4)); + assertPlanCacheRecoversSolution(BSON("b" << 4), + "{cscan: {filter: {b: 4}, dir: 1}}"); + } + + TEST_F(CachePlanSelectionTest, CollscanOrWithoutEnoughIndices) { + addIndex(BSON("a" << 1)); + BSONObj query =fromjson("{$or: [{a: 20}, {b: 21}]}"); + runQuery(query); + assertPlanCacheRecoversSolution(query, + "{cscan: {filter: {$or:[{a:20},{b:21}]}, dir: 1}}"); + } + + TEST_F(CachePlanSelectionTest, CollscanMergeSort) { + addIndex(BSON("a" << 1 << "c" << 1)); + addIndex(BSON("b" << 1 << "c" << 1)); + + BSONObj query = fromjson("{$or: [{a:1}, {b:1}]}"); + BSONObj sort = BSON("c" << 1); + runQuerySortProj(query, sort, BSONObj()); + + assertPlanCacheRecoversSolution(query, sort, BSONObj(), + "{sort: {pattern: {c: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + } + + // + // Check queries that, at least for now, are not cached. + // + + TEST_F(CachePlanSelectionTest, GeoNear2DNotCached) { + addIndex(BSON("a" << "2d")); + runQuery(fromjson("{a: {$near: [0,0], $maxDistance:0.3 }}")); + assertNotCached("{geoNear2d: {a: '2d'}}"); + } + + TEST_F(CachePlanSelectionTest, MinNotCached) { + addIndex(BSON("a" << 1)); + runQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), BSONObj()); + assertNotCached("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(CachePlanSelectionTest, MaxNotCached) { + addIndex(BSON("a" << 1)); + runQueryHintMinMax(BSONObj(), BSONObj(), BSONObj(), fromjson("{a: 1}")); + assertNotCached("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(CachePlanSelectionTest, NaturalHintNotCached) { + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuerySortHint(BSON("a" << 1), BSON("b" << 1), BSON("$natural" << 1)); + assertNotCached("{sort: {pattern: {b: 1}, limit: 0, node: " + "{cscan: {filter: {a: 1}, dir: 1}}}}"); + } + + TEST_F(CachePlanSelectionTest, HintValidNotCached) { + addIndex(BSON("a" << 1)); + runQueryHint(BSONObj(), fromjson("{a: 1}")); + assertNotCached("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + // + // Queries using '2d' indices are not cached. + // + + TEST_F(CachePlanSelectionTest, Basic2DNonNearNotCached) { + addIndex(BSON("a" << "2d")); + BSONObj query; + + // Polygon + query = fromjson("{a : { $within: { $polygon : [[0,0], [2,0], [4,0]] } }}"); + runQuery(query); + assertNotCached("{fetch: {node: {geo2d: {a: '2d'}}}}"); + + // Center + query = fromjson("{a : { $within : { $center : [[ 5, 5 ], 7 ] } }}"); + runQuery(query); + assertNotCached("{fetch: {node: {geo2d: {a: '2d'}}}}"); + + // Centersphere + query = fromjson("{a : { $within : { $centerSphere : [[ 10, 20 ], 0.01 ] } }}"); + runQuery(query); + assertNotCached("{fetch: {node: {geo2d: {a: '2d'}}}}"); + + // Within box. + query = fromjson("{a : {$within: {$box : [[0,0],[9,9]]}}}"); + runQuery(query); + assertNotCached("{fetch: {node: {geo2d: {a: '2d'}}}}"); + } + + TEST_F(CachePlanSelectionTest, Or2DNonNearNotCached) { + addIndex(BSON("a" << "2d")); + addIndex(BSON("b" << "2d")); + BSONObj query = fromjson("{$or: [ {a : { $within : { $polygon : [[0,0], [2,0], [4,0]] } }}," + " {b : { $within : { $center : [[ 5, 5 ], 7 ] } }} ]}"); + + runQuery(query); + assertNotCached("{fetch: {node: {or: {nodes: [{geo2d: {a: '2d'}}, {geo2d: {b: '2d'}}]}}}}"); + } + +} // namespace diff --git a/src/mongo/db/query/plan_enumerator.cpp b/src/mongo/db/query/plan_enumerator.cpp new file mode 100644 index 00000000000..35c629771ea --- /dev/null +++ b/src/mongo/db/query/plan_enumerator.cpp @@ -0,0 +1,1259 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/query/plan_enumerator.h" + +#include + +#include "mongo/db/query/indexability.h" +#include "mongo/db/query/index_tag.h" +#include "mongo/db/query/qlog.h" + +namespace { + + using namespace mongo; + + std::string getPathPrefix(std::string path) { + if (mongoutils::str::contains(path, '.')) { + return mongoutils::str::before(path, '.'); + } + else { + return path; + } + } + + /** + * Returns true if either 'node' or a descendent of 'node' + * is a predicate that is required to use an index. + */ + bool expressionRequiresIndex(const MatchExpression* node) { + return CanonicalQuery::countNodes(node, MatchExpression::GEO_NEAR) > 0 + || CanonicalQuery::countNodes(node, MatchExpression::TEXT) > 0; + } + +} // namespace + + +namespace mongo { + + PlanEnumerator::PlanEnumerator(const PlanEnumeratorParams& params) + : _root(params.root), + _indices(params.indices), + _ixisect(params.intersect), + _orLimit(params.maxSolutionsPerOr), + _intersectLimit(params.maxIntersectPerAnd) { } + + PlanEnumerator::~PlanEnumerator() { + typedef unordered_map MemoMap; + for (MemoMap::iterator it = _memo.begin(); it != _memo.end(); ++it) { + delete it->second; + } + } + + Status PlanEnumerator::init() { + // Fill out our memo structure from the tagged _root. + _done = !prepMemo(_root, PrepMemoContext()); + + // Dump the tags. We replace them with IndexTag instances. + _root->resetTag(); + + return Status::OK(); + } + + std::string PlanEnumerator::dumpMemo() { + mongoutils::str::stream ss; + + // Note that this needs to be kept in sync with allocateAssignment which assigns memo IDs. + for (size_t i = 1; i < _memo.size(); ++i) { + ss << "[Node #" << i << "]: " << _memo[i]->toString() << "\n"; + } + return ss; + } + + string PlanEnumerator::NodeAssignment::toString() const { + if (NULL != pred) { + mongoutils::str::stream ss; + ss << "predicate\n"; + ss << "\tfirst indices: ["; + for (size_t i = 0; i < pred->first.size(); ++i) { + ss << pred->first[i]; + if (i < pred->first.size() - 1) + ss << ", "; + } + ss << "]\n"; + ss << "\tpred: " << pred->expr->toString(); + ss << "\tindexToAssign: " << pred->indexToAssign; + return ss; + } + else if (NULL != andAssignment) { + mongoutils::str::stream ss; + ss << "AND enumstate counter " << andAssignment->counter; + for (size_t i = 0; i < andAssignment->choices.size(); ++i) { + ss << "\n\tchoice " << i << ":\n"; + const AndEnumerableState& state = andAssignment->choices[i]; + ss << "\t\tsubnodes: "; + for (size_t j = 0; j < state.subnodesToIndex.size(); ++j) { + ss << state.subnodesToIndex[j] << " "; + } + ss << '\n'; + for (size_t j = 0; j < state.assignments.size(); ++j) { + const OneIndexAssignment& oie = state.assignments[j]; + ss << "\t\tidx[" << oie.index << "]\n"; + + for (size_t k = 0; k < oie.preds.size(); ++k) { + ss << "\t\t\tpos " << oie.positions[k] + << " pred " << oie.preds[k]->toString(); + } + } + } + return ss; + } + else if (NULL != arrayAssignment) { + mongoutils::str::stream ss; + ss << "ARRAY SUBNODES enumstate " << arrayAssignment->counter << "/ ONE OF: [ "; + for (size_t i = 0; i < arrayAssignment->subnodes.size(); ++i) { + ss << arrayAssignment->subnodes[i] << " "; + } + ss << "]"; + return ss; + } + else { + verify(NULL != orAssignment); + mongoutils::str::stream ss; + ss << "ALL OF: [ "; + for (size_t i = 0; i < orAssignment->subnodes.size(); ++i) { + ss << orAssignment->subnodes[i] << " "; + } + ss << "]"; + return ss; + } + } + + PlanEnumerator::MemoID PlanEnumerator::memoIDForNode(MatchExpression* node) { + unordered_map::iterator it = _nodeToId.find(node); + + if (_nodeToId.end() == it) { + error() << "Trying to look up memo entry for node, none found."; + invariant(0); + } + + return it->second; + } + + bool PlanEnumerator::getNext(MatchExpression** tree) { + if (_done) { return false; } + + // Tag with our first solution. + tagMemo(memoIDForNode(_root)); + + *tree = _root->shallowClone(); + tagForSort(*tree); + sortUsingTags(*tree); + + _root->resetTag(); + QLOG() << "Enumerator: memo just before moving:" << endl << dumpMemo(); + _done = nextMemo(memoIDForNode(_root)); + return true; + } + + // + // Structure creation + // + + void PlanEnumerator::allocateAssignment(MatchExpression* expr, + NodeAssignment** assign, + MemoID* id) { + // We start at 1 so that the lookup of any entries not explicitly allocated + // will refer to an invalid memo slot. + size_t newID = _memo.size() + 1; + + // Shouldn't be anything there already. + verify(_nodeToId.end() == _nodeToId.find(expr)); + _nodeToId[expr] = newID; + verify(_memo.end() == _memo.find(newID)); + NodeAssignment* newAssignment = new NodeAssignment(); + _memo[newID] = newAssignment; + *assign = newAssignment; + *id = newID; + } + + bool PlanEnumerator::prepMemo(MatchExpression* node, PrepMemoContext context) { + PrepMemoContext childContext; + childContext.elemMatchExpr = context.elemMatchExpr; + if (Indexability::nodeCanUseIndexOnOwnField(node)) { + // We only get here if our parent is an OR, an array operator, or we're the root. + + // If we have no index tag there are no indices we can use. + if (NULL == node->getTag()) { return false; } + + RelevantTag* rt = static_cast(node->getTag()); + // In order to definitely use an index it must be prefixed with our field. + // We don't consider notFirst indices here because we must be AND-related to a node + // that uses the first spot in that index, and we currently do not know that + // unless we're in an AND node. + if (0 == rt->first.size()) { return false; } + + // We know we can use an index, so grab a memo spot. + size_t myMemoID; + NodeAssignment* assign; + allocateAssignment(node, &assign, &myMemoID); + + assign->pred.reset(new PredicateAssignment()); + assign->pred->expr = node; + assign->pred->first.swap(rt->first); + return true; + } + else if (Indexability::isBoundsGeneratingNot(node)) { + bool childIndexable = prepMemo(node->getChild(0), childContext); + // If the child isn't indexable then bail out now. + if (!childIndexable) { + return false; + } + + // Our parent node, if any exists, will expect a memo entry keyed on 'node'. As such we + // have the node ID for 'node' just point to the memo created for the child that + // actually generates the bounds. + size_t myMemoID; + NodeAssignment* assign; + allocateAssignment(node, &assign, &myMemoID); + OrAssignment* orAssignment = new OrAssignment(); + orAssignment->subnodes.push_back(memoIDForNode(node->getChild(0))); + assign->orAssignment.reset(orAssignment); + return true; + } + else if (MatchExpression::OR == node->matchType()) { + // For an OR to be indexed, all its children must be indexed. + for (size_t i = 0; i < node->numChildren(); ++i) { + if (!prepMemo(node->getChild(i), childContext)) { + return false; + } + } + + // If we're here we're fully indexed and can be in the memo. + size_t myMemoID; + NodeAssignment* assign; + allocateAssignment(node, &assign, &myMemoID); + + OrAssignment* orAssignment = new OrAssignment(); + for (size_t i = 0; i < node->numChildren(); ++i) { + orAssignment->subnodes.push_back(memoIDForNode(node->getChild(i))); + } + assign->orAssignment.reset(orAssignment); + return true; + } + else if (Indexability::arrayUsesIndexOnChildren(node)) { + // Add each of our children as a subnode. We enumerate through each subnode one at a + // time until it's exhausted then we move on. + auto_ptr aa(new ArrayAssignment()); + + if (MatchExpression::ELEM_MATCH_OBJECT == node->matchType()) { + childContext.elemMatchExpr = node; + } + + // For an OR to be indexed, all its children must be indexed. + for (size_t i = 0; i < node->numChildren(); ++i) { + if (prepMemo(node->getChild(i), childContext)) { + aa->subnodes.push_back(memoIDForNode(node->getChild(i))); + } + } + + if (0 == aa->subnodes.size()) { return false; } + + size_t myMemoID; + NodeAssignment* assign; + allocateAssignment(node, &assign, &myMemoID); + + assign->arrayAssignment.reset(aa.release()); + return true; + } + else if (MatchExpression::AND == node->matchType()) { + // Map from idx id to children that have a pred over it. + + // TODO: The index intersection logic could be simplified if we could iterate over these + // maps in a known order. Currently when iterating over these maps we have to impose an + // ordering on each individual pair of indices in order to make sure that the + // enumeration results are order-independent. See SERVER-12196. + IndexToPredMap idxToFirst; + IndexToPredMap idxToNotFirst; + + // Children that aren't predicates, and which do not necessarily need + // to use an index. + vector subnodes; + + // Children that aren't predicates, but which *must* use an index. + // (e.g. an OR which contains a TEXT child). + vector mandatorySubnodes; + + // A list of predicates contained in the subtree rooted at 'node' + // obtained by traversing deeply through $and and $elemMatch children. + vector indexedPreds; + + // Partition the childen into the children that aren't predicates which may or may + // not be indexed ('subnodes'), children that aren't predicates which must use the + // index ('mandatorySubnodes'). and children that are predicates ('indexedPreds'). + // + // We have to get the subnodes with mandatory assignments rather than adding the + // mandatory preds to 'indexedPreds'. Adding the mandatory preds directly to + // 'indexedPreds' would lead to problems such as pulling a predicate beneath an OR + // into a set joined by an AND. + if (!partitionPreds(node, childContext, &indexedPreds, + &subnodes, &mandatorySubnodes)) { + return false; + } + + if (mandatorySubnodes.size() > 1) { + return false; + } + + // There can only be one mandatory predicate (at most one $text, at most one + // $geoNear, can't combine $text/$geoNear). + MatchExpression* mandatoryPred = NULL; + + // There could be multiple indices which we could use to satisfy the mandatory + // predicate. Keep the set of such indices. Currently only one text index is + // allowed per collection, but there could be multiple 2d or 2dsphere indices + // available to answer a $geoNear predicate. + set mandatoryIndices; + + // Go through 'indexedPreds' and add the predicates to the + // 'idxToFirst' and 'idxToNotFirst' maps. + for (size_t i = 0; i < indexedPreds.size(); ++i) { + MatchExpression* child = indexedPreds[i]; + + invariant(Indexability::nodeCanUseIndexOnOwnField(child)); + + RelevantTag* rt = static_cast(child->getTag()); + + if (expressionRequiresIndex(child)) { + // 'child' is a predicate which *must* be tagged with an index. + // This should include only TEXT and GEO_NEAR preds. + + // We expect either 0 or 1 mandatory predicates. + invariant(NULL == mandatoryPred); + + // Mandatory predicates are TEXT or GEO_NEAR. + invariant(MatchExpression::TEXT == child->matchType() || + MatchExpression::GEO_NEAR == child->matchType()); + + // The mandatory predicate must have a corresponding "mandatory index". + invariant(rt->first.size() != 0 || rt->notFirst.size() != 0); + + mandatoryPred = child; + + // Find all of the indices that could be used to satisfy the pred, + // and add them to the 'mandatoryIndices' set. + mandatoryIndices.insert(rt->first.begin(), rt->first.end()); + mandatoryIndices.insert(rt->notFirst.begin(), rt->notFirst.end()); + } + + for (size_t j = 0; j < rt->first.size(); ++j) { + idxToFirst[rt->first[j]].push_back(child); + } + + for (size_t j = 0 ; j< rt->notFirst.size(); ++j) { + idxToNotFirst[rt->notFirst[j]].push_back(child); + } + } + + // If none of our children can use indices, bail out. + if (idxToFirst.empty() + && (subnodes.size() == 0) + && (mandatorySubnodes.size() == 0)) { + return false; + } + + // At least one child can use an index, so we can create a memo entry. + AndAssignment* andAssignment = new AndAssignment(); + + size_t myMemoID; + NodeAssignment* nodeAssignment; + allocateAssignment(node, &nodeAssignment, &myMemoID); + // Takes ownership. + nodeAssignment->andAssignment.reset(andAssignment); + + // Predicates which must use an index might be buried inside + // a subnode. Handle that case here. + if (1 == mandatorySubnodes.size()) { + AndEnumerableState aes; + aes.subnodesToIndex.push_back(mandatorySubnodes[0]); + andAssignment->choices.push_back(aes); + return true; + } + + if (NULL != mandatoryPred) { + // We must have at least one index which can be used to answer 'mandatoryPred'. + invariant(!mandatoryIndices.empty()); + return enumerateMandatoryIndex(idxToFirst, idxToNotFirst, mandatoryPred, + mandatoryIndices, andAssignment); + } + + enumerateOneIndex(idxToFirst, idxToNotFirst, subnodes, andAssignment); + + if (_ixisect) { + enumerateAndIntersect(idxToFirst, idxToNotFirst, subnodes, andAssignment); + } + + return true; + } + + // Don't know what the node is at this point. + return false; + } + + bool PlanEnumerator::enumerateMandatoryIndex(const IndexToPredMap& idxToFirst, + const IndexToPredMap& idxToNotFirst, + MatchExpression* mandatoryPred, + const set& mandatoryIndices, + AndAssignment* andAssignment) { + // Generate index assignments for each index in 'mandatoryIndices'. We + // must assign 'mandatoryPred' to one of these indices, but we try all + // possibilities in 'mandatoryIndices' because some might be better than + // others for this query. + for (set::const_iterator indexIt = mandatoryIndices.begin(); + indexIt != mandatoryIndices.end(); + ++indexIt) { + + // We have a predicate which *must* be tagged to use an index. + // Get the index entry for the index it should use. + const IndexEntry& thisIndex = (*_indices)[*indexIt]; + + // Only text, 2d, and 2dsphere index types should be able to satisfy + // mandatory predicates. + invariant(INDEX_TEXT == thisIndex.type || + INDEX_2D == thisIndex.type || + INDEX_2DSPHERE == thisIndex.type); + + OneIndexAssignment indexAssign; + indexAssign.index = *indexIt; + + IndexToPredMap::const_iterator it = idxToFirst.find(*indexIt); + if (idxToFirst.end() == it) { + // We don't have any predicate to assign to the leading field of this index. + // This means that we cannot generate a solution using this index, so we + // just move on to the next index. + continue; + } + + const vector& predsOverLeadingField = it->second; + + if (thisIndex.multikey) { + // Special handling for multikey mandatory indices. + if (predsOverLeadingField.end() != std::find(predsOverLeadingField.begin(), + predsOverLeadingField.end(), + mandatoryPred)) { + // The mandatory predicate is over the first field of the index. Assign + // it now. + indexAssign.preds.push_back(mandatoryPred); + indexAssign.positions.push_back(0); + } + else { + // The mandatory pred is notFirst. Assign an arbitrary predicate + // over the first position. + invariant(!predsOverLeadingField.empty()); + indexAssign.preds.push_back(predsOverLeadingField[0]); + indexAssign.positions.push_back(0); + + // Assign the mandatory predicate at the matching position in the compound + // index. We do this in order to ensure that the mandatory predicate (and not + // some other predicate over the same position in the compound index) gets + // assigned. + // + // The bad thing that could happen otherwise: A non-mandatory predicate gets + // chosen by getMultikeyCompoundablePreds(...) instead of 'mandatoryPred'. + // We would then fail to assign the mandatory predicate, and hence generate + // a bad data access plan. + // + // The mandatory predicate is assigned by calling compound(...) because + // compound(...) has logic for matching up a predicate with the proper + // position in the compound index. + vector mandatoryToCompound; + mandatoryToCompound.push_back(mandatoryPred); + compound(mandatoryToCompound, thisIndex, &indexAssign); + + // At this point we have assigned a predicate over the leading field and + // we have assigned the mandatory predicate to a trailing field. + // + // Ex: + // Say we have index {a: 1, b: 1, c: "2dsphere", d: 1}. Also suppose that + // there is a $near predicate over "c", with additional predicates over + // "a", "b", "c", and "d". We will have assigned the $near predicate at + // position 2 and a predicate with path "a" at position 0. + } + + // Compound remaining predicates in a multikey-safe way. + IndexToPredMap::const_iterator compIt = idxToNotFirst.find(indexAssign.index); + if (compIt != idxToNotFirst.end()) { + const vector& couldCompound = compIt->second; + vector tryCompound; + + getMultikeyCompoundablePreds(indexAssign.preds, couldCompound, &tryCompound); + if (tryCompound.size()) { + compound(tryCompound, thisIndex, &indexAssign); + } + } + } + else { + // For non-multikey, we don't have to do anything too special. + // Just assign all "first" predicates and try to compound like usual. + indexAssign.preds = it->second; + + // Since everything in assign.preds prefixes the index, they all go + // at position '0' in the index, the first position. + indexAssign.positions.resize(indexAssign.preds.size(), 0); + + // And now we begin compound analysis. + + // Find everything that could use assign.index but isn't a pred over + // the first field of that index. + IndexToPredMap::const_iterator compIt = idxToNotFirst.find(indexAssign.index); + if (compIt != idxToNotFirst.end()) { + compound(compIt->second, thisIndex, &indexAssign); + } + } + + // The mandatory predicate must be assigned. + invariant(indexAssign.preds.end() != std::find(indexAssign.preds.begin(), + indexAssign.preds.end(), + mandatoryPred)); + + // Output the assignments for this index. + AndEnumerableState state; + state.assignments.push_back(indexAssign); + andAssignment->choices.push_back(state); + } + + return andAssignment->choices.size() > 0; + } + + void PlanEnumerator::enumerateOneIndex(const IndexToPredMap& idxToFirst, + const IndexToPredMap& idxToNotFirst, + const vector& subnodes, + AndAssignment* andAssignment) { + // In the simplest case, an AndAssignment picks indices like a PredicateAssignment. To + // be indexed we must only pick one index + // + // Complications: + // + // Some of our child predicates cannot be answered without an index. As such, the + // indices that those predicates require must always be outputted. We store these + // mandatory index assignments in 'mandatoryIndices'. + // + // Some of our children may not be predicates. We may have ORs (or array operators) as + // children. If one of these subtrees provides an index, the AND is indexed. We store + // these subtree choices in 'subnodes'. + // + // With the above two cases out of the way, we can focus on the remaining case: what to + // do with our children that are leaf predicates. + // + // Guiding principles for index assignment to leaf predicates: + // + // 1. If we assign an index to {x:{$gt: 5}} we should assign the same index to + // {x:{$lt: 50}}. That is, an index assignment should include all predicates + // over its leading field. + // + // 2. If we have the index {a:1, b:1} and we assign it to {a: 5} we should assign it + // to {b:7}, since with a predicate over the first field of the compound index, + // the second field can be bounded as well. We may only assign indices to predicates + // if all fields to the left of the index field are constrained. + + // First, add the state of using each subnode. + for (size_t i = 0; i < subnodes.size(); ++i) { + AndEnumerableState aes; + aes.subnodesToIndex.push_back(subnodes[i]); + andAssignment->choices.push_back(aes); + } + + // For each FIRST, we assign nodes to it. + for (IndexToPredMap::const_iterator it = idxToFirst.begin(); it != idxToFirst.end(); ++it) { + // The assignment we're filling out. + OneIndexAssignment indexAssign; + + // This is the index we assign to. + indexAssign.index = it->first; + + const IndexEntry& thisIndex = (*_indices)[it->first]; + + // If the index is multikey, we only assign one pred to it. We also skip + // compounding. TODO: is this also true for 2d and 2dsphere indices? can they be + // multikey but still compoundable? + if (thisIndex.multikey) { + // TODO: could pick better pred than first but not too worried since we should + // really be isecting indices here. Just take the first pred. We don't assign + // any other preds to this index. The planner will intersect the preds and this + // enumeration strategy is just one index at a time. + indexAssign.preds.push_back(it->second[0]); + indexAssign.positions.push_back(0); + + // If there are any preds that could possibly be compounded with this + // index... + IndexToPredMap::const_iterator compIt = idxToNotFirst.find(indexAssign.index); + if (compIt != idxToNotFirst.end()) { + const vector& couldCompound = compIt->second; + vector tryCompound; + + // ...select the predicates that are safe to compound and try to + // compound them. + getMultikeyCompoundablePreds(indexAssign.preds, couldCompound, &tryCompound); + if (tryCompound.size()) { + compound(tryCompound, thisIndex, &indexAssign); + } + } + } + else { + // The index isn't multikey. Assign all preds to it. The planner will + // intersect the bounds. + indexAssign.preds = it->second; + + // Since everything in assign.preds prefixes the index, they all go + // at position '0' in the index, the first position. + indexAssign.positions.resize(indexAssign.preds.size(), 0); + + // Find everything that could use assign.index but isn't a pred over + // the first field of that index. + IndexToPredMap::const_iterator compIt = idxToNotFirst.find(indexAssign.index); + if (compIt != idxToNotFirst.end()) { + compound(compIt->second, thisIndex, &indexAssign); + } + } + + AndEnumerableState state; + state.assignments.push_back(indexAssign); + andAssignment->choices.push_back(state); + } + } + + void PlanEnumerator::enumerateAndIntersect(const IndexToPredMap& idxToFirst, + const IndexToPredMap& idxToNotFirst, + const vector& subnodes, + AndAssignment* andAssignment) { + // Hardcoded "look at all members of the power set of size 2" search, + // a.k.a. "consider all pairs of indices". + // + // For each unordered pair of indices do the following: + // 0. Impose an ordering (idx1, idx2) using the key patterns. + // (*See note below.) + // 1. Assign predicates which prefix idx1 to idx1. + // 2. Add assigned predicates to a set of predicates---the "already + // assigned set". + // 3. Assign predicates which prefix idx2 to idx2, as long as they + // been assigned to idx1 already. Add newly assigned predicates to + // the "already assigned set". + // 4. Try to assign predicates to idx1 by compounding. + // 5. Add any predicates assigned to idx1 by compounding to the + // "already assigned set", + // 6. Try to assign predicates to idx2 by compounding. + // 7. Determine if we have already assigned all predicates in + // the "already assigned set" to a single index. If so, then + // don't generate an ixisect solution, as compounding will + // be better. Otherwise, output the ixisect assignments. + // + // *NOTE on ordering. Suppose we have two indices A and B, and a + // predicate P1 which is over the prefix of both indices A and B. + // If we order the indices (A, B) then P1 will get assigned to A, + // but if we order the indices (B, A) then P1 will get assigned to + // B. In order to make sure that we get the same result for the unordered + // pair {A, B} we have to begin by imposing an ordering. As a more concrete + // example, if we have indices {x: 1, y: 1} and {x: 1, z: 1} with predicate + // {x: 3}, we want to make sure that {x: 3} gets assigned to the same index + // irrespective of ordering. + + size_t sizeBefore = andAssignment->choices.size(); + + for (IndexToPredMap::const_iterator firstIt = idxToFirst.begin(); + firstIt != idxToFirst.end(); ++firstIt) { + + const IndexEntry& oneIndex = (*_indices)[firstIt->first]; + + // 'oneAssign' is used to assign indices and subnodes or to + // make assignments for the first index when it's multikey. + // It is NOT used in the inner loop that considers pairs of + // indices. + OneIndexAssignment oneAssign; + oneAssign.index = firstIt->first; + oneAssign.preds = firstIt->second; + // Since everything in assign.preds prefixes the index, they all go + // at position '0' in the index, the first position. + oneAssign.positions.resize(oneAssign.preds.size(), 0); + + // We create a scan per predicate so if we have >1 predicate we'll already + // have at least 2 scans (one predicate per scan as the planner can't + // intersect bounds when the index is multikey), so we stop here. + if (oneIndex.multikey && oneAssign.preds.size() > 1) { + // One could imagine an enormous auto-generated $all query with too many clauses to + // have an ixscan per clause. + static const size_t kMaxSelfIntersections = 10; + if (oneAssign.preds.size() > kMaxSelfIntersections) { + // Only take the first kMaxSelfIntersections preds. + oneAssign.preds.resize(kMaxSelfIntersections); + oneAssign.positions.resize(kMaxSelfIntersections); + } + AndEnumerableState state; + state.assignments.push_back(oneAssign); + andAssignment->choices.push_back(state); + continue; + } + + // Output (subnode, firstAssign) pairs. + for (size_t i = 0; i < subnodes.size(); ++i) { + AndEnumerableState indexAndSubnode; + indexAndSubnode.assignments.push_back(oneAssign); + indexAndSubnode.subnodesToIndex.push_back(subnodes[i]); + andAssignment->choices.push_back(indexAndSubnode); + // Limit n^2. + if (andAssignment->choices.size() - sizeBefore > _intersectLimit) { + return; + } + } + + // Start looking at all other indices to find one that we want to bundle + // with firstAssign. + IndexToPredMap::const_iterator secondIt = firstIt; + secondIt++; + for (; secondIt != idxToFirst.end(); secondIt++) { + const IndexEntry& firstIndex = (*_indices)[secondIt->first]; + const IndexEntry& secondIndex = (*_indices)[secondIt->first]; + + // Limit n^2. + if (andAssignment->choices.size() - sizeBefore > _intersectLimit) { + return; + } + + // If the other index we're considering is multikey with >1 pred, we don't + // want to have it as an additional assignment. Eventually, it1 will be + // equal to the current value of secondIt and we'll assign every pred for + // this mapping to the index. + if (secondIndex.multikey && secondIt->second.size() > 1) { + continue; + } + + // + // Step #0: + // Impose an ordering (idx1, idx2) using the key patterns. + // + IndexToPredMap::const_iterator it1, it2; + int ordering = firstIndex.keyPattern.woCompare(secondIndex.keyPattern); + it1 = (ordering > 0) ? firstIt : secondIt; + it2 = (ordering > 0) ? secondIt : firstIt; + const IndexEntry& ie1 = (*_indices)[it1->first]; + const IndexEntry& ie2 = (*_indices)[it2->first]; + + // + // Step #1: + // Assign predicates which prefix firstIndex to firstAssign. + // + OneIndexAssignment firstAssign; + firstAssign.index = it1->first; + firstAssign.preds = it1->second; + // Since everything in assign.preds prefixes the index, they all go + // at position '0' in the index, the first position. + firstAssign.positions.resize(firstAssign.preds.size(), 0); + + // We keep track of what preds are assigned to indices either because they + // prefix the index or have been assigned through compounding. We make sure + // that these predicates DO NOT become additional index assignments. + // Example: what if firstAssign is the index (x, y) and we're trying to + // compound? We want to make sure not to compound if the predicate is + // already assigned to index y. + set predsAssigned; + + // + // Step #2: + // Add indices assigned in 'firstAssign' to 'predsAssigned'. + // + for (size_t i = 0; i < firstAssign.preds.size(); ++i) { + predsAssigned.insert(firstAssign.preds[i]); + } + + // + // Step #3: + // Assign predicates which prefix secondIndex to secondAssign and + // have not already been assigned to firstAssign. Any newly + // assigned predicates are added to 'predsAssigned'. + // + OneIndexAssignment secondAssign; + secondAssign.index = it2->first; + const vector& preds = it2->second; + for (size_t i = 0; i < preds.size(); ++i) { + if (predsAssigned.end() == predsAssigned.find(preds[i])) { + secondAssign.preds.push_back(preds[i]); + secondAssign.positions.push_back(0); + predsAssigned.insert(preds[i]); + } + } + + // Every predicate that would use this index is already assigned in + // firstAssign. + if (0 == secondAssign.preds.size()) { continue; } + + // + // Step #4: + // Compound on firstAssign, if applicable. + // + IndexToPredMap::const_iterator firstIndexCompound = + idxToNotFirst.find(firstAssign.index); + + // Can't compound with multikey indices. + if (!ie1.multikey && firstIndexCompound != idxToNotFirst.end()) { + // We must remove any elements of 'predsAssigned' from consideration. + vector tryCompound; + const vector& couldCompound + = firstIndexCompound->second; + for (size_t i = 0; i < couldCompound.size(); ++i) { + if (predsAssigned.end() == predsAssigned.find(couldCompound[i])) { + tryCompound.push_back(couldCompound[i]); + } + } + if (tryCompound.size()) { + compound(tryCompound, ie1, &firstAssign); + } + } + + // + // Step #5: + // Make sure predicates assigned by compounding in step #4 do not get + // assigned again. + // + for (size_t i = 0; i < firstAssign.preds.size(); ++i) { + if (predsAssigned.end() == predsAssigned.find(firstAssign.preds[i])) { + predsAssigned.insert(firstAssign.preds[i]); + } + } + + // + // Step #6: + // Compound on firstAssign, if applicable. + // + IndexToPredMap::const_iterator secondIndexCompound = + idxToNotFirst.find(secondAssign.index); + + if (!ie2.multikey && secondIndexCompound != idxToNotFirst.end()) { + // We must remove any elements of 'predsAssigned' from consideration. + vector tryCompound; + const vector& couldCompound + = secondIndexCompound->second; + for (size_t i = 0; i < couldCompound.size(); ++i) { + if (predsAssigned.end() == predsAssigned.find(couldCompound[i])) { + tryCompound.push_back(couldCompound[i]); + } + } + if (tryCompound.size()) { + compound(tryCompound, ie2, &secondAssign); + } + } + + // Add predicates in 'secondAssign' to the set of all assigned predicates. + for (size_t i = 0; i < secondAssign.preds.size(); ++i) { + if (predsAssigned.end() == predsAssigned.find(secondAssign.preds[i])) { + predsAssigned.insert(secondAssign.preds[i]); + } + } + + // + // Step #7: + // Make sure we haven't already assigned this set of predicates by compounding. + // If we have, then bail out for this pair of indices. + // + if (alreadyCompounded(predsAssigned, andAssignment)) { + // There is no need to add either 'firstAssign' or 'secondAssign' + // to 'andAssignment' in this case because we have already performed + // assignments to single indices in enumerateOneIndex(...). + continue; + } + + // We're done with this particular pair of indices; output + // the resulting assignments. + AndEnumerableState state; + state.assignments.push_back(firstAssign); + state.assignments.push_back(secondAssign); + andAssignment->choices.push_back(state); + } + } + + // TODO: Do we just want one subnode at a time? We can use far more than 2 indices at once + // doing this very easily. If we want to restrict the # of indices the children use, when + // we memoize the subtree above we can restrict it to 1 index at a time. This can get + // tricky if we want both an intersection and a 1-index memo entry, since our state change + // is simple and we don't traverse the memo in any targeted way. Should also verify that + // having a one-to-many mapping of MatchExpression to MemoID doesn't break anything. This + // approach errors on the side of "too much indexing." + for (size_t i = 0; i < subnodes.size(); ++i) { + for (size_t j = i + 1; j < subnodes.size(); ++j) { + AndEnumerableState state; + state.subnodesToIndex.push_back(subnodes[i]); + state.subnodesToIndex.push_back(subnodes[j]); + andAssignment->choices.push_back(state); + } + } + } + + bool PlanEnumerator::partitionPreds(MatchExpression* node, + PrepMemoContext context, + vector* indexOut, + vector* subnodesOut, + vector* mandatorySubnodes) { + for (size_t i = 0; i < node->numChildren(); ++i) { + MatchExpression* child = node->getChild(i); + if (Indexability::nodeCanUseIndexOnOwnField(child)) { + RelevantTag* rt = static_cast(child->getTag()); + if (NULL != context.elemMatchExpr) { + // If we're in an $elemMatch context, store the + // innermost parent $elemMatch, as well as the + // inner path prefix. + rt->elemMatchExpr = context.elemMatchExpr; + rt->pathPrefix = getPathPrefix(child->path().toString()); + } + else { + // We're not an $elemMatch context, so we should store + // the prefix of the full path. + rt->pathPrefix = getPathPrefix(rt->path); + } + + // Output this as a pred that can use the index. + indexOut->push_back(child); + } + else if (Indexability::isBoundsGeneratingNot(child)) { + partitionPreds(child, context, indexOut, subnodesOut, mandatorySubnodes); + } + else if (MatchExpression::ELEM_MATCH_OBJECT == child->matchType()) { + PrepMemoContext childContext; + childContext.elemMatchExpr = child; + partitionPreds(child, childContext, indexOut, subnodesOut, mandatorySubnodes); + } + else if (MatchExpression::AND == child->matchType()) { + partitionPreds(child, context, indexOut, subnodesOut, mandatorySubnodes); + } + else { + bool mandatory = expressionRequiresIndex(child); + + // Recursively prepMemo for the subnode. We fall through + // to this case for logical nodes other than AND (e.g. OR). + if (prepMemo(child, context)) { + size_t childID = memoIDForNode(child); + + // Output the subnode. + if (mandatory) { + mandatorySubnodes->push_back(childID); + } + else { + subnodesOut->push_back(childID); + } + } + else if (mandatory) { + // The subnode is mandatory but cannot be indexed. This means + // that the entire AND cannot be indexed either. + return false; + } + } + } + + return true; + } + + void PlanEnumerator::getMultikeyCompoundablePreds(const vector& assigned, + const vector& couldCompound, + vector* out) { + // Map from a particular $elemMatch expression to the set of prefixes + // used so far by the predicates inside the $elemMatch. For example, + // {a: {$elemMatch: {b: 1, c: 2}}} would map to the set {'b', 'c'} at + // the end of this function's execution. + // + // NULL maps to the set of prefixes used so far outside of an $elemMatch + // context. + // + // As we iterate over the available indexed predicates, we keep track + // of the used prefixes both inside and outside of an $elemMatch context. + unordered_map > used; + + // Initialize 'used' with the starting predicates in 'assigned'. Begin by + // initializing the top-level scope with the prefix of the full path. + for (size_t i = 0; i < assigned.size(); i++) { + const MatchExpression* assignedPred = assigned[i]; + invariant(NULL != assignedPred->getTag()); + RelevantTag* usedRt = static_cast(assignedPred->getTag()); + set usedPrefixes; + usedPrefixes.insert(getPathPrefix(usedRt->path)); + used[NULL] = usedPrefixes; + + // If 'assigned' is a predicate inside an $elemMatch, we have to + // add the prefix not only to the top-level context, but also to the + // the $elemMatch context. For example, if 'assigned' is {a: {$elemMatch: {b: 1}}}, + // then we will have already added "a" to the set for NULL. We now + // also need to add "b" to the set for the $elemMatch. + if (NULL != usedRt->elemMatchExpr) { + set elemMatchUsed; + // Whereas getPathPrefix(usedRt->path) is the prefix of the full path, + // usedRt->pathPrefix contains the prefix of the portion of the + // path that is inside the $elemMatch. These two prefixes are the same + // in the top-level context, but here must be different because 'usedRt' + // is in an $elemMatch context. + elemMatchUsed.insert(usedRt->pathPrefix); + used[usedRt->elemMatchExpr] = elemMatchUsed; + } + } + + for (size_t i = 0; i < couldCompound.size(); ++i) { + invariant(Indexability::nodeCanUseIndexOnOwnField(couldCompound[i])); + RelevantTag* rt = static_cast(couldCompound[i]->getTag()); + + if (used.end() == used.find(rt->elemMatchExpr)) { + // This is a new $elemMatch that we haven't seen before. + invariant(used.end() != used.find(NULL)); + set& topLevelUsed = used.find(NULL)->second; + + // If the top-level path prefix of the $elemMatch hasn't been + // used yet, couldCompound[i] is safe to compound. + if (topLevelUsed.end() == topLevelUsed.find(getPathPrefix(rt->path))) { + topLevelUsed.insert(getPathPrefix(rt->path)); + set usedPrefixes; + usedPrefixes.insert(rt->pathPrefix); + used[rt->elemMatchExpr] = usedPrefixes; + + // Output the predicate. + out->push_back(couldCompound[i]); + } + + } + else { + // We've seen this $elemMatch before, or the predicate is + // top-level (not in an $elemMatch context). If the prefix stored + // in the tag has not been used yet, then couldCompound[i] is + // safe to compound. + set& usedPrefixes = used.find(rt->elemMatchExpr)->second; + if (usedPrefixes.end() == usedPrefixes.find(rt->pathPrefix)) { + usedPrefixes.insert(rt->pathPrefix); + + // Output the predicate. + out->push_back(couldCompound[i]); + } + } + } + } + + bool PlanEnumerator::alreadyCompounded(const set& ixisectAssigned, + const AndAssignment* andAssignment) { + for (size_t i = 0; i < andAssignment->choices.size(); ++i) { + const AndEnumerableState& state = andAssignment->choices[i]; + + // We cannot have assigned this set of predicates already by + // compounding unless this is an assignment to a single index. + if (state.assignments.size() != 1) { + continue; + } + + // If the set of preds in 'ixisectAssigned' is a subset of 'oneAssign.preds', + // then all the preds can be used by compounding on a single index. + const OneIndexAssignment& oneAssign = state.assignments[0]; + + // If 'ixisectAssigned' is larger than 'oneAssign.preds', then + // it can't be a subset. + if (ixisectAssigned.size() > oneAssign.preds.size()) { + continue; + } + + // Check for subset by counting the number of elements in 'oneAssign.preds' + // that are contained in 'ixisectAssigned'. The elements of both 'oneAssign.preds' + // and 'ixisectAssigned' are unique (no repeated elements). + size_t count = 0; + for (size_t j = 0; j < oneAssign.preds.size(); ++j) { + if (ixisectAssigned.end() != ixisectAssigned.find(oneAssign.preds[j])) { + ++count; + } + } + + if (ixisectAssigned.size() == count) { + return true; + } + + // We cannot assign the preds by compounding on 'oneAssign'. + // Move on to the next index. + } + + return false; + } + + void PlanEnumerator::compound(const vector& tryCompound, + const IndexEntry& thisIndex, + OneIndexAssignment* assign) { + // Let's try to match up the expressions in 'compExprs' with the + // fields in the index key pattern. + BSONObjIterator kpIt(thisIndex.keyPattern); + + // Skip the first elt as it's already assigned. + kpIt.next(); + + // When we compound we store the field number that the predicate + // goes over in order to avoid having to iterate again and compare + // field names. + size_t posInIdx = 0; + + while (kpIt.more()) { + BSONElement keyElt = kpIt.next(); + ++posInIdx; + + // Go through 'tryCompound' to see if there is a compoundable + // predicate for 'keyElt'. If there is nothing to compound, then + // simply move on to the next field in the compound index. We + // do not enforce that fields are assigned contiguously from + // right to left, i.e. for compound index {a: 1, b: 1, c: 1} + // it is okay to compound predicates over "a" and "c", skipping "b". + for (size_t j = 0; j < tryCompound.size(); ++j) { + MatchExpression* maybe = tryCompound[j]; + // Sigh we grab the full path from the relevant tag. + RelevantTag* rt = static_cast(maybe->getTag()); + if (keyElt.fieldName() == rt->path) { + // preds and positions are parallel arrays. + assign->preds.push_back(maybe); + assign->positions.push_back(posInIdx); + } + } + } + } + + // + // Structure navigation + // + + void PlanEnumerator::tagMemo(size_t id) { + QLOG() << "Tagging memoID " << id << endl; + NodeAssignment* assign = _memo[id]; + verify(NULL != assign); + + if (NULL != assign->pred) { + PredicateAssignment* pa = assign->pred.get(); + verify(NULL == pa->expr->getTag()); + verify(pa->indexToAssign < pa->first.size()); + pa->expr->setTag(new IndexTag(pa->first[pa->indexToAssign])); + } + else if (NULL != assign->orAssignment) { + OrAssignment* oa = assign->orAssignment.get(); + for (size_t i = 0; i < oa->subnodes.size(); ++i) { + tagMemo(oa->subnodes[i]); + } + } + else if (NULL != assign->arrayAssignment) { + ArrayAssignment* aa = assign->arrayAssignment.get(); + tagMemo(aa->subnodes[aa->counter]); + } + else if (NULL != assign->andAssignment) { + AndAssignment* aa = assign->andAssignment.get(); + verify(aa->counter < aa->choices.size()); + + const AndEnumerableState& aes = aa->choices[aa->counter]; + + for (size_t j = 0; j < aes.subnodesToIndex.size(); ++j) { + tagMemo(aes.subnodesToIndex[j]); + } + + for (size_t i = 0; i < aes.assignments.size(); ++i) { + const OneIndexAssignment& assign = aes.assignments[i]; + + for (size_t j = 0; j < assign.preds.size(); ++j) { + MatchExpression* pred = assign.preds[j]; + verify(NULL == pred->getTag()); + pred->setTag(new IndexTag(assign.index, assign.positions[j])); + } + } + } + else { + verify(0); + } + } + + bool PlanEnumerator::nextMemo(size_t id) { + NodeAssignment* assign = _memo[id]; + verify(NULL != assign); + + if (NULL != assign->pred) { + PredicateAssignment* pa = assign->pred.get(); + pa->indexToAssign++; + if (pa->indexToAssign >= pa->first.size()) { + pa->indexToAssign = 0; + return true; + } + return false; + } + else if (NULL != assign->orAssignment) { + OrAssignment* oa = assign->orAssignment.get(); + + // Limit the number of OR enumerations + oa->counter++; + if (oa->counter >= _orLimit) { + return true; + } + + // OR just walks through telling its children to + // move forward. + for (size_t i = 0; i < oa->subnodes.size(); ++i) { + // If there's no carry, we just stop. If there's a carry, we move the next child + // forward. + if (!nextMemo(oa->subnodes[i])) { + return false; + } + } + // If we're here, the last subnode had a carry, therefore the OR has a carry. + return true; + } + else if (NULL != assign->arrayAssignment) { + ArrayAssignment* aa = assign->arrayAssignment.get(); + // moving to next on current subnode is OK + if (!nextMemo(aa->subnodes[aa->counter])) { return false; } + // Move to next subnode. + ++aa->counter; + if (aa->counter < aa->subnodes.size()) { + return false; + } + aa->counter = 0; + return true; + } + else if (NULL != assign->andAssignment) { + AndAssignment* aa = assign->andAssignment.get(); + + // One of our subnodes might have to move on to its next enumeration state. + const AndEnumerableState& aes = aa->choices[aa->counter]; + for (size_t i = 0; i < aes.subnodesToIndex.size(); ++i) { + if (!nextMemo(aes.subnodesToIndex[i])) { + return false; + } + } + + // None of the subnodes had another enumeration state, so we move on to the + // next top-level choice. + ++aa->counter; + if (aa->counter < aa->choices.size()) { + return false; + } + aa->counter = 0; + return true; + } + + // This shouldn't happen. + verify(0); + return false; + } + +} // namespace mongo diff --git a/src/mongo/db/query/plan_enumerator.h b/src/mongo/db/query/plan_enumerator.h new file mode 100644 index 00000000000..bd3f9c0a534 --- /dev/null +++ b/src/mongo/db/query/plan_enumerator.h @@ -0,0 +1,447 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/index_entry.h" +#include "mongo/db/query/index_tag.h" +#include "mongo/db/query/query_knobs.h" + +namespace mongo { + + struct PlanEnumeratorParams { + + PlanEnumeratorParams() : intersect(false), + maxSolutionsPerOr(internalQueryEnumerationMaxOrSolutions), + maxIntersectPerAnd(internalQueryEnumerationMaxIntersectPerAnd) { } + + // Do we provide solutions that use more indices than the minimum required to provide + // an indexed solution? + bool intersect; + + // Not owned here. + MatchExpression* root; + + // Not owned here. + const vector* indices; + + // How many plans are we willing to ouput from an OR? We currently consider + // all possibly OR plans, which means the product of the number of possibilities + // for each clause of the OR. This could grow disastrously large. + size_t maxSolutionsPerOr; + + // How many intersect plans are we willing to output from an AND? Given that we pursue an + // all-pairs approach, we could wind up creating a lot of enumeration possibilities for + // certain inputs. + size_t maxIntersectPerAnd; + }; + + /** + * Provides elements from the power set of possible indices to use. Uses the available + * predicate information to make better decisions about what indices are best. + */ + class PlanEnumerator { + MONGO_DISALLOW_COPYING(PlanEnumerator); + public: + /** + * Constructs an enumerator for the query specified in 'root' which is tagged with + * RelevantTag(s). The index patterns mentioned in the tags are described by 'indices'. + * + * Does not take ownership of any arguments. They must outlive any calls to getNext(...). + */ + PlanEnumerator(const PlanEnumeratorParams& params); + + ~PlanEnumerator(); + + /** + * Returns OK and performs a sanity check on the input parameters and prepares the + * internal state so that getNext() can be called. Returns an error status with a + * description if the sanity check failed. + */ + Status init(); + + /** + * Outputs a possible plan. Leaves in the plan are tagged with an index to use. + * Returns true if a plan was outputted, false if no more plans will be outputted. + * + * 'tree' is set to point to the query tree. A QueryAssignment is built from this tree. + * Caller owns the pointer. Note that 'tree' itself points into data owned by the + * provided CanonicalQuery. + * + * Nodes in 'tree' are tagged with indices that should be used to answer the tagged nodes. + * Only nodes that have a field name (isLogical() == false) will be tagged. + */ + bool getNext(MatchExpression** tree); + + private: + + // + // Memoization strategy + // + + + // Everything is really a size_t but it's far more readable to impose a type via typedef. + + // An ID we use to index into _memo. An entry in _memo is a NodeAssignment. + typedef size_t MemoID; + + // An index in _indices. + typedef size_t IndexID; + + // The position of a field in a possibly compound index. + typedef size_t IndexPosition; + + struct PrepMemoContext { + PrepMemoContext() : elemMatchExpr(NULL) { } + MatchExpression* elemMatchExpr; + }; + + /** + * Traverses the match expression and generates the memo structure from it. + * Returns true if the provided node uses an index, false otherwise. + */ + bool prepMemo(MatchExpression* node, PrepMemoContext context); + + /** + * Traverses the memo structure and annotates the tree with IndexTags for the chosen + * indices. + */ + void tagMemo(MemoID id); + + /** + * Move to the next enumeration state. Each assignment stores its own enumeration state. + * See the various ____Assignment classes below for details on enumeration state. + * + * Returns true if the memo subtree with root 'node' has no further enumeration states. In + * this case, that subtree restarts its enumeration at the beginning state. This implies + * that the parent of node should move to the next state. If 'node' is the root of the + * tree, we are done with enumeration. + * + * The return of this function can be thought of like a 'carry' in addition. + * + * Returns false if the memo subtree has moved to the next state. + */ + bool nextMemo(MemoID id); + + /** + * A short word on the memo structure. + * + * The PlanEnumerator is interested in matching predicates and indices. Predicates + * are leaf nodes in the parse tree. {x:5}, {x: {$geoWithin:...}} are both predicates. + * + * When we have simple predicates, like {x:5}, the task is easy: any indices prefixed + * with 'x' can be used to answer the predicate. This is where the PredicateAssignment + * is used. + * + * With logical operators, things are more complicated. Let's start with OR, the simplest. + * Since the output of an OR is the union of its results, each of its children must be + * indexed for the entire OR to be indexed. If each subtree of an OR is indexable, the + * OR is as well. + * + * For an AND to be indexed, only one of its children must be indexed. AND is an + * intersection of its children, so each of its children describes a superset of the + * produced results. + */ + + struct PredicateAssignment { + PredicateAssignment() : indexToAssign(0) { } + + vector first; + // Not owned here. + MatchExpression* expr; + + // Enumeration state. An indexed predicate's possible states are the indices that the + // predicate can directly use (the 'first' indices). As such this value ranges from 0 + // to first.size()-1 inclusive. + size_t indexToAssign; + }; + + struct OrAssignment { + OrAssignment() : counter(0) { } + + // Each child of an OR must be indexed for the OR to be indexed. When an OR moves to a + // subsequent state it just asks all its children to move their states forward. + + // Must use all of subnodes. + vector subnodes; + + // The number of OR states that we've enumerated so far. + size_t counter; + }; + + // This is used by AndAssignment and is not an actual assignment. + struct OneIndexAssignment { + // 'preds[i]' is uses index 'index' at position 'positions[i]' + vector preds; + vector positions; + IndexID index; + }; + + struct AndEnumerableState { + vector assignments; + vector subnodesToIndex; + }; + + struct AndAssignment { + AndAssignment() : counter(0) { } + + vector choices; + + // We're on the counter-th member of state. + size_t counter; + }; + + struct ArrayAssignment { + ArrayAssignment() : counter(0) { } + vector subnodes; + size_t counter; + }; + + /** + * Associates indices with predicates. + */ + struct NodeAssignment { + scoped_ptr pred; + scoped_ptr orAssignment; + scoped_ptr andAssignment; + scoped_ptr arrayAssignment; + string toString() const; + }; + + /** + * Allocates a NodeAssignment and associates it with the provided 'expr'. + * + * The unique MemoID of the new assignment is outputted in '*id'. + * The out parameter '*slot' points to the newly allocated NodeAssignment. + */ + void allocateAssignment(MatchExpression* expr, NodeAssignment** slot, MemoID* id); + + /** + * Predicates inside $elemMatch's that are semantically "$and of $and" + * predicates are not rewritten to the top-level during normalization. + * However, we would like to make predicates inside $elemMatch available + * for combining index bounds with the top-level $and predicates. + * + * This function deeply traverses $and and $elemMatch expressions of + * the tree rooted at 'node', adding all preds that can use an index + * to the output vector 'indexOut'. At the same time, $elemMatch + * context information is stashed in the tags so that we don't lose + * information due to flattening. + * + * Nodes that cannot be deeply traversed are returned via the output + * vectors 'subnodesOut' and 'mandatorySubnodes'. Subnodes are "mandatory" + * if they *must* use an index (TEXT and GEO). + * + * Does not take ownership of arguments. + * + * Returns false if the AND cannot be indexed. Otherwise returns true. + */ + bool partitionPreds(MatchExpression* node, + PrepMemoContext context, + vector* indexOut, + vector* subnodesOut, + vector* mandatorySubnodes); + + /** + * Finds a set of predicates that can be safely compounded with the set + * of predicates in 'assigned', under the assumption that we are assigning + * predicates to a compound, multikey index. + * + * The list of candidate predicates that we could compound is passed + * in 'couldCompound'. A subset of these predicates that is safe to + * combine by compounding is returned in the out-parameter 'out'. + * + * Does not take ownership of its arguments. + * + * The rules for when to compound for multikey indices are reasonably + * complex, and are dependent on the structure of $elemMatch's used + * in the query. Ignoring $elemMatch for the time being, the rule is this: + * + * "Any set of predicates for which no two predicates share a path + * prefix can be compounded." + * + * Suppose we have predicates over paths 'a.b' and 'a.c'. These cannot + * be compounded because they share the prefix 'a'. Similarly, the bounds + * for 'a' and 'a.b' cannot be compounded (in the case of multikey index + * {a: 1, 'a.b': 1}). You *can* compound predicates over the paths 'a.b.c', + * 'd', and 'e.b.c', because there is no shared prefix. + * + * The rules are different in the presence of $elemMatch. For $elemMatch + * {a: {$elemMatch: {, ..., }}}, we are allowed to compound + * bounds for pred1 through predN, even though these predicates share the + * path prefix 'a'. However, we still cannot compound in the case of + * {a: {$elemMatch: {'b.c': {$gt: 1}, 'b.d': 5}}} because 'b.c' and 'b.d' + * share a prefix. In other words, what matters inside an $elemMatch is not + * the absolute prefix, but rather the "relative prefix" after the shared + * $elemMatch part of the path. + * + * A few more examples: + * 1) {'a.b': {$elemMatch: {c: {$gt: 1}, d: 5}}}. In this case, we can + * compound, because the $elemMatch is applied to the shared part of + * the path 'a.b'. + * + * 2) {'a.b': 1, a: {$elemMatch: {b: {$gt: 0}}}}. We cannot combine the + * bounds here because the prefix 'a' is shared by two predicates which + * are not joined together by an $elemMatch. + * + * NOTE: + * Usually 'assigned' has just one predicate. However, in order to support + * mandatory predicate assignment (TEXT and GEO_NEAR), we allow multiple + * already-assigned predicates to be passed. If a mandatory predicate is over + * a trailing field in a multikey compound index, then we assign both a predicate + * over the leading field as well as the mandatory predicate prior to calling + * this function. + * + * Ex: + * Say we have index {a: 1, b: 1, c: "2dsphere", d: 1} as well as a $near + * predicate and a $within predicate over "c". The $near predicate is mandatory + * and must be assigned. The $within predicate is not mandatory. Furthermore, + * it cannot be assigned in addition to the $near predicate because the index + * is multikey. + * + * In this case the enumerator must assign the $near predicate, and pass it in + * in 'assigned'. Otherwise it would be possible to assign the $within predicate, + * and then not assign the $near because the $within is already assigned (and + * has the same path). + */ + void getMultikeyCompoundablePreds(const vector& assigned, + const vector& couldCompound, + vector* out); + + /** + * 'andAssignment' contains assignments that we've already committed to outputting, + * including both single index assignments and ixisect assignments. + * + * 'ixisectAssigned' is a set of predicates that we are about to add to 'andAssignment' + * as an index intersection assignment. + * + * Returns true if an single index assignment which is already in 'andAssignment' + * contains a superset of the predicates in 'ixisectAssigned'. This means that we + * can assign the same preds to a compound index rather than using index intersection. + * + * Ex. + * Suppose we have indices {a: 1}, {b: 1}, and {a: 1, b: 1} with query + * {a: 2, b: 2}. When we try to intersect {a: 1} and {b: 1} the predicates + * a==2 and b==2 will get assigned to respective indices. But then we will + * call this function with ixisectAssigned equal to the set {'a==2', 'b==2'}, + * and notice that we have already assigned this same set of predicates to + * the single index {a: 1, b: 1} via compounding. + */ + bool alreadyCompounded(const set& ixisectAssigned, + const AndAssignment* andAssignment); + /** + * Output index intersection assignments inside of an AND node. + */ + typedef unordered_map > IndexToPredMap; + + /** + * Generate index intersection assignments given the predicate/index structure in idxToFirst + * and idxToNotFirst (and the sub-trees in 'subnodes'). Outputs the assignments in + * 'andAssignment'. + */ + void enumerateAndIntersect(const IndexToPredMap& idxToFirst, + const IndexToPredMap& idxToNotFirst, + const vector& subnodes, + AndAssignment* andAssignment); + + /** + * Generate one-index-at-once assignments given the predicate/index structure in idxToFirst + * and idxToNotFirst (and the sub-trees in 'subnodes'). Outputs the assignments into + * 'andAssignment'. + */ + void enumerateOneIndex(const IndexToPredMap& idxToFirst, + const IndexToPredMap& idxToNotFirst, + const vector& subnodes, + AndAssignment* andAssignment); + + /** + * Generate single-index assignments for queries which contain mandatory + * predicates (TEXT and GEO_NEAR, which are required to use a compatible index). + * Outputs these assignments into 'andAssignment'. + * + * Returns true if it generated at least one assignment, and false if no assignment + * of 'mandatoryPred' is possible. + */ + bool enumerateMandatoryIndex(const IndexToPredMap& idxToFirst, + const IndexToPredMap& idxToNotFirst, + MatchExpression* mandatoryPred, + const set& mandatoryIndices, + AndAssignment* andAssignment); + + /** + * Try to assign predicates in 'tryCompound' to 'thisIndex' as compound assignments. + * Output the assignments in 'assign'. + */ + void compound(const vector& tryCompound, + const IndexEntry& thisIndex, + OneIndexAssignment* assign); + + /** + * Return the memo entry for 'node'. Does some sanity checking to ensure that a memo entry + * actually exists. + */ + MemoID memoIDForNode(MatchExpression* node); + + std::string dumpMemo(); + + // Map from expression to its MemoID. + unordered_map _nodeToId; + + // Map from MemoID to its precomputed solution info. + unordered_map _memo; + + // If true, there are no further enumeration states, and getNext should return false. + // We could be _done immediately after init if we're unable to output an indexed plan. + bool _done; + + // + // Data used by all enumeration strategies + // + + // Match expression we're planning for. Not owned by us. + MatchExpression* _root; + + // Indices we're allowed to enumerate with. Not owned here. + const vector* _indices; + + // Do we output >1 index per AND (index intersection)? + bool _ixisect; + + // How many enumerations are we willing to produce from each OR? + size_t _orLimit; + + // How many things do we want from each AND? + size_t _intersectLimit; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/plan_executor.cpp b/src/mongo/db/query/plan_executor.cpp new file mode 100644 index 00000000000..11717fbe643 --- /dev/null +++ b/src/mongo/db/query/plan_executor.cpp @@ -0,0 +1,198 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/plan_executor.h" + +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/plan_stats.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/exec/working_set_common.h" +#include "mongo/db/pdfile.h" + +namespace mongo { + + PlanExecutor::PlanExecutor(WorkingSet* ws, PlanStage* rt) + : _workingSet(ws) , _root(rt) , _killed(false) { } + + PlanExecutor::~PlanExecutor() { } + + WorkingSet* PlanExecutor::getWorkingSet() { + return _workingSet.get(); + } + + PlanStageStats* PlanExecutor::getStats() const { + return _root->getStats(); + } + + void PlanExecutor::saveState() { + if (!_killed) { _root->prepareToYield(); } + } + + bool PlanExecutor::restoreState() { + if (!_killed) { + _root->recoverFromYield(); + } + return !_killed; + } + + void PlanExecutor::invalidate(const DiskLoc& dl, InvalidationType type) { + if (!_killed) { _root->invalidate(dl, type); } + } + + void PlanExecutor::setYieldPolicy(Runner::YieldPolicy policy) { + if (Runner::YIELD_MANUAL == policy) { + _yieldPolicy.reset(); + } + else { + _yieldPolicy.reset(new RunnerYieldPolicy()); + } + } + + Runner::RunnerState PlanExecutor::getNext(BSONObj* objOut, DiskLoc* dlOut) { + if (_killed) { return Runner::RUNNER_DEAD; } + + for (;;) { + // Yield, if we can yield ourselves. + if (NULL != _yieldPolicy.get() && _yieldPolicy->shouldYield()) { + saveState(); + _yieldPolicy->yield(); + if (_killed) { return Runner::RUNNER_DEAD; } + restoreState(); + } + + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState code = _root->work(&id); + + if (PlanStage::ADVANCED == code) { + // Fast count. + if (WorkingSet::INVALID_ID == id) { + invariant(NULL == objOut); + invariant(NULL == dlOut); + return Runner::RUNNER_ADVANCED; + } + + WorkingSetMember* member = _workingSet->get(id); + bool hasRequestedData = true; + + if (NULL != objOut) { + if (WorkingSetMember::LOC_AND_IDX == member->state) { + if (1 != member->keyData.size()) { + _workingSet->free(id); + hasRequestedData = false; + } + else { + *objOut = member->keyData[0].keyData; + } + } + else if (member->hasObj()) { + *objOut = member->obj; + } + else { + _workingSet->free(id); + hasRequestedData = false; + } + } + + if (NULL != dlOut) { + if (member->hasLoc()) { + *dlOut = member->loc; + } + else { + _workingSet->free(id); + hasRequestedData = false; + } + } + + if (hasRequestedData) { + _workingSet->free(id); + return Runner::RUNNER_ADVANCED; + } + // This result didn't have the data the caller wanted, try again. + } + else if (PlanStage::NEED_TIME == code) { + // Fall through to yield check at end of large conditional. + } + else if (PlanStage::NEED_FETCH == code) { + // id has a loc and refers to an obj we need to fetch. + WorkingSetMember* member = _workingSet->get(id); + + // This must be true for somebody to request a fetch and can only change when an + // invalidation happens, which is when we give up a lock. Don't give up the + // lock between receiving the NEED_FETCH and actually fetching(?). + verify(member->hasLoc()); + + // Actually bring record into memory. + Record* record = member->loc.rec(); + + // If we're allowed to, go to disk outside of the lock. + if (NULL != _yieldPolicy.get()) { + saveState(); + _yieldPolicy->yield(record); + if (_killed) { return Runner::RUNNER_DEAD; } + restoreState(); + } + else { + // We're set to manually yield. We go to disk in the lock. + record->touch(); + } + + // Record should be in memory now. Log if it's not. + if (!Record::likelyInPhysicalMemory(record->dataNoThrowing())) { + OCCASIONALLY { + warning() << "Record wasn't in memory immediately after fetch: " + << member->loc.toString() << endl; + } + } + + // Note that we're not freeing id. Fetch semantics say that we shouldn't. + } + else if (PlanStage::IS_EOF == code) { + return Runner::RUNNER_EOF; + } + else if (PlanStage::DEAD == code) { + return Runner::RUNNER_DEAD; + } + else { + verify(PlanStage::FAILURE == code); + if (NULL != objOut) { + WorkingSetCommon::getStatusMemberObject(*_workingSet, id, objOut); + } + return Runner::RUNNER_ERROR; + } + } + } + + bool PlanExecutor::isEOF() { + return _killed || _root->isEOF(); + } + + void PlanExecutor::kill() { + _killed = true; + } + +} // namespace mongo diff --git a/src/mongo/db/query/plan_executor.h b/src/mongo/db/query/plan_executor.h new file mode 100644 index 00000000000..06214d7cdc2 --- /dev/null +++ b/src/mongo/db/query/plan_executor.h @@ -0,0 +1,111 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/db/query/runner.h" +#include "mongo/db/query/runner_yield_policy.h" + +namespace mongo { + + class BSONObj; + class DiskLoc; + class PlanStage; + struct PlanStageStats; + class WorkingSet; + + /** + * A PlanExecutor is the abstraction that knows how to crank a tree of stages into execution. + * The executor is usually part of a larger abstraction that is interacting with the cache + * and/or the query optimizer. + * + * Executes a plan. Used by a runner. Calls work() on a plan until a result is produced. + * Stops when the plan is EOF or if the plan errors. + */ + class PlanExecutor { + public: + PlanExecutor(WorkingSet* ws, PlanStage* rt); + ~PlanExecutor(); + + // + // Accessors + // + + /** TODO document me */ + WorkingSet* getWorkingSet(); + + /** This is OK even if we were killed */ + PlanStageStats* getStats() const; + + // + // Methods that just pass down to the PlanStage tree. + // + + /** TODO document me */ + void saveState(); + + /** TODO document me */ + bool restoreState(); + + /** TODO document me */ + void invalidate(const DiskLoc& dl, InvalidationType type); + + // + // Running Support + // + + /** TODO document me */ + void setYieldPolicy(Runner::YieldPolicy policy); + + /** TODO document me */ + Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + /** TOOD document me */ + bool isEOF(); + + /** + * During the yield, the database we're operating over or any collection we're relying on + * may be dropped. When this happens all cursors and runners on that database and + * collection are killed or deleted in some fashion. (This is how the _killed gets set.) + */ + void kill(); + + private: + boost::scoped_ptr _workingSet; + boost::scoped_ptr _root; + boost::scoped_ptr _yieldPolicy; + + // Did somebody drop an index we care about or the namespace we're looking at? If so, + // we'll be killed. + bool _killed; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/plan_ranker.cpp b/src/mongo/db/query/plan_ranker.cpp new file mode 100644 index 00000000000..89cf62182ee --- /dev/null +++ b/src/mongo/db/query/plan_ranker.cpp @@ -0,0 +1,272 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include +#include +#include +#include + +#include "mongo/db/query/plan_ranker.h" + +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/server_options.h" +#include "mongo/db/server_parameters.h" + +namespace { + + /** + * Comparator for (scores, candidateIndex) in pickBestPlan(). + */ + bool scoreComparator(const std::pair& lhs, + const std::pair& rhs) { + // Just compare score in lhs.first and rhs.first; + // Ignore candidate array index in lhs.second and rhs.second. + return lhs.first > rhs.first; + } + +} // namespace + +namespace mongo { + + using std::vector; + + // static + size_t PlanRanker::pickBestPlan(const vector& candidates, + PlanRankingDecision* why) { + invariant(!candidates.empty()); + invariant(why); + + // A plan that hits EOF is automatically scored above + // its peers. If multiple plans hit EOF during the same + // set of round-robin calls to work(), then all such plans + // receive the bonus. + double eofBonus = 1.0; + + // Each plan will have a stat tree. + vector statTrees; + + // Get stat trees from each plan. + // Copy stats trees instead of transferring ownership + // because multi plan runner will need its own stats + // trees for explain. + for (size_t i = 0; i < candidates.size(); ++i) { + statTrees.push_back(candidates[i].root->getStats()); + } + + // Holds (score, candidateInndex). + // Used to derive scores and candidate ordering. + vector > scoresAndCandidateindices; + + // Compute score for each tree. Record the best. + for (size_t i = 0; i < statTrees.size(); ++i) { + QLOG() << "Scoring plan " << i << ":" << endl + << candidates[i].solution->toString() << "Stats:\n" + << statsToBSON(*statTrees[i]).jsonString(Strict, true); + LOG(2) << "Scoring query plan: " << getPlanSummary(*candidates[i].solution) + << " planHitEOF=" << statTrees[i]->common.isEOF; + + double score = scoreTree(statTrees[i]); + QLOG() << "score = " << score << endl; + if (statTrees[i]->common.isEOF) { + QLOG() << "Adding +" << eofBonus << " EOF bonus to score." << endl; + score += 1; + } + scoresAndCandidateindices.push_back(std::make_pair(score, i)); + } + + // Sort (scores, candidateIndex). Get best child and populate candidate ordering. + std::stable_sort(scoresAndCandidateindices.begin(), scoresAndCandidateindices.end(), + scoreComparator); + + // Determine whether plans tied for the win. + if (scoresAndCandidateindices.size() > 1) { + double bestScore = scoresAndCandidateindices[0].first; + double runnerUpScore = scoresAndCandidateindices[1].first; + static const double epsilon = 1e-10; + why->tieForBest = fabs(bestScore - runnerUpScore) < epsilon; + } + + // Update results in 'why' + // Stats and scores in 'why' are sorted in descending order by score. + why->stats.clear(); + why->scores.clear(); + why->candidateOrder.clear(); + for (size_t i = 0; i < scoresAndCandidateindices.size(); ++i) { + double score = scoresAndCandidateindices[i].first; + size_t candidateIndex = scoresAndCandidateindices[i].second; + + // We shouldn't cache the scores with the EOF bonus included, + // as this is just a tie-breaking measure for plan selection. + // Plans not run through the multi plan runner will not receive + // the bonus. + // + // An example of a bad thing that could happen if we stored scores + // with the EOF bonus included: + // + // Let's say Plan A hits EOF, is the highest ranking plan, and gets + // cached as such. On subsequent runs it will not receive the bonus. + // Eventually the plan cache feedback mechanism will evict the cache + // entry---the scores will appear to have fallen due to the missing + // EOF bonus. + // + // This begs the question, why don't we include the EOF bonus in + // scoring of cached plans as well? The problem here is that the cached + // plan runner always runs plans to completion before scoring. Queries + // that don't get the bonus in the multi plan runner might get the bonus + // after being run from the plan cache. + if (statTrees[candidateIndex]->common.isEOF) { + score -= eofBonus; + } + + why->stats.mutableVector().push_back(statTrees[candidateIndex]); + why->scores.push_back(score); + why->candidateOrder.push_back(candidateIndex); + } + + size_t bestChild = scoresAndCandidateindices[0].second; + return bestChild; + } + + // TODO: Move this out. This is a signal for ranking but will become its own complicated + // stats-collecting beast. + double computeSelectivity(const PlanStageStats* stats) { + if (STAGE_IXSCAN == stats->stageType) { + IndexScanStats* iss = static_cast(stats->specific.get()); + return iss->keyPattern.nFields(); + } + else { + double sum = 0; + for (size_t i = 0; i < stats->children.size(); ++i) { + sum += computeSelectivity(stats->children[i]); + } + return sum; + } + } + + bool hasStage(const StageType type, const PlanStageStats* stats) { + if (type == stats->stageType) { + return true; + } + for (size_t i = 0; i < stats->children.size(); ++i) { + if (hasStage(type, stats->children[i])) { + return true; + } + } + return false; + } + + // static + double PlanRanker::scoreTree(const PlanStageStats* stats) { + // We start all scores at 1. Our "no plan selected" score is 0 and we want all plans to + // be greater than that. + double baseScore = 1; + + // How many "units of work" did the plan perform. Each call to work(...) + // counts as one unit, and each NEED_FETCH is penalized as an additional work unit. + size_t workUnits = stats->common.works + stats->common.needFetch; + + // How much did a plan produce? + // Range: [0, 1] + double productivity = static_cast(stats->common.advanced) + / static_cast(workUnits); + + // Just enough to break a tie. Must be small enough to ensure that a more productive + // plan doesn't lose to a less productive plan due to tie breaking. + const double epsilon = std::min(1.0 / static_cast(10 * workUnits), 1e-4); + + // We prefer covered projections. + // + // We only do this when we have a projection stage because we have so many jstests that + // check bounds even when a collscan plan is just as good as the ixscan'd plan :( + double noFetchBonus = epsilon; + if (hasStage(STAGE_PROJECTION, stats) && hasStage(STAGE_FETCH, stats)) { + noFetchBonus = 0; + } + + // In the case of ties, prefer solutions without a blocking sort + // to solutions with a blocking sort. + double noSortBonus = epsilon; + if (hasStage(STAGE_SORT, stats)) { + noSortBonus = 0; + } + + // In the case of ties, prefer single index solutions to ixisect. Index + // intersection solutions are often slower than single-index solutions + // because they require examining a superset of index keys that would be + // examined by a single index scan. + // + // On the other hand, index intersection solutions examine the same + // number or fewer of documents. In the case that index intersection + // allows us to examine fewer documents, the penalty given to ixisect + // can be made up via the no fetch bonus. + double noIxisectBonus = epsilon; + if (hasStage(STAGE_AND_HASH, stats) || hasStage(STAGE_AND_SORTED, stats)) { + noIxisectBonus = 0; + } + + double tieBreakers = noFetchBonus + noSortBonus + noIxisectBonus; + double score = baseScore + productivity + tieBreakers; + + mongoutils::str::stream ss; + ss << "score(" << score << ") = baseScore(" << baseScore << ")" + << " + productivity((" << stats->common.advanced + << " advanced)/(" + << stats->common.works + << " works + " + << stats->common.needFetch + << " needFetch) = " + << productivity << ")" + << " + tieBreakers(" << noFetchBonus + << " noFetchBonus + " + << noSortBonus + << " noSortBonus + " + << noIxisectBonus + << " noIxisectBonus = " + << tieBreakers << ")"; + std::string scoreStr = ss; + QLOG() << scoreStr << endl; + LOG(2) << scoreStr; + + if (internalQueryForceIntersectionPlans) { + if (hasStage(STAGE_AND_HASH, stats) || hasStage(STAGE_AND_SORTED, stats)) { + // The boost should be >2.001 to make absolutely sure the ixisect plan will win due + // to the combination of 1) productivity, 2) eof bonus, and 3) no ixisect bonus. + score += 3; + QLOG() << "Score boosted to " << score << " due to intersection forcing." << endl; + } + } + + return score; + } + +} // namespace mongo diff --git a/src/mongo/db/query/plan_ranker.h b/src/mongo/db/query/plan_ranker.h new file mode 100644 index 00000000000..973429c0609 --- /dev/null +++ b/src/mongo/db/query/plan_ranker.h @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/owned_pointer_vector.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/plan_stats.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + struct CandidatePlan; + struct PlanRankingDecision; + + /** + * Ranks 2 or more plans. + */ + class PlanRanker { + public: + /** + * Returns index in 'candidates' of which plan is best. + * Populates 'why' with information relevant to how each plan fared in the ranking process. + * Caller owns pointers in 'why'. + * 'candidateOrder' holds indices into candidates ordered by score (winner in first element). + */ + static size_t pickBestPlan(const vector& candidates, + PlanRankingDecision* why); + + /** + * Assign the stats tree a 'goodness' score. The higher the score, the better + * the plan. The exact value isn't meaningful except for imposing a ranking. + */ + static double scoreTree(const PlanStageStats* stats); + }; + + /** + * A container holding one to-be-ranked plan and its associated/relevant data. + * Does not own any of its pointers. + */ + struct CandidatePlan { + CandidatePlan(QuerySolution* s, PlanStage* r, WorkingSet* w) + : solution(s), root(r), ws(w), failed(false) { } + + QuerySolution* solution; + PlanStage* root; + WorkingSet* ws; + + // Any results produced during the plan's execution prior to ranking are retained here. + std::list results; + + bool failed; + }; + + /** + * Information about why a plan was picked to be the best. Data here is placed into the cache + * and used by the CachedPlanRunner to compare expected performance with actual. + */ + struct PlanRankingDecision { + + PlanRankingDecision() : tieForBest(false) { } + + /** + * Make a deep copy. + */ + PlanRankingDecision* clone() const { + PlanRankingDecision* decision = new PlanRankingDecision(); + for (size_t i = 0; i < stats.size(); ++i) { + PlanStageStats* s = stats.vector()[i]; + invariant(s); + decision->stats.mutableVector().push_back(s->clone()); + } + decision->scores = scores; + decision->candidateOrder = candidateOrder; + decision->tieForBest = tieForBest; + return decision; + } + + // Stats of all plans sorted in descending order by score. + // Owned by us. + OwnedPointerVector stats; + + // The "goodness" score corresponding to 'stats'. + // Sorted in descending order. + std::vector scores; + + // Ordering of original plans in descending of score. + // Filled in by PlanRanker::pickBestPlan(candidates, ...) + // so that candidates[candidateOrder[0]] refers to the best plan + // with corresponding cores[0] and stats[0]. Runner-up would be + // candidates[candidateOrder[1]] followed by + // candidates[candidateOrder[2]], ... + std::vector candidateOrder; + + // Did two plans tie for best? + // + // NOTE: Reading this is the only reliable way to determine if there was a tie, + // because the scores kept inside the PlanRankingDecision do not incorporate + // the EOF bonus. + bool tieForBest; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/planner_access.cpp b/src/mongo/db/query/planner_access.cpp new file mode 100644 index 00000000000..972cf778229 --- /dev/null +++ b/src/mongo/db/query/planner_access.cpp @@ -0,0 +1,1295 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/planner_access.h" + +#include +#include + +#include "mongo/db/matcher/expression_array.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/matcher/expression_text.h" +#include "mongo/db/query/indexability.h" +#include "mongo/db/query/index_bounds_builder.h" +#include "mongo/db/query/index_tag.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_planner_common.h" + +namespace { + + using namespace mongo; + + /** + * Text node functors. + */ + bool isTextNode(const QuerySolutionNode* node) { + return STAGE_TEXT == node->getType(); + } + +} // namespace + +namespace mongo { + + using std::vector; + + // static + QuerySolutionNode* QueryPlannerAccess::makeCollectionScan(const CanonicalQuery& query, + bool tailable, + const QueryPlannerParams& params) { + // Make the (only) node, a collection scan. + CollectionScanNode* csn = new CollectionScanNode(); + csn->name = query.ns(); + csn->filter.reset(query.root()->shallowClone()); + csn->tailable = tailable; + csn->maxScan = query.getParsed().getMaxScan(); + + // If the hint is {$natural: +-1} this changes the direction of the collection scan. + if (!query.getParsed().getHint().isEmpty()) { + BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural"); + if (!natural.eoo()) { + csn->direction = natural.numberInt() >= 0 ? 1 : -1; + } + } + + // The sort can specify $natural as well. The sort direction should override the hint + // direction if both are specified. + const BSONObj& sortObj = query.getParsed().getSort(); + if (!sortObj.isEmpty()) { + BSONElement natural = sortObj.getFieldDotted("$natural"); + if (!natural.eoo()) { + csn->direction = natural.numberInt() >= 0 ? 1 : -1; + } + } + + return csn; + } + + // static + QuerySolutionNode* QueryPlannerAccess::makeLeafNode(const CanonicalQuery& query, + const IndexEntry& index, + size_t pos, + MatchExpression* expr, + IndexBoundsBuilder::BoundsTightness* tightnessOut) { + // We're guaranteed that all GEO_NEARs are first. This slightly violates the "sort index + // predicates by their position in the compound index" rule but GEO_NEAR isn't an ixscan. + // This saves our bacon when we have {foo: 1, bar: "2dsphere"} and the predicate on bar is a + // $near. If we didn't get the GEO_NEAR first we'd create an IndexScanNode and later cast + // it to a GeoNear2DSphereNode + // + // This should gracefully deal with the case where we have a pred over foo but no geo clause + // over bar. In that case there is no GEO_NEAR to appear first and it's treated like a + // straight ixscan. + BSONElement elt = index.keyPattern.firstElement(); + bool indexIs2D = (String == elt.type() && "2d" == elt.String()); + + if (MatchExpression::GEO_NEAR == expr->matchType()) { + // We must not keep the expression node around. + *tightnessOut = IndexBoundsBuilder::EXACT; + GeoNearMatchExpression* nearExpr = static_cast(expr); + // 2d geoNear requires a hard limit and as such we take it out before it gets here. If + // this happens it's a bug. + verify(!indexIs2D); + GeoNear2DSphereNode* ret = new GeoNear2DSphereNode(); + ret->indexKeyPattern = index.keyPattern; + ret->nq = nearExpr->getData(); + ret->baseBounds.fields.resize(index.keyPattern.nFields()); + if (NULL != query.getProj()) { + ret->addPointMeta = query.getProj()->wantGeoNearPoint(); + ret->addDistMeta = query.getProj()->wantGeoNearDistance(); + } + return ret; + } + else if (indexIs2D) { + // We must not keep the expression node around. + *tightnessOut = IndexBoundsBuilder::EXACT; + verify(MatchExpression::GEO == expr->matchType()); + GeoMatchExpression* nearExpr = static_cast(expr); + verify(indexIs2D); + Geo2DNode* ret = new Geo2DNode(); + ret->indexKeyPattern = index.keyPattern; + ret->gq = nearExpr->getGeoQuery(); + return ret; + } + else if (MatchExpression::TEXT == expr->matchType()) { + // We must not keep the expression node around. + *tightnessOut = IndexBoundsBuilder::EXACT; + TextMatchExpression* textExpr = static_cast(expr); + TextNode* ret = new TextNode(); + ret->indexKeyPattern = index.keyPattern; + ret->query = textExpr->getQuery(); + ret->language = textExpr->getLanguage(); + return ret; + } + else { + // Note that indexKeyPattern.firstElement().fieldName() may not equal expr->path() + // because expr might be inside an array operator that provides a path prefix. + IndexScanNode* isn = new IndexScanNode(); + isn->indexKeyPattern = index.keyPattern; + isn->indexIsMultiKey = index.multikey; + isn->bounds.fields.resize(index.keyPattern.nFields()); + isn->maxScan = query.getParsed().getMaxScan(); + isn->addKeyMetadata = query.getParsed().returnKey(); + + // Get the ixtag->pos-th element of the index key pattern. + // TODO: cache this instead/with ixtag->pos? + BSONObjIterator it(index.keyPattern); + BSONElement keyElt = it.next(); + for (size_t i = 0; i < pos; ++i) { + verify(it.more()); + keyElt = it.next(); + } + verify(!keyElt.eoo()); + + IndexBoundsBuilder::translate(expr, keyElt, index, &isn->bounds.fields[pos], + tightnessOut); + + return isn; + } + } + + bool QueryPlannerAccess::shouldMergeWithLeaf(const MatchExpression* expr, + const ScanBuildingState& scanState) { + const QuerySolutionNode* node = scanState.currentScan.get(); + if (NULL == node || NULL == expr) { + return false; + } + + if (NULL == scanState.ixtag) { + return false; + } + + if (scanState.currentIndexNumber != scanState.ixtag->index) { + return false; + } + + size_t pos = scanState.ixtag->pos; + const IndexEntry& index = scanState.indices[scanState.currentIndexNumber]; + const MatchExpression::MatchType mergeType = scanState.root->matchType(); + + const StageType type = node->getType(); + verify(STAGE_GEO_NEAR_2D != type); + + const MatchExpression::MatchType exprType = expr->matchType(); + + // + // First handle special solution tree leaf types. In general, normal index bounds + // building is not used for special leaf types, and hence we cannot merge leaves. + // + // This rule is always true for OR, but there are exceptions for AND. + // Specifically, we can often merge a predicate with a special leaf type + // by adding a filter to the special leaf type. + // + + if (STAGE_GEO_2D == type) { + // Don't merge GEO with a geo leaf. Instead, we will generate an AND_HASH solution + // with two separate leaves. + return MatchExpression::AND == mergeType + && MatchExpression::GEO != exprType; + } + + if (STAGE_TEXT == type) { + // Currently only one text predicate is allowed, but to be safe, make sure that we + // do not try to merge two text predicates. + return MatchExpression::AND == mergeType + && MatchExpression::TEXT != exprType; + } + + if (STAGE_GEO_NEAR_2DSPHERE == type) { + // Currently only one GEO_NEAR is allowed, but to be safe, make sure that we + // do not try to merge two GEO_NEAR predicates. + return MatchExpression::AND == mergeType + && MatchExpression::GEO_NEAR != exprType; + } + + // + // If we're here, then we're done checking for special leaf nodes, and the leaf + // must be a regular index scan. + // + + invariant(type == STAGE_IXSCAN); + const IndexScanNode* scan = static_cast(node); + const IndexBounds* boundsToFillOut = &scan->bounds; + + if (boundsToFillOut->fields[pos].name.empty()) { + // The bounds will be compounded. This is OK because the + // plan enumerator told us that it is OK. + return true; + } + else { + if (MatchExpression::AND == mergeType) { + // The bounds will be intersected. This is OK provided + // that the index is NOT multikey. + return !index.multikey; + } + else { + // The bounds will be unionized. + return true; + } + } + + + } + + void QueryPlannerAccess::mergeWithLeafNode(MatchExpression* expr, + ScanBuildingState* scanState) { + QuerySolutionNode* node = scanState->currentScan.get(); + invariant(NULL != node); + + const MatchExpression::MatchType mergeType = scanState->root->matchType(); + size_t pos = scanState->ixtag->pos; + const IndexEntry& index = scanState->indices[scanState->currentIndexNumber]; + + const StageType type = node->getType(); + verify(STAGE_GEO_NEAR_2D != type); + + if (STAGE_GEO_2D == type) { + scanState->tightness = IndexBoundsBuilder::INEXACT_FETCH; + return; + } + + // Text data is covered, but not exactly. Text covering is unlike any other covering + // so we deal with it in addFilterToSolutionNode. + if (STAGE_TEXT == type) { + scanState->tightness = IndexBoundsBuilder::INEXACT_COVERED; + return; + } + + IndexBounds* boundsToFillOut = NULL; + + if (STAGE_GEO_NEAR_2DSPHERE == type) { + GeoNear2DSphereNode* gn = static_cast(node); + boundsToFillOut = &gn->baseBounds; + } + else { + verify(type == STAGE_IXSCAN); + IndexScanNode* scan = static_cast(node); + boundsToFillOut = &scan->bounds; + } + + // Get the ixtag->pos-th element of the index key pattern. + // TODO: cache this instead/with ixtag->pos? + BSONObjIterator it(index.keyPattern); + BSONElement keyElt = it.next(); + for (size_t i = 0; i < pos; ++i) { + verify(it.more()); + keyElt = it.next(); + } + verify(!keyElt.eoo()); + scanState->tightness = IndexBoundsBuilder::INEXACT_FETCH; + + verify(boundsToFillOut->fields.size() > pos); + + OrderedIntervalList* oil = &boundsToFillOut->fields[pos]; + + if (boundsToFillOut->fields[pos].name.empty()) { + IndexBoundsBuilder::translate(expr, keyElt, index, oil, &scanState->tightness); + } + else { + if (MatchExpression::AND == mergeType) { + IndexBoundsBuilder::translateAndIntersect(expr, keyElt, index, oil, + &scanState->tightness); + } + else { + verify(MatchExpression::OR == mergeType); + IndexBoundsBuilder::translateAndUnion(expr, keyElt, index, oil, + &scanState->tightness); + } + } + } + + // static + void QueryPlannerAccess::finishTextNode(QuerySolutionNode* node, const IndexEntry& index) { + TextNode* tn = static_cast(node); + + // Figure out what positions are prefix positions. We build an index key prefix from + // the predicates over the text index prefix keys. + // For example, say keyPattern = { a: 1, _fts: "text", _ftsx: 1, b: 1 } + // prefixEnd should be 1. + size_t prefixEnd = 0; + BSONObjIterator it(tn->indexKeyPattern); + // Count how many prefix terms we have. + while (it.more()) { + // We know that the only key pattern with a type of String is the _fts field + // which is immediately after all prefix fields. + if (String == it.next().type()) { + break; + } + ++prefixEnd; + } + + // If there's no prefix, the filter is already on the node and the index prefix is null. + // We can just return. + if (!prefixEnd) { + return; + } + + // We can't create a text stage if there aren't EQ predicates on its prefix terms. So + // if we've made it this far, we should have collected the prefix predicates in the + // filter. + invariant(NULL != tn->filter.get()); + MatchExpression* textFilterMe = tn->filter.get(); + + BSONObjBuilder prefixBob; + + if (MatchExpression::AND != textFilterMe->matchType()) { + // Only one prefix term. + invariant(1 == prefixEnd); + // Sanity check: must be an EQ. + invariant(MatchExpression::EQ == textFilterMe->matchType()); + + EqualityMatchExpression* eqExpr = static_cast(textFilterMe); + prefixBob.append(eqExpr->getData()); + tn->filter.reset(); + } + else { + invariant(MatchExpression::AND == textFilterMe->matchType()); + + // Indexed by the keyPattern position index assignment. We want to add + // prefixes in order but we must order them first. + vector prefixExprs(prefixEnd, NULL); + + AndMatchExpression* amExpr = static_cast(textFilterMe); + invariant(amExpr->numChildren() >= prefixEnd); + + // Look through the AND children. The prefix children we want to + // stash in prefixExprs. + size_t curChild = 0; + while (curChild < amExpr->numChildren()) { + MatchExpression* child = amExpr->getChild(curChild); + IndexTag* ixtag = static_cast(child->getTag()); + invariant(NULL != ixtag); + // Skip this child if it's not part of a prefix, or if we've already assigned a + // predicate to this prefix position. + if (ixtag->pos >= prefixEnd || prefixExprs[ixtag->pos] != NULL) { + ++curChild; + continue; + } + // prefixExprs takes ownership of 'child'. + prefixExprs[ixtag->pos] = child; + amExpr->getChildVector()->erase(amExpr->getChildVector()->begin() + curChild); + // Don't increment curChild. + } + + // Go through the prefix equalities in order and create an index prefix out of them. + for (size_t i = 0; i < prefixExprs.size(); ++i) { + MatchExpression* prefixMe = prefixExprs[i]; + invariant(NULL != prefixMe); + invariant(MatchExpression::EQ == prefixMe->matchType()); + EqualityMatchExpression* eqExpr = static_cast(prefixMe); + prefixBob.append(eqExpr->getData()); + // We removed this from the AND expression that owned it, so we must clean it + // up ourselves. + delete prefixMe; + } + + // Clear out an empty $and. + if (0 == amExpr->numChildren()) { + tn->filter.reset(); + } + else if (1 == amExpr->numChildren()) { + // Clear out unsightly only child of $and + MatchExpression* child = amExpr->getChild(0); + amExpr->getChildVector()->clear(); + // Deletes current filter which is amExpr. + tn->filter.reset(child); + } + } + + tn->indexPrefix = prefixBob.obj(); + } + + // static + bool QueryPlannerAccess::orNeedsFetch(const ScanBuildingState* scanState) { + if (scanState->loosestBounds == IndexBoundsBuilder::EXACT) { + return false; + } + else if (scanState->loosestBounds == IndexBoundsBuilder::INEXACT_FETCH) { + return true; + } + else { + invariant(scanState->loosestBounds == IndexBoundsBuilder::INEXACT_COVERED); + const IndexEntry& index = scanState->indices[scanState->currentIndexNumber]; + return index.multikey; + } + } + + // static + void QueryPlannerAccess::finishAndOutputLeaf(ScanBuildingState* scanState, + vector* out) { + finishLeafNode(scanState->currentScan.get(), + scanState->indices[scanState->currentIndexNumber]); + + if (MatchExpression::OR == scanState->root->matchType()) { + if (orNeedsFetch(scanState)) { + // In order to correctly evaluate the predicates for this index, we have to + // fetch the full documents. Add a fetch node above the index scan whose filter + // includes *all* of the predicates used to generate the ixscan. + FetchNode* fetch = new FetchNode(); + // Takes ownership. + fetch->filter.reset(scanState->curOr.release()); + // Takes ownership. + fetch->children.push_back(scanState->currentScan.release()); + + scanState->currentScan.reset(fetch); + } + else if (scanState->loosestBounds == IndexBoundsBuilder::INEXACT_COVERED) { + // This an OR, at least one of the predicates used to generate 'currentScan' + // is inexact covered, but none is inexact fetch. This means that we can put + // these predicates, joined by an $or, as filters on the index scan. This avoids + // a fetch and allows the predicates to be covered by the index. + // + // Ex. + // Say we have index {a: 1} and query {$or: [{a: /foo/}, {a: /bar/}]}. + // The entire query, {$or: [{a: /foo/}, {a: /bar/}]}, should be a filter + // in the index scan stage itself. + scanState->currentScan->filter.reset(scanState->curOr.release()); + } + } + + out->push_back(scanState->currentScan.release()); + } + + // static + void QueryPlannerAccess::finishLeafNode(QuerySolutionNode* node, const IndexEntry& index) { + const StageType type = node->getType(); + verify(STAGE_GEO_NEAR_2D != type); + + if (STAGE_GEO_2D == type) { + return; + } + + if (STAGE_TEXT == type) { + finishTextNode(node, index); + return; + } + + IndexBounds* bounds = NULL; + + if (STAGE_GEO_NEAR_2DSPHERE == type) { + GeoNear2DSphereNode* gnode = static_cast(node); + bounds = &gnode->baseBounds; + } + else { + verify(type == STAGE_IXSCAN); + IndexScanNode* scan = static_cast(node); + bounds = &scan->bounds; + } + + // Find the first field in the scan's bounds that was not filled out. + // TODO: could cache this. + size_t firstEmptyField = 0; + for (firstEmptyField = 0; firstEmptyField < bounds->fields.size(); ++firstEmptyField) { + if ("" == bounds->fields[firstEmptyField].name) { + verify(bounds->fields[firstEmptyField].intervals.empty()); + break; + } + } + + // All fields are filled out with bounds, nothing to do. + if (firstEmptyField == bounds->fields.size()) { + IndexBoundsBuilder::alignBounds(bounds, index.keyPattern); + return; + } + + // Skip ahead to the firstEmptyField-th element, where we begin filling in bounds. + BSONObjIterator it(index.keyPattern); + for (size_t i = 0; i < firstEmptyField; ++i) { + verify(it.more()); + it.next(); + } + + // For each field in the key... + while (it.more()) { + BSONElement kpElt = it.next(); + // There may be filled-in fields to the right of the firstEmptyField. + // Example: + // The index {loc:"2dsphere", x:1} + // With a predicate over x and a near search over loc. + if ("" == bounds->fields[firstEmptyField].name) { + verify(bounds->fields[firstEmptyField].intervals.empty()); + // ...build the "all values" interval. + IndexBoundsBuilder::allValuesForField(kpElt, + &bounds->fields[firstEmptyField]); + } + ++firstEmptyField; + } + + // Make sure that the length of the key is the length of the bounds we started. + verify(firstEmptyField == bounds->fields.size()); + + // We create bounds assuming a forward direction but can easily reverse bounds to align + // according to our desired direction. + IndexBoundsBuilder::alignBounds(bounds, index.keyPattern); + } + + // static + void QueryPlannerAccess::findElemMatchChildren(const MatchExpression* node, + vector* out, + vector* subnodesOut) { + for (size_t i = 0; i < node->numChildren(); ++i) { + MatchExpression* child = node->getChild(i); + if (Indexability::isBoundsGenerating(child) && + NULL != child->getTag()) { + out->push_back(child); + } + else if (MatchExpression::AND == child->matchType() || + Indexability::arrayUsesIndexOnChildren(child)) { + findElemMatchChildren(child, out, subnodesOut); + } + else if (NULL != child->getTag()) { + subnodesOut->push_back(child); + } + } + } + + // static + bool QueryPlannerAccess::processIndexScans(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const std::vector& indices, + std::vector* out) { + // Initialize the ScanBuildingState. + ScanBuildingState scanState(root, inArrayOperator, indices); + + while (scanState.curChild < root->numChildren()) { + MatchExpression* child = root->getChild(scanState.curChild); + + // If there is no tag, it's not using an index. We've sorted our children such that the + // children with tags are first, so we stop now. + if (NULL == child->getTag()) { break; } + + scanState.ixtag = static_cast(child->getTag()); + // If there's a tag it must be valid. + verify(IndexTag::kNoIndex != scanState.ixtag->index); + + // If the child can't use an index on its own field (and the child is not a negation + // of a bounds-generating expression), then it's indexed by virtue of one of + // its children having an index. + // + // NOTE: If the child is logical, it could possibly collapse into a single ixscan. we + // ignore this for now. + if (!Indexability::isBoundsGenerating(child)) { + // If we're here, then the child is indexed by virtue of its children. + // In most cases this means that we recursively build indexed data + // access on 'child'. + if (!processIndexScansSubnode(query, &scanState, out)) { + return false; + } + continue; + } + + // If we're here, we now know that 'child' can use an index directly and the index is + // over the child's field. + + // If 'child' is a NOT, then the tag we're interested in is on the NOT's + // child node. + if (MatchExpression::NOT == child->matchType()) { + scanState.ixtag = static_cast(child->getChild(0)->getTag()); + invariant(IndexTag::kNoIndex != scanState.ixtag->index); + } + + // If the child we're looking at uses a different index than the current index scan, add + // the current index scan to the output as we're done with it. The index scan created + // by the child then becomes our new current index scan. Note that the current scan + // could be NULL, in which case we don't output it. The rest of the logic is identical. + // + // If the child uses the same index as the current index scan, we may be able to merge + // the bounds for the two scans. + // + // Guiding principle: must the values we're testing come from the same array in the + // document? If so, we can combine bounds (via intersection or compounding). If not, + // we can't. + // + // If the index is NOT multikey, it's always semantically correct to combine bounds, + // as there are no arrays to worry about. + // + // If the index is multikey, there are arrays of values. There are several + // complications in the multikey case that have to be obeyed both by the enumerator + // and here as we try to merge predicates into query solution leaves. The hairy + // details of these rules are documented near the top of planner_access.h. + if (shouldMergeWithLeaf(child, scanState)) { + // The child uses the same index we're currently building a scan for. Merge + // the bounds and filters. + verify(scanState.currentIndexNumber == scanState.ixtag->index); + scanState.tightness = IndexBoundsBuilder::INEXACT_FETCH; + mergeWithLeafNode(child, &scanState); + handleFilter(&scanState); + } + else { + if (NULL != scanState.currentScan.get()) { + // Output the current scan before starting to construct a new out. + finishAndOutputLeaf(&scanState, out); + } + else { + verify(IndexTag::kNoIndex == scanState.currentIndexNumber); + } + + // Reset state before producing a new leaf. + scanState.resetForNextScan(scanState.ixtag); + + scanState.currentScan.reset(makeLeafNode(query, + indices[scanState.currentIndexNumber], + scanState.ixtag->pos, child, + &scanState.tightness)); + + handleFilter(&scanState); + } + } + + // Output the scan we're done with, if it exists. + if (NULL != scanState.currentScan.get()) { + finishAndOutputLeaf(&scanState, out); + } + + return true; + } + + // static + bool QueryPlannerAccess::processIndexScansElemMatch(const CanonicalQuery& query, + ScanBuildingState* scanState, + std::vector* out) { + MatchExpression* root = scanState->root; + MatchExpression* child = root->getChild(scanState->curChild); + const vector& indices = scanState->indices; + + // We have an AND with an ELEM_MATCH_OBJECT child. The plan enumerator produces + // index taggings which indicate that we should try to compound with + // predicates retrieved from inside the subtree rooted at the ELEM_MATCH. + // In order to obey the enumerator's tagging, we need to retrieve these + // predicates from inside the $elemMatch, and try to merge them with + // the current index scan. + + // Contains tagged predicates from inside the tree rooted at 'child' + // which are logically part of the AND. + vector emChildren; + + // Contains tagged nodes that are not logically part of the AND and + // cannot use the index directly (e.g. OR nodes which are tagged to + // be indexed). + vector emSubnodes; + + // Populate 'emChildren' and 'emSubnodes'. + findElemMatchChildren(child, &emChildren, &emSubnodes); + + // Recursively build data access for the nodes inside 'emSubnodes'. + for (size_t i = 0; i < emSubnodes.size(); ++i) { + MatchExpression* subnode = emSubnodes[i]; + + if (!Indexability::isBoundsGenerating(subnode)) { + // Must pass true for 'inArrayOperator' because the subnode is + // beneath an ELEM_MATCH_OBJECT. + QuerySolutionNode* childSolution = buildIndexedDataAccess(query, + subnode, + true, + indices); + + // buildIndexedDataAccess(...) returns NULL in error conditions, when + // it is unable to construct a query solution from a tagged match + // expression tree. If we are unable to construct a solution according + // to the instructions from the enumerator, then we bail out early + // (by returning false) rather than continuing on and potentially + // constructing an invalid solution tree. + if (NULL == childSolution) { return false; } + + // Output the resulting solution tree. + out->push_back(childSolution); + } + } + + // For each predicate in 'emChildren', try to merge it with the current index scan. + // + // This loop is similar to that in processIndexScans(...), except it does not call into + // handleFilters(...). Instead, we leave the entire $elemMatch filter intact. This way, + // the complete $elemMatch expression will be affixed as a filter later on. + for (size_t i = 0; i < emChildren.size(); ++i) { + MatchExpression* emChild = emChildren[i]; + invariant(NULL != emChild->getTag()); + scanState->ixtag = static_cast(emChild->getTag()); + + // If 'emChild' is a NOT, then the tag we're interested in is on the NOT's + // child node. + if (MatchExpression::NOT == emChild->matchType()) { + invariant(NULL != emChild->getChild(0)->getTag()); + scanState->ixtag = static_cast(emChild->getChild(0)->getTag()); + invariant(IndexTag::kNoIndex != scanState->ixtag->index); + } + + if (shouldMergeWithLeaf(emChild, *scanState)) { + // The child uses the same index we're currently building a scan for. Merge + // the bounds and filters. + verify(scanState->currentIndexNumber == scanState->ixtag->index); + + scanState->tightness = IndexBoundsBuilder::INEXACT_FETCH; + mergeWithLeafNode(emChild, scanState); + } + else { + if (NULL != scanState->currentScan.get()) { + finishAndOutputLeaf(scanState, out); + } + else { + verify(IndexTag::kNoIndex == scanState->currentIndexNumber); + } + + scanState->currentIndexNumber = scanState->ixtag->index; + + scanState->tightness = IndexBoundsBuilder::INEXACT_FETCH; + scanState->currentScan.reset(makeLeafNode(query, indices[scanState->currentIndexNumber], + scanState->ixtag->pos, + emChild, &scanState->tightness)); + } + } + + // We're done processing the $elemMatch child. We leave it hanging off + // it's AND parent so that it will be affixed as a filter later on, + // and move on to the next child of the AND. + ++scanState->curChild; + return true; + } + + // static + bool QueryPlannerAccess::processIndexScansSubnode(const CanonicalQuery& query, + ScanBuildingState* scanState, + std::vector* out) { + MatchExpression* root = scanState->root; + MatchExpression* child = root->getChild(scanState->curChild); + const vector& indices = scanState->indices; + bool inArrayOperator = scanState->inArrayOperator; + + if (MatchExpression::AND == root->matchType() && + MatchExpression::ELEM_MATCH_OBJECT == child->matchType()) { + return processIndexScansElemMatch(query, scanState, out); + } + else if (!inArrayOperator) { + // The logical sub-tree is responsible for fully evaluating itself. Any + // required filters or fetches are already hung on it. As such, we remove the + // filter branch from our tree. buildIndexedDataAccess takes ownership of the + // child. + root->getChildVector()->erase(root->getChildVector()->begin() + scanState->curChild); + // The curChild of today is the curChild+1 of yesterday. + } + else { + ++scanState->curChild; + } + + // If inArrayOperator: takes ownership of child, which is OK, since we detached + // child from root. + QuerySolutionNode* childSolution = buildIndexedDataAccess(query, + child, + inArrayOperator, + indices); + if (NULL == childSolution) { return false; } + out->push_back(childSolution); + return true; + } + + // static + QuerySolutionNode* QueryPlannerAccess::buildIndexedAnd(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const vector& indices) { + auto_ptr autoRoot; + if (!inArrayOperator) { + autoRoot.reset(root); + } + + vector ixscanNodes; + if (!processIndexScans(query, root, inArrayOperator, indices, &ixscanNodes)) { + return NULL; + } + + // + // Process all non-indexed predicates. We hang these above the AND with a fetch and + // filter. + // + + // This is the node we're about to return. + QuerySolutionNode* andResult; + + // We must use an index for at least one child of the AND. We shouldn't be here if this + // isn't the case. + verify(ixscanNodes.size() >= 1); + + // Short-circuit: an AND of one child is just the child. + if (ixscanNodes.size() == 1) { + andResult = ixscanNodes[0]; + } + else { + // Figure out if we want AndHashNode or AndSortedNode. + bool allSortedByDiskLoc = true; + for (size_t i = 0; i < ixscanNodes.size(); ++i) { + if (!ixscanNodes[i]->sortedByDiskLoc()) { + allSortedByDiskLoc = false; + break; + } + } + if (allSortedByDiskLoc) { + AndSortedNode* asn = new AndSortedNode(); + asn->children.swap(ixscanNodes); + andResult = asn; + } + else if (internalQueryPlannerEnableHashIntersection) { + AndHashNode* ahn = new AndHashNode(); + ahn->children.swap(ixscanNodes); + andResult = ahn; + // The AndHashNode provides the sort order of its last child. If any of the + // possible subnodes of AndHashNode provides the sort order we care about, we put + // that one last. + for (size_t i = 0; i < ahn->children.size(); ++i) { + ahn->children[i]->computeProperties(); + const BSONObjSet& sorts = ahn->children[i]->getSort(); + if (sorts.end() != sorts.find(query.getParsed().getSort())) { + std::swap(ahn->children[i], ahn->children.back()); + break; + } + } + } + else { + // We can't use sort-based intersection, and hash-based intersection is disabled. + // Clean up the index scans and bail out by returning NULL. + QLOG() << "Can't build index intersection solution: " + << "AND_SORTED is not possible and AND_HASH is disabled."; + + for (size_t i = 0; i < ixscanNodes.size(); i++) { + delete ixscanNodes[i]; + } + return NULL; + } + } + + // Don't bother doing any kind of fetch analysis lite if we're doing it anyway above us. + if (inArrayOperator) { + return andResult; + } + + // If there are any nodes still attached to the AND, we can't answer them using the + // index, so we put a fetch with filter. + if (root->numChildren() > 0) { + FetchNode* fetch = new FetchNode(); + verify(NULL != autoRoot.get()); + if (autoRoot->numChildren() == 1) { + // An $and of one thing is that thing. + MatchExpression* child = autoRoot->getChild(0); + autoRoot->getChildVector()->clear(); + // Takes ownership. + fetch->filter.reset(child); + // 'autoRoot' will delete the empty $and. + } + else { // root->numChildren() > 1 + // Takes ownership. + fetch->filter.reset(autoRoot.release()); + } + // takes ownership + fetch->children.push_back(andResult); + andResult = fetch; + } + else { + // root has no children, let autoRoot get rid of it when it goes out of scope. + } + + return andResult; + } + + // static + QuerySolutionNode* QueryPlannerAccess::buildIndexedOr(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const vector& indices) { + auto_ptr autoRoot; + if (!inArrayOperator) { + autoRoot.reset(root); + } + + vector ixscanNodes; + if (!processIndexScans(query, root, inArrayOperator, indices, &ixscanNodes)) { + return NULL; + } + + // Unlike an AND, an OR cannot have filters hanging off of it. We stop processing + // when any of our children lack index tags. If a node lacks an index tag it cannot + // be answered via an index. + if (!inArrayOperator && 0 != root->numChildren()) { + warning() << "planner OR error, non-indexed child of OR."; + // We won't enumerate an OR without indices for each child, so this isn't an issue, even + // if we have an AND with an OR child -- we won't get here unless the OR is fully + // indexed. + return NULL; + } + + QuerySolutionNode* orResult = NULL; + + // An OR of one node is just that node. + if (1 == ixscanNodes.size()) { + orResult = ixscanNodes[0]; + } + else { + bool shouldMergeSort = false; + + if (!query.getParsed().getSort().isEmpty()) { + const BSONObj& desiredSort = query.getParsed().getSort(); + + // If there exists a sort order that is present in each child, we can merge them and + // maintain that sort order / those sort orders. + ixscanNodes[0]->computeProperties(); + BSONObjSet sharedSortOrders = ixscanNodes[0]->getSort(); + + if (!sharedSortOrders.empty()) { + for (size_t i = 1; i < ixscanNodes.size(); ++i) { + ixscanNodes[i]->computeProperties(); + BSONObjSet isect; + set_intersection(sharedSortOrders.begin(), + sharedSortOrders.end(), + ixscanNodes[i]->getSort().begin(), + ixscanNodes[i]->getSort().end(), + std::inserter(isect, isect.end()), + BSONObjCmp()); + sharedSortOrders = isect; + if (sharedSortOrders.empty()) { + break; + } + } + } + + // TODO: If we're looking for the reverse of one of these sort orders we could + // possibly reverse the ixscan nodes. + shouldMergeSort = (sharedSortOrders.end() != sharedSortOrders.find(desiredSort)); + } + + if (shouldMergeSort) { + MergeSortNode* msn = new MergeSortNode(); + msn->sort = query.getParsed().getSort(); + msn->children.swap(ixscanNodes); + orResult = msn; + } + else { + OrNode* orn = new OrNode(); + orn->children.swap(ixscanNodes); + orResult = orn; + } + } + + // Evaluate text nodes first to ensure that text scores are available. + // Move text nodes to front of vector. + std::stable_partition(orResult->children.begin(), orResult->children.end(), isTextNode); + + // OR must have an index for each child, so we should have detached all children from + // 'root', and there's nothing useful to do with an empty or MatchExpression. We let it die + // via autoRoot. + + return orResult; + } + + // static + QuerySolutionNode* QueryPlannerAccess::buildIndexedDataAccess(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const vector& indices) { + if (root->isLogical() && !Indexability::isBoundsGeneratingNot(root)) { + if (MatchExpression::AND == root->matchType()) { + // Takes ownership of root. + return buildIndexedAnd(query, root, inArrayOperator, indices); + } + else if (MatchExpression::OR == root->matchType()) { + // Takes ownership of root. + return buildIndexedOr(query, root, inArrayOperator, indices); + } + else { + // Can't do anything with negated logical nodes index-wise. + if (!inArrayOperator) { + delete root; + } + return NULL; + } + } + else { + auto_ptr autoRoot; + if (!inArrayOperator) { + autoRoot.reset(root); + } + + // isArray or isLeaf is true. Either way, it's over one field, and the bounds builder + // deals with it. + if (NULL == root->getTag()) { + // No index to use here, not in the context of logical operator, so we're SOL. + return NULL; + } + else if (Indexability::isBoundsGenerating(root)) { + // Make an index scan over the tagged index #. + IndexTag* tag = static_cast(root->getTag()); + + IndexBoundsBuilder::BoundsTightness tightness = IndexBoundsBuilder::EXACT; + QuerySolutionNode* soln = makeLeafNode(query, indices[tag->index], tag->pos, + root, &tightness); + verify(NULL != soln); + finishLeafNode(soln, indices[tag->index]); + + if (inArrayOperator) { + return soln; + } + + // If the bounds are exact, the set of documents that satisfy the predicate is + // exactly equal to the set of documents that the scan provides. + // + // If the bounds are not exact, the set of documents returned from the scan is a + // superset of documents that satisfy the predicate, and we must check the + // predicate. + + if (tightness == IndexBoundsBuilder::EXACT) { + return soln; + } + else if (tightness == IndexBoundsBuilder::INEXACT_COVERED + && !indices[tag->index].multikey) { + verify(NULL == soln->filter.get()); + soln->filter.reset(autoRoot.release()); + return soln; + } + else { + FetchNode* fetch = new FetchNode(); + verify(NULL != autoRoot.get()); + fetch->filter.reset(autoRoot.release()); + fetch->children.push_back(soln); + return fetch; + } + } + else if (Indexability::arrayUsesIndexOnChildren(root)) { + QuerySolutionNode* solution = NULL; + + invariant(MatchExpression::ELEM_MATCH_OBJECT); + // The child is an AND. + invariant(1 == root->numChildren()); + solution = buildIndexedDataAccess(query, root->getChild(0), true, indices); + if (NULL == solution) { + return NULL; + } + + // There may be an array operator above us. + if (inArrayOperator) { return solution; } + + FetchNode* fetch = new FetchNode(); + // Takes ownership of 'root'. + verify(NULL != autoRoot.get()); + fetch->filter.reset(autoRoot.release()); + fetch->children.push_back(solution); + return fetch; + } + } + + if (!inArrayOperator) { + delete root; + } + + return NULL; + } + + QuerySolutionNode* QueryPlannerAccess::scanWholeIndex(const IndexEntry& index, + const CanonicalQuery& query, + const QueryPlannerParams& params, + int direction) { + QuerySolutionNode* solnRoot = NULL; + + // Build an ixscan over the id index, use it, and return it. + IndexScanNode* isn = new IndexScanNode(); + isn->indexKeyPattern = index.keyPattern; + isn->indexIsMultiKey = index.multikey; + isn->maxScan = query.getParsed().getMaxScan(); + isn->addKeyMetadata = query.getParsed().returnKey(); + + IndexBoundsBuilder::allValuesBounds(index.keyPattern, &isn->bounds); + + if (-1 == direction) { + QueryPlannerCommon::reverseScans(isn); + isn->direction = -1; + } + + MatchExpression* filter = query.root()->shallowClone(); + + // If it's find({}) remove the no-op root. + if (MatchExpression::AND == filter->matchType() && (0 == filter->numChildren())) { + delete filter; + solnRoot = isn; + } + else { + // TODO: We may not need to do the fetch if the predicates in root are covered. But + // for now it's safe (though *maybe* slower). + FetchNode* fetch = new FetchNode(); + fetch->filter.reset(filter); + fetch->children.push_back(isn); + solnRoot = fetch; + } + + return solnRoot; + } + + // static + void QueryPlannerAccess::addFilterToSolutionNode(QuerySolutionNode* node, + MatchExpression* match, + MatchExpression::MatchType type) { + if (NULL == node->filter) { + node->filter.reset(match); + } + else if (type == node->filter->matchType()) { + // The 'node' already has either an AND or OR filter that matches 'type'. Add 'match' as + // another branch of the filter. + ListOfMatchExpression* listFilter = + static_cast(node->filter.get()); + listFilter->add(match); + } + else { + // The 'node' already has a filter that does not match 'type'. If 'type' is AND, then + // combine 'match' with the existing filter by adding an AND. If 'type' is OR, combine + // by adding an OR node. + ListOfMatchExpression* listFilter; + if (MatchExpression::AND == type) { + listFilter = new AndMatchExpression(); + } + else { + verify(MatchExpression::OR == type); + listFilter = new OrMatchExpression(); + } + MatchExpression* oldFilter = node->filter->shallowClone(); + listFilter->add(oldFilter); + listFilter->add(match); + node->filter.reset(listFilter); + } + } + + // static + void QueryPlannerAccess::handleFilter(ScanBuildingState* scanState) { + if (MatchExpression::OR == scanState->root->matchType()) { + handleFilterOr(scanState); + } + else if (MatchExpression::AND == scanState->root->matchType()) { + handleFilterAnd(scanState); + } + else { + // We must be building leaves for either and AND or an OR. + invariant(0); + } + } + + // static + void QueryPlannerAccess::handleFilterOr(ScanBuildingState* scanState) { + MatchExpression* root = scanState->root; + MatchExpression* child = root->getChild(scanState->curChild); + + if (scanState->inArrayOperator) { + // We're inside an array operator. The entire array operator expression + // should always be affixed as a filter. We keep 'curChild' in the $and + // for affixing later. + ++scanState->curChild; + } + else { + if (scanState->tightness < scanState->loosestBounds) { + scanState->loosestBounds = scanState->tightness; + } + + // Detach 'child' and add it to 'curOr'. + root->getChildVector()->erase(root->getChildVector()->begin() + scanState->curChild); + scanState->curOr->getChildVector()->push_back(child); + } + } + + // static + void QueryPlannerAccess::handleFilterAnd(ScanBuildingState* scanState) { + MatchExpression* root = scanState->root; + MatchExpression* child = root->getChild(scanState->curChild); + const IndexEntry& index = scanState->indices[scanState->currentIndexNumber]; + + if (scanState->inArrayOperator) { + // We're inside an array operator. The entire array operator expression + // should always be affixed as a filter. We keep 'curChild' in the $and + // for affixing later. + ++scanState->curChild; + } + else if (scanState->tightness == IndexBoundsBuilder::EXACT) { + root->getChildVector()->erase(root->getChildVector()->begin() + scanState->curChild); + delete child; + } + else if (scanState->tightness == IndexBoundsBuilder::INEXACT_COVERED + && (INDEX_TEXT == index.type || !index.multikey)) { + // The bounds are not exact, but the information needed to + // evaluate the predicate is in the index key. Remove the + // MatchExpression from its parent and attach it to the filter + // of the index scan we're building. + // + // We can only use this optimization if the index is NOT multikey. + // Suppose that we had the multikey index {x: 1} and a document + // {x: ["a", "b"]}. Now if we query for {x: /b/} the filter might + // ever only be applied to the index key "a". We'd incorrectly + // conclude that the document does not match the query :( so we + // gotta stick to non-multikey indices. + root->getChildVector()->erase(root->getChildVector()->begin() + scanState->curChild); + + addFilterToSolutionNode(scanState->currentScan.get(), child, root->matchType()); + } + else { + // We keep curChild in the AND for affixing later. + ++scanState->curChild; + } + } + + QuerySolutionNode* QueryPlannerAccess::makeIndexScan(const IndexEntry& index, + const CanonicalQuery& query, + const QueryPlannerParams& params, + const BSONObj& startKey, + const BSONObj& endKey) { + QuerySolutionNode* solnRoot = NULL; + + // Build an ixscan over the id index, use it, and return it. + IndexScanNode* isn = new IndexScanNode(); + isn->indexKeyPattern = index.keyPattern; + isn->indexIsMultiKey = index.multikey; + isn->direction = 1; + isn->maxScan = query.getParsed().getMaxScan(); + isn->addKeyMetadata = query.getParsed().returnKey(); + isn->bounds.isSimpleRange = true; + isn->bounds.startKey = startKey; + isn->bounds.endKey = endKey; + isn->bounds.endKeyInclusive = false; + + MatchExpression* filter = query.root()->shallowClone(); + + // If it's find({}) remove the no-op root. + if (MatchExpression::AND == filter->matchType() && (0 == filter->numChildren())) { + delete filter; + solnRoot = isn; + } + else { + // TODO: We may not need to do the fetch if the predicates in root are covered. But + // for now it's safe (though *maybe* slower). + FetchNode* fetch = new FetchNode(); + fetch->filter.reset(filter); + fetch->children.push_back(isn); + solnRoot = fetch; + } + + return solnRoot; + } + +} // namespace mongo diff --git a/src/mongo/db/query/planner_access.h b/src/mongo/db/query/planner_access.h new file mode 100644 index 00000000000..2031f866190 --- /dev/null +++ b/src/mongo/db/query/planner_access.h @@ -0,0 +1,404 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/index_bounds_builder.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + /** + * MULTIKEY INDEX BOUNDS RULES + * + * 1. In general for a multikey index, we cannot intersect bounds + * even if the index is not compound. + * Example: + * Let's say we have the document {a: [5, 7]}. + * This document satisfies the query {$and: [ {a: 5}, {a: 7} ] } + * For the index {a:1} we have the keys {"": 5} and {"": 7}. + * Each child of the AND is tagged with the index {a: 1} + * The interval for the {a: 5} branch is [5, 5]. It is exact. + * The interval for the {a: 7} branch is [7, 7]. It is exact. + * The intersection of the intervals is {}. + * If we scan over {}, the intersection of the intervals, we will retrieve nothing. + * + * 2. In general for a multikey compound index, we *can* compound the bounds. + * For example, if we have multikey index {a: 1, b: 1} and query {a: 2, b: 3}, + * we can use the bounds {a: [[2, 2]], b: [[3, 3]]}. + * + * 3. Despite rule #2, if fields in the compound index share a prefix, then it + * is not safe to compound the bounds. We can only specify bounds for the first + * field. + * Example: + * Let's say we have the document {a: [ {b: 3}, {c: 4} ] } + * This document satisfies the query {'a.b': 3, 'a.c': 4}. + * For the index {'a.b': 1, 'a.c': 1} we have the keys {"": 3, "": null} and + * {"": null, "": 4}. + * Let's use the aforementioned index to answer the query. + * The bounds for 'a.b' are [3,3], and the bounds for 'a.c' are [4,4]. + * If we combine the bounds, we would only look at keys {"": 3, "":4 }. + * Therefore we wouldn't look at the document's keys in the index. + * Therefore we don't combine bounds. + * + * 4. There is an exception to rule #1, and that is when we're evaluating + * an $elemMatch. + * Example: + * Let's say that we have the same document from (1), {a: [5, 7]}. + * This document satisfies {a: {$lte: 5, $gte: 7}}, but it does not + * satisfy {a: {$elemMatch: {$lte: 5, $gte: 7}}}. The $elemMatch indicates + * that we are allowed to intersect the bounds, which means that we will + * scan over the empty interval {} and retrieve nothing. This is the + * expected result because there is no entry in the array "a" that + * simultaneously satisfies the predicates a<=5 and a>=7. + * + * 5. There is also an exception to rule #3, and that is when we're evaluating + * an $elemMatch. The bounds can be compounded for predicates that share a prefix + * so long as the shared prefix is the path for which there is an $elemMatch. + * Example: + * Suppose we have the same document from (3), {a: [{b: 3}, {c: 4}]}. As discussed + * above, we cannot compound the index bounds for query {'a.b': 1, 'a.c': 1}. + * However, for the query {a: {$elemMatch: {b: 1, c: 1}} we can compound the + * bounds because the $elemMatch is applied to the shared prefix "a". + */ + + /** + * Methods for creating a QuerySolutionNode tree that accesses the data required by the query. + */ + class QueryPlannerAccess { + public: + /** + * Building the leaves (i.e. the index scans) is done by looping through + * predicates one at a time. During the process, there is a fair amount of state + * information to keep track of, which we consolidate into this data structure. + */ + struct ScanBuildingState { + + ScanBuildingState(MatchExpression* theRoot, + bool inArrayOp, + const std::vector& indexList) + : root(theRoot), + inArrayOperator(inArrayOp), + indices(indexList), + currentScan(NULL), + curChild(0), + currentIndexNumber(IndexTag::kNoIndex), + ixtag(NULL), + tightness(IndexBoundsBuilder::INEXACT_FETCH), + curOr(NULL), + loosestBounds(IndexBoundsBuilder::EXACT) { + } + + /** + * Reset the scan building state in preparation for building a new scan. + * + * This always should be called prior to allocating a new 'currentScan'. + */ + void resetForNextScan(IndexTag* newTag) { + currentScan.reset(NULL); + currentIndexNumber = newTag->index; + tightness = IndexBoundsBuilder::INEXACT_FETCH; + loosestBounds = IndexBoundsBuilder::EXACT; + + if (MatchExpression::OR == root->matchType()) { + curOr.reset(new OrMatchExpression()); + } + } + + // The root of the MatchExpression tree for which we are currently building index + // scans. Should be either an AND node or an OR node. + MatchExpression* root; + + // Are we inside an array operator such as $elemMatch or $all? + bool inArrayOperator; + + // A list of relevant indices which 'root' may be tagged to use. + const std::vector& indices; + + // The index access node that we are currently constructing. We may merge + // multiple tagged predicates into a single index scan. + std::auto_ptr currentScan; + + // An index into the child vector of 'root'. Indicates the child MatchExpression + // for which we are currently either constructing a new scan or which we are about + // to merge with 'currentScan'. + size_t curChild; + + // An index into the 'indices', so that 'indices[currentIndexNumber]' gives the + // index used by 'currentScan'. If there is no currentScan, this should be set + // to 'IndexTag::kNoIndex'. + size_t currentIndexNumber; + + // The tag on 'curChild'. + IndexTag* ixtag; + + // Whether the bounds for predicate 'curChild' are exact, inexact and covered by + // the index, or inexact with a fetch required. + IndexBoundsBuilder::BoundsTightness tightness; + + // If 'root' is an $or, the child predicates which are tagged with the same index are + // detached from the original root and added here. 'curOr' may be attached as a filter + // later on, or ignored and cleaned up by the auto_ptr. + std::auto_ptr curOr; + + // The values of BoundsTightness range from loosest to tightest in this order: + // + // INEXACT_FETCH < INEXACT_COVERED < EXACT + // + // 'loosestBounds' stores the smallest of these three values encountered so far for + // the current scan. If at least one of the child predicates assigned to the current + // index is INEXACT_FETCH, then 'loosestBounds' is INEXACT_FETCH. If at least one of + // the child predicates assigned to the current index is INEXACT_COVERED but none are + // INEXACT_FETCH, then 'loosestBounds' is INEXACT_COVERED. + IndexBoundsBuilder::BoundsTightness loosestBounds; + + private: + // Default constructor is not allowed. + ScanBuildingState(); + }; + + /** + * Return a CollectionScanNode that scans as requested in 'query'. + */ + static QuerySolutionNode* makeCollectionScan(const CanonicalQuery& query, + bool tailable, + const QueryPlannerParams& params); + + /** + * Return a plan that uses the provided index as a proxy for a collection scan. + */ + static QuerySolutionNode* scanWholeIndex(const IndexEntry& index, + const CanonicalQuery& query, + const QueryPlannerParams& params, + int direction = 1); + + /** + * Return a plan that scans the provided index from [startKey to endKey). + */ + static QuerySolutionNode* makeIndexScan(const IndexEntry& index, + const CanonicalQuery& query, + const QueryPlannerParams& params, + const BSONObj& startKey, + const BSONObj& endKey); + + // + // Indexed Data Access methods. + // + // The inArrayOperator flag deserves some attention. It is set when we're processing a + // child of an MatchExpression::ELEM_MATCH_OBJECT. + // + // When true, the following behavior changes for all methods below that take it as an argument: + // 0. No deletion of MatchExpression(s). In fact, + // 1. No mutation of the MatchExpression at all. We need the tree as-is in order to perform + // a filter on the entire tree. + // 2. No fetches performed. There will be a final fetch by the caller of buildIndexedDataAccess + // who set the value of inArrayOperator to true. + // 3. No compound indices are used and no bounds are combined. These are incorrect in the context + // of these operators. + // + + /** + * If 'inArrayOperator' is false, takes ownership of 'root'. + */ + static QuerySolutionNode* buildIndexedDataAccess(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const vector& indices); + + /** + * Takes ownership of 'root'. + */ + static QuerySolutionNode* buildIndexedAnd(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const vector& indices); + + /** + * Takes ownership of 'root'. + */ + static QuerySolutionNode* buildIndexedOr(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const vector& indices); + + /** + * Traverses the tree rooted at the $elemMatch expression 'node', + * finding all predicates that can use an index directly and returning + * them in the out-parameter vector 'out'. + * + * Traverses only through AND and ELEM_MATCH_OBJECT nodes. + * + * Other nodes (i.e. nodes which cannot use an index directly, and which are + * neither AND nor ELEM_MATCH_OBJECT) are returned in 'subnodesOut' if they are + * tagged to use an index. + */ + static void findElemMatchChildren(const MatchExpression* node, + vector* out, + vector* subnodesOut); + + /** + * Helper used by buildIndexedAnd and buildIndexedOr. + * + * The children of AND and OR nodes are sorted by the index that the subtree rooted at + * that node uses. Child nodes that use the same index are adjacent to one another to + * facilitate grouping of index scans. As such, the processing for AND and OR is + * almost identical. + * + * See tagForSort and sortUsingTags in index_tag.h for details on ordering the children + * of OR and AND. + * + * Does not take ownership of 'root' but may remove children from it. + */ + static bool processIndexScans(const CanonicalQuery& query, + MatchExpression* root, + bool inArrayOperator, + const vector& indices, + vector* out); + + /** + * Used by processIndexScans(...) in order to recursively build a data access + * plan for a "subnode", a node in the MatchExpression tree which is indexed by + * virtue of its children. + * + * The resulting scans are outputted in the out-parameter 'out'. + */ + static bool processIndexScansSubnode(const CanonicalQuery& query, + ScanBuildingState* scanState, + std::vector* out); + + /** + * Used by processIndexScansSubnode(...) to build the leaves of the solution tree for an + * ELEM_MATCH_OBJECT node beneath an AND. + * + * The resulting scans are outputted in the out-parameter 'out'. + */ + static bool processIndexScansElemMatch(const CanonicalQuery& query, + ScanBuildingState* scanState, + std::vector* out); + + // + // Helpers for creating an index scan. + // + + /** + * Create a new data access node. + * + * If the node is an index scan, the bounds for 'expr' are computed and placed into the + * first field's OIL position. The rest of the OILs are allocated but uninitialized. + * + * If the node is a geo node, grab the geo data from 'expr' and stuff it into the + * geo solution node of the appropriate type. + */ + static QuerySolutionNode* makeLeafNode(const CanonicalQuery& query, + const IndexEntry& index, + size_t pos, + MatchExpression* expr, + IndexBoundsBuilder::BoundsTightness* tightnessOut); + + /** + * Merge the predicate 'expr' with the leaf node 'node'. + */ + static void mergeWithLeafNode(MatchExpression* expr, ScanBuildingState* scanState); + + /** + * Determines whether it is safe to merge the expression 'expr' with + * the leaf node of the query solution contained in 'scanState'. + * + * Does not take ownership of its arguments. + */ + static bool shouldMergeWithLeaf(const MatchExpression* expr, + const ScanBuildingState& scanState); + + /** + * If index scan (regular or expression index), fill in any bounds that are missing in + * 'node' with the "all values for this field" interval. + * + * If geo, do nothing. + * If text, punt to finishTextNode. + */ + static void finishLeafNode(QuerySolutionNode* node, const IndexEntry& index); + + /** + * Fills in any missing bounds by calling finishLeafNode(...) for the scan contained in + * 'scanState'. The resulting scan is outputted in the out-parameter 'out', transferring + * ownership in the process. + * + * If 'scanState' is building an index scan for OR-related predicates, filters + * may be affixed to the scan as necessary. + */ + static void finishAndOutputLeaf(ScanBuildingState* scanState, + std::vector* out); + + /** + * Returns true if the current scan in 'scanState' requires a FetchNode. + */ + static bool orNeedsFetch(const ScanBuildingState* scanState); + + static void finishTextNode(QuerySolutionNode* node, const IndexEntry& index); + + /** + * Add the filter 'match' to the query solution node 'node'. Takes + * ownership of 'match'. + * + * The MatchType, 'type', indicates whether 'match' is a child of an + * AND or an OR match expression. + */ + static void addFilterToSolutionNode(QuerySolutionNode* node, MatchExpression* match, + MatchExpression::MatchType type); + + /** + * Once a predicate is merged into the current scan, there are a few things we might + * want to do with the filter: + * 1) Detach the filter from its parent and delete it because the predicate is + * answered by exact index bounds. + * 2) Leave the filter alone so that it can be affixed as part of a fetch node later. + * 3) Detach the filter from its parent and attach it directly to an index scan node. + * We can sometimes due this for INEXACT_COVERED predicates which are not answered exactly + * by the bounds, but can be answered by examing the data in the index key. + * 4) Detach the filter from its parent and attach it as a child of a separate + * MatchExpression tree. This is done for proper handling of inexact bounds for $or + * queries. + * + * This executes one of the four options above, according to the data in 'scanState'. + */ + static void handleFilter(ScanBuildingState* scanState); + + /** + * Implements handleFilter(...) for OR queries. + */ + static void handleFilterAnd(ScanBuildingState* scanState); + + /** + * Implements handleFilter(...) for AND queries. + */ + static void handleFilterOr(ScanBuildingState* scanState); + }; + +} // namespace mongo diff --git a/src/mongo/db/query/planner_analysis.cpp b/src/mongo/db/query/planner_analysis.cpp new file mode 100644 index 00000000000..0237eb3caf5 --- /dev/null +++ b/src/mongo/db/query/planner_analysis.cpp @@ -0,0 +1,725 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/planner_analysis.h" + +#include + +#include "mongo/db/jsobj.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/qlog.h" + +namespace mongo { + + // + // Helpers for bounds explosion AKA quick-and-dirty SERVER-1205. + // + + namespace { + + /** + * Walk the tree 'root' and output all leaf nodes into 'leafNodes'. + */ + void getLeafNodes(QuerySolutionNode* root, vector* leafNodes) { + if (0 == root->children.size()) { + leafNodes->push_back(root); + } + else { + for (size_t i = 0; i < root->children.size(); ++i) { + getLeafNodes(root->children[i], leafNodes); + } + } + } + + /** + * Returns true if every interval in 'oil' is a point, false otherwise. + */ + bool isUnionOfPoints(const OrderedIntervalList& oil) { + // We can't explode if there are empty bounds. Don't consider the + // oil a union of points if there are no intervals. + if (0 == oil.intervals.size()) { + return false; + } + + for (size_t i = 0; i < oil.intervals.size(); ++i) { + if (!oil.intervals[i].isPoint()) { + return false; + } + } + + return true; + } + + /** + * Should we try to expand the index scan(s) in 'solnRoot' to pull out an indexed sort? + * + * Returns the node which should be replaced by the merge sort of exploded scans + * in the out-parameter 'toReplace'. + */ + bool structureOKForExplode(QuerySolutionNode* solnRoot, QuerySolutionNode** toReplace) { + // For now we only explode if we *know* we will pull the sort out. We can look at + // more structure (or just explode and recalculate properties and see what happens) + // but for now we just explode if it's a sure bet. + // + // TODO: Can also try exploding if root is AND_HASH (last child dictates order.), + // or other less obvious cases... + if (STAGE_IXSCAN == solnRoot->getType()) { + *toReplace = solnRoot; + return true; + } + + if (STAGE_FETCH == solnRoot->getType()) { + if (STAGE_IXSCAN == solnRoot->children[0]->getType()) { + *toReplace = solnRoot->children[0]; + return true; + } + } + + if (STAGE_OR == solnRoot->getType()) { + for (size_t i = 0; i < solnRoot->children.size(); ++i) { + if (STAGE_IXSCAN != solnRoot->children[i]->getType()) { + return false; + } + } + *toReplace = solnRoot; + return true; + } + + return false; + } + + // vectors of vectors can be > > annoying. + typedef vector PointPrefix; + + /** + * The first 'fieldsToExplode' fields of 'bounds' are points. Compute the Cartesian product + * of those fields and place it in 'prefixOut'. + */ + void makeCartesianProduct(const IndexBounds& bounds, + size_t fieldsToExplode, + vector* prefixOut) { + + vector prefixForScans; + + // We dump the Cartesian product of bounds into prefixForScans, starting w/the first + // field's points. + verify(fieldsToExplode >= 1); + const OrderedIntervalList& firstOil = bounds.fields[0]; + verify(firstOil.intervals.size() >= 1); + for (size_t i = 0; i < firstOil.intervals.size(); ++i) { + const Interval& ival = firstOil.intervals[i]; + verify(ival.isPoint()); + PointPrefix pfix; + pfix.push_back(ival); + prefixForScans.push_back(pfix); + } + + // For each subsequent field... + for (size_t i = 1; i < fieldsToExplode; ++i) { + vector newPrefixForScans; + const OrderedIntervalList& oil = bounds.fields[i]; + verify(oil.intervals.size() >= 1); + // For each point interval in that field (all ivals must be points)... + for (size_t j = 0; j < oil.intervals.size(); ++j) { + const Interval& ival = oil.intervals[j]; + verify(ival.isPoint()); + // Make a new scan by appending it to all scans in prefixForScans. + for (size_t k = 0; k < prefixForScans.size(); ++k) { + PointPrefix pfix = prefixForScans[k]; + pfix.push_back(ival); + newPrefixForScans.push_back(pfix); + } + } + // And update prefixForScans. + newPrefixForScans.swap(prefixForScans); + } + + prefixOut->swap(prefixForScans); + } + + /** + * Take the provided index scan node 'isn'. Returns a list of index scans which are + * logically equivalent to 'isn' if joined by a MergeSort through the out-parameter + * 'explosionResult'. These index scan instances are owned by the caller. + * + * fieldsToExplode is a count of how many fields in the scan's bounds are the union of point + * intervals. This is computed beforehand and provided as a small optimization. + * + * Example: + * + * For the query find({a: {$in: [1,2]}}).sort({b: 1}) using the index {a:1, b:1}: + * 'isn' will be scan with bounds a:[[1,1],[2,2]] & b: [MinKey, MaxKey] + * 'sort' will be {b: 1} + * 'fieldsToExplode' will be 1 (as only one field isUnionOfPoints). + * + * On return, 'explosionResult' will contain the following two scans: + * a:[[1,1]], b:[MinKey, MaxKey] + * a:[[2,2]], b:[MinKey, MaxKey] + */ + void explodeScan(IndexScanNode* isn, + const BSONObj& sort, + size_t fieldsToExplode, + vector* explosionResult) { + + // Turn the compact bounds in 'isn' into a bunch of points... + vector prefixForScans; + makeCartesianProduct(isn->bounds, fieldsToExplode, &prefixForScans); + + for (size_t i = 0; i < prefixForScans.size(); ++i) { + const PointPrefix& prefix = prefixForScans[i]; + verify(prefix.size() == fieldsToExplode); + + // Copy boring fields into new child. + IndexScanNode* child = new IndexScanNode(); + child->indexKeyPattern = isn->indexKeyPattern; + child->direction = isn->direction; + child->maxScan = isn->maxScan; + child->addKeyMetadata = isn->addKeyMetadata; + child->indexIsMultiKey = isn->indexIsMultiKey; + + // Copy the filter, if there is one. + if (isn->filter.get()) { + child->filter.reset(isn->filter->shallowClone()); + } + + // Create child bounds. + child->bounds.fields.resize(isn->bounds.fields.size()); + for (size_t j = 0; j < fieldsToExplode; ++j) { + child->bounds.fields[j].intervals.push_back(prefix[j]); + child->bounds.fields[j].name = isn->bounds.fields[j].name; + } + for (size_t j = fieldsToExplode; j < isn->bounds.fields.size(); ++j) { + child->bounds.fields[j] = isn->bounds.fields[j]; + } + explosionResult->push_back(child); + } + } + + /** + * In the tree '*root', replace 'oldNode' with 'newNode'. + */ + void replaceNodeInTree(QuerySolutionNode** root, + QuerySolutionNode* oldNode, + QuerySolutionNode* newNode) { + if (*root == oldNode) { + *root = newNode; + } + else { + for (size_t i = 0 ; i < (*root)->children.size(); ++i) { + replaceNodeInTree(&(*root)->children[i], oldNode, newNode); + } + } + } + + bool hasNode(QuerySolutionNode* root, StageType type) { + if (type == root->getType()) { + return true; + } + + for (size_t i = 0; i < root->children.size(); ++i) { + if (hasNode(root->children[i], type)) { + return true; + } + } + + return false; + } + + } // namespace + + // static + BSONObj QueryPlannerAnalysis::getSortPattern(const BSONObj& indexKeyPattern) { + BSONObjBuilder sortBob; + BSONObjIterator kpIt(indexKeyPattern); + while (kpIt.more()) { + BSONElement elt = kpIt.next(); + if (elt.type() == mongo::String) { + break; + } + long long val = elt.safeNumberLong(); + int sortOrder = val >= 0 ? 1 : -1; + sortBob.append(elt.fieldName(), sortOrder); + } + return sortBob.obj(); + } + + // static + bool QueryPlannerAnalysis::explodeForSort(const CanonicalQuery& query, + const QueryPlannerParams& params, + QuerySolutionNode** solnRoot) { + vector leafNodes; + + QuerySolutionNode* toReplace; + if (!structureOKForExplode(*solnRoot, &toReplace)) { + return false; + } + + getLeafNodes(*solnRoot, &leafNodes); + + const BSONObj& desiredSort = query.getParsed().getSort(); + + // How many scan leaves will result from our expansion? + size_t totalNumScans = 0; + + // The value of entry i is how many scans we want to blow up for leafNodes[i]. + // We calculate this in the loop below and might as well reuse it if we blow up + // that scan. + vector fieldsToExplode; + + // The sort order we're looking for has to possibly be provided by each of the index scans + // upon explosion. + for (size_t i = 0; i < leafNodes.size(); ++i) { + // We can do this because structureOKForExplode is only true if the leaves are index + // scans. + IndexScanNode* isn = static_cast(leafNodes[i]); + const IndexBounds& bounds = isn->bounds; + + // Not a point interval prefix, can't try to rewrite. + if (bounds.isSimpleRange) { + return false; + } + + // How many scans will we create if we blow up this ixscan? + size_t numScans = 1; + + // Skip every field that is a union of point intervals and build the resulting sort + // order from the remaining fields. + BSONObjIterator kpIt(isn->indexKeyPattern); + size_t boundsIdx = 0; + while (kpIt.more()) { + const OrderedIntervalList& oil = bounds.fields[boundsIdx]; + if (!isUnionOfPoints(oil)) { + break; + } + numScans *= oil.intervals.size(); + kpIt.next(); + ++boundsIdx; + } + + // There's no sort order left to gain by exploding. Just go home. TODO: verify nothing + // clever we can do here. + if (!kpIt.more()) { + return false; + } + + // Only explode if there's at least one field to explode for this scan. + if (0 == boundsIdx) { + return false; + } + + // The rest of the fields define the sort order we could obtain by exploding + // the bounds. + BSONObjBuilder resultingSortBob; + while (kpIt.more()) { + resultingSortBob.append(kpIt.next()); + } + + // See if it's the order we're looking for. + BSONObj possibleSort = resultingSortBob.obj(); + if (!desiredSort.isPrefixOf(possibleSort)) { + // We can't get the sort order from the index scan. See if we can + // get the sort by reversing the scan. + BSONObj reversePossibleSort = QueryPlannerCommon::reverseSortObj(possibleSort); + if (!desiredSort.isPrefixOf(reversePossibleSort)) { + // Can't get the sort order from the reversed index scan either. Give up. + return false; + } + else { + // We can get the sort order we need if we reverse the scan. + QueryPlannerCommon::reverseScans(isn); + } + } + + // Do some bookkeeping to see how many ixscans we'll create total. + totalNumScans += numScans; + + // And for this scan how many fields we expand. + fieldsToExplode.push_back(boundsIdx); + } + + // Too many ixscans spoil the performance. + if (totalNumScans > (size_t)internalQueryMaxScansToExplode) { + QLOG() << "Could expand ixscans to pull out sort order but resulting scan count" + << "(" << totalNumScans << ") is too high."; + return false; + } + + // If we're here, we can (probably? depends on how restrictive the structure check is) + // get our sort order via ixscan blow-up. + MergeSortNode* merge = new MergeSortNode(); + merge->sort = desiredSort; + for (size_t i = 0; i < leafNodes.size(); ++i) { + IndexScanNode* isn = static_cast(leafNodes[i]); + explodeScan(isn, desiredSort, fieldsToExplode[i], &merge->children); + } + + merge->computeProperties(); + + // Replace 'toReplace' with the new merge sort node. + replaceNodeInTree(solnRoot, toReplace, merge); + // And get rid of the node that got replaced. + delete toReplace; + + return true; + } + + // static + QuerySolutionNode* QueryPlannerAnalysis::analyzeSort(const CanonicalQuery& query, + const QueryPlannerParams& params, + QuerySolutionNode* solnRoot, + bool* blockingSortOut) { + *blockingSortOut = false; + + const BSONObj& sortObj = query.getParsed().getSort(); + + if (sortObj.isEmpty()) { + return solnRoot; + } + + // TODO: We could check sortObj for any projections other than :1 and :-1 + // and short-cut some of this. + + // If the sort is $natural, we ignore it, assuming that the caller has detected that and + // outputted a collscan to satisfy the desired order. + BSONElement natural = sortObj.getFieldDotted("$natural"); + if (!natural.eoo()) { + return solnRoot; + } + + // See if solnRoot gives us the sort. If so, we're done. + BSONObjSet sorts = solnRoot->getSort(); + + // If the sort we want is in the set of sort orders provided already, bail out. + if (sorts.end() != sorts.find(sortObj)) { + return solnRoot; + } + + // Sort is not provided. See if we provide the reverse of our sort pattern. + // If so, we can reverse the scan direction(s). + BSONObj reverseSort = QueryPlannerCommon::reverseSortObj(sortObj); + if (sorts.end() != sorts.find(reverseSort)) { + QueryPlannerCommon::reverseScans(solnRoot); + QLOG() << "Reversing ixscan to provide sort. Result: " + << solnRoot->toString() << endl; + return solnRoot; + } + + // Sort not provided, can't reverse scans to get the sort. One last trick: We can "explode" + // index scans over point intervals to an OR of sub-scans in order to pull out a sort. + // Let's try this. + if (explodeForSort(query, params, &solnRoot)) { + return solnRoot; + } + + // If we're here, we need to add a sort stage. + + // If we're not allowed to put a blocking sort in, bail out. + if (params.options & QueryPlannerParams::NO_BLOCKING_SORT) { + delete solnRoot; + return NULL; + } + + // Add a fetch stage so we have the full object when we hit the sort stage. TODO: Can we + // pull the values that we sort by out of the key and if so in what cases? Perhaps we can + // avoid a fetch. + if (!solnRoot->fetched()) { + FetchNode* fetch = new FetchNode(); + fetch->children.push_back(solnRoot); + solnRoot = fetch; + } + + // And build the full sort stage. + SortNode* sort = new SortNode(); + sort->pattern = sortObj; + sort->query = query.getParsed().getFilter(); + sort->children.push_back(solnRoot); + solnRoot = sort; + // When setting the limit on the sort, we need to consider both + // the limit N and skip count M. The sort should return an ordered list + // N + M items so that the skip stage can discard the first M results. + if (0 != query.getParsed().getNumToReturn()) { + // Overflow here would be bad and could cause a nonsense limit. Cast + // skip and limit values to unsigned ints to make sure that the + // sum is never stored as signed. (See SERVER-13537). + sort->limit = size_t(query.getParsed().getNumToReturn()) + + size_t(query.getParsed().getSkip()); + + // This is a SORT with a limit. The wire protocol has a single quantity + // called "numToReturn" which could mean either limit or batchSize. + // We have no idea what the client intended. One way to handle the ambiguity + // of a limited OR stage is to use the SPLIT_LIMITED_SORT hack. + // + // If numToReturn is really a limit, then we want to add a limit to this + // SORT stage, and hence perform a topK. + // + // If numToReturn is really a batchSize, then we want to perform a regular + // blocking sort. + // + // Since we don't know which to use, just join the two options with an OR, + // with the topK first. If the client wants a limit, they'll get the efficiency + // of topK. If they want a batchSize, the other OR branch will deliver the missing + // results. The OR stage handles deduping. + if (params.options & QueryPlannerParams::SPLIT_LIMITED_SORT + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO) + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)) { + // If we're here then the SPLIT_LIMITED_SORT hack is turned on, + // and the query is of a type that allows the hack. + // + // Not allowed for geo or text, because we assume elsewhere that those + // stages appear just once. + OrNode* orn = new OrNode(); + orn->children.push_back(sort); + SortNode* sortClone = static_cast(sort->clone()); + sortClone->limit = 0; + orn->children.push_back(sortClone); + solnRoot = orn; + } + } + else { + sort->limit = 0; + } + + *blockingSortOut = true; + + return solnRoot; + } + + // static + QuerySolution* QueryPlannerAnalysis::analyzeDataAccess(const CanonicalQuery& query, + const QueryPlannerParams& params, + QuerySolutionNode* solnRoot) { + auto_ptr soln(new QuerySolution()); + soln->filterData = query.getQueryObj(); + verify(soln->filterData.isOwned()); + soln->ns = query.ns(); + soln->indexFilterApplied = params.indexFiltersApplied; + + solnRoot->computeProperties(); + + // solnRoot finds all our results. Let's see what transformations we must perform to the + // data. + + // If we're answering a query on a sharded system, we need to drop documents that aren't + // logically part of our shard. + if (params.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) { + // TODO: We could use params.shardKey to do fetch analysis instead of always fetching. + if (!solnRoot->fetched()) { + FetchNode* fetch = new FetchNode(); + fetch->children.push_back(solnRoot); + solnRoot = fetch; + } + ShardingFilterNode* sfn = new ShardingFilterNode(); + sfn->children.push_back(solnRoot); + solnRoot = sfn; + } + + bool hasSortStage = false; + solnRoot = analyzeSort(query, params, solnRoot, &hasSortStage); + + // This can happen if we need to create a blocking sort stage and we're not allowed to. + if (NULL == solnRoot) { return NULL; } + + // A solution can be blocking if it has a blocking sort stage or + // a hashed AND stage. + bool hasAndHashStage = hasNode(solnRoot, STAGE_AND_HASH); + soln->hasBlockingStage = hasSortStage || hasAndHashStage; + + // If we can (and should), add the keep mutations stage. + + // We cannot keep mutated documents if: + // + // 1. The query requires an index to evaluate the predicate ($text). We can't tell whether + // or not the doc actually satisfies the $text predicate since we can't evaluate a + // text MatchExpression. + // + // 2. The query implies a sort ($geoNear). It would be rather expensive and hacky to merge + // the document at the right place. + // + // 3. There is an index-provided sort. Ditto above comment about merging. + // + // TODO: do we want some kind of pre-planning step where we look for certain nodes and cache + // them? We do lookups in the tree a few times. This may not matter as most trees are + // shallow in terms of query nodes. + bool cannotKeepFlagged = hasNode(solnRoot, STAGE_TEXT) + || hasNode(solnRoot, STAGE_GEO_NEAR_2D) + || hasNode(solnRoot, STAGE_GEO_NEAR_2DSPHERE) + || (!query.getParsed().getSort().isEmpty() && !hasSortStage); + + // Only these stages can produce flagged results. A stage has to hold state past one call + // to work(...) in order to possibly flag a result. + bool couldProduceFlagged = hasNode(solnRoot, STAGE_GEO_2D) + || hasAndHashStage + || hasNode(solnRoot, STAGE_AND_SORTED) + || hasNode(solnRoot, STAGE_FETCH); + + bool shouldAddMutation = !cannotKeepFlagged && couldProduceFlagged; + + if (shouldAddMutation && (params.options & QueryPlannerParams::KEEP_MUTATIONS)) { + KeepMutationsNode* keep = new KeepMutationsNode(); + + // We must run the entire expression tree to make sure the document is still valid. + keep->filter.reset(query.root()->shallowClone()); + + if (STAGE_SORT == solnRoot->getType()) { + // We want to insert the invalidated results before the sort stage, if there is one. + verify(1 == solnRoot->children.size()); + keep->children.push_back(solnRoot->children[0]); + solnRoot->children[0] = keep; + } + else { + keep->children.push_back(solnRoot); + solnRoot = keep; + } + } + + // Project the results. + if (NULL != query.getProj()) { + QLOG() << "PROJECTION: fetched status: " << solnRoot->fetched() << endl; + QLOG() << "PROJECTION: Current plan is:\n" << solnRoot->toString() << endl; + + ProjectionNode::ProjectionType projType = ProjectionNode::DEFAULT; + BSONObj coveredKeyObj; + + if (query.getProj()->requiresDocument()) { + QLOG() << "PROJECTION: claims to require doc adding fetch.\n"; + // If the projection requires the entire document, somebody must fetch. + if (!solnRoot->fetched()) { + FetchNode* fetch = new FetchNode(); + fetch->children.push_back(solnRoot); + solnRoot = fetch; + } + } + else if (!query.getProj()->wantIndexKey()) { + // The only way we're here is if it's a simple projection. That is, we can pick out + // the fields we want to include and they're not dotted. So we want to execute the + // projection in the fast-path simple fashion. Just don't know which fast path yet. + QLOG() << "PROJECTION: requires fields\n"; + const vector& fields = query.getProj()->getRequiredFields(); + bool covered = true; + for (size_t i = 0; i < fields.size(); ++i) { + if (!solnRoot->hasField(fields[i])) { + QLOG() << "PROJECTION: not covered due to field " + << fields[i] << endl; + covered = false; + break; + } + } + + QLOG() << "PROJECTION: is covered?: = " << covered << endl; + + // If any field is missing from the list of fields the projection wants, + // a fetch is required. + if (!covered) { + FetchNode* fetch = new FetchNode(); + fetch->children.push_back(solnRoot); + solnRoot = fetch; + + // It's simple but we'll have the full document and we should just iterate + // over that. + projType = ProjectionNode::SIMPLE_DOC; + QLOG() << "PROJECTION: not covered, fetching."; + } + else { + if (solnRoot->fetched()) { + // Fetched implies hasObj() so let's run with that. + projType = ProjectionNode::SIMPLE_DOC; + QLOG() << "PROJECTION: covered via FETCH, using SIMPLE_DOC fast path"; + } + else { + // If we're here we're not fetched so we're covered. Let's see if we can + // get out of using the default projType. If there's only one leaf + // underneath and it's giving us index data we can use the faster covered + // impl. + vector leafNodes; + getLeafNodes(solnRoot, &leafNodes); + + if (1 == leafNodes.size()) { + // Both the IXSCAN and DISTINCT stages provide covered key data. + if (STAGE_IXSCAN == leafNodes[0]->getType()) { + projType = ProjectionNode::COVERED_ONE_INDEX; + IndexScanNode* ixn = static_cast(leafNodes[0]); + coveredKeyObj = ixn->indexKeyPattern; + QLOG() << "PROJECTION: covered via IXSCAN, using COVERED fast path"; + } + else if (STAGE_DISTINCT == leafNodes[0]->getType()) { + projType = ProjectionNode::COVERED_ONE_INDEX; + DistinctNode* dn = static_cast(leafNodes[0]); + coveredKeyObj = dn->indexKeyPattern; + QLOG() << "PROJECTION: covered via DISTINCT, using COVERED fast path"; + } + } + } + } + } + + // We now know we have whatever data is required for the projection. + ProjectionNode* projNode = new ProjectionNode(); + projNode->children.push_back(solnRoot); + projNode->fullExpression = query.root(); + projNode->projection = query.getParsed().getProj(); + projNode->projType = projType; + projNode->coveredKeyObj = coveredKeyObj; + solnRoot = projNode; + } + else { + // If there's no projection, we must fetch, as the user wants the entire doc. + if (!solnRoot->fetched()) { + FetchNode* fetch = new FetchNode(); + fetch->children.push_back(solnRoot); + solnRoot = fetch; + } + } + + if (0 != query.getParsed().getSkip()) { + SkipNode* skip = new SkipNode(); + skip->skip = query.getParsed().getSkip(); + skip->children.push_back(solnRoot); + solnRoot = skip; + } + + // When there is both a blocking sort and a limit, the limit will + // be enforced by the blocking sort. + // Otherwise, we need to limit the results in the case of a hard limit + // (ie. limit in raw query is negative) + if (0 != query.getParsed().getNumToReturn() && + !hasSortStage && + !query.getParsed().wantMore()) { + + LimitNode* limit = new LimitNode(); + limit->limit = query.getParsed().getNumToReturn(); + limit->children.push_back(solnRoot); + solnRoot = limit; + } + + soln->root.reset(solnRoot); + return soln.release(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/planner_analysis.h b/src/mongo/db/query/planner_analysis.h new file mode 100644 index 00000000000..4e1ea2ad8bd --- /dev/null +++ b/src/mongo/db/query/planner_analysis.h @@ -0,0 +1,108 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + class QueryPlannerAnalysis { + public: + /** + * Takes an index key pattern and returns an object describing the "maximal sort" that this + * index can provide. Returned object is in normalized sort form (all elements have value 1 + * or -1). + * + * Examples: + * - {a: 1, b: -1} => {a: 1, b: -1} + * - {a: true} => {a: 1} + * - {a: "hashed"} => {} + * - {a: 1, b: "text", c: 1} => {a: 1} + */ + static BSONObj getSortPattern(const BSONObj& indexKeyPattern); + + /** + * In brief: performs sort and covering analysis. + * + * The solution rooted at 'solnRoot' provides data for the query, whether through some + * configuration of indices or through a collection scan. Additional stages may be required + * to perform sorting, projection, or other operations that are independent of the source + * of the data. These stages are added atop 'solnRoot'. + * + * 'taggedRoot' is a copy of the parse tree. Nodes in 'solnRoot' may point into it. + * + * Takes ownership of 'solnRoot' and 'taggedRoot'. + * + * Returns NULL if a solution cannot be constructed given the requirements in 'params'. + * + * Caller owns the returned QuerySolution. + */ + static QuerySolution* analyzeDataAccess(const CanonicalQuery& query, + const QueryPlannerParams& params, + QuerySolutionNode* solnRoot); + + /** + * Sort the results, if there is a sort required. + */ + static QuerySolutionNode* analyzeSort(const CanonicalQuery& query, + const QueryPlannerParams& params, + QuerySolutionNode* solnRoot, + bool* blockingSortOut); + + /** + * Internal helper function used by analyzeSort. + * + * Rewrites an index scan over many point intervals as an OR of many index scans in order to + * obtain an indexed sort. For full details, see SERVER-1205. + * + * Here is an example: + * + * Consider the query find({a: {$in: [1,2]}}).sort({b: 1}) with using the index {a:1, b:1}. + * + * Our default solution will be to construct one index scan with the bounds a:[[1,1],[2,2]] + * and b: [MinKey, MaxKey]. + * + * However, this is logically equivalent to the union of the following scans: + * a:[1,1], b:[MinKey, MaxKey] + * a:[2,2], b:[MinKey, MaxKey] + * + * Since the bounds on 'a' are a point, each scan provides the sort order {b:1} in addition + * to {a:1, b:1}. + * + * If we union these scans with a merge sort instead of a normal hashing OR, we can preserve + * the sort order that each scan provides. + */ + static bool explodeForSort(const CanonicalQuery& query, + const QueryPlannerParams& params, + QuerySolutionNode** solnRoot); + }; + +} // namespace mongo diff --git a/src/mongo/db/query/planner_analysis_test.cpp b/src/mongo/db/query/planner_analysis_test.cpp new file mode 100644 index 00000000000..10e23bfe032 --- /dev/null +++ b/src/mongo/db/query/planner_analysis_test.cpp @@ -0,0 +1,165 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/planner_analysis.h" + +#include "mongo/db/json.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" + +using namespace mongo; + +namespace { + + TEST(QueryPlannerAnalysis, GetSortPatternBasic) { + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1}"))); + ASSERT_EQUALS(fromjson("{a: -1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: -1}"))); + ASSERT_EQUALS(fromjson("{a: 1, b: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: 1}"))); + ASSERT_EQUALS(fromjson("{a: 1, b: -1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: -1}"))); + ASSERT_EQUALS(fromjson("{a: -1, b: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: -1, b: 1}"))); + ASSERT_EQUALS(fromjson("{a: -1, b: -1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: -1, b: -1}"))); + } + + TEST(QueryPlannerAnalysis, GetSortPatternOtherElements) { + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 0}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 100}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: Infinity}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: true}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: false}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: []}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: {}}"))); + + ASSERT_EQUALS(fromjson("{a: -1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: -100}"))); + ASSERT_EQUALS(fromjson("{a: -1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: -Infinity}"))); + + ASSERT_EQUALS(fromjson("{}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{}"))); + } + + TEST(QueryPlannerAnalysis, GetSortPatternSpecialIndexTypes) { + ASSERT_EQUALS(fromjson("{}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 'hashed'}"))); + ASSERT_EQUALS(fromjson("{}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 'text'}"))); + ASSERT_EQUALS(fromjson("{}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: '2dsphere'}"))); + ASSERT_EQUALS(fromjson("{}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: ''}"))); + ASSERT_EQUALS(fromjson("{}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 'foo'}"))); + + ASSERT_EQUALS(fromjson("{a: -1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: -1, b: 'text'}"))); + ASSERT_EQUALS(fromjson("{a: -1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: -1, b: '2dsphere'}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: 'text'}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: '2dsphere'}"))); + + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: 'text', c: 1}"))); + ASSERT_EQUALS(fromjson("{a: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: '2dsphere'," + " c: 1}"))); + + ASSERT_EQUALS(fromjson("{a: 1, b: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: 1, c: 'text'}"))); + ASSERT_EQUALS(fromjson("{a: 1, b: 1}"), + QueryPlannerAnalysis::getSortPattern(fromjson("{a: 1, b: 1, c: 'text'," + " d: 1}"))); + } + + // Test the generation of sort orders provided by an index scan done by + // IndexScanNode::computeProperties(). + TEST(QueryPlannerAnalysis, IxscanSortOrdersBasic) { + IndexScanNode ixscan; + ixscan.indexKeyPattern = fromjson("{a: 1, b: 1, c: 1, d: 1, e: 1}"); + + // Bounds are {a: [[1,1]], b: [[2,2]], c: [[3,3]], d: [[1,5]], e:[[1,1],[2,2]]}, + // all inclusive. + OrderedIntervalList oil1("a"); + oil1.intervals.push_back(Interval(fromjson("{'': 1, '': 1}"), true, true)); + ixscan.bounds.fields.push_back(oil1); + + OrderedIntervalList oil2("b"); + oil2.intervals.push_back(Interval(fromjson("{'': 2, '': 2}"), true, true)); + ixscan.bounds.fields.push_back(oil2); + + OrderedIntervalList oil3("c"); + oil3.intervals.push_back(Interval(fromjson("{'': 3, '': 3}"), true, true)); + ixscan.bounds.fields.push_back(oil3); + + OrderedIntervalList oil4("d"); + oil4.intervals.push_back(Interval(fromjson("{'': 1, '': 5}"), true, true)); + ixscan.bounds.fields.push_back(oil4); + + OrderedIntervalList oil5("e"); + oil5.intervals.push_back(Interval(fromjson("{'': 1, '': 1}"), true, true)); + oil5.intervals.push_back(Interval(fromjson("{'': 2, '': 2}"), true, true)); + ixscan.bounds.fields.push_back(oil5); + + // Compute and retrieve the set of sorts. + ixscan.computeProperties(); + const BSONObjSet& sorts = ixscan.getSort(); + + // One possible sort is the index key pattern. + ASSERT(sorts.find(fromjson("{a: 1, b: 1, c: 1, d: 1, e: 1}")) != sorts.end()); + + // All prefixes of the key pattern. + ASSERT(sorts.find(fromjson("{a: 1}")) != sorts.end()); + ASSERT(sorts.find(fromjson("{a: 1, b: 1}")) != sorts.end()); + ASSERT(sorts.find(fromjson("{a: 1, b: 1, c: 1}")) != sorts.end()); + ASSERT(sorts.find(fromjson("{a: 1, b: 1, c: 1, d: 1}")) != sorts.end()); + + // Additional sorts considered due to point intervals on 'a', 'b', and 'c'. + ASSERT(sorts.find(fromjson("{b: 1, c: 1, d: 1, e: 1}")) != sorts.end()); + ASSERT(sorts.find(fromjson("{c: 1, d: 1, e: 1}")) != sorts.end()); + ASSERT(sorts.find(fromjson("{d: 1, e: 1}")) != sorts.end()); + ASSERT(sorts.find(fromjson("{d: 1}")) != sorts.end()); + + // There should be 9 total sorts: make sure no other ones snuck their way in. + ASSERT_EQUALS(9U, sorts.size()); + } + +} // namespace diff --git a/src/mongo/db/query/planner_ixselect.cpp b/src/mongo/db/query/planner_ixselect.cpp new file mode 100644 index 00000000000..8f4be1f8432 --- /dev/null +++ b/src/mongo/db/query/planner_ixselect.cpp @@ -0,0 +1,731 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/planner_ixselect.h" + +#include + +#include "mongo/db/geo/core.h" +#include "mongo/db/geo/hash.h" +#include "mongo/db/index_names.h" +#include "mongo/db/matcher/expression_array.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/matcher/expression_text.h" +#include "mongo/db/query/indexability.h" +#include "mongo/db/query/index_tag.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner_common.h" + +namespace mongo { + + static double fieldWithDefault(const BSONObj& infoObj, const string& name, double def) { + BSONElement e = infoObj[name]; + if (e.isNumber()) { return e.numberDouble(); } + return def; + } + + /** + * 2d indices don't handle wrapping so we can't use them for queries that wrap. + */ + static bool twoDWontWrap(const Circle& circle, const IndexEntry& index) { + GeoHashConverter::Parameters params; + params.bits = static_cast(fieldWithDefault(index.infoObj, "bits", 26)); + params.max = fieldWithDefault(index.infoObj, "max", 180.0); + params.min = fieldWithDefault(index.infoObj, "min", -180.0); + double numBuckets = (1024 * 1024 * 1024 * 4.0); + params.scaling = numBuckets / (params.max - params.min); + + GeoHashConverter conv(params); + + // FYI: old code used flat not spherical error. + double yscandist = rad2deg(circle.radius) + conv.getErrorSphere(); + double xscandist = computeXScanDistance(circle.center.y, yscandist); + bool ret = circle.center.x + xscandist < 180 + && circle.center.x - xscandist > -180 + && circle.center.y + yscandist < 90 + && circle.center.y - yscandist > -90; + return ret; + } + + // static + void QueryPlannerIXSelect::getFields(MatchExpression* node, + string prefix, + unordered_set* out) { + // Do not traverse tree beyond a NOR negation node + MatchExpression::MatchType exprtype = node->matchType(); + if (exprtype == MatchExpression::NOR) { + return; + } + + // Leaf nodes with a path and some array operators. + if (Indexability::nodeCanUseIndexOnOwnField(node)) { + out->insert(prefix + node->path().toString()); + } + else if (Indexability::arrayUsesIndexOnChildren(node)) { + // If the array uses an index on its children, it's something like + // {foo : {$elemMatch: { bar: 1}}}, in which case the predicate is really over + // foo.bar. + // + // When we have {foo: {$all: [{$elemMatch: {a:1}}], the path of the embedded elemMatch + // is empty. We don't want to append a dot in that case as the field would be foo..a. + if (!node->path().empty()) { + prefix += node->path().toString() + "."; + } + + for (size_t i = 0; i < node->numChildren(); ++i) { + getFields(node->getChild(i), prefix, out); + } + } + else if (node->isLogical()) { + for (size_t i = 0; i < node->numChildren(); ++i) { + getFields(node->getChild(i), prefix, out); + } + } + } + + // static + void QueryPlannerIXSelect::findRelevantIndices(const unordered_set& fields, + const vector& allIndices, + vector* out) { + for (size_t i = 0; i < allIndices.size(); ++i) { + BSONObjIterator it(allIndices[i].keyPattern); + verify(it.more()); + BSONElement elt = it.next(); + if (fields.end() != fields.find(elt.fieldName())) { + out->push_back(allIndices[i]); + } + } + } + + // static + bool QueryPlannerIXSelect::compatible(const BSONElement& elt, + const IndexEntry& index, + MatchExpression* node) { + // Historically one could create indices with any particular value for the index spec, + // including values that now indicate a special index. As such we have to make sure the + // index type wasn't overridden before we pay attention to the string in the index key + // pattern element. + // + // e.g. long ago we could have created an index {a: "2dsphere"} and it would + // be treated as a btree index by an ancient version of MongoDB. To try to run + // 2dsphere queries over it would be folly. + string indexedFieldType; + if (String != elt.type() || (INDEX_BTREE == index.type)) { + indexedFieldType = ""; + } + else { + indexedFieldType = elt.String(); + } + + // We know elt.fieldname() == node->path(). + MatchExpression::MatchType exprtype = node->matchType(); + + if (indexedFieldType.empty()) { + // Can't check for null w/a sparse index. + if (exprtype == MatchExpression::EQ && index.sparse) { + const EqualityMatchExpression* expr + = static_cast(node); + if (expr->getData().isNull()) { + return false; + } + } + + // We can't use a btree-indexed field for geo expressions. + if (exprtype == MatchExpression::GEO || exprtype == MatchExpression::GEO_NEAR) { + return false; + } + + // There are restrictions on when we can use the index if + // the expression is a NOT. + if (exprtype == MatchExpression::NOT) { + // Don't allow indexed NOT on special index types such as geo or text indices. + if (INDEX_BTREE != index.type) { + return false; + } + + // Prevent negated preds from using sparse indices. Doing so would cause us to + // miss documents which do not contain the indexed fields. + if (index.sparse) { + return false; + } + + // Can't index negations of MOD, REGEX, TYPE_OPERATOR, or ELEM_MATCH_VALUE. + MatchExpression::MatchType childtype = node->getChild(0)->matchType(); + if (MatchExpression::REGEX == childtype || + MatchExpression::MOD == childtype || + MatchExpression::TYPE_OPERATOR == childtype || + MatchExpression::ELEM_MATCH_VALUE == childtype) { + return false; + } + + // If it's a negated $in, it can't have any REGEX's inside. + if (MatchExpression::MATCH_IN == childtype) { + InMatchExpression* ime = static_cast(node->getChild(0)); + if (ime->getData().numRegexes() != 0) { + return false; + } + } + } + + // We can only index EQ using text indices. This is an artificial limitation imposed by + // FTSSpec::getIndexPrefix() which will fail if there is not an EQ predicate on each + // index prefix field of the text index. + // + // Example for key pattern {a: 1, b: "text"}: + // - Allowed: node = {a: 7} + // - Not allowed: node = {a: {$gt: 7}} + + if (INDEX_TEXT != index.type) { + return true; + } + + // If we're here we know it's a text index. Equalities are OK anywhere in a text index. + if (MatchExpression::EQ == exprtype) { + return true; + } + + // Not-equalities can only go in a suffix field of an index kp. We look through the key + // pattern to see if the field we're looking at now appears as a prefix. If so, we + // can't use this index for it. + BSONObjIterator specIt(index.keyPattern); + while (specIt.more()) { + BSONElement elt = specIt.next(); + // We hit the dividing mark between prefix and suffix, so whatever field we're + // looking at is a suffix, since it appears *after* the dividing mark between the + // two. As such, we can use the index. + if (String == elt.type()) { + return true; + } + + // If we're here, we're still looking at prefix elements. We know that exprtype + // isn't EQ so we can't use this index. + if (node->path() == elt.fieldNameStringData()) { + return false; + } + } + + // NOTE: This shouldn't be reached. Text index implies there is a separator implies we + // will always hit the 'return true' above. + invariant(0); + return true; + } + else if (IndexNames::HASHED == indexedFieldType) { + return exprtype == MatchExpression::MATCH_IN || exprtype == MatchExpression::EQ; + } + else if (IndexNames::GEO_2DSPHERE == indexedFieldType) { + if (exprtype == MatchExpression::GEO) { + // within or intersect. + GeoMatchExpression* gme = static_cast(node); + const GeoQuery& gq = gme->getGeoQuery(); + const GeometryContainer& gc = gq.getGeometry(); + return gc.hasS2Region(); + } + else if (exprtype == MatchExpression::GEO_NEAR) { + GeoNearMatchExpression* gnme = static_cast(node); + // Make sure the near query is compatible with 2dsphere. + if (gnme->getData().centroid.crs == SPHERE || gnme->getData().isNearSphere) { + return true; + } + } + return false; + } + else if (IndexNames::GEO_2D == indexedFieldType) { + if (exprtype == MatchExpression::GEO_NEAR) { + GeoNearMatchExpression* gnme = static_cast(node); + return gnme->getData().centroid.crs == FLAT; + } + else if (exprtype == MatchExpression::GEO) { + // 2d only supports within. + GeoMatchExpression* gme = static_cast(node); + const GeoQuery& gq = gme->getGeoQuery(); + if (GeoQuery::WITHIN != gq.getPred()) { + return false; + } + + const GeometryContainer& gc = gq.getGeometry(); + + // 2d indices answer flat queries. + if (gc.hasFlatRegion()) { + return true; + } + + // 2d indices can answer centerSphere queries. + if (NULL == gc._cap.get()) { + return false; + } + + verify(SPHERE == gc._cap->crs); + const Circle& circle = gc._cap->circle; + + // No wrapping around the edge of the world is allowed in 2d centerSphere. + return twoDWontWrap(circle, index); + } + return false; + } + else if (IndexNames::TEXT == indexedFieldType) { + return (exprtype == MatchExpression::TEXT); + } + else if (IndexNames::GEO_HAYSTACK == indexedFieldType) { + return false; + } + else { + warning() << "Unknown indexing for node " << node->toString() + << " and field " << elt.toString() << endl; + verify(0); + } + } + + // static + void QueryPlannerIXSelect::rateIndices(MatchExpression* node, + string prefix, + const vector& indices) { + // Do not traverse tree beyond logical NOR node + MatchExpression::MatchType exprtype = node->matchType(); + if (exprtype == MatchExpression::NOR) { + return; + } + + // Every indexable node is tagged even when no compatible index is + // available. + if (Indexability::isBoundsGenerating(node)) { + string fullPath; + if (MatchExpression::NOT == node->matchType()) { + fullPath = prefix + node->getChild(0)->path().toString(); + } + else { + fullPath = prefix + node->path().toString(); + } + + verify(NULL == node->getTag()); + RelevantTag* rt = new RelevantTag(); + node->setTag(rt); + rt->path = fullPath; + + // TODO: This is slow, with all the string compares. + for (size_t i = 0; i < indices.size(); ++i) { + BSONObjIterator it(indices[i].keyPattern); + BSONElement elt = it.next(); + if (elt.fieldName() == fullPath && compatible(elt, indices[i], node)) { + rt->first.push_back(i); + } + while (it.more()) { + elt = it.next(); + if (elt.fieldName() == fullPath && compatible(elt, indices[i], node)) { + rt->notFirst.push_back(i); + } + } + } + + // If this is a NOT, we have to clone the tag and attach + // it to the NOT's child. + if (MatchExpression::NOT == node->matchType()) { + RelevantTag* childRt = static_cast(rt->clone()); + childRt->path = rt->path; + node->getChild(0)->setTag(childRt); + } + } + else if (Indexability::arrayUsesIndexOnChildren(node)) { + // See comment in getFields about all/elemMatch and paths. + if (!node->path().empty()) { + prefix += node->path().toString() + "."; + } + for (size_t i = 0; i < node->numChildren(); ++i) { + rateIndices(node->getChild(i), prefix, indices); + } + } + else if (node->isLogical()) { + for (size_t i = 0; i < node->numChildren(); ++i) { + rateIndices(node->getChild(i), prefix, indices); + } + } + } + + // static + void QueryPlannerIXSelect::stripInvalidAssignments(MatchExpression* node, + const vector& indices) { + + stripInvalidAssignmentsToTextIndexes(node, indices); + + if (MatchExpression::GEO != node->matchType() && + MatchExpression::GEO_NEAR != node->matchType()) { + + stripInvalidAssignmentsTo2dsphereIndices(node, indices); + } + } + + namespace { + + /** + * For every node in the subtree rooted at 'node' that has a RelevantTag, removes index + * assignments from that tag. + * + * Used as a helper for stripUnneededAssignments(). + */ + void clearAssignments(MatchExpression* node) { + if (node->getTag()) { + RelevantTag* rt = static_cast(node->getTag()); + rt->first.clear(); + rt->notFirst.clear(); + } + + for (size_t i = 0; i < node->numChildren(); i++) { + clearAssignments(node->getChild(i)); + } + } + + } // namespace + + // static + void QueryPlannerIXSelect::stripUnneededAssignments(MatchExpression* node, + const std::vector& indices) { + if (MatchExpression::AND == node->matchType()) { + for (size_t i = 0; i < node->numChildren(); i++) { + MatchExpression* child = node->getChild(i); + + if (MatchExpression::EQ != child->matchType()) { + continue; + } + + if (!child->getTag()) { + continue; + } + + // We found a EQ child of an AND which is tagged. + RelevantTag* rt = static_cast(child->getTag()); + + // Look through all of the indices for which this predicate can be answered with + // the leading field of the index. + for (std::vector::const_iterator i = rt->first.begin(); + i != rt->first.end(); ++i) { + size_t index = *i; + + if (indices[index].unique && 1 == indices[index].keyPattern.nFields()) { + // Found an EQ predicate which can use a single-field unique index. + // Clear assignments from the entire tree, and add back a single assignment + // for 'child' to the unique index. + clearAssignments(node); + RelevantTag* newRt = static_cast(child->getTag()); + newRt->first.push_back(index); + + // Tag state has been reset in the entire subtree at 'root'; nothing + // else for us to do. + return; + } + } + } + } + + for (size_t i = 0; i < node->numChildren(); i++) { + stripUnneededAssignments(node->getChild(i), indices); + } + } + + // + // Helpers used by stripInvalidAssignments + // + + /** + * Remove 'idx' from the RelevantTag lists for 'node'. 'node' must be a leaf. + */ + static void removeIndexRelevantTag(MatchExpression* node, size_t idx) { + RelevantTag* tag = static_cast(node->getTag()); + verify(tag); + vector::iterator firstIt = std::find(tag->first.begin(), + tag->first.end(), + idx); + if (firstIt != tag->first.end()) { + tag->first.erase(firstIt); + } + + vector::iterator notFirstIt = std::find(tag->notFirst.begin(), + tag->notFirst.end(), + idx); + if (notFirstIt != tag->notFirst.end()) { + tag->notFirst.erase(notFirstIt); + } + } + + // + // Text index quirks + // + + /** + * Traverse the subtree rooted at 'node' to remove invalid RelevantTag assignments to text index + * 'idx', which has prefix paths 'prefixPaths'. + */ + static void stripInvalidAssignmentsToTextIndex(MatchExpression* node, + size_t idx, + const unordered_set& prefixPaths) { + + // If we're here, there are prefixPaths and node is either: + // 1. a text pred which we can't use as we have nothing over its prefix, or + // 2. a non-text pred which we can't use as we don't have a text pred AND-related. + if (Indexability::nodeCanUseIndexOnOwnField(node)) { + removeIndexRelevantTag(node, idx); + return; + } + + // Do not traverse tree beyond negation node. + if (node->matchType() == MatchExpression::NOT + || node->matchType() == MatchExpression::NOR) { + + return; + } + + // For anything to use a text index with prefixes, we require that: + // 1. The text pred exists in an AND, + // 2. The non-text preds that use the text index's prefixes are also in that AND. + + if (node->matchType() != MatchExpression::AND) { + // It's an OR or some kind of array operator. + for (size_t i = 0; i < node->numChildren(); ++i) { + stripInvalidAssignmentsToTextIndex(node->getChild(i), idx, prefixPaths); + } + return; + } + + // If we're here, we're an AND. Determine whether the children satisfy the index prefix for + // the text index. + invariant(node->matchType() == MatchExpression::AND); + + bool hasText = false; + + // The AND must have an EQ predicate for each prefix path. When we encounter a child with a + // tag we remove it from childrenPrefixPaths. All children exist if this set is empty at + // the end. + unordered_set childrenPrefixPaths = prefixPaths; + + for (size_t i = 0; i < node->numChildren(); ++i) { + MatchExpression* child = node->getChild(i); + RelevantTag* tag = static_cast(child->getTag()); + + if (NULL == tag) { + // 'child' could be a logical operator. Maybe there are some assignments hiding + // inside. + stripInvalidAssignmentsToTextIndex(child, idx, prefixPaths); + continue; + } + + bool inFirst = tag->first.end() != std::find(tag->first.begin(), + tag->first.end(), + idx); + + bool inNotFirst = tag->notFirst.end() != std::find(tag->notFirst.begin(), + tag->notFirst.end(), + idx); + + if (inFirst || inNotFirst) { + // Great! 'child' was assigned to our index. + if (child->matchType() == MatchExpression::TEXT) { + hasText = true; + } + else { + childrenPrefixPaths.erase(child->path()); + // One fewer prefix we're looking for, possibly. Note that we could have a + // suffix assignment on the index and wind up here. In this case the erase + // above won't do anything since a suffix isn't a prefix. + } + } + else { + // Recurse on the children to ensure that they're not hiding any assignments + // to idx. + stripInvalidAssignmentsToTextIndex(child, idx, prefixPaths); + } + } + + // Our prereqs for using the text index were not satisfied so we remove the assignments from + // all children of the AND. + if (!hasText || !childrenPrefixPaths.empty()) { + for (size_t i = 0; i < node->numChildren(); ++i) { + stripInvalidAssignmentsToTextIndex(node->getChild(i), idx, prefixPaths); + } + } + } + + // static + void QueryPlannerIXSelect::stripInvalidAssignmentsToTextIndexes( + MatchExpression* node, + const vector& indices) { + + for (size_t i = 0; i < indices.size(); ++i) { + const IndexEntry& index = indices[i]; + + // We only care about text indices. + if (INDEX_TEXT != index.type) { + continue; + } + + // Gather the set of paths that comprise the index prefix for this text index. + // Each of those paths must have an equality assignment, otherwise we can't assign + // *anything* to this index. + unordered_set textIndexPrefixPaths; + BSONObjIterator it(index.keyPattern); + + // We stop when we see the first string in the key pattern. We know that + // the prefix precedes "text". + for (BSONElement elt = it.next(); elt.type() != String; elt = it.next()) { + textIndexPrefixPaths.insert(elt.fieldName()); + verify(it.more()); + } + + // If the index prefix is non-empty, remove invalid assignments to it. + if (!textIndexPrefixPaths.empty()) { + stripInvalidAssignmentsToTextIndex(node, i, textIndexPrefixPaths); + } + } + } + + // + // 2dsphere V2 sparse quirks + // + + static void stripInvalidAssignmentsTo2dsphereIndex( + MatchExpression* node, + size_t idx, + const unordered_set& geoFields) { + + if (Indexability::nodeCanUseIndexOnOwnField(node) + && MatchExpression::GEO != node->matchType() + && MatchExpression::GEO_NEAR != node->matchType()) { + // We found a non-geo predicate tagged to use a V2 2dsphere index which is not + // and-related to a geo predicate that can use the index. + removeIndexRelevantTag(node, idx); + return; + } + + const MatchExpression::MatchType nodeType = node->matchType(); + + // Don't bother peeking inside of negations. + if (MatchExpression::NOT == nodeType || MatchExpression::NOR == nodeType) { + return; + } + + if (MatchExpression::AND != nodeType) { + // It's an OR or some kind of array operator. + for (size_t i = 0; i < node->numChildren(); ++i) { + stripInvalidAssignmentsTo2dsphereIndex(node->getChild(i), idx, geoFields); + } + return; + } + + bool hasGeoField = false; + + for (size_t i = 0; i < node->numChildren(); ++i) { + MatchExpression* child = node->getChild(i); + RelevantTag* tag = static_cast(child->getTag()); + + if (NULL == tag) { + // 'child' could be a logical operator. Maybe there are some assignments hiding + // inside. + stripInvalidAssignmentsTo2dsphereIndex(child, idx, geoFields); + continue; + } + + bool inFirst = tag->first.end() != std::find(tag->first.begin(), + tag->first.end(), + idx); + + bool inNotFirst = tag->notFirst.end() != std::find(tag->notFirst.begin(), + tag->notFirst.end(), + idx); + + // If there is an index assignment... + if (inFirst || inNotFirst) { + // And it's a geo predicate... + if (MatchExpression::GEO == child->matchType() || + MatchExpression::GEO_NEAR == child->matchType()) { + + hasGeoField = true; + } + } + else { + // Recurse on the children to ensure that they're not hiding any assignments + // to idx. + stripInvalidAssignmentsTo2dsphereIndex(child, idx, geoFields); + } + } + + // If there isn't a geo predicate our results aren't a subset of what's in the geo index, so + // if we use the index we'll miss results. + if (!hasGeoField) { + for (size_t i = 0; i < node->numChildren(); ++i) { + stripInvalidAssignmentsTo2dsphereIndex(node->getChild(i), idx, geoFields); + } + } + } + + // static + void QueryPlannerIXSelect::stripInvalidAssignmentsTo2dsphereIndices( + MatchExpression* node, + const vector& indices) { + + for (size_t i = 0; i < indices.size(); ++i) { + const IndexEntry& index = indices[i]; + + // We only worry about 2dsphere indices. + if (INDEX_2DSPHERE != index.type) { + continue; + } + + // They also have to be V2. Both ignore the sparse flag but V1 is + // never-sparse, V2 geo-sparse. + BSONElement elt = index.infoObj["2dsphereIndexVersion"]; + if (elt.eoo()) { + continue; + } + if (!elt.isNumber()) { + continue; + } + if (2 != elt.numberInt()) { + continue; + } + + // Gather the set of geo fields in this index. + unordered_set geoFields; + BSONObjIterator it(index.keyPattern); + while (it.more()) { + BSONElement elt = it.next(); + if (String == elt.type()) { + geoFields.insert(elt.fieldName()); + } + } + + // If every field is geo don't bother doing anything. + if (geoFields.size() == static_cast(index.keyPattern.nFields())) { + continue; + } + + // You can't have a 2dsphere index without a 2dsphere field. + invariant(!geoFields.empty()); + + // Remove bad assignments from this index. + stripInvalidAssignmentsTo2dsphereIndex(node, i, geoFields); + } + } + +} // namespace mongo diff --git a/src/mongo/db/query/planner_ixselect.h b/src/mongo/db/query/planner_ixselect.h new file mode 100644 index 00000000000..30f2f1f0196 --- /dev/null +++ b/src/mongo/db/query/planner_ixselect.h @@ -0,0 +1,182 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/index_entry.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + /** + * Methods for determining what fields and predicates can use indices. + */ + class QueryPlannerIXSelect { + public: + /** + * Return all the fields in the tree rooted at 'node' that we can use an index on + * in order to answer the query. + * + * The 'prefix' argument is a path prefix to be prepended to any fields mentioned in + * predicates encountered. Some array operators specify a path prefix. + */ + static void getFields(MatchExpression* node, string prefix, unordered_set* out); + + /** + * Find all indices prefixed by fields we have predicates over. Only these indices are + * useful in answering the query. + */ + static void findRelevantIndices(const unordered_set& fields, + const vector& indices, + vector* out); + + /** + * Return true if the index key pattern field 'elt' (which belongs to 'index') can be used + * to answer the predicate 'node'. + * + * For example, {field: "hashed"} can only be used with sets of equalities. + * {field: "2d"} can only be used with some geo predicates. + * {field: "2dsphere"} can only be used with some other geo predicates. + */ + static bool compatible(const BSONElement& elt, + const IndexEntry& index, + MatchExpression* node); + + /** + * Determine how useful all of our relevant 'indices' are to all predicates in the subtree + * rooted at 'node'. Affixes a RelevantTag to all predicate nodes which can use an index. + * + * 'prefix' is a path prefix that should be prepended to any path (certain array operators + * imply a path prefix). + * + * For an index to be useful to a predicate, the index must be compatible (see above). + * + * If an index is prefixed by the predicate's path, it's always useful. + * + * If an index is compound but not prefixed by a predicate's path, it's only useful if + * there exists another predicate that 1. will use that index and 2. is related to the + * original predicate by having an AND as a parent. + */ + static void rateIndices(MatchExpression* node, + string prefix, + const vector& indices); + + /** + * Amend the RelevantTag lists for all predicates in the subtree rooted at 'node' to remove + * invalid assignments to text and geo indices. + * + * See the body of this function and the specific stripInvalidAssignments functions for details. + */ + static void stripInvalidAssignments(MatchExpression* node, + const vector& indices); + + /** + * In some special cases, we can strip most of the index assignments from the tree early + * on. Specifically, if we find an AND which has a child tagged for equality over a + * single-field unique index, then all other predicate-to-index assignments can be + * stripped off the subtree rooted at 'node'. + * + * This is used to ensure that we always favor key-value lookup plans over any + * more complex plan. + * + * Example: + * Suppose you have match expression OR (AND (a==1, b==2), AND (c==3, d==4)). + * There are indices on fields, 'a', 'b', 'c', and 'd'. The index on 'd' is + * the only unique index. + * + * This code will find that the subtree AND (c==3, d==4) can be answered by + * looking up the value of 'd' in the unique index. Since no better plan than + * a single key lookup is ever available, all assignments in this subtree + * are stripped, except for the assignment of d==4 to the unique 'd' index. + * + * Stripping the assignment for 'c' causes the planner to generate just two + * possible plans: + * 1) an OR of an index scan over 'a' and an index scan over 'd' + * 2) an OR of an index scan over 'b' and an index scan over 'd' + */ + static void stripUnneededAssignments(MatchExpression* node, + const std::vector& indices); + + private: + /** + * Amend the RelevantTag lists for all predicates in the subtree rooted at 'node' to remove + * invalid assignments to text indexes. + * + * A predicate on a field from a compound text index with a non-empty index prefix + * (e.g. pred {a: 1, b: 1} on index {a: 1, b: 1, c: "text"}) is only considered valid to + * assign to the text index if it is a direct child of an AND with the following properties: + * - it has a TEXT child + * - for every index prefix component, it has an EQ child on that component's path + * + * Note that compatible() enforces the precondition that only EQ nodes are considered + * relevant to text index prefixes. + * If there is a relevant compound text index with a non-empty "index prefix" (e.g. the + * prefix {a: 1, b: 1} for the index {a: 1, b: 1, c: "text"}), amend the RelevantTag(s) + * created above to remove assignments to the text index where the query does not have + * predicates over each indexed field of the prefix. + * + * This is necessary because text indices do not obey the normal rules of sparseness, in + * that they generate no index keys for documents without indexable text data in at least + * one text field (in fact, text indices ignore the sparse option entirely). For example, + * given the text index {a: 1, b: 1, c: "text"}: + * + * - Document {a: 1, b: 6, c: "hello world"} generates 2 index keys + * - Document {a: 1, b: 7, c: {d: 1}} generates 0 index keys + * - Document {a: 1, b: 8} generates 0 index keys + * + * As a result, the query {a: 1} *cannot* be satisfied by the text index {a: 1, b: 1, c: + * "text"}, since documents without indexed text data would not be returned by the query. + * rateIndices() above will eagerly annotate the pred {a: 1} as relevant to the text index; + * those annotations get removed here. + */ + static void stripInvalidAssignmentsToTextIndexes(MatchExpression* node, + const vector& indices); + + /** + * For V1 2dsphere indices we ignore the sparse option. As such we can use an index + * like {nongeo: 1, geo: "2dsphere"} to answer queries only involving nongeo. + * + * For V2 2dsphere indices also ignore the sparse flag but indexing behavior as compared to + * V1 is different. If all of the geo fields are missing from the document we do not index + * it. As such we cannot use V2 sparse indices unless we have a predicate over a geo + * field. + * + * 2dsphere indices V2 are "geo-sparse." That is, if there aren't any geo-indexed fields in + * a document it won't be indexed. As such we can't use an index like {foo:1, geo: + * "2dsphere"} to answer a query on 'foo' if the index is V2 as it will not contain the + * document {foo:1}. + * + * We *can* use it to answer a query on 'foo' if the predicate on 'foo' is AND-related to a + * predicate on every geo field in the index. + */ + static void stripInvalidAssignmentsTo2dsphereIndices(MatchExpression* node, + const vector& indices); + }; + +} // namespace mongo diff --git a/src/mongo/db/query/planner_ixselect_test.cpp b/src/mongo/db/query/planner_ixselect_test.cpp new file mode 100644 index 00000000000..041167771e8 --- /dev/null +++ b/src/mongo/db/query/planner_ixselect_test.cpp @@ -0,0 +1,269 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/planner_ixselect.cpp + */ + +#include "mongo/db/query/planner_ixselect.h" + +#include +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/index_tag.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/text.h" + +using namespace mongo; + +namespace { + + using std::auto_ptr; + using std::string; + using std::vector; + + /** + * Utility function to create MatchExpression + */ + MatchExpression* parseMatchExpression(const BSONObj& obj) { + StatusWithMatchExpression status = MatchExpressionParser::parse(obj); + ASSERT_TRUE(status.isOK()); + MatchExpression* expr(status.getValue()); + return expr; + } + + /** + * Utility function to join elements in iterator range with comma + */ + template string toString(Iter begin, Iter end) { + mongoutils::str::stream ss; + ss << "["; + for (Iter i = begin; i != end; i++) { + if (i != begin) { + ss << " "; + } + ss << *i; + } + ss << "]"; + return ss; + } + + /** + * Test function for getFields() + * Parses query string to obtain MatchExpression which is passed together with prefix + * to QueryPlannerIXSelect::getFields() + * Results are compared with expected fields (parsed from expectedFieldsStr) + */ + void testGetFields(const char* query, const char* prefix, const char* expectedFieldsStr) { + BSONObj obj = fromjson(query); + auto_ptr expr(parseMatchExpression(obj)); + unordered_set fields; + QueryPlannerIXSelect::getFields(expr.get(), prefix, &fields); + + // Verify results + // First, check that results contain a superset of expected fields. + vector expectedFields = StringSplitter::split(expectedFieldsStr, ","); + for (vector::const_iterator i = expectedFields.begin(); i != expectedFields.end(); + i++) { + if (fields.find(*i) == fields.end()) { + mongoutils::str::stream ss; + ss << "getFields(query=" << query << ", prefix=" << prefix << "): unable to find " + << *i << " in result: " << toString(fields.begin(), fields.end()); + FAIL(ss); + } + } + + // Next, confirm that results do not contain any unexpected fields. + if (fields.size() != expectedFields.size()) { + mongoutils::str::stream ss; + ss << "getFields(query=" << query << ", prefix=" << prefix + << "): unexpected fields in result. expected: " + << toString(expectedFields.begin(), expectedFields.end()) + << ". actual: " << toString(fields.begin(), fields.end()); + FAIL(ss); + } + } + + /** + * Basic test cases for getFields() + * Includes logical operators + */ + TEST(QueryPlannerIXSelectTest, GetFieldsBasic) { + // Arguments to test function: query, prefix, comma-delimited list of expected fields + testGetFields("{}", "", ""); + testGetFields("{a: 1}", "", "a"); + testGetFields("{a: 1}", "c.", "c.a"); + testGetFields("{a: 1, b: 1}", "", "a,b"); + testGetFields("{a: {$in: [1]}}", "", "a"); + testGetFields("{$or: [{a: 1}, {b: 1}]}", "", "a,b"); + } + + /** + * Array test cases for getFields + */ + TEST(QueryPlannerIXSelectTest, GetFieldsArray) { + testGetFields("{a: {$elemMatch: {b: 1}}}", "", "a.b"); + testGetFields("{a: {$all: [{$elemMatch: {b: 1}}]}}", "", "a.b"); + } + + /** + * Negation test cases for getFields() + * $ne, $nin, $nor + */ + TEST(QueryPlannerIXSelectTest, GetFieldsNegation) { + testGetFields("{a: {$ne: 1}}", "", "a"); + testGetFields("{a: {$nin: [1]}}", "", "a"); + testGetFields("{$nor: [{a: 1}, {b: 1}]}", "", ""); + testGetFields("{$and: [{a: 1}, {a: {$ne: 2}}]}", "", "a"); + } + + /** + * Array negation test cases for getFields + */ + TEST(QueryPlannerIXSelectTest, GetFieldsArrayNegation) { + testGetFields("{a: {$elemMatch: {b: {$ne: 1}}}}", "", "a.b"); + testGetFields("{a: {$all: [{$elemMatch: {b: {$ne: 1}}}]}}", "", "a.b"); + } + + /** + * Performs a pre-order traversal of expression tree. Validates + * that all tagged nodes contain an instance of RelevantTag. + */ + void findRelevantTaggedNodePaths(MatchExpression* root, vector* paths) { + MatchExpression::TagData* tag = root->getTag(); + if (tag) { + StringBuilder buf; + tag->debugString(&buf); + RelevantTag* r = dynamic_cast(tag); + if (!r) { + mongoutils::str::stream ss; + ss << "tag is not instance of RelevantTag. tree: " << root->toString() + << "; tag: " << buf.str(); + FAIL(ss); + } + paths->push_back(r->path); + } + for (size_t i = 0; i < root->numChildren(); ++i) { + findRelevantTaggedNodePaths(root->getChild(i), paths); + } + } + + /** + * Parses a MatchExpression from query string and passes that along with + * prefix to rateIndices. + * Verifies results against list of expected paths. + * For now, we're only interested in which nodes are tagged. + * In future, we may expand this test function to include + * validate which indices are assigned to a node. + */ + void testRateIndicesTaggedNodePaths(const char* query, const char* prefix, + const char* expectedPathsStr) { + // Parse and rate query. Some of the nodes in the rated tree + // will be tagged after the rating process. + BSONObj obj = fromjson(query); + auto_ptr expr(parseMatchExpression(obj)); + + // Currently, we tag every indexable node even when no compatible + // index is available. Hence, it is fine to pass an empty vector of + // indices to rateIndices(). + vector indices; + QueryPlannerIXSelect::rateIndices(expr.get(), prefix, indices); + + // Retrieve a list of paths embedded in + // tagged nodes. + vector paths; + findRelevantTaggedNodePaths(expr.get(), &paths); + + // Compare with expected list of paths. + // First verify number of paths retrieved. + vector expectedPaths = StringSplitter::split(expectedPathsStr, ","); + if (paths.size() != expectedPaths.size()) { + mongoutils::str::stream ss; + ss << "rateIndices(query=" << query << ", prefix=" << prefix + << "): unexpected number of tagged nodes found. expected: " + << toString(expectedPaths.begin(), expectedPaths.end()) << ". actual: " + << toString(paths.begin(), paths.end()); + FAIL(ss); + } + + // Next, check that value and order of each element match between the two lists. + for (vector::const_iterator i = paths.begin(), j = expectedPaths.begin(); + i != paths.end(); i++, j++) { + if (*i == *j) { + continue; + } + mongoutils::str::stream ss; + ss << "rateIndices(query=" << query << ", prefix=" << prefix + << "): unexpected path found. expected: " << *j << " " + << toString(expectedPaths.begin(), expectedPaths.end()) << ". actual: " + << *i << " " << toString(paths.begin(), paths.end()); + FAIL(ss); + } + } + + /** + * Basic test cases for rateIndices(). + * Includes logical operators. + */ + TEST(QueryPlannerIXSelectTest, RateIndicesTaggedNodePathsBasic) { + // Test arguments: query, prefix, comma-delimited list of expected paths + testRateIndicesTaggedNodePaths("{}", "", ""); + testRateIndicesTaggedNodePaths("{a: 1}", "", "a"); + testRateIndicesTaggedNodePaths("{a: 1}", "c.", "c.a"); + testRateIndicesTaggedNodePaths("{a: 1, b: 1}", "", "a,b"); + testRateIndicesTaggedNodePaths("{a: {$in: [1]}}", "", "a"); + testRateIndicesTaggedNodePaths("{$or: [{a: 1}, {b: 1}]}", "", "a,b"); + } + + /** + * Array test cases for rateIndices(). + */ + TEST(QueryPlannerIXSelectTest, RateIndicesTaggedNodePathArray) { + testRateIndicesTaggedNodePaths("{a: {$elemMatch: {b: 1}}}", "", "a.b"); + testRateIndicesTaggedNodePaths("{a: {$all: [{$elemMatch: {b: 1}}]}}", "", "a.b"); + } + + /** + * Negation test cases for rateIndices(). + */ + TEST(QueryPlannerIXSelectTest, RateIndicesTaggedNodePathsNegation) { + testRateIndicesTaggedNodePaths("{a: {$ne: 1}}", "", "a,a"); + testRateIndicesTaggedNodePaths("{a: {$nin: [1]}}", "", "a,a"); + testRateIndicesTaggedNodePaths("{$nor: [{a: 1}, {b: 1}]}", "", ""); + testRateIndicesTaggedNodePaths("{$and: [{a: 1}, {a: {$ne: 2}}]}", "", "a,a,a"); + } + + /** + * Array negation test cases for rateIndices(). + */ + TEST(QueryPlannerIXSelectTest, RateIndicesTaggedNodePathArrayNegation) { + testRateIndicesTaggedNodePaths("{a: {$elemMatch: {b: {$ne: 1}}}}", "", "a.b,a.b"); + testRateIndicesTaggedNodePaths("{a: {$all: [{$elemMatch: {b: {$ne: 1}}}]}}", "", "a.b,a.b"); + } + +} // namespace diff --git a/src/mongo/db/query/qlog.cpp b/src/mongo/db/query/qlog.cpp new file mode 100644 index 00000000000..f818f3cbd2e --- /dev/null +++ b/src/mongo/db/query/qlog.cpp @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/qlog.h" +#include "mongo/db/server_options.h" +#include "mongo/db/server_parameters.h" + +namespace mongo { + + MONGO_EXPORT_SERVER_PARAMETER(verboseQueryLogging, bool, false); + + bool qlogOff() { + bool old = verboseQueryLogging; + verboseQueryLogging = false; + return old; + } + + bool qlogOn() { + bool old = verboseQueryLogging; + verboseQueryLogging = true; + return old; + } + +} // namespace mongo diff --git a/src/mongo/db/query/qlog.h b/src/mongo/db/query/qlog.h new file mode 100644 index 00000000000..a7d83f7d52b --- /dev/null +++ b/src/mongo/db/query/qlog.h @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +namespace mongo { + + extern bool verboseQueryLogging; + +// With a #define like this, we don't evaluate the costly toString()s that are QLOG'd +#define QLOG() if (verboseQueryLogging) log() << "[QLOG] " + + bool qlogOff(); + bool qlogOn(); + +} // namespace mongo diff --git a/src/mongo/db/query/query_knobs.cpp b/src/mongo/db/query/query_knobs.cpp new file mode 100644 index 00000000000..30efd8d15e9 --- /dev/null +++ b/src/mongo/db/query/query_knobs.cpp @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/server_options.h" +#include "mongo/db/server_parameters.h" + +namespace mongo { + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlanEvaluationWorks, int, 10000); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlanEvaluationCollFraction, double, 0.3); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlanEvaluationMaxResults, int, 101); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheSize, int, 5000); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheFeedbacksStored, int, 20); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheStdDeviations, double, 2.0); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryCacheWriteOpsBetweenFlush, int, 1000); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlannerMaxIndexedSolutions, int, 64); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryEnumerationMaxOrSolutions, int, 10); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryEnumerationMaxIntersectPerAnd, int, 3); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryForceIntersectionPlans, bool, false); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlannerEnableIndexIntersection, bool, true); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlannerEnableHashIntersection, bool, false); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryPlanOrChildrenIndependently, bool, true); + + MONGO_EXPORT_SERVER_PARAMETER(internalQueryMaxScansToExplode, int, 200); + +} // namespace mongo diff --git a/src/mongo/db/query/query_knobs.h b/src/mongo/db/query/query_knobs.h new file mode 100644 index 00000000000..913362558d7 --- /dev/null +++ b/src/mongo/db/query/query_knobs.h @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { + + // + // multi-plan ranking + // + + // Max number of times we call work() on plans before comparing them, + // for small collections. + extern int internalQueryPlanEvaluationWorks; + + // For large collections, the number times we work() candidate plans is + // taken as this fraction of the collection size. + extern double internalQueryPlanEvaluationCollFraction; + + // Stop working plans once a plan returns this many results. + extern int internalQueryPlanEvaluationMaxResults; + + // Do we give a big ranking bonus to intersection plans? + extern bool internalQueryForceIntersectionPlans; + + // Do we have ixisect on at all? + extern bool internalQueryPlannerEnableIndexIntersection; + + // Do we use hash-based intersection for rooted $and queries? + extern bool internalQueryPlannerEnableHashIntersection; + + // + // plan cache + // + + // How many entries in the cache? + extern int internalQueryCacheSize; + + // How many feedback entries do we collect before possibly evicting from the cache based on bad + // performance? + extern int internalQueryCacheFeedbacksStored; + + // How many stddevs must a feedback be from the 'reference' performance for us to evict the + // entry from the cache? + extern double internalQueryCacheStdDeviations; + + // How many write ops should we allow in a collection before tossing all cache entries? + extern int internalQueryCacheWriteOpsBetweenFlush; + + // + // Planning and enumeration. + // + + // How many indexed solutions will QueryPlanner::plan output? + extern int internalQueryPlannerMaxIndexedSolutions; + + // How many solutions will the enumerator consider at each OR? + extern int internalQueryEnumerationMaxOrSolutions; + + // How many intersections will the enumerator consider at each AND? + extern int internalQueryEnumerationMaxIntersectPerAnd; + + // Do we want to plan each child of the OR independently? + extern bool internalQueryPlanOrChildrenIndependently; + + // How many index scans are we willing to produce in order to obtain a sort order + // during explodeForSort? + extern int internalQueryMaxScansToExplode; + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner.cpp b/src/mongo/db/query/query_planner.cpp new file mode 100644 index 00000000000..2ebd0b51834 --- /dev/null +++ b/src/mongo/db/query/query_planner.cpp @@ -0,0 +1,1003 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_planner.h" + +#include + +#include "mongo/client/dbclientinterface.h" // For QueryOption_foobar +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/matcher/expression_text.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/planner_access.h" +#include "mongo/db/query/planner_analysis.h" +#include "mongo/db/query/planner_ixselect.h" +#include "mongo/db/query/plan_enumerator.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + // Copied verbatim from db/index.h + static bool isIdIndex( const BSONObj &pattern ) { + BSONObjIterator i(pattern); + BSONElement e = i.next(); + //_id index must have form exactly {_id : 1} or {_id : -1}. + //Allows an index of form {_id : "hashed"} to exist but + //do not consider it to be the primary _id index + if(! ( strcmp(e.fieldName(), "_id") == 0 + && (e.numberInt() == 1 || e.numberInt() == -1))) + return false; + return i.next().eoo(); + } + + static bool is2DIndex(const BSONObj& pattern) { + BSONObjIterator it(pattern); + while (it.more()) { + BSONElement e = it.next(); + if (String == e.type() && str::equals("2d", e.valuestr())) { + return true; + } + } + return false; + } + + string optionString(size_t options) { + mongoutils::str::stream ss; + + // These options are all currently mutually exclusive. + if (QueryPlannerParams::DEFAULT == options) { + ss << "DEFAULT "; + } + if (options & QueryPlannerParams::NO_TABLE_SCAN) { + ss << "NO_TABLE_SCAN "; + } + if (options & QueryPlannerParams::INCLUDE_COLLSCAN) { + ss << "INCLUDE_COLLSCAN "; + } + if (options & QueryPlannerParams::INCLUDE_SHARD_FILTER) { + ss << "INCLUDE_SHARD_FILTER "; + } + if (options & QueryPlannerParams::NO_BLOCKING_SORT) { + ss << "NO_BLOCKING_SORT "; + } + if (options & QueryPlannerParams::INDEX_INTERSECTION) { + ss << "INDEX_INTERSECTION "; + } + if (options & QueryPlannerParams::KEEP_MUTATIONS) { + ss << "KEEP_MUTATIONS"; + } + + return ss; + } + + static BSONObj getKeyFromQuery(const BSONObj& keyPattern, const BSONObj& query) { + return query.extractFieldsUnDotted(keyPattern); + } + + static bool indexCompatibleMaxMin(const BSONObj& obj, const BSONObj& keyPattern) { + BSONObjIterator kpIt(keyPattern); + BSONObjIterator objIt(obj); + + for (;;) { + // Every element up to this point has matched so the KP matches + if (!kpIt.more() && !objIt.more()) { + return true; + } + + // If only one iterator is done, it's not a match. + if (!kpIt.more() || !objIt.more()) { + return false; + } + + // Field names must match and be in the same order. + BSONElement kpElt = kpIt.next(); + BSONElement objElt = objIt.next(); + if (!mongoutils::str::equals(kpElt.fieldName(), objElt.fieldName())) { + return false; + } + } + } + + static BSONObj stripFieldNames(const BSONObj& obj) { + BSONObjIterator it(obj); + BSONObjBuilder bob; + while (it.more()) { + bob.appendAs(it.next(), ""); + } + return bob.obj(); + } + + /** + * "Finishes" the min object for the $min query option by filling in an empty object with + * MinKey/MaxKey and stripping field names. + * + * In the case that 'minObj' is empty, we "finish" it by filling in either MinKey or MaxKey + * instead. Choosing whether to use MinKey or MaxKey is done by comparing against 'maxObj'. + * For instance, suppose 'minObj' is empty, 'maxObj' is { a: 3 }, and the key pattern is + * { a: -1 }. According to the key pattern ordering, { a: 3 } < MinKey. This means that the + * proper resulting bounds are + * + * start: { '': MaxKey }, end: { '': 3 } + * + * as opposed to + * + * start: { '': MinKey }, end: { '': 3 } + * + * Suppose instead that the key pattern is { a: 1 }, with the same 'minObj' and 'maxObj' + * (that is, an empty object and { a: 3 } respectively). In this case, { a: 3 } > MinKey, + * which means that we use range [{'': MinKey}, {'': 3}]. The proper 'minObj' in this case is + * MinKey, whereas in the previous example it was MaxKey. + * + * If 'minObj' is non-empty, then all we do is strip its field names (because index keys always + * have empty field names). + */ + static BSONObj finishMinObj(const BSONObj& kp, const BSONObj& minObj, const BSONObj& maxObj) { + BSONObjBuilder bob; + bob.appendMinKey(""); + BSONObj minKey = bob.obj(); + + if (minObj.isEmpty()) { + if (0 > minKey.woCompare(maxObj, kp, false)) { + BSONObjBuilder minKeyBuilder; + minKeyBuilder.appendMinKey(""); + return minKeyBuilder.obj(); + } + else { + BSONObjBuilder maxKeyBuilder; + maxKeyBuilder.appendMaxKey(""); + return maxKeyBuilder.obj(); + } + } + else { + return stripFieldNames(minObj); + } + } + + /** + * "Finishes" the max object for the $max query option by filling in an empty object with + * MinKey/MaxKey and stripping field names. + * + * See comment for finishMinObj() for why we need both 'minObj' and 'maxObj'. + */ + static BSONObj finishMaxObj(const BSONObj& kp, const BSONObj& minObj, const BSONObj& maxObj) { + BSONObjBuilder bob; + bob.appendMaxKey(""); + BSONObj maxKey = bob.obj(); + + if (maxObj.isEmpty()) { + if (0 < maxKey.woCompare(minObj, kp, false)) { + BSONObjBuilder maxKeyBuilder; + maxKeyBuilder.appendMaxKey(""); + return maxKeyBuilder.obj(); + } + else { + BSONObjBuilder minKeyBuilder; + minKeyBuilder.appendMinKey(""); + return minKeyBuilder.obj(); + } + } + else { + return stripFieldNames(maxObj); + } + } + + QuerySolution* buildCollscanSoln(const CanonicalQuery& query, + bool tailable, + const QueryPlannerParams& params) { + + QuerySolutionNode* solnRoot = QueryPlannerAccess::makeCollectionScan(query, tailable, params); + return QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); + } + + QuerySolution* buildWholeIXSoln(const IndexEntry& index, + const CanonicalQuery& query, + const QueryPlannerParams& params, + int direction = 1) { + + QuerySolutionNode* solnRoot = QueryPlannerAccess::scanWholeIndex(index, query, params, direction); + return QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); + } + + bool providesSort(const CanonicalQuery& query, const BSONObj& kp) { + return query.getParsed().getSort().isPrefixOf(kp); + } + + Status QueryPlanner::cacheDataFromTaggedTree(const MatchExpression* const taggedTree, + const vector& relevantIndices, + PlanCacheIndexTree** out) { + // On any early return, the out-parameter must contain NULL. + *out = NULL; + + if (NULL == taggedTree) { + return Status(ErrorCodes::BadValue, "Cannot produce cache data: tree is NULL."); + } + + auto_ptr indexTree(new PlanCacheIndexTree()); + + if (NULL != taggedTree->getTag()) { + IndexTag* itag = static_cast(taggedTree->getTag()); + if (itag->index >= relevantIndices.size()) { + mongoutils::str::stream ss; + ss << "Index number is " << itag->index + << " but there are only " << relevantIndices.size() + << " relevant indices."; + return Status(ErrorCodes::BadValue, ss); + } + + // Make sure not to cache solutions which use '2d' indices. + // A 2d index that doesn't wrap on one query may wrap on another, so we have to + // check that the index is OK with the predicate. The only thing we have to do + // this for is 2d. For now it's easier to move ahead if we don't cache 2d. + // + // TODO: revisit with a post-cached-index-assignment compatibility check + if (is2DIndex(relevantIndices[itag->index].keyPattern)) { + return Status(ErrorCodes::BadValue, "can't cache '2d' index"); + } + + IndexEntry* ientry = new IndexEntry(relevantIndices[itag->index]); + indexTree->entry.reset(ientry); + indexTree->index_pos = itag->pos; + } + + for (size_t i = 0; i < taggedTree->numChildren(); ++i) { + MatchExpression* taggedChild = taggedTree->getChild(i); + PlanCacheIndexTree* indexTreeChild; + Status s = cacheDataFromTaggedTree(taggedChild, relevantIndices, &indexTreeChild); + if (!s.isOK()) { + return s; + } + indexTree->children.push_back(indexTreeChild); + } + + *out = indexTree.release(); + return Status::OK(); + } + + // static + Status QueryPlanner::tagAccordingToCache(MatchExpression* filter, + const PlanCacheIndexTree* const indexTree, + const map& indexMap) { + if (NULL == filter) { + return Status(ErrorCodes::BadValue, "Cannot tag tree: filter is NULL."); + } + if (NULL == indexTree) { + return Status(ErrorCodes::BadValue, "Cannot tag tree: indexTree is NULL."); + } + + // We're tagging the tree here, so it shouldn't have + // any tags hanging off yet. + verify(NULL == filter->getTag()); + + if (filter->numChildren() != indexTree->children.size()) { + mongoutils::str::stream ss; + ss << "Cache topology and query did not match: " + << "query has " << filter->numChildren() << " children " + << "and cache has " << indexTree->children.size() << " children."; + return Status(ErrorCodes::BadValue, ss); + } + + // Continue the depth-first tree traversal. + for (size_t i = 0; i < filter->numChildren(); ++i) { + Status s = tagAccordingToCache(filter->getChild(i), indexTree->children[i], indexMap); + if (!s.isOK()) { + return s; + } + } + + if (NULL != indexTree->entry.get()) { + map::const_iterator got = indexMap.find(indexTree->entry->keyPattern); + if (got == indexMap.end()) { + mongoutils::str::stream ss; + ss << "Did not find index with keyPattern: " << indexTree->entry->keyPattern.toString(); + return Status(ErrorCodes::BadValue, ss); + } + filter->setTag(new IndexTag(got->second, indexTree->index_pos)); + } + + return Status::OK(); + } + + // static + Status QueryPlanner::planFromCache(const CanonicalQuery& query, + const QueryPlannerParams& params, + const SolutionCacheData& cacheData, + QuerySolution** out) { + if (SolutionCacheData::WHOLE_IXSCAN_SOLN == cacheData.solnType) { + // The solution can be constructed by a scan over the entire index. + QuerySolution* soln = buildWholeIXSoln(*cacheData.tree->entry, + query, params, cacheData.wholeIXSolnDir); + if (soln == NULL) { + return Status(ErrorCodes::BadValue, + "plan cache error: soln that uses index to provide sort"); + } + else { + *out = soln; + return Status::OK(); + } + } + else if (SolutionCacheData::COLLSCAN_SOLN == cacheData.solnType) { + // The cached solution is a collection scan. We don't cache collscans + // with tailable==true, hence the false below. + QuerySolution* soln = buildCollscanSoln(query, false, params); + if (soln == NULL) { + return Status(ErrorCodes::BadValue, "plan cache error: collection scan soln"); + } + else { + *out = soln; + return Status::OK(); + } + } + + // SolutionCacheData::USE_TAGS_SOLN == cacheData->solnType + // If we're here then this is neither the whole index scan or collection scan + // cases, and we proceed by using the PlanCacheIndexTree to tag the query tree. + + // Create a copy of the expression tree. We use cachedSoln to annotate this with indices. + auto_ptr clone(query.root()->shallowClone()); + + QLOG() << "Tagging the match expression according to cache data: " << endl + << "Filter:" << endl << clone->toString() + << "Cache data:" << endl << cacheData.toString(); + + // Map from index name to index number. + // TODO: can we assume that the index numbering has the same lifetime + // as the cache state? + map indexMap; + for (size_t i = 0; i < params.indices.size(); ++i) { + const IndexEntry& ie = params.indices[i]; + indexMap[ie.keyPattern] = i; + QLOG() << "Index " << i << ": " << ie.keyPattern.toString() << endl; + } + + Status s = tagAccordingToCache(clone.get(), cacheData.tree.get(), indexMap); + if (!s.isOK()) { + return s; + } + + // The planner requires a defined sort order. + sortUsingTags(clone.get()); + + QLOG() << "Tagged tree:" << endl << clone->toString(); + + // Use the cached index assignments to build solnRoot. + QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, + clone.release(), + false, + params.indices); + + if (NULL != solnRoot) { + // Takes ownership of 'solnRoot'. + QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); + if (NULL != soln) { + QLOG() << "Planner: solution constructed from the cache:\n" << soln->toString() << endl; + *out = soln; + return Status::OK(); + } + } + + return Status(ErrorCodes::BadValue, "couldn't plan from cache"); + } + + // static + Status QueryPlanner::planFromCache(const CanonicalQuery& query, + const QueryPlannerParams& params, + const CachedSolution& cachedSoln, + QuerySolution** out, + QuerySolution** backupOut) { + verify(!cachedSoln.plannerData.empty()); + verify(out); + verify(backupOut); + verify(PlanCache::shouldCacheQuery(query)); + + // If there is no backup solution, then return NULL through + // the 'backupOut' out-parameter. + *backupOut = NULL; + + // Queries not suitable for caching are filtered + // in multi plan runner using PlanCache::shouldCacheQuery(). + + // Look up winning solution in cached solution's array. + SolutionCacheData* winnerCacheData = cachedSoln.plannerData[0]; + Status s = planFromCache(query, params, *winnerCacheData, out); + if (!s.isOK()) { + return s; + } + + if (cachedSoln.backupSoln) { + SolutionCacheData* backupCacheData = cachedSoln.plannerData[*cachedSoln.backupSoln]; + Status backupStatus = planFromCache(query, params, *backupCacheData, backupOut); + if (!backupStatus.isOK()) { + return backupStatus; + } + } + + return Status::OK(); + } + + // static + Status QueryPlanner::plan(const CanonicalQuery& query, + const QueryPlannerParams& params, + std::vector* out) { + + QLOG() << "Beginning planning..." << endl + << "=============================" << endl + << "Options = " << optionString(params.options) << endl + << "Canonical query:" << endl << query.toString() + << "=============================" << endl; + + for (size_t i = 0; i < params.indices.size(); ++i) { + QLOG() << "Index " << i << " is " << params.indices[i].toString() << endl; + } + + bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); + + // If the query requests a tailable cursor, the only solution is a collscan + filter with + // tailable set on the collscan. TODO: This is a policy departure. Previously I think you + // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we + // can't provide one. Is this what we want? + if (query.getParsed().hasOption(QueryOption_CursorTailable)) { + if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) + && canTableScan) { + QuerySolution* soln = buildCollscanSoln(query, true, params); + if (NULL != soln) { + out->push_back(soln); + } + } + return Status::OK(); + } + + // The hint or sort can be $natural: 1. If this happens, output a collscan. If both + // a $natural hint and a $natural sort are specified, then the direction of the collscan + // is determined by the sign of the sort (not the sign of the hint). + if (!query.getParsed().getHint().isEmpty() || !query.getParsed().getSort().isEmpty()) { + BSONObj hintObj = query.getParsed().getHint(); + BSONObj sortObj = query.getParsed().getSort(); + BSONElement naturalHint = hintObj.getFieldDotted("$natural"); + BSONElement naturalSort = sortObj.getFieldDotted("$natural"); + + // A hint overrides a $natural sort. This means that we don't force a table + // scan if there is a $natural sort with a non-$natural hint. + if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) { + QLOG() << "Forcing a table scan due to hinted $natural\n"; + // min/max are incompatible with $natural. + if (canTableScan && query.getParsed().getMin().isEmpty() + && query.getParsed().getMax().isEmpty()) { + QuerySolution* soln = buildCollscanSoln(query, false, params); + if (NULL != soln) { + out->push_back(soln); + } + } + return Status::OK(); + } + } + + // Figure out what fields we care about. + unordered_set fields; + QueryPlannerIXSelect::getFields(query.root(), "", &fields); + + for (unordered_set::const_iterator it = fields.begin(); it != fields.end(); ++it) { + QLOG() << "Predicate over field '" << *it << "'" << endl; + } + + // Filter our indices so we only look at indices that are over our predicates. + vector relevantIndices; + + // Hints require us to only consider the hinted index. + // If index filters in the query settings were used to override + // the allowed indices for planning, we should not use the hinted index + // requested in the query. + BSONObj hintIndex; + if (!params.indexFiltersApplied) { + hintIndex = query.getParsed().getHint(); + } + + // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real + // plan. If that fails, just scan the _id index. + if (query.getParsed().isSnapshot()) { + // Find the ID index in indexKeyPatterns. It's our hint. + for (size_t i = 0; i < params.indices.size(); ++i) { + if (isIdIndex(params.indices[i].keyPattern)) { + hintIndex = params.indices[i].keyPattern; + break; + } + } + } + + size_t hintIndexNumber = numeric_limits::max(); + + if (hintIndex.isEmpty()) { + QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); + } + else { + // Sigh. If the hint is specified it might be using the index name. + BSONElement firstHintElt = hintIndex.firstElement(); + if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { + string hintName = firstHintElt.String(); + for (size_t i = 0; i < params.indices.size(); ++i) { + if (params.indices[i].name == hintName) { + QLOG() << "Hint by name specified, restricting indices to " + << params.indices[i].keyPattern.toString() << endl; + relevantIndices.clear(); + relevantIndices.push_back(params.indices[i]); + hintIndexNumber = i; + hintIndex = params.indices[i].keyPattern; + break; + } + } + } + else { + for (size_t i = 0; i < params.indices.size(); ++i) { + if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { + relevantIndices.clear(); + relevantIndices.push_back(params.indices[i]); + QLOG() << "Hint specified, restricting indices to " << hintIndex.toString() + << endl; + hintIndexNumber = i; + break; + } + } + } + + if (hintIndexNumber == numeric_limits::max()) { + return Status(ErrorCodes::BadValue, "bad hint"); + } + } + + // Deal with the .min() and .max() query options. If either exist we can only use an index + // that matches the object inside. + if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) { + BSONObj minObj = query.getParsed().getMin(); + BSONObj maxObj = query.getParsed().getMax(); + + // The unfinished siblings of these objects may not be proper index keys because they + // may be empty objects or have field names. When an index is picked to use for the + // min/max query, these "finished" objects will always be valid index keys for the + // index's key pattern. + BSONObj finishedMinObj; + BSONObj finishedMaxObj; + + // This is the index into params.indices[...] that we use. + size_t idxNo = numeric_limits::max(); + + // If there's an index hinted we need to be able to use it. + if (!hintIndex.isEmpty()) { + if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) { + QLOG() << "Minobj doesn't work with hint"; + return Status(ErrorCodes::BadValue, + "hint provided does not work with min query"); + } + + if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) { + QLOG() << "Maxobj doesn't work with hint"; + return Status(ErrorCodes::BadValue, + "hint provided does not work with max query"); + } + + const BSONObj& kp = params.indices[hintIndexNumber].keyPattern; + finishedMinObj = finishMinObj(kp, minObj, maxObj); + finishedMaxObj = finishMaxObj(kp, minObj, maxObj); + + // The min must be less than the max for the hinted index ordering. + if (0 <= finishedMinObj.woCompare(finishedMaxObj, kp, false)) { + QLOG() << "Minobj/Maxobj don't work with hint"; + return Status(ErrorCodes::BadValue, + "hint provided does not work with min/max query"); + } + + idxNo = hintIndexNumber; + } + else { + // No hinted index, look for one that is compatible (has same field names and + // ordering thereof). + for (size_t i = 0; i < params.indices.size(); ++i) { + const BSONObj& kp = params.indices[i].keyPattern; + + BSONObj toUse = minObj.isEmpty() ? maxObj : minObj; + if (indexCompatibleMaxMin(toUse, kp)) { + // In order to be fully compatible, the min has to be less than the max + // according to the index key pattern ordering. The first step in verifying + // this is "finish" the min and max by replacing empty objects and stripping + // field names. + finishedMinObj = finishMinObj(kp, minObj, maxObj); + finishedMaxObj = finishMaxObj(kp, minObj, maxObj); + + // Now we have the final min and max. This index is only relevant for + // the min/max query if min < max. + if (0 >= finishedMinObj.woCompare(finishedMaxObj, kp, false)) { + // Found a relevant index. + idxNo = i; + break; + } + + // This index is not relevant; move on to the next. + } + } + } + + if (idxNo == numeric_limits::max()) { + QLOG() << "Can't find relevant index to use for max/min query"; + // Can't find an index to use, bail out. + return Status(ErrorCodes::BadValue, + "unable to find relevant index for max/min query"); + } + + QLOG() << "Max/min query using index " << params.indices[idxNo].toString() << endl; + + // Make our scan and output. + QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(params.indices[idxNo], + query, + params, + finishedMinObj, + finishedMaxObj); + + QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); + if (NULL != soln) { + out->push_back(soln); + } + + return Status::OK(); + } + + for (size_t i = 0; i < relevantIndices.size(); ++i) { + QLOG() << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl; + LOG(2) << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl; + } + + // Figure out how useful each index is to each predicate. + QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); + QueryPlannerIXSelect::stripInvalidAssignments(query.root(), relevantIndices); + + // Unless we have GEO_NEAR, TEXT, or a projection, we may be able to apply an optimization + // in which we strip unnecessary index assignments. + // + // Disallowed with projection because assignment to a non-unique index can allow the plan + // to be covered. + // + // TEXT and GEO_NEAR are special because they require the use of a text/geo index in order + // to be evaluated correctly. Stripping these "mandatory assignments" is therefore invalid. + if (query.getParsed().getProj().isEmpty() + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { + QueryPlannerIXSelect::stripUnneededAssignments(query.root(), relevantIndices); + } + + // query.root() is now annotated with RelevantTag(s). + QLOG() << "Rated tree:" << endl << query.root()->toString(); + + // If there is a GEO_NEAR it must have an index it can use directly. + MatchExpression* gnNode = NULL; + if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { + // No index for GEO_NEAR? No query. + RelevantTag* tag = static_cast(gnNode->getTag()); + if (0 == tag->first.size() && 0 == tag->notFirst.size()) { + QLOG() << "Unable to find index for $geoNear query." << endl; + // Don't leave tags on query tree. + query.root()->resetTag(); + return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query"); + } + + GeoNearMatchExpression* gnme = static_cast(gnNode); + + vector newFirst; + + // 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed + // the full query tree inside it as a matcher. + for (size_t i = 0; i < tag->first.size(); ++i) { + // GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning. + if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) { + newFirst.push_back(tag->first[i]); + continue; + } + + // If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the + // entire tree as a filter, if possible. + + GeoNear2DNode* solnRoot = new GeoNear2DNode(); + solnRoot->nq = gnme->getData(); + if (NULL != query.getProj()) { + solnRoot->addPointMeta = query.getProj()->wantGeoNearPoint(); + solnRoot->addDistMeta = query.getProj()->wantGeoNearDistance(); + } + + if (MatchExpression::GEO_NEAR != query.root()->matchType()) { + // root is an AND, clone and delete the GEO_NEAR child. + MatchExpression* filterTree = query.root()->shallowClone(); + verify(MatchExpression::AND == filterTree->matchType()); + + bool foundChild = false; + for (size_t i = 0; i < filterTree->numChildren(); ++i) { + if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) { + foundChild = true; + scoped_ptr holder(filterTree->getChild(i)); + filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i); + break; + } + } + verify(foundChild); + solnRoot->filter.reset(filterTree); + } + + solnRoot->numWanted = query.getParsed().getNumToReturn(); + if (0 == solnRoot->numWanted) { + solnRoot->numWanted = 100; + } + solnRoot->numWanted += query.getParsed().getSkip(); + solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern; + + // Remove the 2d index. 2d can only be the first field, and we know there is + // only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll + // only be first for gnNode. + tag->first.erase(tag->first.begin() + i); + + QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); + + if (NULL != soln) { + out->push_back(soln); + } + } + + // Continue planning w/non-2d indices tagged for this pred. + tag->first.swap(newFirst); + + if (0 == tag->first.size() && 0 == tag->notFirst.size()) { + // Don't leave tags on query tree. + query.root()->resetTag(); + return Status::OK(); + } + + QLOG() << "Rated tree after geonear processing:" << query.root()->toString(); + } + + // Likewise, if there is a TEXT it must have an index it can use directly. + MatchExpression* textNode = NULL; + if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { + RelevantTag* tag = static_cast(textNode->getTag()); + + // Exactly one text index required for TEXT. We need to check this explicitly because + // the text stage can't be built if no text index exists or there is an ambiguity as to + // which one to use. + size_t textIndexCount = 0; + for (size_t i = 0; i < params.indices.size(); i++) { + if (INDEX_TEXT == params.indices[i].type) { + textIndexCount++; + } + } + if (textIndexCount != 1) { + // Don't leave tags on query tree. + query.root()->resetTag(); + return Status(ErrorCodes::BadValue, "need exactly one text index for $text query"); + } + + // Error if the text node is tagged with zero indices. + if (0 == tag->first.size() && 0 == tag->notFirst.size()) { + // Don't leave tags on query tree. + query.root()->resetTag(); + return Status(ErrorCodes::BadValue, + "failed to use text index to satisfy $text query (if text index is " + "compound, are equality predicates given for all prefix fields?)"); + } + + // At this point, we know that there is only one text index and that the TEXT node is + // assigned to it. + invariant(1 == tag->first.size() + tag->notFirst.size()); + + QLOG() << "Rated tree after text processing:" << query.root()->toString(); + } + + // If we have any relevant indices, we try to create indexed plans. + if (0 < relevantIndices.size()) { + // The enumerator spits out trees tagged with IndexTag(s). + PlanEnumeratorParams enumParams; + enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION; + enumParams.root = query.root(); + enumParams.indices = &relevantIndices; + + PlanEnumerator isp(enumParams); + isp.init(); + + MatchExpression* rawTree; + while (isp.getNext(&rawTree) && (out->size() < params.maxIndexedSolutions)) { + QLOG() << "About to build solntree from tagged tree:" << endl + << rawTree->toString(); + + // The tagged tree produced by the plan enumerator is not guaranteed + // to be canonically sorted. In order to be compatible with the cached + // data, sort the tagged tree according to CanonicalQuery ordering. + boost::scoped_ptr clone(rawTree->shallowClone()); + CanonicalQuery::sortTree(clone.get()); + + PlanCacheIndexTree* cacheData; + Status indexTreeStatus = cacheDataFromTaggedTree(clone.get(), relevantIndices, &cacheData); + if (!indexTreeStatus.isOK()) { + QLOG() << "Query is not cachable: " << indexTreeStatus.reason() << endl; + } + auto_ptr autoData(cacheData); + + // This can fail if enumeration makes a mistake. + QuerySolutionNode* solnRoot = + QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices); + + if (NULL == solnRoot) { continue; } + + QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); + if (NULL != soln) { + QLOG() << "Planner: adding solution:" << endl << soln->toString(); + if (indexTreeStatus.isOK()) { + SolutionCacheData* scd = new SolutionCacheData(); + scd->tree.reset(autoData.release()); + soln->cacheData.reset(scd); + } + out->push_back(soln); + } + } + } + + // Don't leave tags on query tree. + query.root()->resetTag(); + + QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n"; + + // Produce legible error message for failed OR planning with a TEXT child. + // TODO: support collection scan for non-TEXT children of OR. + if (out->size() == 0 && textNode != NULL && + MatchExpression::OR == query.root()->matchType()) { + MatchExpression* root = query.root(); + for (size_t i = 0; i < root->numChildren(); ++i) { + if (textNode == root->getChild(i)) { + return Status(ErrorCodes::BadValue, + "Failed to produce a solution for TEXT under OR - " + "other non-TEXT clauses under OR have to be indexed as well."); + } + } + } + + // An index was hinted. If there are any solutions, they use the hinted index. If not, we + // scan the entire index to provide results and output that as our plan. This is the + // desired behavior when an index is hinted that is not relevant to the query. + if (!hintIndex.isEmpty()) { + if (0 == out->size()) { + QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); + verify(NULL != soln); + QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl; + out->push_back(soln); + } + return Status::OK(); + } + + // If a sort order is requested, there may be an index that provides it, even if that + // index is not over any predicates in the query. + // + if (!query.getParsed().getSort().isEmpty() + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { + + // See if we have a sort provided from an index already. + // This is implied by the presence of a non-blocking solution. + bool usingIndexToSort = false; + for (size_t i = 0; i < out->size(); ++i) { + QuerySolution* soln = (*out)[i]; + if (!soln->hasBlockingStage) { + usingIndexToSort = true; + break; + } + } + + if (!usingIndexToSort) { + for (size_t i = 0; i < params.indices.size(); ++i) { + const IndexEntry& index = params.indices[i]; + // Only regular (non-plugin) indexes can be used to provide a sort. + if (index.type != INDEX_BTREE) { + continue; + } + // Only non-sparse indexes can be used to provide a sort. + if (index.sparse) { + continue; + } + + // TODO: Sparse indexes can't normally provide a sort, because non-indexed + // documents could potentially be missing from the result set. However, if the + // query predicate can be used to guarantee that all documents to be returned + // are indexed, then the index should be able to provide the sort. + // + // For example: + // - Sparse index {a: 1, b: 1} should be able to provide a sort for + // find({b: 1}).sort({a: 1}). SERVER-13908. + // - Index {a: 1, b: "2dsphere"} (which is "geo-sparse", if + // 2dsphereIndexVersion=2) should be able to provide a sort for + // find({b: GEO}).sort({a:1}). SERVER-10801. + + const BSONObj kp = QueryPlannerAnalysis::getSortPattern(index.keyPattern); + if (providesSort(query, kp)) { + QLOG() << "Planner: outputting soln that uses index to provide sort." + << endl; + QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); + if (NULL != soln) { + PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); + indexTree->setIndexEntry(params.indices[i]); + SolutionCacheData* scd = new SolutionCacheData(); + scd->tree.reset(indexTree); + scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; + scd->wholeIXSolnDir = 1; + + soln->cacheData.reset(scd); + out->push_back(soln); + break; + } + } + if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { + QLOG() << "Planner: outputting soln that uses (reverse) index " + << "to provide sort." << endl; + QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); + if (NULL != soln) { + PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); + indexTree->setIndexEntry(params.indices[i]); + SolutionCacheData* scd = new SolutionCacheData(); + scd->tree.reset(indexTree); + scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; + scd->wholeIXSolnDir = -1; + + soln->cacheData.reset(scd); + out->push_back(soln); + break; + } + } + } + } + } + + // geoNear and text queries *require* an index. + // Also, if a hint is specified it indicates that we MUST use it. + bool possibleToCollscan = !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) + && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) + && hintIndex.isEmpty(); + + // The caller can explicitly ask for a collscan. + bool collscanRequested = (params.options & QueryPlannerParams::INCLUDE_COLLSCAN); + + // No indexed plans? We must provide a collscan if possible or else we can't run the query. + bool collscanNeeded = (0 == out->size() && canTableScan); + + if (possibleToCollscan && (collscanRequested || collscanNeeded)) { + QuerySolution* collscan = buildCollscanSoln(query, false, params); + if (NULL != collscan) { + SolutionCacheData* scd = new SolutionCacheData(); + scd->solnType = SolutionCacheData::COLLSCAN_SOLN; + collscan->cacheData.reset(scd); + out->push_back(collscan); + QLOG() << "Planner: outputting a collscan:" << endl + << collscan->toString(); + } + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner.h b/src/mongo/db/query/query_planner.h new file mode 100644 index 00000000000..a8534c69f62 --- /dev/null +++ b/src/mongo/db/query/query_planner.h @@ -0,0 +1,129 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + class CachedSolution; + + /** + * QueryPlanner's job is to provide an entry point to the query planning and optimization + * process. + */ + class QueryPlanner { + public: + /** + * Outputs a series of possible solutions for the provided 'query' into 'out'. Uses the + * indices and other data in 'params' to plan with. + * + * Caller owns pointers in *out. + */ + static Status plan(const CanonicalQuery& query, + const QueryPlannerParams& params, + std::vector* out); + + /** + * Helper that does most of the heavy lifting for the planFromCache + * method which this overloads. Whereas the overloaded version plans + * from cache twice (once for the winning solution and once from the + * backup solution), this version plans from cache once. + * + * It requires a single SolutionCacheData, rather than a CachedSolution, which + * owns a vector of SolutionCacheData instances. + */ + static Status planFromCache(const CanonicalQuery& query, + const QueryPlannerParams& params, + const SolutionCacheData& cacheData, + QuerySolution** out); + + /** + * Attempt to generate a query solution, given data retrieved + * from the plan cache. + * + * @param query -- query for which we are generating a plan + * @param params -- planning parameters + * @param cachedSoln -- the CachedSolution retrieved from the plan cache. + * @param out -- an out-parameter which will be filled in with the solution + * generated from the cache data + * @param backupOut -- if 'out' contains a blocking sort, then backoutOut may + * contain an alternative solution with no blocking sort; otherwise it will + * contain NULL on return. + */ + static Status planFromCache(const CanonicalQuery& query, + const QueryPlannerParams& params, + const CachedSolution& cachedSoln, + QuerySolution** out, + QuerySolution** backupOut); + + /** + * Used to generated the index tag tree that will be inserted + * into the plan cache. This data gets stashed inside a QuerySolution + * until it can be inserted into the cache proper. + * + * @param taggedTree -- a MatchExpression with index tags that has been + * produced by the enumerator. + * @param relevantIndices -- a list of the index entries used to tag + * the tree (i.e. index numbers in the tags refer to entries in this vector) + * + * On success, a new tagged tree is returned through the out-parameter 'out'. + * The caller has ownership of both taggedTree and *out. + * + * On failure, 'out' is set to NULL. + */ + static Status cacheDataFromTaggedTree(const MatchExpression* const taggedTree, + const vector& relevantIndices, + PlanCacheIndexTree** out); + + /** + * @param filter -- an untagged MatchExpression + * @param indexTree -- a tree structure retrieved from the + * cache with index tags that indicates how 'filter' should + * be tagged. + * @param indexMap -- needed in order to put the proper index + * numbers inside the index tags + * + * On success, 'filter' is mutated so that it has all the + * index tags needed in order for the access planner to recreate + * the cached plan. + * + * On failure, the tag state attached to the nodes of 'filter' + * is invalid. Planning from the cache should be aborted. + * + * Does not take ownership of either filter or indexTree. + */ + static Status tagAccordingToCache(MatchExpression* filter, + const PlanCacheIndexTree* const indexTree, + const map& indexMap); + }; + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner_common.h b/src/mongo/db/query/query_planner_common.h new file mode 100644 index 00000000000..e1c858ab98c --- /dev/null +++ b/src/mongo/db/query/query_planner_common.h @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/db/query/qlog.h" + +namespace mongo { + + /** + * Methods used by several parts of the planning process. + */ + class QueryPlannerCommon { + public: + /** + * Does the tree rooted at 'root' have a node with matchType 'type'? + * + * If 'out' is not NULL, sets 'out' to the first node of type 'type' encountered. + */ + static bool hasNode(MatchExpression* root, MatchExpression::MatchType type, + MatchExpression** out = NULL) { + if (type == root->matchType()) { + if (NULL != out) { + *out = root; + } + return true; + } + + for (size_t i = 0; i < root->numChildren(); ++i) { + if (hasNode(root->getChild(i), type, out)) { + return true; + } + } + return false; + } + + /** + * Assumes the provided BSONObj is of the form {field1: -+1, ..., field2: -+1} + * Returns a BSONObj with the values negated. + */ + static BSONObj reverseSortObj(const BSONObj& sortObj) { + BSONObjBuilder reverseBob; + BSONObjIterator it(sortObj); + while (it.more()) { + BSONElement elt = it.next(); + reverseBob.append(elt.fieldName(), elt.numberInt() * -1); + } + return reverseBob.obj(); + } + + /** + * Traverses the tree rooted at 'node'. For every STAGE_IXSCAN encountered, reverse + * the scan direction and index bounds. + */ + static void reverseScans(QuerySolutionNode* node) { + StageType type = node->getType(); + + if (STAGE_IXSCAN == type) { + IndexScanNode* isn = static_cast(node); + isn->direction *= -1; + + if (isn->bounds.isSimpleRange) { + std::swap(isn->bounds.startKey, isn->bounds.endKey); + // XXX: Not having a startKeyInclusive means that if we reverse a max/min query + // we have different results with and without the reverse... + isn->bounds.endKeyInclusive = true; + } + else { + for (size_t i = 0; i < isn->bounds.fields.size(); ++i) { + vector& iv = isn->bounds.fields[i].intervals; + // Step 1: reverse the list. + std::reverse(iv.begin(), iv.end()); + // Step 2: reverse each interval. + for (size_t j = 0; j < iv.size(); ++j) { + iv[j].reverse(); + } + } + } + + if (!isn->bounds.isValidFor(isn->indexKeyPattern, isn->direction)) { + QLOG() << "Invalid bounds: " << isn->bounds.toString() << endl; + verify(0); + } + + // TODO: we can just negate every value in the already computed properties. + isn->computeProperties(); + } + else if (STAGE_SORT_MERGE == type) { + // reverse direction of comparison for merge + MergeSortNode* msn = static_cast(node); + msn->sort = reverseSortObj(msn->sort); + } + else { + verify(STAGE_SORT != type); + // This shouldn't be here... + } + + for (size_t i = 0; i < node->children.size(); ++i) { + reverseScans(node->children[i]); + } + } + }; + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner_params.h b/src/mongo/db/query/query_planner_params.h new file mode 100644 index 00000000000..6d51291f1fb --- /dev/null +++ b/src/mongo/db/query/query_planner_params.h @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/jsobj.h" +#include "mongo/db/query/index_entry.h" +#include "mongo/db/query/query_knobs.h" + +namespace mongo { + + struct QueryPlannerParams { + + QueryPlannerParams() : options(DEFAULT), + indexFiltersApplied(false), + maxIndexedSolutions(internalQueryPlannerMaxIndexedSolutions) { } + + enum Options { + // You probably want to set this. + DEFAULT = 0, + + // Set this if you don't want a table scan. + // See http://docs.mongodb.org/manual/reference/parameters/ + NO_TABLE_SCAN = 1, + + // Set this if you *always* want a collscan outputted, even if there's an ixscan. This + // makes ranking less accurate, especially in the presence of blocking stages. + INCLUDE_COLLSCAN = 1 << 1, + + // Set this if you're running on a sharded cluster. We'll add a "drop all docs that + // shouldn't be on this shard" stage before projection. + // + // In order to set this, you must check + // shardingState.needCollectionMetadata(current_namespace) in the same lock that you use + // to build the query runner. You must also wrap the Runner in a ClientCursor within the + // same lock. See the comment on ShardFilterStage for details. + INCLUDE_SHARD_FILTER = 1 << 2, + + // Set this if you don't want any plans with a blocking sort stage. All sorts must be + // provided by an index. + NO_BLOCKING_SORT = 1 << 3, + + // Set this if you want to turn on index intersection. + INDEX_INTERSECTION = 1 << 4, + + // Set this if you want to try to keep documents deleted or mutated during the execution + // of the query in the query results. + KEEP_MUTATIONS = 1 << 5, + + // Nobody should set this above the getRunner interface. Internal flag set as a hint to + // the planner that the caller is actually the count command. + PRIVATE_IS_COUNT = 1 << 6, + + // Set this if you want to handle batchSize properly with sort(). If limits on SORT + // stages are always actually limits, then this should be left off. If they are + // sometimes to be interpreted as batchSize, then this should be turned on. + SPLIT_LIMITED_SORT = 1 << 7 + }; + + // See Options enum above. + size_t options; + + // What indices are available for planning? + vector indices; + + // What's our shard key? If INCLUDE_SHARD_FILTER is set we will create a shard filtering + // stage. If we know the shard key, we can perform covering analysis instead of always + // forcing a fetch. + BSONObj shardKey; + + // Were index filters applied to indices? + bool indexFiltersApplied; + + // What's the max number of indexed solutions we want to output? It's expensive to compare + // plans via the MultiPlanRunner, and the set of possible plans is very large for certain + // index+query combinations. + size_t maxIndexedSolutions; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner_test.cpp b/src/mongo/db/query/query_planner_test.cpp new file mode 100644 index 00000000000..b4bdac93a50 --- /dev/null +++ b/src/mongo/db/query/query_planner_test.cpp @@ -0,0 +1,4778 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/query_planner.cpp + */ + +#include "mongo/db/query/query_planner_test_lib.h" + +#include +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_knobs.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +using namespace mongo; + +namespace { + + static const char* ns = "somebogusns"; + + class QueryPlannerTest : public mongo::unittest::Test { + protected: + void setUp() { + internalQueryPlannerEnableHashIntersection = true; + params.options = QueryPlannerParams::INCLUDE_COLLSCAN; + addIndex(BSON("_id" << 1)); + } + + void tearDown() { + delete cq; + + for (vector::iterator it = solns.begin(); it != solns.end(); ++it) { + delete *it; + } + } + + // + // Build up test. + // + + void addIndex(BSONObj keyPattern, bool multikey = false) { + // The first false means not multikey. + // The second false means not sparse. + params.indices.push_back(IndexEntry(keyPattern, + multikey, + false, + false, + "hari_king_of_the_stove", + BSONObj())); + } + + void addIndex(BSONObj keyPattern, bool multikey, bool sparse) { + params.indices.push_back(IndexEntry(keyPattern, + multikey, + sparse, + false, + "note_to_self_dont_break_build", + BSONObj())); + } + + void addIndex(BSONObj keyPattern, bool multikey, bool sparse, bool unique) { + params.indices.push_back(IndexEntry(keyPattern, + multikey, + sparse, + unique, + "sql_query_walks_into_bar_and_says_can_i_join_you?", + BSONObj())); + } + + void addIndex(BSONObj keyPattern, BSONObj infoObj) { + params.indices.push_back(IndexEntry(keyPattern, false, false, false, "foo", infoObj)); + } + + // + // Execute planner. + // + + void runQuery(BSONObj query) { + runQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), 0, 0); + } + + void runQuerySortProj(const BSONObj& query, const BSONObj& sort, const BSONObj& proj) { + runQuerySortProjSkipLimit(query, sort, proj, 0, 0); + } + + void runQuerySkipLimit(const BSONObj& query, long long skip, long long limit) { + runQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), skip, limit); + } + + void runQueryHint(const BSONObj& query, const BSONObj& hint) { + runQuerySortProjSkipLimitHint(query, BSONObj(), BSONObj(), 0, 0, hint); + } + + void runQuerySortProjSkipLimit(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit) { + runQuerySortProjSkipLimitHint(query, sort, proj, skip, limit, BSONObj()); + } + + void runQuerySortHint(const BSONObj& query, const BSONObj& sort, const BSONObj& hint) { + runQuerySortProjSkipLimitHint(query, sort, BSONObj(), 0, 0, hint); + } + + void runQueryHintMinMax(const BSONObj& query, const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj) { + + runQueryFull(query, BSONObj(), BSONObj(), 0, 0, hint, minObj, maxObj, false); + } + + void runQuerySortProjSkipLimitHint(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint) { + runQueryFull(query, sort, proj, skip, limit, hint, BSONObj(), BSONObj(), false); + } + + void runQuerySnapshot(const BSONObj& query) { + runQueryFull(query, BSONObj(), BSONObj(), 0, 0, BSONObj(), BSONObj(), + BSONObj(), true); + } + + void runQueryFull(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + const BSONObj& minObj, + const BSONObj& maxObj, + bool snapshot) { + solns.clear(); + Status s = CanonicalQuery::canonicalize(ns, query, sort, proj, skip, limit, hint, + minObj, maxObj, snapshot, + false, // explain + &cq); + if (!s.isOK()) { cq = NULL; } + ASSERT_OK(s); + s = QueryPlanner::plan(*cq, params, &solns); + ASSERT_OK(s); + } + + /** + * Same as runQuery* functions except we expect a failed status from the planning stage. + */ + void runInvalidQuery(const BSONObj& query) { + runInvalidQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), 0, 0); + } + + void runInvalidQuerySortProj(const BSONObj& query, const BSONObj& sort, + const BSONObj& proj) { + runInvalidQuerySortProjSkipLimit(query, sort, proj, 0, 0); + } + + void runInvalidQuerySortProjSkipLimit(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit) { + runInvalidQuerySortProjSkipLimitHint(query, sort, proj, skip, limit, BSONObj()); + } + + void runInvalidQueryHint(const BSONObj& query, const BSONObj& hint) { + runInvalidQuerySortProjSkipLimitHint(query, BSONObj(), BSONObj(), 0, 0, hint); + } + + void runInvalidQueryHintMinMax(const BSONObj& query, const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj) { + runInvalidQueryFull(query, BSONObj(), BSONObj(), 0, 0, hint, minObj, maxObj, false); + } + + void runInvalidQuerySortProjSkipLimitHint(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint) { + runInvalidQueryFull(query, sort, proj, skip, limit, hint, BSONObj(), BSONObj(), false); + } + + void runInvalidQueryFull(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + const BSONObj& minObj, + const BSONObj& maxObj, + bool snapshot) { + solns.clear(); + Status s = CanonicalQuery::canonicalize(ns, query, sort, proj, skip, limit, hint, + minObj, maxObj, snapshot, + false, // explain + &cq); + if (!s.isOK()) { cq = NULL; } + ASSERT_OK(s); + s = QueryPlanner::plan(*cq, params, &solns); + ASSERT_NOT_OK(s); + } + + // + // Introspect solutions. + // + + size_t getNumSolutions() const { + return solns.size(); + } + + void dumpSolutions() { + mongoutils::str::stream ost; + dumpSolutions(ost); + log() << string(ost); + } + + void dumpSolutions(mongoutils::str::stream& ost) const { + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + ost << (*it)->toString() << '\n'; + } + } + + /** + * Checks number solutions. Generates assertion message + * containing solution dump if applicable. + */ + void assertNumSolutions(size_t expectSolutions) const { + if (getNumSolutions() == expectSolutions) { + return; + } + mongoutils::str::stream ss; + ss << "expected " << expectSolutions << " solutions but got " << getNumSolutions() + << " instead. solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + } + + size_t numSolutionMatches(const string& solnJson) const { + BSONObj testSoln = fromjson(solnJson); + size_t matches = 0; + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + QuerySolutionNode* root = (*it)->root.get(); + if (QueryPlannerTestLib::solutionMatches(testSoln, root)) { + ++matches; + } + } + return matches; + } + + /** + * Verifies that the solution tree represented in json by 'solnJson' is + * one of the solutions generated by QueryPlanner. + * + * The number of expected matches, 'numMatches', could be greater than + * 1 if solutions differ only by the pattern of index tags on a filter. + */ + void assertSolutionExists(const string& solnJson, size_t numMatches = 1) const { + size_t matches = numSolutionMatches(solnJson); + if (numMatches == matches) { + return; + } + mongoutils::str::stream ss; + ss << "expected " << numMatches << " matches for solution " << solnJson + << " but got " << matches + << " instead. all solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + } + + /** + * Given a vector of string-based solution tree representations 'solnStrs', + * verifies that the query planner generated exactly one of these solutions. + */ + void assertHasOneSolutionOf(const vector& solnStrs) const { + size_t matches = 0; + for (vector::const_iterator it = solnStrs.begin(); + it != solnStrs.end(); + ++it) { + if (1U == numSolutionMatches(*it)) { + ++matches; + } + } + if (1U == matches) { + return; + } + mongoutils::str::stream ss; + ss << "assertHasOneSolutionOf expected one matching solution" + << " but got " << matches + << " instead. all solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + } + + BSONObj queryObj; + CanonicalQuery* cq; + QueryPlannerParams params; + vector solns; + }; + + // + // Equality + // + + TEST_F(QueryPlannerTest, EqualityIndexScan) { + addIndex(BSON("x" << 1)); + + runQuery(BSON("x" << 5)); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: 5}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, EqualityIndexScanWithTrailingFields) { + addIndex(BSON("x" << 1 << "y" << 1)); + + runQuery(BSON("x" << 5)); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: 5}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {x: 1, y: 1}}}}}"); + } + + // + // indexFilterApplied + // Check that index filter flag is passed from planner params + // to generated query solution. + // + + TEST_F(QueryPlannerTest, IndexFilterAppliedDefault) { + addIndex(BSON("x" << 1)); + + runQuery(BSON("x" << 5)); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: 5}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {x: 1}}}}}"); + + // Check indexFilterApplied in query solutions; + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + QuerySolution* soln = *it; + ASSERT_FALSE(soln->indexFilterApplied); + } + } + + TEST_F(QueryPlannerTest, IndexFilterAppliedTrue) { + params.indexFiltersApplied = true; + + addIndex(BSON("x" << 1)); + + runQuery(BSON("x" << 5)); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: 5}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {x: 1}}}}}"); + + // Check indexFilterApplied in query solutions; + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + QuerySolution* soln = *it; + ASSERT_EQUALS(params.indexFiltersApplied, soln->indexFilterApplied); + } + } + + // + // < + // + + TEST_F(QueryPlannerTest, LessThan) { + addIndex(BSON("x" << 1)); + + runQuery(BSON("x" << BSON("$lt" << 5))); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: {$lt: 5}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {x: 1}}}}}"); + } + + // + // <= + // + + TEST_F(QueryPlannerTest, LessThanEqual) { + addIndex(BSON("x" << 1)); + + runQuery(BSON("x" << BSON("$lte" << 5))); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: {$lte: 5}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + // + // > + // + + TEST_F(QueryPlannerTest, GreaterThan) { + addIndex(BSON("x" << 1)); + + runQuery(BSON("x" << BSON("$gt" << 5))); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: {$gt: 5}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + // + // >= + // + + TEST_F(QueryPlannerTest, GreaterThanEqual) { + addIndex(BSON("x" << 1)); + + runQuery(BSON("x" << BSON("$gte" << 5))); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: {$gte: 5}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + // + // Mod + // + + TEST_F(QueryPlannerTest, Mod) { + addIndex(BSON("a" << 1)); + + runQuery(fromjson("{a: {$mod: [2, 0]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a: {$mod: [2, 0]}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: {a: {$mod: [2, 0]}}, pattern: {a: 1}}}}}"); + } + + // + // Exists + // + + TEST_F(QueryPlannerTest, ExistsTrue) { + addIndex(BSON("x" << 1)); + + runQuery(fromjson("{x: {$exists: true}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ExistsFalse) { + addIndex(BSON("x" << 1)); + + runQuery(fromjson("{x: {$exists: false}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ExistsTrueSparseIndex) { + addIndex(BSON("x" << 1), false, true); + + runQuery(fromjson("{x: {$exists: true}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ExistsFalseSparseIndex) { + addIndex(BSON("x" << 1), false, true); + + runQuery(fromjson("{x: {$exists: false}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + TEST_F(QueryPlannerTest, ExistsTrueOnUnindexedField) { + addIndex(BSON("x" << 1)); + + runQuery(fromjson("{x: 1, y: {$exists: true}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ExistsFalseOnUnindexedField) { + addIndex(BSON("x" << 1)); + + runQuery(fromjson("{x: 1, y: {$exists: false}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ExistsTrueSparseIndexOnOtherField) { + addIndex(BSON("x" << 1), false, true); + + runQuery(fromjson("{x: 1, y: {$exists: true}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ExistsFalseSparseIndexOnOtherField) { + addIndex(BSON("x" << 1), false, true); + + runQuery(fromjson("{x: 1, y: {$exists: false}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1}}}}}"); + } + + // + // skip and limit + // + + TEST_F(QueryPlannerTest, BasicSkipNoIndex) { + addIndex(BSON("a" << 1)); + + runQuerySkipLimit(BSON("x" << 5), 3, 0); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{skip: {n: 3, node: {cscan: {dir: 1, filter: {x: 5}}}}}"); + } + + TEST_F(QueryPlannerTest, BasicSkipWithIndex) { + addIndex(BSON("a" << 1 << "b" << 1)); + + runQuerySkipLimit(BSON("a" << 5), 8, 0); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{skip: {n: 8, node: {cscan: {dir: 1, filter: {a: 5}}}}}"); + assertSolutionExists("{skip: {n: 8, node: {fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a: 1, b: 1}}}}}}}"); + } + + TEST_F(QueryPlannerTest, BasicLimitNoIndex) { + addIndex(BSON("a" << 1)); + + runQuerySkipLimit(BSON("x" << 5), 0, -3); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{limit: {n: 3, node: {cscan: {dir: 1, filter: {x: 5}}}}}"); + } + + TEST_F(QueryPlannerTest, BasicSoftLimitNoIndex) { + addIndex(BSON("a" << 1)); + + runQuerySkipLimit(BSON("x" << 5), 0, 3); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{cscan: {dir: 1, filter: {x: 5}}}"); + } + + TEST_F(QueryPlannerTest, BasicLimitWithIndex) { + addIndex(BSON("a" << 1 << "b" << 1)); + + runQuerySkipLimit(BSON("a" << 5), 0, -5); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{limit: {n: 5, node: {cscan: {dir: 1, filter: {a: 5}}}}}"); + assertSolutionExists("{limit: {n: 5, node: {fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a: 1, b: 1}}}}}}}"); + } + + TEST_F(QueryPlannerTest, BasicSoftLimitWithIndex) { + addIndex(BSON("a" << 1 << "b" << 1)); + + runQuerySkipLimit(BSON("a" << 5), 0, 5); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a: 5}}}}"); + assertSolutionExists("{fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a: 1, b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, SkipAndLimit) { + addIndex(BSON("x" << 1)); + + runQuerySkipLimit(BSON("x" << BSON("$lte" << 4)), 7, -2); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{limit: {n: 2, node: {skip: {n: 7, node: " + "{cscan: {dir: 1, filter: {x: {$lte: 4}}}}}}}}"); + assertSolutionExists("{limit: {n: 2, node: {skip: {n: 7, node: {fetch: " + "{filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}}}}}"); + } + + TEST_F(QueryPlannerTest, SkipAndSoftLimit) { + addIndex(BSON("x" << 1)); + + runQuerySkipLimit(BSON("x" << BSON("$lte" << 4)), 7, 2); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{skip: {n: 7, node: " + "{cscan: {dir: 1, filter: {x: {$lte: 4}}}}}}"); + assertSolutionExists("{skip: {n: 7, node: {fetch: " + "{filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}}}"); + } + + // + // tree operations + // + + TEST_F(QueryPlannerTest, TwoPredicatesAnding) { + addIndex(BSON("x" << 1)); + + runQuery(fromjson("{$and: [ {x: {$gt: 1}}, {x: {$lt: 3}} ] }")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, SimpleOr) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a: 20}, {a: 21}]}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$or: [{a: 20}, {a: 21}]}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a:1}}}}}"); + } + + TEST_F(QueryPlannerTest, OrWithoutEnoughIndices) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a: 20}, {b: 21}]}")); + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{cscan: {dir: 1, filter: {$or: [{a: 20}, {b: 21}]}}}"); + } + + TEST_F(QueryPlannerTest, OrWithAndChild) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a: 20}, {$and: [{a:1}, {b:7}]}]}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a: 1}}}, " + "{fetch: {filter: {b: 7}, node: {ixscan: " + "{filter: null, pattern: {a: 1}}}}}]}}}}"); + } + + TEST_F(QueryPlannerTest, AndWithUnindexedOrChild) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{a:20, $or: [{b:1}, {c:7}]}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + + // Logical rewrite means we could get one of these two outcomes: + size_t matches = 0; + matches += numSolutionMatches("{fetch: {filter: {$or: [{b: 1}, {c: 7}]}, node: " + "{ixscan: {filter: null, pattern: {a: 1}}}}}"); + matches += numSolutionMatches("{or: {filter: null, nodes: [" + "{fetch: {filter: {b:1}, node: {" + "ixscan: {filter: null, pattern: {a:1}}}}}," + "{fetch: {filter: {c:7}, node: {" + "ixscan: {filter: null, pattern: {a:1}}}}}]}}"); + ASSERT_GREATER_THAN_OR_EQUALS(matches, 1U); + } + + + TEST_F(QueryPlannerTest, AndWithOrWithOneIndex) { + addIndex(BSON("b" << 1)); + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{b:1}, {c:7}], a:20}")); + + // Logical rewrite gives us at least one of these: + assertSolutionExists("{cscan: {dir: 1}}"); + size_t matches = 0; + matches += numSolutionMatches("{fetch: {filter: {$or: [{b: 1}, {c: 7}]}, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + matches += numSolutionMatches("{or: {filter: null, nodes: [" + "{fetch: {filter: {b:1}, node: {" + "ixscan: {filter: null, pattern: {a:1}}}}}," + "{fetch: {filter: {c:7}, node: {" + "ixscan: {filter: null, pattern: {a:1}}}}}]}}"); + ASSERT_GREATER_THAN_OR_EQUALS(matches, 1U); + } + + // + // Additional $or tests + // + + TEST_F(QueryPlannerTest, OrCollapsesToSingleScan) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a:{$gt:2}}, {a:{$gt:0}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [[0,Infinity,false,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, OrCollapsesToSingleScan2) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a:{$lt:2}}, {a:{$lt:4}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [[-Infinity,4,true,false]]}}}}}"); + } + + TEST_F(QueryPlannerTest, OrCollapsesToSingleScan3) { + addIndex(BSON("a" << 1)); + runQueryHint(fromjson("{$or: [{a:1},{a:3}]}"), fromjson("{a:1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [[1,1,true,true], [3,3,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, OrOnlyOneBranchCanUseIndex) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a:1}, {b:2}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + TEST_F(QueryPlannerTest, OrOnlyOneBranchCanUseIndexHinted) { + addIndex(BSON("a" << 1)); + runQueryHint(fromjson("{$or: [{a:1}, {b:2}]}"), fromjson("{a:1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {$or:[{a:1},{b:2}]}, node: {ixscan: " + "{pattern: {a:1}, bounds: " + "{a: [['MinKey','MaxKey',true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, OrNaturalHint) { + addIndex(BSON("a" << 1)); + runQueryHint(fromjson("{$or: [{a:1}, {a:3}]}"), fromjson("{$natural:1}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // SERVER-13714. A non-top-level indexable negation exposed a bug in plan enumeration. + TEST_F(QueryPlannerTest, NonTopLevelIndexedNegation) { + addIndex(BSON("state" << 1)); + addIndex(BSON("is_draft" << 1)); + addIndex(BSON("published_date" << 1)); + addIndex(BSON("newsroom_id" << 1)); + + BSONObj queryObj = fromjson("{$and:[{$or:[{is_draft:false},{creator_id:1}]}," + "{$or:[{state:3,is_draft:false}," + "{published_date:{$ne:null}}]}," + "{newsroom_id:{$in:[1]}}]}"); + runQuery(queryObj); + } + + TEST_F(QueryPlannerTest, NonTopLevelIndexedNegationMinQuery) { + addIndex(BSON("state" << 1)); + addIndex(BSON("is_draft" << 1)); + addIndex(BSON("published_date" << 1)); + + // This is the min query to reproduce SERVER-13714 + BSONObj queryObj = fromjson("{$or:[{state:1, is_draft:1}, {published_date:{$ne: 1}}]}"); + runQuery(queryObj); + } + + // SERVER-12594: we don't yet collapse an OR of ANDs into a single ixscan. + TEST_F(QueryPlannerTest, OrOfAnd) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a:{$gt:2,$lt:10}}, {a:{$gt:0,$lt:5}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {pattern: {a:1}, bounds: {a: [[2,10,false,false]]}}}, " + "{ixscan: {pattern: {a:1}, bounds: " + "{a: [[0,5,false,false]]}}}]}}}}"); + } + + // SERVER-12594: we don't yet collapse an OR of ANDs into a single ixscan. + TEST_F(QueryPlannerTest, OrOfAnd2) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a:{$gt:2,$lt:10}}, {a:{$gt:0,$lt:15}}, {a:{$gt:20}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {pattern: {a:1}, bounds: {a: [[2,10,false,false]]}}}, " + "{ixscan: {pattern: {a:1}, bounds: {a: [[0,15,false,false]]}}}, " + "{ixscan: {pattern: {a:1}, bounds: " + "{a: [[20,Infinity,false,true]]}}}]}}}}"); + } + + // SERVER-12594: we don't yet collapse an OR of ANDs into a single ixscan. + TEST_F(QueryPlannerTest, OrOfAnd3) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a:{$gt:1,$lt:5},b:6}, {a:3,b:{$gt:0,$lt:10}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{or: {nodes: [" + "{fetch: {filter: {b:6}, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [[1,5,false,false]]}}}}}, " + "{fetch: {filter: {$and:[{b:{$lt:10}},{b:{$gt:0}}]}, node: " + "{ixscan: {pattern: {a:1}, bounds: {a:[[3,3,true,true]]}}}}}]}}"); + } + + // SERVER-12594: we don't yet collapse an OR of ANDs into a single ixscan. + TEST_F(QueryPlannerTest, OrOfAnd4) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{$or: [{a:{$gt:1,$lt:5}, b:{$gt:0,$lt:3}, c:6}, " + "{a:3, b:{$gt:1,$lt:2}, c:{$gt:0,$lt:10}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{or: {nodes: [" + "{fetch: {filter: {c:6}, node: {ixscan: {pattern: {a:1,b:1}, " + "bounds: {a: [[1,5,false,false]], b: [[0,3,false,false]]}}}}}, " + "{fetch: {filter: {$and:[{c:{$lt:10}},{c:{$gt:0}}]}, node: " + "{ixscan: {pattern: {a:1,b:1}, " + " bounds: {a:[[3,3,true,true]], b:[[1,2,false,false]]}}}}}]}}"); + } + + // SERVER-12594: we don't yet collapse an OR of ANDs into a single ixscan. + TEST_F(QueryPlannerTest, OrOfAnd5) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{$or: [{a:{$gt:1,$lt:5}, c:6}, " + "{a:3, b:{$gt:1,$lt:2}, c:{$gt:0,$lt:10}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{or: {nodes: [" + "{fetch: {filter: {c:6}, node: {ixscan: {pattern: {a:1,b:1}, " + "bounds: {a: [[1,5,false,false]], " + "b: [['MinKey','MaxKey',true,true]]}}}}}, " + "{fetch: {filter: {$and:[{c:{$lt:10}},{c:{$gt:0}}]}, node: " + "{ixscan: {pattern: {a:1,b:1}, " + " bounds: {a:[[3,3,true,true]], b:[[1,2,false,false]]}}}}}]}}"); + } + + // SERVER-12594: we don't yet collapse an OR of ANDs into a single ixscan. + TEST_F(QueryPlannerTest, OrOfAnd6) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{$or: [{a:{$in:[1]},b:{$in:[1]}}, {a:{$in:[1,5]},b:{$in:[1,5]}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {pattern: {a:1,b:1}, bounds: " + "{a: [[1,1,true,true]], b: [[1,1,true,true]]}}}, " + "{ixscan: {pattern: {a:1,b:1}, bounds: " + "{a: [[1,1,true,true], [5,5,true,true]], " + " b: [[1,1,true,true], [5,5,true,true]]}}}]}}}}"); + } + + // SERVER-13960: properly handle $or with a mix of exact and inexact predicates. + TEST_F(QueryPlannerTest, OrInexactWithExact) { + addIndex(BSON("name" << 1)); + runQuery(fromjson("{$or: [{name: 'thomas'}, {name: /^alexand(er|ra)/}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {filter:" + "{$or: [{name: 'thomas'}, {name: /^alexand(er|ra)/}]}," + "pattern: {name: 1}}}}}"); + } + + // SERVER-13960 + TEST_F(QueryPlannerTest, OrInexactWithExact2) { + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuery(fromjson("{$or: [{a: 'foo'}, {a: /bar/}, {b: 'foo'}, {b: /bar/}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {or: {nodes: [" + "{ixscan: {filter: {$or:[{a:'foo'},{a:/bar/}]}," + "pattern: {a: 1}}}," + "{ixscan: {filter: {$or:[{b:'foo'},{b:/bar/}]}," + "pattern: {b: 1}}}]}}}}"); + } + + // + // Min/Max + // + + TEST_F(QueryPlannerTest, MinValid) { + addIndex(BSON("a" << 1)); + runQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), BSONObj()); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, MinWithoutIndex) { + runInvalidQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), BSONObj()); + } + + TEST_F(QueryPlannerTest, MinBadHint) { + addIndex(BSON("b" << 1)); + runInvalidQueryHintMinMax(BSONObj(), fromjson("{b: 1}"), fromjson("{a: 1}"), BSONObj()); + } + + TEST_F(QueryPlannerTest, MaxValid) { + addIndex(BSON("a" << 1)); + runQueryHintMinMax(BSONObj(), BSONObj(), BSONObj(), fromjson("{a: 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, MinMaxSameValue) { + addIndex(BSON("a" << 1)); + runQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), fromjson("{a: 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, MaxWithoutIndex) { + runInvalidQueryHintMinMax(BSONObj(), BSONObj(), BSONObj(), fromjson("{a: 1}")); + } + + TEST_F(QueryPlannerTest, MaxBadHint) { + addIndex(BSON("b" << 1)); + runInvalidQueryHintMinMax(BSONObj(), fromjson("{b: 1}"), BSONObj(), fromjson("{a: 1}")); + } + + TEST_F(QueryPlannerTest, MaxMinSort) { + addIndex(BSON("a" << 1)); + + // Run an empty query, sort {a: 1}, max/min arguments. + runQueryFull(BSONObj(), fromjson("{a: 1}"), BSONObj(), 0, 0, BSONObj(), + fromjson("{a: 2}"), fromjson("{a: 8}"), false); + + assertNumSolutions(1); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, MaxMinReverseSort) { + addIndex(BSON("a" << 1)); + + // Run an empty query, sort {a: -1}, max/min arguments. + runQueryFull(BSONObj(), fromjson("{a: -1}"), BSONObj(), 0, 0, BSONObj(), + fromjson("{a: 2}"), fromjson("{a: 8}"), false); + + assertNumSolutions(1); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, dir: -1, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, MaxMinReverseIndexDir) { + addIndex(BSON("a" << -1)); + + // Because the index is descending, the min is numerically larger than the max. + runQueryFull(BSONObj(), fromjson("{a: -1}"), BSONObj(), 0, 0, BSONObj(), + fromjson("{a: 8}"), fromjson("{a: 2}"), false); + + assertNumSolutions(1); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, dir: 1, pattern: {a: -1}}}}}"); + } + + TEST_F(QueryPlannerTest, MaxMinReverseIndexDirSort) { + addIndex(BSON("a" << -1)); + + // Min/max specifies a forward scan with bounds [{a: 8}, {a: 2}]. Asking for + // an ascending sort reverses the direction of the scan to [{a: 2}, {a: 8}]. + runQueryFull(BSONObj(), fromjson("{a: 1}"), BSONObj(), 0, 0, BSONObj(), + fromjson("{a: 8}"), fromjson("{a: 2}"), false); + + assertNumSolutions(1); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, dir: -1," + "pattern: {a: -1}}}}}"); + } + + TEST_F(QueryPlannerTest, MaxMinNoMatchingIndexDir) { + addIndex(BSON("a" << -1)); + runInvalidQueryHintMinMax(BSONObj(), fromjson("{a: 2}"), BSONObj(), fromjson("{a: 8}")); + } + + TEST_F(QueryPlannerTest, MaxMinSelectCorrectlyOrderedIndex) { + // There are both ascending and descending indices on 'a'. + addIndex(BSON("a" << 1)); + addIndex(BSON("a" << -1)); + + // The ordering of min and max means that we *must* use the descending index. + runQueryFull(BSONObj(), BSONObj(), BSONObj(), 0, 0, BSONObj(), + fromjson("{a: 8}"), fromjson("{a: 2}"), false); + + assertNumSolutions(1); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, dir: 1, pattern: {a: -1}}}}}"); + + // If we switch the ordering, then we use the ascending index. + // The ordering of min and max means that we *must* use the descending index. + runQueryFull(BSONObj(), BSONObj(), BSONObj(), 0, 0, BSONObj(), + fromjson("{a: 2}"), fromjson("{a: 8}"), false); + + assertNumSolutions(1); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, dir: 1, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, MaxMinBadHintSelectsReverseIndex) { + // There are both ascending and descending indices on 'a'. + addIndex(BSON("a" << 1)); + addIndex(BSON("a" << -1)); + + // A query hinting on {a: 1} is bad if min is {a: 8} and {a: 2} because this + // min/max pairing requires a descending index. + runInvalidQueryFull(BSONObj(), BSONObj(), BSONObj(), 0, 0, fromjson("{a: 1}"), + fromjson("{a: 8}"), fromjson("{a: 2}"), false); + } + + + // + // $snapshot + // + + TEST_F(QueryPlannerTest, Snapshot) { + addIndex(BSON("a" << 1)); + runQuerySnapshot(fromjson("{a: {$gt: 0}}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a:{$gt:0}}, node: " + "{ixscan: {filter: null, pattern: {_id: 1}}}}}"); + } + + // + // Tree operations that require simple tree rewriting. + // + + TEST_F(QueryPlannerTest, AndOfAnd) { + addIndex(BSON("x" << 1)); + runQuery(fromjson("{$and: [ {$and: [ {x: 2.5}]}, {x: {$gt: 1}}, {x: {$lt: 3}} ] }")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + // + // Logically equivalent queries + // + + TEST_F(QueryPlannerTest, EquivalentAndsOne) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{$and: [{a: 1}, {b: {$all: [10, 20]}}]}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$and:[{a:1},{b:10},{b:20}]}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1, b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, EquivalentAndsTwo) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{$and: [{a: 1, b: 10}, {a: 1, b: 20}]}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$and:[{a:1},{a:1},{b:10},{b:20}]}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1, b: 1}}}}}"); + } + + // + // Covering + // + + TEST_F(QueryPlannerTest, BasicCovering) { + addIndex(BSON("x" << 1)); + // query, sort, proj + runQuerySortProj(fromjson("{ x : {$gt: 1}}"), BSONObj(), fromjson("{_id: 0, x: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, x: 1}, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, x: 1}, node: " + "{cscan: {dir: 1, filter: {x:{$gt:1}}}}}}"); + } + + TEST_F(QueryPlannerTest, DottedFieldCovering) { + addIndex(BSON("a.b" << 1)); + runQuerySortProj(fromjson("{'a.b': 5}"), BSONObj(), fromjson("{_id: 0, 'a.b': 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, 'a.b': 1}, node: " + "{cscan: {dir: 1, filter: {'a.b': 5}}}}}"); + // SERVER-2104 + //assertSolutionExists("{proj: {spec: {_id: 0, 'a.b': 1}, node: {'a.b': 1}}}"); + } + + TEST_F(QueryPlannerTest, IdCovering) { + runQuerySortProj(fromjson("{_id: {$gt: 10}}"), BSONObj(), fromjson("{_id: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 1}, node: " + "{cscan: {dir: 1, filter: {_id: {$gt: 10}}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 1}, node: {ixscan: " + "{filter: null, pattern: {_id: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ProjNonCovering) { + addIndex(BSON("x" << 1)); + runQuerySortProj(fromjson("{ x : {$gt: 1}}"), BSONObj(), fromjson("{x: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {x: 1}, node: {cscan: " + "{dir: 1, filter: {x: {$gt: 1}}}}}}"); + assertSolutionExists("{proj: {spec: {x: 1}, node: {fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {x: 1}}}}}}}"); + } + + // + // Basic sort + // + + TEST_F(QueryPlannerTest, BasicSort) { + addIndex(BSON("x" << 1)); + runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {}}}}}"); + } + + TEST_F(QueryPlannerTest, CantUseHashedIndexToProvideSort) { + addIndex(BSON("x" << "hashed")); + runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {}}}}}"); + } + + TEST_F(QueryPlannerTest, CantUseHashedIndexToProvideSortWithIndexablePred) { + addIndex(BSON("x" << "hashed")); + runQuerySortProj(BSON("x" << BSON("$in" << BSON_ARRAY(0 << 1))), BSON("x" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, node: " + "{fetch: {node: " + "{ixscan: {pattern: {x: 'hashed'}}}}}}}"); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, node: " + "{cscan: {dir: 1, filter: {x: {$in: [0, 1]}}}}}}"); + } + + + TEST_F(QueryPlannerTest, CantUseTextIndexToProvideSort) { + addIndex(BSON("x" << 1 << "_fts" << "text" << "_ftsx" << 1)); + runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {}}}}}"); + } + + TEST_F(QueryPlannerTest, CantUseNonCompoundGeoIndexToProvideSort) { + addIndex(BSON("x" << "2dsphere")); + runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {}}}}}"); + } + + TEST_F(QueryPlannerTest, CantUseNonCompoundGeoIndexToProvideSortWithIndexablePred) { + addIndex(BSON("x" << "2dsphere")); + runQuerySortProj(fromjson("{x: {$geoIntersects: {$geometry: {type: 'Point'," + " coordinates: [0, 0]}}}}"), + BSON("x" << 1), + BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, node: " + "{fetch: {node: " + "{ixscan: {pattern: {x: '2dsphere'}}}}}}}"); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, node: " + "{cscan: {dir: 1}}}}"); + } + + TEST_F(QueryPlannerTest, CantUseCompoundGeoIndexToProvideSortIfNoGeoPred) { + addIndex(BSON("x" << 1 << "y" << "2dsphere")); + runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {}}}}}"); + } + + TEST_F(QueryPlannerTest, CanUseCompoundGeoIndexToProvideSortWithGeoPred) { + addIndex(BSON("x" << 1 << "y" << "2dsphere")); + runQuerySortProj(fromjson("{x: 1, y: {$geoIntersects: {$geometry: {type: 'Point'," + " coordinates: [0, 0]}}}}"), + BSON("x" << 1), + BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{fetch: {node: " + "{ixscan: {pattern: {x: 1, y: '2dsphere'}}}}}"); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, node: " + "{cscan: {dir: 1}}}}"); + } + + TEST_F(QueryPlannerTest, BasicSortWithIndexablePred) { + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuerySortProj(fromjson("{ a : 5 }"), BSON("b" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 3U); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {a: 5}}}}}"); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, " + "node: {fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a: 1}}}}}}}"); + assertSolutionExists("{fetch: {filter: {a: 5}, node: {ixscan: " + "{filter: null, pattern: {b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, BasicSortBooleanIndexKeyPattern) { + addIndex(BSON("a" << true)); + runQuerySortProj(fromjson("{ a : 5 }"), BSON("a" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {a: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {a: 5}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: true}}}}}"); + } + + // SERVER-14070 + TEST_F(QueryPlannerTest, CompoundIndexWithEqualityPredicatesProvidesSort) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProj(fromjson("{a: 1, b: 1}"), fromjson("{b: 1}"), BSONObj()); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {filter: null," + "pattern: {a: 1, b: 1}, " + "bounds: {a:[[1,1,true,true]], b:[[1,1,true,true]]}}}}}"); + } + + // + // Sort with limit and/or skip + // + + TEST_F(QueryPlannerTest, SortLimit) { + // Negative limit indicates hard limit - see lite_parsed_query.cpp + runQuerySortProjSkipLimit(BSONObj(), fromjson("{a: 1}"), BSONObj(), 0, -3); + assertNumSolutions(1U); + assertSolutionExists("{sort: {pattern: {a: 1}, limit: 3, " + "node: {cscan: {dir: 1}}}}"); + } + + TEST_F(QueryPlannerTest, SortSkip) { + runQuerySortProjSkipLimit(BSONObj(), fromjson("{a: 1}"), BSONObj(), 2, 0); + assertNumSolutions(1U); + // If only skip is provided, do not limit sort. + assertSolutionExists("{skip: {n: 2, node: " + "{sort: {pattern: {a: 1}, limit: 0, " + "node: {cscan: {dir: 1}}}}}}"); + } + + TEST_F(QueryPlannerTest, SortSkipLimit) { + runQuerySortProjSkipLimit(BSONObj(), fromjson("{a: 1}"), BSONObj(), 2, -3); + assertNumSolutions(1U); + // Limit in sort node should be adjusted by skip count + assertSolutionExists("{skip: {n: 2, node: " + "{sort: {pattern: {a: 1}, limit: 5, " + "node: {cscan: {dir: 1}}}}}}"); + } + + TEST_F(QueryPlannerTest, SortSoftLimit) { + runQuerySortProjSkipLimit(BSONObj(), fromjson("{a: 1}"), BSONObj(), 0, 3); + assertNumSolutions(1U); + assertSolutionExists("{sort: {pattern: {a: 1}, limit: 3, " + "node: {cscan: {dir: 1}}}}"); + } + + TEST_F(QueryPlannerTest, SortSkipSoftLimit) { + runQuerySortProjSkipLimit(BSONObj(), fromjson("{a: 1}"), BSONObj(), 2, 3); + assertNumSolutions(1U); + assertSolutionExists("{skip: {n: 2, node: " + "{sort: {pattern: {a: 1}, limit: 5, " + "node: {cscan: {dir: 1}}}}}}"); + } + + // + // Sort elimination + // + + TEST_F(QueryPlannerTest, BasicSortElim) { + addIndex(BSON("x" << 1)); + // query, sort, proj + runQuerySortProj(fromjson("{ x : {$gt: 1}}"), fromjson("{x: 1}"), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {x: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {x: {$gt: 1}}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {filter: null, pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, SortElimCompound) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProj(fromjson("{ a : 5 }"), BSON("b" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {a: 5}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1, b: 1}}}}}"); + } + + // SERVER-13611: test that sort elimination still works if there are + // trailing fields in the index. + TEST_F(QueryPlannerTest, SortElimTrailingFields) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + runQuerySortProj(fromjson("{a: 5}"), BSON("b" << 1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {a: 5}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1, b: 1, c: 1}}}}}"); + } + + // Sort elimination with trailing fields where the sort direction is descending. + TEST_F(QueryPlannerTest, SortElimTrailingFieldsReverse) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1 << "d" << 1)); + runQuerySortProj(fromjson("{a: 5, b: 6}"), BSON("c" << -1), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {c: -1}, limit: 0, " + "node: {cscan: {dir: 1, filter: {a: 5, b: 6}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, dir: -1, pattern: {a: 1, b: 1, c: 1, d: 1}}}}}"); + } + + // + // Basic compound + // + + TEST_F(QueryPlannerTest, BasicCompound) { + addIndex(BSON("x" << 1 << "y" << 1)); + runQuery(fromjson("{ x : 5, y: 10}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1, y: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundMissingField) { + addIndex(BSON("x" << 1 << "y" << 1 << "z" << 1)); + runQuery(fromjson("{ x : 5, z: 10}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {x: 1, y: 1, z: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundFieldsOrder) { + addIndex(BSON("x" << 1 << "y" << 1 << "z" << 1)); + runQuery(fromjson("{ x : 5, z: 10, y:1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1, y: 1, z: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, CantUseCompound) { + addIndex(BSON("x" << 1 << "y" << 1)); + runQuery(fromjson("{ y: 10}")); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{cscan: {dir: 1, filter: {y: 10}}}"); + } + + // + // Array operators + // + + TEST_F(QueryPlannerTest, ElemMatchOneField) { + addIndex(BSON("a.b" << 1)); + runQuery(fromjson("{a : {$elemMatch: {b:1}}}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a:{$elemMatch:{b:1}}}}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:1}}}, node: " + "{ixscan: {filter: null, pattern: {'a.b': 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ElemMatchTwoFields) { + addIndex(BSON("a.b" << 1)); + addIndex(BSON("a.c" << 1)); + runQuery(fromjson("{a : {$elemMatch: {b:1, c:1}}}")); + + ASSERT_EQUALS(getNumSolutions(), 3U); + assertSolutionExists("{cscan: {dir: 1, filter: {a:{$elemMatch:{b:1,c:1}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, pattern: {'a.b': 1}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, pattern: {'a.c': 1}}}}}"); + } + + TEST_F(QueryPlannerTest, BasicAllElemMatch) { + addIndex(BSON("foo.a" << 1)); + addIndex(BSON("foo.b" << 1)); + runQuery(fromjson("{foo: {$all: [ {$elemMatch: {a:1, b:1}}, {$elemMatch: {a:2, b:2}}]}}")); + + assertNumSolutions(3U); + assertSolutionExists("{cscan: {dir: 1, filter: {foo:{$all:" + "[{$elemMatch:{a:1,b:1}},{$elemMatch:{a:2,b:2}}]}}}}"); + + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, pattern: {'foo.a': 1}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {filter: null, pattern: {'foo.b': 1}}}}}"); + } + + TEST_F(QueryPlannerTest, BasicAllElemMatch2) { + // true means multikey + addIndex(BSON("a.x" << 1), true); + + runQuery(fromjson("{a: {$all: [{$elemMatch: {x: 3}}, {$elemMatch: {y: 5}}]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$all:[{$elemMatch:{x:3}},{$elemMatch:{y:5}}]}}," + "node: {ixscan: {pattern: {'a.x': 1}," + "bounds: {'a.x': [[3,3,true,true]]}}}}}"); + } + + // SERVER-16256 + TEST_F(QueryPlannerTest, AllElemMatchCompound) { + // true means multikey + addIndex(BSON("d" << 1 << "a.b" << 1 << "a.c" << 1), true); + + runQuery(fromjson("{d: 1, a: {$all: [{$elemMatch: {b: 2, c: 2}}," + "{$elemMatch: {b: 3, c: 3}}]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and: [{a: {$elemMatch: {b: 2, c: 2}}}," + "{a: {$elemMatch: {b: 3, c: 3}}}]}," + "node: {ixscan: {filter: null, pattern: {d:1,'a.b':1,'a.c':1}," + "bounds: {d: [[1,1,true,true]]," + "'a.b': [[2,2,true,true]]," + "'a.c': [[2,2,true,true]]}}}}}"); + } + + // SERVER-13677 + TEST_F(QueryPlannerTest, ElemMatchWithAllElemMatchChild) { + addIndex(BSON("a.b.c.d" << 1)); + runQuery(fromjson("{z: 1, 'a.b': {$elemMatch: {c: {$all: [{$elemMatch: {d: 0}}]}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c.d': 1}}}}}"); + } + + // SERVER-13677 + TEST_F(QueryPlannerTest, ElemMatchWithAllElemMatchChild2) { + // true means multikey + addIndex(BSON("a.b.c.d" << 1), true); + runQuery(fromjson("{'a.b': {$elemMatch: {c: {$all: " + "[{$elemMatch: {d: {$gt: 1, $lt: 3}}}]}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c.d': 1}, " + "bounds: {'a.b.c.d': [[-Infinity,3,true,false]]}}}}}"); + } + + // SERVER-13677 + TEST_F(QueryPlannerTest, ElemMatchWithAllChild) { + // true means multikey + addIndex(BSON("a.b.c" << 1), true); + runQuery(fromjson("{z: 1, 'a.b': {$elemMatch: {c: {$all: [4, 5, 6]}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, " + "bounds: {'a.b.c': [[4,4,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, ElemMatchValueMatch) { + addIndex(BSON("foo" << 1)); + addIndex(BSON("foo" << 1 << "bar" << 1)); + runQuery(fromjson("{foo: {$elemMatch: {$gt: 5, $lt: 10}}}")); + + ASSERT_EQUALS(getNumSolutions(), 3U); + assertSolutionExists("{cscan: {dir: 1, filter: {foo:{$elemMatch:{$gt:5,$lt:10}}}}}"); + assertSolutionExists("{fetch: {filter: {foo: {$elemMatch: {$gt: 5, $lt: 10}}}, node: " + "{ixscan: {filter: null, pattern: {foo: 1}}}}}"); + assertSolutionExists("{fetch: {filter: {foo: {$elemMatch: {$gt: 5, $lt: 10}}}, node: " + "{ixscan: {filter: null, pattern: {foo: 1, bar: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ElemMatchValueIndexability) { + addIndex(BSON("foo" << 1)); + + // An ELEM_MATCH_VALUE can be indexed if all of its child predicates + // are "index bounds generating". + runQuery(fromjson("{foo: {$elemMatch: {$gt: 5, $lt: 10}}}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {foo:{$elemMatch:{$gt:5,$lt:10}}}}}"); + assertSolutionExists("{fetch: {filter: {foo: {$elemMatch: {$gt: 5, $lt: 10}}}, node: " + "{ixscan: {filter: null, pattern: {foo: 1}}}}}"); + + // We cannot build index bounds for the $size predicate. This means that the + // ELEM_MATCH_VALUE is not indexable, and we get no indexed solutions. + runQuery(fromjson("{foo: {$elemMatch: {$gt: 5, $size: 10}}}")); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{cscan: {dir: 1, filter: {foo:{$elemMatch:{$gt:5,$size:10}}}}}"); + } + + TEST_F(QueryPlannerTest, ElemMatchNested) { + addIndex(BSON("a.b.c" << 1)); + runQuery(fromjson("{ a:{ $elemMatch:{ b:{ $elemMatch:{ c:{ $gte:1, $lte:1 } } } } }}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}}}}}"); + } + + TEST_F(QueryPlannerTest, TwoElemMatchNested) { + addIndex(BSON("a.d.e" << 1)); + addIndex(BSON("a.b.c" << 1)); + runQuery(fromjson("{ a:{ $elemMatch:{ d:{ $elemMatch:{ e:{ $lte:1 } } }," + "b:{ $elemMatch:{ c:{ $gte:1 } } } } } }")); + + ASSERT_EQUALS(getNumSolutions(), 3U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.d.e': 1}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ElemMatchCompoundTwoFields) { + addIndex(BSON("a.b" << 1 << "a.c" << 1)); + runQuery(fromjson("{a : {$elemMatch: {b:1, c:1}}}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': 1, 'a.c': 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ArrayEquality) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{a : [1, 2, 3]}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a:[1,2,3]}}}"); + assertSolutionExists("{fetch: {filter: {a:[1,2,3]}, node: " + "{ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + // SERVER-13664 + TEST_F(QueryPlannerTest, ElemMatchEmbeddedAnd) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: {$gte: 2, $lt: 4}, c: 25}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:{$gte:2,$lt: 4},c:25}}}, node: " + "{ixscan: {filter: null, pattern: {'a.b': 1, 'a.c': 1}, " + "bounds: {'a.b': [[-Infinity,4,true,false]], " + "'a.c': [[25,25,true,true]]}}}}}"); + } + + // SERVER-13664 + TEST_F(QueryPlannerTest, ElemMatchEmbeddedOr) { + // true means multikey + addIndex(BSON("a.b" << 1), true); + // true means multikey + addIndex(BSON("a.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {$or: [{b: 3}, {c: 4}]}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{$or:[{b:3},{c:4}]}}}, " + "node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {'a.b': 1}}}, " + "{ixscan: {filter: null, pattern: {'a.c': 1}}}]}}}}"); + } + + // SERVER-13664 + TEST_F(QueryPlannerTest, ElemMatchEmbeddedRegex) { + addIndex(BSON("a.b" << 1)); + runQuery(fromjson("{a: {$elemMatch: {b: /foo/}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:/foo/}}}, node: " + "{ixscan: {filter: null, pattern: {'a.b': 1}}}}}"); + } + + // SERVER-14180 + TEST_F(QueryPlannerTest, ElemMatchEmbeddedRegexAnd) { + addIndex(BSON("a.b" << 1)); + runQuery(fromjson("{a: {$elemMatch: {b: /foo/}}, z: 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:/foo/}}, z:1}, node: " + "{ixscan: {filter: null, pattern: {'a.b': 1}}}}}"); + } + + // SERVER-14180 + TEST_F(QueryPlannerTest, ElemMatchEmbeddedRegexAnd2) { + addIndex(BSON("a.b" << 1)); + runQuery(fromjson("{a: {$elemMatch: {b: /foo/, b: 3}}, z: 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:/foo/,b:3}}, z:1}, node: " + "{ixscan: {filter: null, pattern: {'a.b': 1}}}}}"); + } + + // $not can appear as a value operator inside of an elemMatch (value). We shouldn't crash if we + // see it. + TEST_F(QueryPlannerTest, ElemMatchWithNotInside) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{a: {$elemMatch: {$not: {$gte: 6}}}}")); + } + + // SERVER-14625: Make sure we construct bounds properly for $elemMatch object with a + // negation inside. + TEST_F(QueryPlannerTest, ElemMatchWithNotInside2) { + addIndex(BSON("a.b" << 1 << "a.c" << 1)); + runQuery(fromjson("{d: 1, a: {$elemMatch: {c: {$ne: 3}, b: 4}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {d: 1, a: {$elemMatch: {c: {$ne: 3}, b: 4}}}, node:" + "{ixscan: {filter: null, pattern: {'a.b': 1, 'a.c': 1}, bounds:" + "{'a.b': [[4,4,true,true]]," + " 'a.c': [['MinKey',3,true,false]," + "[3,'MaxKey',false,true]]}}}}}"); + } + + // SERVER-13789 + TEST_F(QueryPlannerTest, ElemMatchIndexedNestedOr) { + addIndex(BSON("bar.baz" << 1)); + runQuery(fromjson("{foo: 1, $and: [{bar: {$elemMatch: {$or: [{baz: 2}]}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and: [{foo:1}," + "{bar:{$elemMatch:{$or:[{baz:2}]}}}]}, " + "node: {ixscan: {pattern: {'bar.baz': 1}, " + "bounds: {'bar.baz': [[2,2,true,true]]}}}}}"); + } + + // SERVER-13789 + TEST_F(QueryPlannerTest, ElemMatchIndexedNestedOrMultiplePreds) { + addIndex(BSON("bar.baz" << 1)); + addIndex(BSON("bar.z" << 1)); + runQuery(fromjson("{foo: 1, $and: [{bar: {$elemMatch: {$or: [{baz: 2}, {z: 3}]}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and: [{foo:1}," + "{bar:{$elemMatch:{$or:[{baz:2},{z:3}]}}}]}, " + "node: {or: {nodes: [" + "{ixscan: {pattern: {'bar.baz': 1}, " + "bounds: {'bar.baz': [[2,2,true,true]]}}}," + "{ixscan: {pattern: {'bar.z': 1}, " + "bounds: {'bar.z': [[3,3,true,true]]}}}]}}}}"); + } + + // SERVER-13789: Ensure that we properly compound in the multikey case when an + // $or is beneath an $elemMatch. + TEST_F(QueryPlannerTest, ElemMatchIndexedNestedOrMultikey) { + // true means multikey + addIndex(BSON("bar.baz" << 1 << "bar.z" << 1), true); + runQuery(fromjson("{foo: 1, $and: [{bar: {$elemMatch: {$or: [{baz: 2, z: 3}]}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and: [{foo:1}," + "{bar: {$elemMatch: {$or: [{$and: [{baz:2}, {z:3}]}]}}}]}," + "node: {ixscan: {pattern: {'bar.baz': 1, 'bar.z': 1}, " + "bounds: {'bar.baz': [[2,2,true,true]]," + "'bar.z': [[3,3,true,true]]}}}}}"); + } + + // SERVER-13789: Right now we don't index $nor, but make sure that the planner + // doesn't get confused by a $nor beneath an $elemMatch. + TEST_F(QueryPlannerTest, ElemMatchIndexedNestedNor) { + addIndex(BSON("bar.baz" << 1)); + runQuery(fromjson("{foo: 1, $and: [{bar: {$elemMatch: {$nor: [{baz: 2}, {baz: 3}]}}}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // SERVER-13789 + TEST_F(QueryPlannerTest, ElemMatchIndexedNestedNE) { + addIndex(BSON("bar.baz" << 1)); + runQuery(fromjson("{foo: 1, $and: [{bar: {$elemMatch: {baz: {$ne: 2}}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and: [{foo:1}," + "{bar:{$elemMatch:{baz:{$ne:2}}}}]}, " + "node: {ixscan: {pattern: {'bar.baz': 1}, " + "bounds: {'bar.baz': [['MinKey',2,true,false], " + "[2,'MaxKey',false,true]]}}}}}"); + } + + // SERVER-13789: Make sure we properly handle an $or below $elemMatch that is not + // tagged by the enumerator to use an index. + TEST_F(QueryPlannerTest, ElemMatchNestedOrNotIndexed) { + addIndex(BSON("a.b" << 1)); + runQuery(fromjson("{c: 1, a: {$elemMatch: {b: 3, $or: [{c: 4}, {c: 5}]}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': 1}, bounds: " + "{'a.b': [[3,3,true,true]]}}}}}"); + } + + // + // Geo + // http://docs.mongodb.org/manual/reference/operator/query-geospatial/#geospatial-query-compatibility-chart + // + + TEST_F(QueryPlannerTest, Basic2DNonNear) { + // 2d can answer: within poly, within center, within centersphere, within box. + // And it can use an index (or not) for each of them. As such, 2 solns expected. + addIndex(BSON("a" << "2d")); + + // Polygon + runQuery(fromjson("{a : { $within: { $polygon : [[0,0], [2,0], [4,0]] } }}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {geo2d: {a: '2d'}}}}"); + + // Center + runQuery(fromjson("{a : { $within : { $center : [[ 5, 5 ], 7 ] } }}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {geo2d: {a: '2d'}}}}"); + + // Centersphere + runQuery(fromjson("{a : { $within : { $centerSphere : [[ 10, 20 ], 0.01 ] } }}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {geo2d: {a: '2d'}}}}"); + + // Within box. + runQuery(fromjson("{a : {$within: {$box : [[0,0],[9,9]]}}}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {geo2d: {a: '2d'}}}}"); + + // TODO: test that we *don't* annotate for things we shouldn't. + } + + TEST_F(QueryPlannerTest, Basic2DSphereCompound) { + addIndex(BSON("a" << 1 << "b" << 1)); + addIndex(BSON("loc" << "2dsphere")); + + runQuery(fromjson("{loc:{$near:{$geometry:{type:'Point'," + "coordinates : [-81.513743,28.369947] }," + " $maxDistance :100}},a: 'mouse'}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {loc: '2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, Multikey2DSphereCompound) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << 1), true); + addIndex(BSON("loc" << "2dsphere"), true); + + runQuery(fromjson("{loc:{$near:{$geometry:{type:'Point'," + "coordinates : [-81.513743,28.369947] }," + " $maxDistance :100}},a: 'mouse'}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {loc: '2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, Basic2DSphereNonNear) { + // 2dsphere can do: within+geometry, intersects+geometry + addIndex(BSON("a" << "2dsphere")); + + runQuery(fromjson("{a: {$geoIntersects: {$geometry: {type: 'Point'," + "coordinates: [10.0, 10.0]}}}}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + + runQuery(fromjson("{a : { $geoWithin : { $centerSphere : [[ 10, 20 ], 0.01 ] } }}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + + // TODO: test that we *don't* annotate for things we shouldn't. + } + + TEST_F(QueryPlannerTest, Multikey2DSphereNonNear) { + // 2dsphere can do: within+geometry, intersects+geometry + // true means multikey + addIndex(BSON("a" << "2dsphere"), true); + + runQuery(fromjson("{a: {$geoIntersects: {$geometry: {type: 'Point'," + "coordinates: [10.0, 10.0]}}}}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + + runQuery(fromjson("{a : { $geoWithin : { $centerSphere : [[ 10, 20 ], 0.01 ] } }}")); + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + + // TODO: test that we *don't* annotate for things we shouldn't. + } + + TEST_F(QueryPlannerTest, Basic2DGeoNear) { + // Can only do near + old point. + addIndex(BSON("a" << "2d")); + runQuery(fromjson("{a: {$near: [0,0], $maxDistance:0.3 }}")); + assertNumSolutions(1U); + assertSolutionExists("{geoNear2d: {a: '2d'}}"); + } + + TEST_F(QueryPlannerTest, Basic2DSphereGeoNear) { + // Can do nearSphere + old point, near + new point. + addIndex(BSON("a" << "2dsphere")); + + runQuery(fromjson("{a: {$nearSphere: [0,0], $maxDistance: 0.31 }}")); + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{geoNear2dsphere: {a: '2dsphere'}}"); + + runQuery(fromjson("{a: {$geoNear: {$geometry: {type: 'Point', coordinates: [0,0]}," + "$maxDistance:100}}}")); + assertNumSolutions(1U); + assertSolutionExists("{geoNear2dsphere: {a: '2dsphere'}}"); + } + + TEST_F(QueryPlannerTest, Multikey2DSphereGeoNear) { + // Can do nearSphere + old point, near + new point. + // true means multikey + addIndex(BSON("a" << "2dsphere"), true); + + runQuery(fromjson("{a: {$nearSphere: [0,0], $maxDistance: 0.31 }}")); + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{geoNear2dsphere: {a: '2dsphere'}}"); + + runQuery(fromjson("{a: {$geoNear: {$geometry: {type: 'Point', coordinates: [0,0]}," + "$maxDistance:100}}}")); + assertNumSolutions(1U); + assertSolutionExists("{geoNear2dsphere: {a: '2dsphere'}}"); + } + + TEST_F(QueryPlannerTest, Basic2DSphereGeoNearReverseCompound) { + addIndex(BSON("x" << 1)); + addIndex(BSON("x" << 1 << "a" << "2dsphere")); + runQuery(fromjson("{x:1, a: {$nearSphere: [0,0], $maxDistance: 0.31 }}")); + + assertNumSolutions(1U); + assertSolutionExists("{geoNear2dsphere: {x: 1, a: '2dsphere'}}"); + } + + TEST_F(QueryPlannerTest, Multikey2DSphereGeoNearReverseCompound) { + addIndex(BSON("x" << 1), true); + addIndex(BSON("x" << 1 << "a" << "2dsphere"), true); + runQuery(fromjson("{x:1, a: {$nearSphere: [0,0], $maxDistance: 0.31 }}")); + + assertNumSolutions(1U); + assertSolutionExists("{geoNear2dsphere: {x: 1, a: '2dsphere'}}"); + } + + TEST_F(QueryPlannerTest, NearNoIndex) { + addIndex(BSON("x" << 1)); + runInvalidQuery(fromjson("{x:1, a: {$nearSphere: [0,0], $maxDistance: 0.31 }}")); + } + + TEST_F(QueryPlannerTest, TwoDSphereNoGeoPred) { + addIndex(BSON("x" << 1 << "a" << "2dsphere")); + runQuery(fromjson("{x:1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1, a: '2dsphere'}}}}}"); + } + + TEST_F(QueryPlannerTest, TwoDSphereNoGeoPredMultikey) { + addIndex(BSON("x" << 1 << "a" << "2dsphere"), true); + runQuery(fromjson("{x:1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x: 1, a: '2dsphere'}}}}}"); + } + + // SERVER-14723 + TEST_F(QueryPlannerTest, GeoNearMultipleRelevantIndicesButOnlyOneCompatible) { + addIndex(BSON("a" << "2dsphere")); + addIndex(BSON("b" << 1 << "a" << "2dsphere")); + + runQuery(fromjson("{a: {$nearSphere: {$geometry: {type: 'Point', coordinates: [0,0]}}}," + " b: {$exists: false}}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {b: {$exists: false}}, node: " + "{geoNear2dsphere: {a: '2dsphere'}}}}"); + } + + // SERVER-3984, $or 2d index + TEST_F(QueryPlannerTest, Or2DNonNear) { + addIndex(BSON("a" << "2d")); + addIndex(BSON("b" << "2d")); + runQuery(fromjson("{$or: [ {a : { $within : { $polygon : [[0,0], [2,0], [4,0]] } }}," + " {b : { $within : { $center : [[ 5, 5 ], 7 ] } }} ]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {or: {nodes: [{geo2d: {a: '2d'}}, {geo2d: {b: '2d'}}]}}}}"); + } + + // SERVER-3984, $or 2d index + TEST_F(QueryPlannerTest, Or2DSameFieldNonNear) { + addIndex(BSON("a" << "2d")); + runQuery(fromjson("{$or: [ {a : { $within : { $polygon : [[0,0], [2,0], [4,0]] } }}," + " {a : { $within : { $center : [[ 5, 5 ], 7 ] } }} ]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {or: {nodes: [{geo2d: {a: '2d'}}, {geo2d: {a: '2d'}}]}}}}"); + } + + // SERVER-3984, $or 2dsphere index + TEST_F(QueryPlannerTest, Or2DSphereNonNear) { + addIndex(BSON("a" << "2dsphere")); + addIndex(BSON("b" << "2dsphere")); + runQuery(fromjson("{$or: [ {a: {$geoIntersects: {$geometry: {type: 'Point', coordinates: [10.0, 10.0]}}}}," + " {b: {$geoWithin: { $centerSphere: [[ 10, 20 ], 0.01 ] } }} ]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{or: {nodes: [{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}," + "{fetch: {node: {ixscan: {pattern: {b: '2dsphere'}}}}}]}}"); + } + + // SERVER-3984, $or 2dsphere index + TEST_F(QueryPlannerTest, Or2DSphereNonNearMultikey) { + // true means multikey + addIndex(BSON("a" << "2dsphere"), true); + addIndex(BSON("b" << "2dsphere"), true); + runQuery(fromjson("{$or: [ {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [10.0, 10.0]}}}}," + " {b: {$geoWithin: { $centerSphere: [[ 10, 20 ], 0.01 ] } }} ]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{or: {nodes: " + "[{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}," + "{fetch: {node: {ixscan: {pattern: {b: '2dsphere'}}}}}]}}"); + } + + TEST_F(QueryPlannerTest, And2DSameFieldNonNear) { + addIndex(BSON("a" << "2d")); + runQuery(fromjson("{$and: [ {a : { $within : { $polygon : [[0,0], [2,0], [4,0]] } }}," + " {a : { $within : { $center : [[ 5, 5 ], 7 ] } }} ]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {andHash: {nodes: [" + "{geo2d: {a: '2d'}}, {geo2d: {a: '2d'}}]}}}}"); + } + + TEST_F(QueryPlannerTest, And2DWith2DNearSameField) { + addIndex(BSON("a" << "2d")); + runQuery(fromjson("{$and: [ {a : { $within : { $polygon : [[0,0], [2,0], [4,0]] } }}," + " {a : { $near : [ 5, 5 ] } } ]}")); + + // GEO_NEAR must use the index, and GEO predicate becomes a filter. + assertNumSolutions(1U); + assertSolutionExists("{geoNear2d: {a: '2d'}}"); + } + + TEST_F(QueryPlannerTest, And2DSphereSameFieldNonNear) { + addIndex(BSON("a" << "2dsphere")); + runQuery(fromjson("{$and: [ {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [3.0, 1.0]}}}}," + " {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [4.0, 1.0]}}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + // Bounds of the two 2dsphere geo predicates are combined into + // a single index scan. + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + } + + TEST_F(QueryPlannerTest, And2DSphereSameFieldNonNearMultikey) { + // true means multikey + addIndex(BSON("a" << "2dsphere"), true); + runQuery(fromjson("{$and: [ {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [3.0, 1.0]}}}}," + " {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [4.0, 1.0]}}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + // Bounds of the two 2dsphere geo predicates are combined into + // a single index scan. + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + } + + TEST_F(QueryPlannerTest, And2DSphereWithNearSameField) { + addIndex(BSON("a" << "2dsphere")); + runQuery(fromjson("{$and: [{a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [3.0, 1.0]}}}}," + "{a: {$near: {$geometry: " + "{type: 'Point', coordinates: [10.0, 10.0]}}}}]}")); + + // GEO_NEAR must use the index, and GEO predicate becomes a filter. + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {a: '2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, And2DSphereWithNearSameFieldMultikey) { + // true means multikey + addIndex(BSON("a" << "2dsphere"), true); + runQuery(fromjson("{$and: [{a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [3.0, 1.0]}}}}," + "{a: {$near: {$geometry: " + "{type: 'Point', coordinates: [10.0, 10.0]}}}}]}")); + + // GEO_NEAR must use the index, and GEO predicate becomes a filter. + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {a: '2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, Or2DSphereSameFieldNonNear) { + addIndex(BSON("a" << "2dsphere")); + runQuery(fromjson("{$or: [ {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [3.0, 1.0]}}}}," + " {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [4.0, 1.0]}}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + } + + TEST_F(QueryPlannerTest, Or2DSphereSameFieldNonNearMultikey) { + // true means multikey + addIndex(BSON("a" << "2dsphere"), true); + runQuery(fromjson("{$or: [ {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [3.0, 1.0]}}}}," + " {a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [4.0, 1.0]}}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundMultikey2DSphereNear) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << "2dsphere"), true); + runQuery(fromjson("{a: {$gte: 0}, b: {$near: {$geometry: " + "{type: 'Point', coordinates: [2, 2]}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{geoNear2dsphere: {a: 1, b: '2dsphere'}}"); + } + + TEST_F(QueryPlannerTest, CompoundMultikey2DSphereNearFetchRequired) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << "2dsphere"), true); + runQuery(fromjson("{a: {$gte: 0, $lt: 5}, b: {$near: {$geometry: " + "{type: 'Point', coordinates: [2, 2]}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a:{$gte:0}}, node: " + "{geoNear2dsphere: {a: 1, b: '2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundMultikey2DSphereNearMultipleIndices) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << "2dsphere"), true); + addIndex(BSON("c" << 1 << "b" << "2dsphere"), true); + runQuery(fromjson("{a: {$gte: 0}, c: 3, b: {$near: {$geometry: " + "{type: 'Point', coordinates: [2, 2]}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{fetch: {filter: {c:3}, node: " + "{geoNear2dsphere: {a: 1, b: '2dsphere'}}}}"); + assertSolutionExists("{fetch: {filter: {a:{$gte:0}}, node: " + "{geoNear2dsphere: {c: 1, b: '2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundMultikey2DSphereNearMultipleLeadingFields) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << 1 << "c" << "2dsphere"), true); + runQuery(fromjson("{a: {$lt: 5, $gt: 1}, b: 6, c: {$near: {$geometry: " + "{type: 'Point', coordinates: [2, 2]}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a:{$gt:1}}, node: " + "{geoNear2dsphere: {a: 1, b: 1, c: '2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundMultikey2DSphereNearMultipleGeoPreds) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << 1 << "c" << "2dsphere"), true); + runQuery(fromjson("{a: 1, b: 6, $and: [" + "{c: {$near: {$geometry: {type: 'Point', coordinates: [2, 2]}}}}," + "{c: {$geoWithin: {$box: [ [1, 1], [3, 3] ] } } } ] }")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {a:1, b:1, c:'2dsphere'}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundMultikey2DSphereNearCompoundTest) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << "2dsphere" << "c" << 1 << "d" << 1), true); + runQuery(fromjson("{a: {$gte: 0}, c: {$gte: 0, $lt: 4}, d: {$gt: 1, $lt: 5}," + "b: {$near: {$geometry: " + "{type: 'Point', coordinates: [2, 2]}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {d:{$gt:1},c:{$gte:0}}, node: " + "{geoNear2dsphere: {a: 1, b: '2dsphere', c: 1, d: 1}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundMultikey2DNear) { + // true means multikey + addIndex(BSON("a" << "2d" << "b" << 1), true); + runQuery(fromjson("{a: {$near: [0, 0]}, b: {$gte: 0}}")); + + assertNumSolutions(1U); + assertSolutionExists("{geoNear2d: {a: '2d', b: 1}}"); + } + + // + // $in + // + + TEST_F(QueryPlannerTest, InBasic) { + addIndex(fromjson("{a: 1}")); + runQuery(fromjson("{a: {$in: [1, 2]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a: {$in: [1, 2]}}}}"); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {pattern: {a: 1}}}}}"); + } + + // Logically equivalent to the preceding $in query. + // Indexed solution should be the same. + TEST_F(QueryPlannerTest, InBasicOrEquivalent) { + addIndex(fromjson("{a: 1}")); + runQuery(fromjson("{$or: [{a: 1}, {a: 2}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$or: [{a: 1}, {a: 2}]}}}"); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, InCompoundIndexFirst) { + addIndex(fromjson("{a: 1, b: 1}")); + runQuery(fromjson("{a: {$in: [1, 2]}, b: 3}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1, filter: {b: 3, a: {$in: [1, 2]}}}}"); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {pattern: {a: 1, b: 1}}}}}"); + } + + // Logically equivalent to the preceding $in query. + // Indexed solution should be the same. + // Currently fails - pre-requisite to SERVER-12024 + /* + TEST_F(QueryPlannerTest, InCompoundIndexFirstOrEquivalent) { + addIndex(fromjson("{a: 1, b: 1}")); + runQuery(fromjson("{$and: [{$or: [{a: 1}, {a: 2}]}, {b: 3}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$and: [{$or: [{a: 1}, {a: 2}]}, {b: 3}]}}}"); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {pattern: {a: 1, b: 1}}}}}"); + } + */ + + TEST_F(QueryPlannerTest, InCompoundIndexLast) { + addIndex(fromjson("{a: 1, b: 1}")); + runQuery(fromjson("{a: 3, b: {$in: [1, 2]}}")); + + assertNumSolutions(2U); + // TODO: update filter in cscan solution when SERVER-12024 is implemented + assertSolutionExists("{cscan: {dir: 1, filter: {a: 3, b: {$in: [1, 2]}}}}"); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {pattern: {a: 1, b: 1}}}}}"); + } + + // Logically equivalent to the preceding $in query. + // Indexed solution should be the same. + // Currently fails - pre-requisite to SERVER-12024 + /* + TEST_F(QueryPlannerTest, InCompoundIndexLastOrEquivalent) { + addIndex(fromjson("{a: 1, b: 1}")); + runQuery(fromjson("{$and: [{a: 3}, {$or: [{b: 1}, {b: 2}]}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$and: [{a: 3}, {$or: [{b: 1}, {b: 2}]}]}}}"); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {pattern: {a: 1, b: 1}}}}}"); + } + */ + + // SERVER-1205 + TEST_F(QueryPlannerTest, InWithSort) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProjSkipLimit(fromjson("{a: {$in: [1, 2]}}"), + BSON("b" << 1), BSONObj(), 0, 1); + + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 1, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a: 1, b: 1}}}, {ixscan: {pattern: {a: 1, b: 1}}}]}}}}"); + } + + // SERVER-1205 + TEST_F(QueryPlannerTest, InWithoutSort) { + addIndex(BSON("a" << 1 << "b" << 1)); + // No sort means we don't bother to blow up the bounds. + runQuerySortProjSkipLimit(fromjson("{a: {$in: [1, 2]}}"), BSONObj(), BSONObj(), 0, 1); + + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1, b: 1}}}}}"); + } + + // SERVER-1205 + TEST_F(QueryPlannerTest, ManyInWithSort) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1 << "d" << 1)); + runQuerySortProjSkipLimit(fromjson("{a: {$in: [1, 2]}, b:{$in:[1,2]}, c:{$in:[1,2]}}"), + BSON("d" << 1), BSONObj(), 0, 1); + + assertSolutionExists("{sort: {pattern: {d: 1}, limit: 1, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a: 1, b: 1, c:1, d:1}}}," + "{ixscan: {pattern: {a: 1, b: 1, c:1, d:1}}}," + "{ixscan: {pattern: {a: 1, b: 1, c:1, d:1}}}," + "{ixscan: {pattern: {a: 1, b: 1, c:1, d:1}}}," + "{ixscan: {pattern: {a: 1, b: 1, c:1, d:1}}}," + "{ixscan: {pattern: {a: 1, b: 1, c:1, d:1}}}]}}}}"); + } + + // SERVER-1205 + TEST_F(QueryPlannerTest, TooManyToExplode) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1 << "d" << 1)); + runQuerySortProjSkipLimit(fromjson("{a: {$in: [1,2,3,4,5,6]}," + "b:{$in:[1,2,3,4,5,6,7,8]}," + "c:{$in:[1,2,3,4,5,6,7,8]}}"), + BSON("d" << 1), BSONObj(), 0, 1); + + // We cap the # of ixscans we're willing to create. + assertNumSolutions(2); + assertSolutionExists("{sort: {pattern: {d: 1}, limit: 1, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{sort: {pattern: {d: 1}, limit: 1, node: " + "{fetch: {node: {ixscan: {pattern: {a: 1, b: 1, c:1, d:1}}}}}}}"); + } + + // SERVER-13618: test that exploding scans for sort works even + // if we must reverse the scan direction. + TEST_F(QueryPlannerTest, ExplodeMustReverseScans) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1 << "d" << 1)); + runQuerySortProj(fromjson("{a: {$in: [1, 2]}, b: {$in: [3, 4]}}"), + BSON("c" << -1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {c: -1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a:1, b:1, c:1, d:1}}}," + "{ixscan: {pattern: {a:1, b:1, c:1, d:1}}}," + "{ixscan: {pattern: {a:1, b:1, c:1, d:1}}}," + "{ixscan: {pattern: {a:1, b:1, c:1, d:1}}}]}}}}"); + } + + // SERVER-13618 + TEST_F(QueryPlannerTest, ExplodeMustReverseScans2) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << -1)); + runQuerySortProj(fromjson("{a: {$in: [1, 2]}, b: {$in: [3, 4]}}"), + BSON("c" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {c: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a:1, b:1, c:-1}}}," + "{ixscan: {pattern: {a:1, b:1, c:-1}}}," + "{ixscan: {pattern: {a:1, b:1, c:-1}}}," + "{ixscan: {pattern: {a:1, b:1, c:-1}}}]}}}}"); + } + + // SERVER-13752: don't try to explode if the ordered interval list for + // the leading field of the compound index is empty. + TEST_F(QueryPlannerTest, CantExplodeWithEmptyBounds) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProj(fromjson("{a: {$in: []}}"), BSON("b" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: " + "{fetch: {node: {ixscan: {pattern: {a: 1, b: 1}}}}}}}"); + } + + // SERVER-13752 + TEST_F(QueryPlannerTest, CantExplodeWithEmptyBounds2) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + runQuerySortProj(fromjson("{a: {$gt: 3, $lt: 0}}"), BSON("b" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: " + "{fetch: {node: {ixscan: {pattern: {a:1,b:1,c:1}}}}}}}"); + } + + // SERVER-13754: exploding an $or + TEST_F(QueryPlannerTest, ExplodeOrForSort) { + addIndex(BSON("a" << 1 << "c" << 1)); + addIndex(BSON("b" << 1 << "c" << 1)); + + runQuerySortProj(fromjson("{$or: [{a: 1}, {a: 2}, {b: 2}]}"), + BSON("c" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {c: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {bounds: {a: [[1,1,true,true]], " + "c: [['MinKey','MaxKey',true,true]]}," + "pattern: {a:1, c:1}}}," + "{ixscan: {bounds: {a: [[2,2,true,true]], " + "c: [['MinKey','MaxKey',true,true]]}," + "pattern: {a:1, c:1}}}," + "{ixscan: {bounds: {b: [[2,2,true,true]], " + "c: [['MinKey','MaxKey',true,true]]}," + "pattern: {b:1, c:1}}}]}}}}"); + } + + // SERVER-13754: exploding an $or + TEST_F(QueryPlannerTest, ExplodeOrForSort2) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + addIndex(BSON("d" << 1 << "c" << 1)); + + runQuerySortProj(fromjson("{$or: [{a: 1, b: {$in: [1, 2]}}, {d: 3}]}"), + BSON("c" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {c: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {bounds: {a: [[1,1,true,true]], b: [[1,1,true,true]]," + "c: [['MinKey','MaxKey',true,true]]}," + "pattern: {a:1, b:1, c:1}}}," + "{ixscan: {bounds: {a: [[1,1,true,true]], b: [[2,2,true,true]]," + "c: [['MinKey','MaxKey',true,true]]}," + "pattern: {a:1, b:1, c:1}}}," + "{ixscan: {bounds: {d: [[3,3,true,true]], " + "c: [['MinKey','MaxKey',true,true]]}," + "pattern: {d:1, c:1}}}]}}}}"); + } + + // SERVER-13754: an $or that can't be exploded, because one clause of the + // $or does provide the sort, even after explosion. + TEST_F(QueryPlannerTest, CantExplodeOrForSort) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + addIndex(BSON("d" << 1 << "c" << 1)); + + runQuerySortProj(fromjson("{$or: [{a: {$in: [1, 2]}}, {d: 3}]}"), + BSON("c" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {c: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{sort: {pattern: {c: 1}, limit: 0, node: " + "{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {pattern: {a: 1, b: 1, c: 1}}}," + "{ixscan: {pattern: {d: 1, c: 1}}}]}}}}}}"); + } + + // SERVER-15286: Make sure that at least the explodeForSort() path bails out + // when it finds that there are no union of point interval fields to explode. + // We could convert this into a MERGE_SORT plan, but we don't yet do this + // optimization. + TEST_F(QueryPlannerTest, CantExplodeOrForSort2) { + addIndex(BSON("a" << 1)); + + runQuerySortProj(fromjson("{$or: [{a: {$gt: 1, $lt: 3}}, {a: {$gt: 6, $lt: 10}}]}"), + BSON("a" << -1), + BSONObj()); + + assertNumSolutions(3U); + assertSolutionExists("{sort: {pattern: {a: -1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1}}}}}"); + assertSolutionExists("{sort: {pattern: {a: -1}, limit: 0, node: " + "{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {pattern: {a: 1}, bounds: " + "{a: [[1,3,false,false]]}}}," + "{ixscan: {pattern: {a: 1}, bounds: " + "{a: [[6,10,false,false]]}}}]}}}}}}"); + } + + // SERVER-13754: too many scans in an $or explosion. + TEST_F(QueryPlannerTest, TooManyToExplodeOr) { + addIndex(BSON("a" << 1 << "e" << 1)); + addIndex(BSON("b" << 1 << "e" << 1)); + addIndex(BSON("c" << 1 << "e" << 1)); + addIndex(BSON("d" << 1 << "e" << 1)); + runQuerySortProj(fromjson("{$or: [{a: {$in: [1,2,3,4,5,6]}," + "b: {$in: [1,2,3,4,5,6]}}," + "{c: {$in: [1,2,3,4,5,6]}," + "d: {$in: [1,2,3,4,5,6]}}]}"), + BSON("e" << 1), BSONObj()); + + // We cap the # of ixscans we're willing to create, so we don't get explosion. Instead + // we get 5 different solutions which all use a blocking sort. + assertNumSolutions(5U); + assertSolutionExists("{sort: {pattern: {e: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{sort: {pattern: {e: 1}, limit: 0, node: " + "{or: {nodes: [" + "{fetch: {node: {ixscan: {pattern: {a: 1, e: 1}}}}}," + "{fetch: {node: {ixscan: {pattern: {c: 1, e: 1}}}}}]}}}}"); + assertSolutionExists("{sort: {pattern: {e: 1}, limit: 0, node: " + "{or: {nodes: [" + "{fetch: {node: {ixscan: {pattern: {b: 1, e: 1}}}}}," + "{fetch: {node: {ixscan: {pattern: {c: 1, e: 1}}}}}]}}}}"); + assertSolutionExists("{sort: {pattern: {e: 1}, limit: 0, node: " + "{or: {nodes: [" + "{fetch: {node: {ixscan: {pattern: {a: 1, e: 1}}}}}," + "{fetch: {node: {ixscan: {pattern: {d: 1, e: 1}}}}}]}}}}"); + assertSolutionExists("{sort: {pattern: {e: 1}, limit: 0, node: " + "{or: {nodes: [" + "{fetch: {node: {ixscan: {pattern: {b: 1, e: 1}}}}}," + "{fetch: {node: {ixscan: {pattern: {d: 1, e: 1}}}}}]}}}}"); + } + + // SERVER-15696: Make sure explodeForSort copies filters on IXSCAN stages to all of the + // scans resulting from the explode. Regex is the easiest way to have the planner create + // an index scan which filters using the index key. + TEST_F(QueryPlannerTest, ExplodeIxscanWithFilter) { + addIndex(BSON("a" << 1 << "b" << 1)); + + runQuerySortProj(fromjson("{$and: [{b: {$regex: 'foo', $options: 'i'}}," + "{a: {$in: [1, 2]}}]}"), + BSON("b" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a:1, b:1}," + "filter: {b: {$regex: 'foo', $options: 'i'}}}}," + "{ixscan: {pattern: {a:1, b:1}," + "filter: {b: {$regex: 'foo', $options: 'i'}}}}]}}}}"); + + } + + TEST_F(QueryPlannerTest, InWithSortAndLimitTrailingField) { + addIndex(BSON("a" << 1 << "b" << -1 << "c" << 1)); + runQuerySortProjSkipLimit(fromjson("{a: {$in: [1, 2]}, b: {$gte: 0}}"), + fromjson("{b: -1}"), + BSONObj(), // no projection + 0, // no skip + -1); // .limit(1) + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {b:-1}, limit: 1, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{limit: {n: 1, node: {fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a:1,b:-1,c:1}}}, " + " {ixscan: {pattern: {a:1,b:-1,c:1}}}]}}}}}}"); + } + + // + // Multiple solutions + // + + TEST_F(QueryPlannerTest, TwoPlans) { + addIndex(BSON("a" << 1)); + addIndex(BSON("a" << 1 << "b" << 1)); + + runQuery(fromjson("{a:1, b:{$gt:2,$lt:2}}")); + + // 2 indexed solns and one non-indexed + ASSERT_EQUALS(getNumSolutions(), 3U); + assertSolutionExists("{cscan: {dir: 1, filter: {$and:[{b:{$lt:2}},{a:1},{b:{$gt:2}}]}}}"); + assertSolutionExists("{fetch: {filter: {$and:[{b:{$lt:2}},{b:{$gt:2}}]}, node: " + "{ixscan: {filter: null, pattern: {a: 1}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1, b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, TwoPlansElemMatch) { + addIndex(BSON("a" << 1 << "b" << 1)); + addIndex(BSON("arr.x" << 1 << "a" << 1)); + + runQuery(fromjson("{arr: { $elemMatch : { x : 5 , y : 5 } }," + " a : 55 , b : { $in : [ 1 , 5 , 8 ] } }")); + + // 2 indexed solns and one non-indexed + ASSERT_EQUALS(getNumSolutions(), 3U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1, b: 1}, bounds: " + "{a: [[55,55,true,true]], b: [[1,1,true,true], " + "[5,5,true,true], [8,8,true,true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {$and: [{arr:{$elemMatch:{x:5,y:5}}}," + "{b:{$in:[1,5,8]}}]}, " + "node: {ixscan: {pattern: {'arr.x':1,a:1}, bounds: " + "{'arr.x': [[5,5,true,true]], 'a':[[55,55,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundAndNonCompoundIndices) { + addIndex(BSON("a" << 1)); + addIndex(BSON("a" << 1 << "b" << 1), true); + runQuery(fromjson("{a: 1, b: {$gt: 2, $lt: 2}}")); + + ASSERT_EQUALS(getNumSolutions(), 3U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and:[{b:{$lt:2}},{b:{$gt:2}}]}, node: " + "{ixscan: {pattern: {a:1}, bounds: {a: [[1,1,true,true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {b:{$gt:2}}, node: " + "{ixscan: {pattern: {a:1,b:1}, bounds: " + "{a: [[1,1,true,true]], b: [[-Infinity,2,true,false]]}}}}}"); + } + + // + // Sort orders + // + + // SERVER-1205. + TEST_F(QueryPlannerTest, MergeSort) { + addIndex(BSON("a" << 1 << "c" << 1)); + addIndex(BSON("b" << 1 << "c" << 1)); + runQuerySortProj(fromjson("{$or: [{a:1}, {b:1}]}"), fromjson("{c:1}"), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {c: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a: 1, c: 1}}}, {ixscan: {pattern: {b: 1, c: 1}}}]}}}}"); + } + + // SERVER-1205 as well. + TEST_F(QueryPlannerTest, NoMergeSortIfNoSortWanted) { + addIndex(BSON("a" << 1 << "c" << 1)); + addIndex(BSON("b" << 1 << "c" << 1)); + runQuerySortProj(fromjson("{$or: [{a:1}, {b:1}]}"), BSONObj(), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$or: [{a:1}, {b:1}]}}}"); + assertSolutionExists("{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a: 1, c: 1}}}, " + "{ixscan: {filter: null, pattern: {b: 1, c: 1}}}]}}}}"); + } + + // Test that a 2dsphere index can satisfy a whole index scan solution if the query has a GEO + // predicate on at least one of the indexed geo fields. + // Currently fails. Tracked by SERVER-10801. + /* + TEST_F(QueryPlannerTest, SortOnGeoQuery) { + addIndex(BSON("timestamp" << -1 << "position" << "2dsphere")); + BSONObj query = fromjson("{position: {$geoWithin: {$geometry: {type: \"Polygon\", coordinates: [[[1, 1], [1, 90], [180, 90], [180, 1], [1, 1]]]}}}}"); + BSONObj sort = fromjson("{timestamp: -1}"); + runQuerySortProj(query, sort, BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {timestamp: -1}, limit: 0, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {timestamp: -1, position: '2dsphere'}}}}}"); + } + + TEST_F(QueryPlannerTest, SortOnGeoQueryMultikey) { + // true means multikey + addIndex(BSON("timestamp" << -1 << "position" << "2dsphere"), true); + BSONObj query = fromjson("{position: {$geoWithin: {$geometry: {type: \"Polygon\", " + "coordinates: [[[1, 1], [1, 90], [180, 90], [180, 1], [1, 1]]]}}}}"); + BSONObj sort = fromjson("{timestamp: -1}"); + runQuerySortProj(query, sort, BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {timestamp: -1}, limit: 0, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: " + "{timestamp: -1, position: '2dsphere'}}}}}"); + } + */ + + // SERVER-9257 + TEST_F(QueryPlannerTest, CompoundGeoNoGeoPredicate) { + addIndex(BSON("creationDate" << 1 << "foo.bar" << "2dsphere")); + runQuerySortProj(fromjson("{creationDate: { $gt: 7}}"), + fromjson("{creationDate: 1}"), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {creationDate: 1}, limit: 0, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {creationDate: 1, 'foo.bar': '2dsphere'}}}}}"); + } + + // SERVER-9257 + TEST_F(QueryPlannerTest, CompoundGeoNoGeoPredicateMultikey) { + // true means multikey + addIndex(BSON("creationDate" << 1 << "foo.bar" << "2dsphere"), true); + runQuerySortProj(fromjson("{creationDate: { $gt: 7}}"), + fromjson("{creationDate: 1}"), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {creationDate: 1}, limit: 0, " + "node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {creationDate: 1, 'foo.bar': '2dsphere'}}}}}"); + } + + // Basic "keep sort in mind with an OR" + TEST_F(QueryPlannerTest, MergeSortEvenIfSameIndex) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProj(fromjson("{$or: [{a:1}, {a:7}]}"), fromjson("{b:1}"), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + // TODO the second solution should be mergeSort rather than just sort + } + + TEST_F(QueryPlannerTest, ReverseScanForSort) { + addIndex(BSON("_id" << 1)); + runQuerySortProj(BSONObj(), fromjson("{_id: -1}"), BSONObj()); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{sort: {pattern: {_id: -1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {_id: 1}}}}}"); + } + + // + // Hint tests + // + + TEST_F(QueryPlannerTest, NaturalHint) { + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuerySortHint(BSON("a" << 1), BSON("b" << 1), BSON("$natural" << 1)); + + assertNumSolutions(1U); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, node: " + "{cscan: {filter: {a: 1}, dir: 1}}}}"); + } + + // Test $natural sort and its interaction with $natural hint. + TEST_F(QueryPlannerTest, NaturalSortAndHint) { + addIndex(BSON("x" << 1)); + + // Non-empty query, -1 sort, no hint. + runQuerySortHint(fromjson("{x: {$exists: true}}"), BSON("$natural" << -1), BSONObj()); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: -1}}"); + + // Non-empty query, 1 sort, no hint. + runQuerySortHint(fromjson("{x: {$exists: true}}"), BSON("$natural" << 1), BSONObj()); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + + // Non-empty query, -1 sort, -1 hint. + runQuerySortHint(fromjson("{x: {$exists: true}}"), BSON("$natural" << -1), + BSON("$natural" << -1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: -1}}"); + + // Non-empty query, 1 sort, -1 hint. + runQuerySortHint(fromjson("{x: {$exists: true}}"), BSON("$natural" << 1), + BSON("$natural" << -1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + + // Non-empty query, -1 sort, 1 hint. + runQuerySortHint(fromjson("{x: {$exists: true}}"), BSON("$natural" << -1), + BSON("$natural" << 1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: -1}}"); + + // Non-empty query, 1 sort, 1 hint. + runQuerySortHint(fromjson("{x: {$exists: true}}"), BSON("$natural" << 1), + BSON("$natural" << 1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + + // Empty query, -1 sort, no hint. + runQuerySortHint(BSONObj(), BSON("$natural" << -1), BSONObj()); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: -1}}"); + + // Empty query, 1 sort, no hint. + runQuerySortHint(BSONObj(), BSON("$natural" << 1), BSONObj()); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + + // Empty query, -1 sort, -1 hint. + runQuerySortHint(BSONObj(), BSON("$natural" << -1), BSON("$natural" << -1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: -1}}"); + + // Empty query, 1 sort, -1 hint. + runQuerySortHint(BSONObj(), BSON("$natural" << 1), BSON("$natural" << -1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + + // Empty query, -1 sort, 1 hint. + runQuerySortHint(BSONObj(), BSON("$natural" << -1), BSON("$natural" << 1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: -1}}"); + + // Empty query, 1 sort, 1 hint. + runQuerySortHint(BSONObj(), BSON("$natural" << 1), BSON("$natural" << 1)); + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + TEST_F(QueryPlannerTest, HintOverridesNaturalSort) { + addIndex(BSON("x" << 1)); + runQuerySortHint(fromjson("{x: {$exists: true}}"), BSON("$natural" << -1), BSON("x" << 1)); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {x:{$exists:true}}, node: " + "{ixscan: {filter: null, pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, HintValid) { + addIndex(BSON("a" << 1)); + runQueryHint(BSONObj(), fromjson("{a: 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, HintValidWithPredicate) { + addIndex(BSON("a" << 1)); + runQueryHint(fromjson("{a: {$gt: 1}}"), fromjson("{a: 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, HintValidWithSort) { + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuerySortHint(fromjson("{a: 100, b: 200}"), fromjson("{b: 1}"), fromjson("{a: 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{sort: {pattern: {b: 1}, limit: 0, node: " + "{fetch: {filter: {b: 200}, " + "node: {ixscan: {filter: null, pattern: {a: 1}}}}}}}"); + } + + TEST_F(QueryPlannerTest, HintElemMatch) { + // true means multikey + addIndex(fromjson("{'a.b': 1}"), true); + runQueryHint(fromjson("{'a.b': 1, a: {$elemMatch: {b: 2}}}"), fromjson("{'a.b': 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {$and: [{a:{$elemMatch:{b:2}}}, {'a.b': 1}]}, " + "node: {ixscan: {filter: null, pattern: {'a.b': 1}, bounds: " + "{'a.b': [[2, 2, true, true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, HintInvalid) { + addIndex(BSON("a" << 1)); + runInvalidQueryHint(BSONObj(), fromjson("{b: 1}")); + } + + // + // Sparse indices, SERVER-8067 + // Each index in this block of tests is sparse. + // + + TEST_F(QueryPlannerTest, SparseIndexIgnoreForSort) { + addIndex(fromjson("{a: 1}"), false, true); + runQuerySortProj(BSONObj(), fromjson("{a: 1}"), BSONObj()); + + assertNumSolutions(1U); + assertSolutionExists("{sort: {pattern: {a: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + } + + TEST_F(QueryPlannerTest, SparseIndexHintForSort) { + addIndex(fromjson("{a: 1}"), false, true); + runQuerySortHint(BSONObj(), fromjson("{a: 1}"), fromjson("{a: 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, SparseIndexPreferCompoundIndexForSort) { + addIndex(fromjson("{a: 1}"), false, true); + addIndex(fromjson("{a: 1, b: 1}")); + runQuerySortProj(BSONObj(), fromjson("{a: 1}"), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {a: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1, b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, SparseIndexForQuery) { + addIndex(fromjson("{a: 1}"), false, true); + runQuerySortProj(fromjson("{a: 1}"), BSONObj(), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a: 1}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1}}}}}"); + } + + // + // Regex + // + + TEST_F(QueryPlannerTest, PrefixRegex) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{a: /^foo/}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a: /^foo/}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, PrefixRegexCovering) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{a: /^foo/}"), BSONObj(), fromjson("{_id: 0, a: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{cscan: {dir: 1, filter: {a: /^foo/}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NonPrefixRegex) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{a: /foo/}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {a: /foo/}}}"); + assertSolutionExists("{fetch: {filter: null, node: " + "{ixscan: {filter: {a: /foo/}, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NonPrefixRegexCovering) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{a: /foo/}"), BSONObj(), fromjson("{_id: 0, a: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{cscan: {dir: 1, filter: {a: /foo/}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{ixscan: {filter: {a: /foo/}, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NonPrefixRegexAnd) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{a: /foo/, b: 2}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {dir: 1, filter: {$and: [{b: 2}, {a: /foo/}]}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: {a: /foo/}, pattern: {a: 1, b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NonPrefixRegexAndCovering) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProj(fromjson("{a: /foo/, b: 2}"), BSONObj(), + fromjson("{_id: 0, a: 1, b: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1, b: 1}, node: " + "{cscan: {dir: 1, filter: {$and: [{b: 2}, {a: /foo/}]}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1, b: 1}, node: " + "{ixscan: {filter: {a: /foo/}, pattern: {a: 1, b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NonPrefixRegexOrCovering) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$or: [{a: /0/}, {a: /1/}]}"), BSONObj(), + fromjson("{_id: 0, a: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{cscan: {dir: 1, filter: {$or: [{a: /0/}, {a: /1/}]}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{ixscan: {filter: {$or: [{a: /0/}, {a: /1/}]}, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NonPrefixRegexInCovering) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{a: {$in: [/foo/, /bar/]}}"), BSONObj(), + fromjson("{_id: 0, a: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{cscan: {dir: 1, filter: {a:{$in:[/foo/,/bar/]}}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{ixscan: {filter: {a:{$in:[/foo/,/bar/]}}, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, TwoRegexCompoundIndexCovering) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProj(fromjson("{a: /0/, b: /1/}"), BSONObj(), + fromjson("{_id: 0, a: 1, b: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1, b: 1}, node: " + "{cscan: {dir: 1, filter: {$and:[{a:/0/},{b:/1/}]}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1, b: 1}, node: " + "{ixscan: {filter: {$and:[{a:/0/},{b:/1/}]}, pattern: {a: 1, b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, TwoRegexSameFieldCovering) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$and: [{a: /0/}, {a: /1/}]}"), BSONObj(), + fromjson("{_id: 0, a: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{cscan: {dir: 1, filter: {$and:[{a:/0/},{a:/1/}]}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{ixscan: {filter: {$and:[{a:/0/},{a:/1/}]}, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, ThreeRegexSameFieldCovering) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$and: [{a: /0/}, {a: /1/}, {a: /2/}]}"), BSONObj(), + fromjson("{_id: 0, a: 1}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{cscan: {dir: 1, filter: {$and:[{a:/0/},{a:/1/},{a:/2/}]}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{ixscan: {filter: {$and:[{a:/0/},{a:/1/},{a:/2/}]}, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NonPrefixRegexMultikey) { + // true means multikey + addIndex(BSON("a" << 1), true); + runQuery(fromjson("{a: /foo/}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {filter: {a: /foo/}, dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a: /foo/}, node: {ixscan: " + "{pattern: {a: 1}, filter: null}}}}"); + } + + TEST_F(QueryPlannerTest, ThreeRegexSameFieldMultikey) { + // true means multikey + addIndex(BSON("a" << 1), true); + runQuery(fromjson("{$and: [{a: /0/}, {a: /1/}, {a: /2/}]}")); + + ASSERT_EQUALS(getNumSolutions(), 2U); + assertSolutionExists("{cscan: {filter: {$and:[{a:/0/},{a:/1/},{a:/2/}]}, dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and:[{a:/0/},{a:/1/},{a:/2/}]}, node: {ixscan: " + "{pattern: {a: 1}, filter: null}}}}"); + } + + // + // Negation + // + + TEST_F(QueryPlannerTest, NegationIndexForSort) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{a: {$ne: 1}}"), fromjson("{a: 1}"), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {a: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1}, " + "bounds: {a: [['MinKey',1,true,false], " + "[1,'MaxKey',false,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegationTopLevel) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{a: {$ne: 1}}"), BSONObj(), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [['MinKey',1,true,false], " + "[1,'MaxKey',false,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegationOr) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$or: [{a: 1}, {b: {$ne: 1}}]}"), BSONObj(), BSONObj()); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + TEST_F(QueryPlannerTest, NegationOrNotIn) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$or: [{a: 1}, {b: {$nin: [1]}}]}"), BSONObj(), BSONObj()); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + TEST_F(QueryPlannerTest, NegationAndIndexOnEquality) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$and: [{a: 1}, {b: {$ne: 1}}]}"), BSONObj(), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1}," + "bounds: {a: [[1,1,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegationAndIndexOnEqualityAndNegationBranches) { + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuerySortProj(fromjson("{$and: [{a: 1}, {b: 2}, {b: {$ne: 1}}]}"), BSONObj(), BSONObj()); + + assertNumSolutions(3U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1}, " + "bounds: {a: [[1,1,true,true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {b: 1}, " + "bounds: {b: [[2,2,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegationAndIndexOnInequality) { + addIndex(BSON("b" << 1)); + runQuerySortProj(fromjson("{$and: [{a: 1}, {b: {$ne: 1}}]}"), BSONObj(), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:1}, node: {ixscan: {pattern: {b:1}, " + "bounds: {b: [['MinKey',1,true,false], " + "[1,'MaxKey',false,true]]}}}}}"); + } + + // Negated regexes don't use the index. + TEST_F(QueryPlannerTest, NegationRegexPrefix) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: /^a/}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // Negated mods don't use the index + TEST_F(QueryPlannerTest, NegationMod) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: {$mod: [2, 1]}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // Negated $type doesn't use the index + TEST_F(QueryPlannerTest, NegationTypeOperator) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: {$type: 16}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // Negated $elemMatch value won't use the index + TEST_F(QueryPlannerTest, NegationElemMatchValue) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: {$elemMatch: {$gt: 3, $lt: 10}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // Negated $elemMatch object won't use the index + TEST_F(QueryPlannerTest, NegationElemMatchObject) { + addIndex(BSON("i.j" << 1)); + runQuery(fromjson("{i: {$not: {$elemMatch: {j: 1}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // Negated $elemMatch object won't use the index + TEST_F(QueryPlannerTest, NegationElemMatchObject2) { + addIndex(BSON("i.j" << 1)); + runQuery(fromjson("{i: {$not: {$elemMatch: {j: {$ne: 1}}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // If there is a negation that can't use the index, + // ANDed with a predicate that can use the index, then + // we can still use the index for the latter predicate. + TEST_F(QueryPlannerTest, NegationRegexWithIndexablePred) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{$and: [{i: {$not: /o/}}, {i: 2}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {i:1}, " + "bounds: {i: [[2,2,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegationCantUseSparseIndex) { + // false means not multikey, true means sparse + addIndex(BSON("i" << 1), false, true); + runQuery(fromjson("{i: {$ne: 4}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + TEST_F(QueryPlannerTest, NegationCantUseSparseIndex2) { + // false means not multikey, true means sparse + addIndex(BSON("i" << 1 << "j" << 1), false, true); + runQuery(fromjson("{i: 4, j: {$ne: 5}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {i:1,j:1}, bounds: " + "{i: [[4,4,true,true]], j: [['MinKey','MaxKey',true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegatedRangeStrGT) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: {$gt: 'a'}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {i:1}, " + "bounds: {i: [['MinKey','a',true,true], " + "[{},'MaxKey',true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegatedRangeStrGTE) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: {$gte: 'a'}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {i:1}, " + "bounds: {i: [['MinKey','a',true,false], " + "[{},'MaxKey',true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegatedRangeIntGT) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: {$gt: 5}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {i:1}, " + "bounds: {i: [['MinKey',5,true,true], " + "[Infinity,'MaxKey',false,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegatedRangeIntGTE) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{i: {$not: {$gte: 5}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {i:1}, " + "bounds: {i: [['MinKey',5,true,false], " + "[Infinity,'MaxKey',false,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, TwoNegatedRanges) { + addIndex(BSON("i" << 1)); + runQuery(fromjson("{$and: [{i: {$not: {$lte: 'b'}}}, " + "{i: {$not: {$gte: 'f'}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {i:1}, " + "bounds: {i: [['MinKey','',true,false], " + "['b','f',false,false], " + "[{},'MaxKey',true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, AndWithNestedNE) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{a: {$gt: -1, $lt: 1, $ne: 0}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [[-1,0,false,false], " + "[0,1,false,false]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NegatePredOnCompoundIndex) { + addIndex(BSON("x" << 1 << "a" << 1)); + runQuery(fromjson("{x: 1, a: {$ne: 1}, b: {$ne: 2}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {x:1,a:1}, bounds: " + "{x: [[1,1,true,true]], " + "a: [['MinKey',1,true,false], [1,'MaxKey',false,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, NEOnMultikeyIndex) { + // true means multikey + addIndex(BSON("a" << 1), true); + runQuery(fromjson("{a: {$ne: 3}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$ne:3}}, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [['MinKey',3,true,false]," + "[3,'MaxKey',false,true]]}}}}}"); + } + + // In general, a negated $nin can make use of an index. + TEST_F(QueryPlannerTest, NinUsesMultikeyIndex) { + // true means multikey + addIndex(BSON("a" << 1), true); + runQuery(fromjson("{a: {$nin: [4, 10]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$nin:[4,10]}}, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [['MinKey',4,true,false]," + "[4,10,false,false]," + "[10,'MaxKey',false,true]]}}}}}"); + } + + // But it can't if the $nin contains a regex because regex bounds can't + // be complemented. + TEST_F(QueryPlannerTest, NinCantUseMultikeyIndex) { + // true means multikey + addIndex(BSON("a" << 1), true); + runQuery(fromjson("{a: {$nin: [4, /foobar/]}}")); + + assertNumSolutions(1U); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // + // 2D geo negation + // The filter b != 1 is embedded in the geoNear2d node. + // Can only do near + old point. + // + TEST_F(QueryPlannerTest, Negation2DGeoNear) { + addIndex(BSON("a" << "2d")); + runQuery(fromjson("{$and: [{a: {$near: [0, 0], $maxDistance: 0.3}}, {b: {$ne: 1}}]}")); + assertNumSolutions(1U); + assertSolutionExists("{geoNear2d: {a: '2d'}}"); + } + + // + // 2DSphere geo negation + // Filter is embedded in a separate fetch node. + // + TEST_F(QueryPlannerTest, Negation2DSphereGeoNear) { + // Can do nearSphere + old point, near + new point. + addIndex(BSON("a" << "2dsphere")); + + runQuery(fromjson("{$and: [{a: {$nearSphere: [0,0], $maxDistance: 0.31}}, " + "{b: {$ne: 1}}]}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {a: '2dsphere'}}}}"); + + runQuery(fromjson("{$and: [{a: {$geoNear: {$geometry: {type: 'Point', " + "coordinates: [0, 0]}," + "$maxDistance: 100}}}," + "{b: {$ne: 1}}]}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {a: '2dsphere'}}}}"); + } + + // + // 2DSphere geo negation + // Filter is embedded in a separate fetch node. + // + TEST_F(QueryPlannerTest, Negation2DSphereGeoNearMultikey) { + // Can do nearSphere + old point, near + new point. + // true means multikey + addIndex(BSON("a" << "2dsphere"), true); + + runQuery(fromjson("{$and: [{a: {$nearSphere: [0,0], $maxDistance: 0.31}}, " + "{b: {$ne: 1}}]}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {a: '2dsphere'}}}}"); + + runQuery(fromjson("{$and: [{a: {$geoNear: {$geometry: {type: 'Point', " + "coordinates: [0, 0]}," + "$maxDistance: 100}}}," + "{b: {$ne: 1}}]}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {geoNear2dsphere: {a: '2dsphere'}}}}"); + } + + // + // Multikey indices + // + + /** + * Index bounds constraints on a field should not be intersected + * if the index is multikey. + */ + TEST_F(QueryPlannerTest, MultikeyTwoConstraintsSameField) { + addIndex(BSON("a" << 1), true); + runQuery(fromjson("{a: {$gt: 0, $lt: 5}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {filter: {$and: [{a: {$lt: 5}}, {a: {$gt: 0}}]}, dir: 1}}"); + + vector alternates; + alternates.push_back("{fetch: {filter: {a: {$lt: 5}}, node: {ixscan: {filter: null, " + "pattern: {a: 1}, bounds: {a: [[0, Infinity, false, true]]}}}}}"); + alternates.push_back("{fetch: {filter: {a: {$gt: 0}}, node: {ixscan: {filter: null, " + "pattern: {a: 1}, bounds: {a: [[-Infinity, 5, true, false]]}}}}}"); + assertHasOneSolutionOf(alternates); + } + + /** + * Constraints on fields with a shared parent should not be intersected + * if the index is multikey. + */ + TEST_F(QueryPlannerTest, MultikeyTwoConstraintsDifferentFields) { + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{'a.b': 2, 'a.c': 3}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {filter: {$and: [{'a.b': 2}, {'a.c': 3}]}, dir: 1}}"); + + vector alternates; + alternates.push_back("{fetch: {filter: {'a.c': 3}, node: {ixscan: {filter: null, " + "pattern: {'a.b': 1, 'a.c': 1}, bounds: " + "{'a.b': [[2,2,true,true]], " + " 'a.c': [['MinKey','MaxKey',true,true]]}}}}}"); + alternates.push_back("{fetch: {filter: {'a.b': 2}, node: {ixscan: {filter: null, " + "pattern: {'a.b': 1, 'a.c': 1}, bounds: " + "{'a.b': [['MinKey','MaxKey',true,true]], " + " 'a.c': [[3,3,true,true]]}}}}}"); + assertHasOneSolutionOf(alternates); + } + + // + // Index bounds related tests + // + + TEST_F(QueryPlannerTest, CompoundIndexBoundsLastFieldMissing) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + runQuery(fromjson("{a: 5, b: {$gt: 7}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1, b: 1, c: 1}, bounds: " + "{a: [[5,5,true,true]], b: [[7,Infinity,false,true]], " + " c: [['MinKey','MaxKey',true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundIndexBoundsMiddleFieldMissing) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + runQuery(fromjson("{a: 1, c: {$lt: 3}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1, b: 1, c: 1}, bounds: " + "{a: [[1,1,true,true]], b: [['MinKey','MaxKey',true,true]], " + " c: [[-Infinity,3,true,false]]}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundIndexBoundsRangeAndEquality) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{a: {$gt: 8}, b: 6}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1, b: 1}, bounds: " + "{a: [[8,Infinity,false,true]], b:[[6,6,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundIndexBoundsEqualityThenIn) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{a: 5, b: {$in: [2,6,11]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {filter: null, pattern: " + "{a: 1, b: 1}, bounds: {a: [[5,5,true,true]], " + "b:[[2,2,true,true],[6,6,true,true],[11,11,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundIndexBoundsStringBounds) { + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{a: {$gt: 'foo'}, b: {$gte: 'bar'}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {filter: null, pattern: " + "{a: 1, b: 1}, bounds: {a: [['foo',{},false,false]], " + "b:[['bar',{},true,false]]}}}}}"); + } + + TEST_F(QueryPlannerTest, IndexBoundsAndWithNestedOr) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$and: [{a: 1, $or: [{a: 2}, {a: 3}]}]}")); + + // Given that the index over 'a' isn't multikey, we ideally won't generate any solutions + // since we know the query describes an empty set if 'a' isn't multikey. Any solutions + // below are "this is how it currently works" instead of "this is how it should work." + + // It's kind of iffy to look for indexed solutions so we don't... + size_t matches = 0; + matches += numSolutionMatches("{cscan: {dir: 1, filter: " + "{$or: [{a: 2, a:1}, {a: 3, a:1}]}}}"); + matches += numSolutionMatches("{cscan: {dir: 1, filter: " + "{$and: [{$or: [{a: 2}, {a: 3}]}, {a: 1}]}}}"); + ASSERT_GREATER_THAN_OR_EQUALS(matches, 1U); + } + + TEST_F(QueryPlannerTest, IndexBoundsIndexedSort) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$or: [{a: 1}, {a: 2}]}"), BSON("a" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {a:1}, limit: 0, node: " + "{cscan: {filter: {$or:[{a:1},{a:2}]}, dir: 1}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {filter: null, " + "pattern: {a:1}, bounds: {a: [[1,1,true,true], [2,2,true,true]]}}}}}"); + } + + TEST_F(QueryPlannerTest, IndexBoundsUnindexedSort) { + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{$or: [{a: 1}, {a: 2}]}"), BSON("b" << 1), BSONObj()); + + assertNumSolutions(2U); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: " + "{cscan: {filter: {$or:[{a:1},{a:2}]}, dir: 1}}}}"); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: {fetch: " + "{filter: null, node: {ixscan: {filter: null, " + "pattern: {a:1}, bounds: {a: [[1,1,true,true], [2,2,true,true]]}}}}}}}"); + } + + TEST_F(QueryPlannerTest, IndexBoundsUnindexedSortHint) { + addIndex(BSON("a" << 1)); + runQuerySortHint(fromjson("{$or: [{a: 1}, {a: 2}]}"), BSON("b" << 1), BSON("a" << 1)); + + assertNumSolutions(1U); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: {fetch: " + "{filter: null, node: {ixscan: {filter: null, " + "pattern: {a:1}, bounds: {a: [[1,1,true,true], [2,2,true,true]]}}}}}}}"); + } + + TEST_F(QueryPlannerTest, CompoundIndexBoundsIntersectRanges) { + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + addIndex(BSON("a" << 1 << "c" << 1)); + runQuery(fromjson("{a: {$gt: 1, $lt: 10}, c: {$gt: 1, $lt: 10}}")); + + assertNumSolutions(3U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1,b:1,c:1}, " + "bounds: {a: [[1,10,false,false]], " + "b: [['MinKey','MaxKey',true,true]], " + "c: [[1,10,false,false]]}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1,c:1}, " + "bounds: {a: [[1,10,false,false]], " + "c: [[1,10,false,false]]}}}}}"); + } + + // Test that planner properly unionizes the index bounds for two negation + // predicates (SERVER-13890). + TEST_F(QueryPlannerTest, IndexBoundsOrOfNegations) { + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a: {$ne: 3}}, {a: {$ne: 4}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {pattern: {a:1}, " + "bounds: {a: [['MinKey','MaxKey',true,true]]}}}}}"); + } + + // + // Tests related to building index bounds for multikey + // indices, combined with compound and $elemMatch + // + + // SERVER-12475: make sure that we compound bounds, even + // for a multikey index. + TEST_F(QueryPlannerTest, CompoundMultikeyBounds) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << 1), true); + runQuery(fromjson("{a: 1, b: 3}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {filter: {$and:[{a:1},{b:3}]}, dir: 1}}"); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: {filter: null, " + "pattern: {a:1,b:1}, bounds: " + "{a: [[1,1,true,true]], b: [[3,3,true,true]]}}}}}"); + } + + // Make sure that we compound bounds but do not intersect bounds + // for a compound multikey index. + TEST_F(QueryPlannerTest, CompoundMultikeyBoundsNoIntersect) { + // true means multikey + addIndex(BSON("a" << 1 << "b" << 1), true); + runQuery(fromjson("{a: 1, b: {$gt: 3, $lte: 5}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {b:{$gt:3}}, node: {ixscan: {filter: null, " + "pattern: {a:1,b:1}, bounds: " + "{a: [[1,1,true,true]], b: [[-Infinity,5,true,true]]}}}}}"); + } + + // The index bounds can be compounded because the index is not multikey. + TEST_F(QueryPlannerTest, CompoundBoundsElemMatchNotMultikey) { + addIndex(BSON("a.x" << 1 << "a.b.c" << 1)); + runQuery(fromjson("{'a.x': 1, a: {$elemMatch: {b: {$elemMatch: {c: {$gte: 1}}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:{$elemMatch:{c:{$gte:1}}}}}}, " + "node: {ixscan: {pattern: {'a.x':1, 'a.b.c':1}, bounds: " + "{'a.x': [[1,1,true,true]], " + " 'a.b.c': [[1,Infinity,true,true]]}}}}}"); + } + + // The index bounds cannot be compounded because the predicates over 'a.x' and + // 'a.b.c' 1) share the prefix "a", and 2) are not conjoined by an $elemMatch + // over the prefix "a". + TEST_F(QueryPlannerTest, CompoundMultikeyBoundsElemMatch) { + // true means multikey + addIndex(BSON("a.x" << 1 << "a.b.c" << 1), true); + runQuery(fromjson("{'a.x': 1, a: {$elemMatch: {b: {$elemMatch: {c: {$gte: 1}}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.x':1, 'a.b.c':1}, bounds: " + "{'a.x': [[1,1,true,true]], " + " 'a.b.c': [['MinKey','MaxKey',true,true]]}}}}}"); + } + + // The index bounds cannot be intersected because the index is multikey. + // The bounds could be intersected if there was an $elemMatch applied to path + // "a.b.c". However, the $elemMatch is applied to the path "a.b" rather than + // the full path of the indexed field. + TEST_F(QueryPlannerTest, MultikeyNestedElemMatch) { + // true means multikey + addIndex(BSON("a.b.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {c: {$gte: 1, $lte: 1}}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, bounds: " + "{'a.b.c': [[-Infinity, 1, true, true]]}}}}}"); + } + + // The index bounds cannot be intersected because the index is multikey. + // The bounds could be intersected if there was an $elemMatch applied to path + // "a.b.c". However, the $elemMatch is applied to the path "a.b" rather than + // the full path of the indexed field. + TEST_F(QueryPlannerTest, MultikeyNestedElemMatchIn) { + // true means multikey + addIndex(BSON("a.b.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {c: {$gte: 1, $in:[2]}}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, bounds: " + "{'a.b.c': [[1, Infinity, true, true]]}}}}}"); + } + + // The bounds can be compounded because the index is not multikey. + TEST_F(QueryPlannerTest, TwoNestedElemMatchBounds) { + addIndex(BSON("a.d.e" << 1 << "a.b.c" << 1)); + runQuery(fromjson("{a: {$elemMatch: {d: {$elemMatch: {e: {$lte: 1}}}," + "b: {$elemMatch: {c: {$gte: 1}}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.d.e': 1, 'a.b.c': 1}, bounds: " + "{'a.d.e': [[-Infinity, 1, true, true]]," + "'a.b.c': [[1, Infinity, true, true]]}}}}}"); + } + + // The bounds cannot be compounded. Although there is an $elemMatch over the + // shared path prefix 'a', the predicates must be conjoined by the same $elemMatch, + // without nested $elemMatch's intervening. The bounds could be compounded if + // the query were rewritten as {a: {$elemMatch: {'d.e': {$lte: 1}, 'b.c': {$gte: 1}}}}. + TEST_F(QueryPlannerTest, MultikeyTwoNestedElemMatchBounds) { + // true means multikey + addIndex(BSON("a.d.e" << 1 << "a.b.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {d: {$elemMatch: {e: {$lte: 1}}}," + "b: {$elemMatch: {c: {$gte: 1}}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.d.e': 1, 'a.b.c': 1}, bounds: " + "{'a.d.e': [[-Infinity, 1, true, true]]," + "'a.b.c': [['MinKey', 'MaxKey', true, true]]}}}}}"); + } + + // Bounds can be intersected for a multikey index when the predicates are + // joined by an $elemMatch over the full path of the index field. + TEST_F(QueryPlannerTest, MultikeyElemMatchValue) { + // true means multikey + addIndex(BSON("a.b" << 1), true); + runQuery(fromjson("{'a.b': {$elemMatch: {$gte: 1, $lte: 1}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': 1}, bounds: " + "{'a.b': [[1, 1, true, true]]}}}}}"); + } + + // We can intersect the bounds for all three predicates because + // the index is not multikey. + TEST_F(QueryPlannerTest, ElemMatchInterectBoundsNotMultikey) { + addIndex(BSON("a.b" << 1)); + runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {$gte: 1, $lte: 4}}}}," + "'a.b': {$in: [2,5]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': 1}, bounds: " + "{'a.b': [[2, 2, true, true]]}}}}}"); + } + + // Bounds can be intersected for a multikey index when the predicates are + // joined by an $elemMatch over the full path of the index field. The bounds + // from the $in predicate are not intersected with the bounds from the + // remaining to predicates because the $in is not joined to the other + // predicates with an $elemMatch. + TEST_F(QueryPlannerTest, ElemMatchInterectBoundsMultikey) { + // true means multikey + addIndex(BSON("a.b" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {$gte: 1, $lte: 4}}}}," + "'a.b': {$in: [2,5]}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': 1}, bounds: " + "{'a.b': [[1, 4, true, true]]}}}}}"); + } + + // Bounds can be intersected because the predicates are joined by an + // $elemMatch over the path "a.b.c", the full path of the multikey + // index field. + TEST_F(QueryPlannerTest, MultikeyNestedElemMatchValue) { + // true means multikey + addIndex(BSON("a.b.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {'b.c': {$elemMatch: {$gte: 1, $lte: 1}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, bounds: " + "{'a.b.c': [[1, 1, true, true]]}}}}}"); + } + + // Bounds cannot be compounded for a multikey compound index when + // the predicates share a prefix (and there is no $elemMatch). + TEST_F(QueryPlannerTest, MultikeySharedPrefixNoElemMatch) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{'a.b': 1, 'a.c': 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], " + " 'a.c': [['MinKey','MaxKey',true,true]]}}}}}"); + } + + // Bounds can be compounded because there is an $elemMatch applied to the + // shared prefix "a". + TEST_F(QueryPlannerTest, MultikeySharedPrefixElemMatch) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: 1, c: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], 'a.c': [[1,1,true,true]]}}}}}"); + } + + // Bounds cannot be compounded for the multikey index even though there is an + // $elemMatch, because the $elemMatch does not join the two predicates. This + // query is semantically indentical to {'a.b': 1, 'a.c': 1}. + TEST_F(QueryPlannerTest, MultikeySharedPrefixElemMatchNotShared) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{'a.b': 1, a: {$elemMatch: {c: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], " + " 'a.c': [['MinKey','MaxKey',true,true]]}}}}}"); + } + + // Bounds cannot be compounded for the multikey index even though there are + // $elemMatch's, because there is not an $elemMatch which joins the two + // predicates. This query is semantically indentical to {'a.b': 1, 'a.c': 1}. + TEST_F(QueryPlannerTest, MultikeySharedPrefixTwoElemMatches) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{$and: [{a: {$elemMatch: {b: 1}}}, {a: {$elemMatch: {c: 1}}}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], " + " 'a.c': [['MinKey','MaxKey',true,true]]}}}}}"); + } + + // Bounds for the predicates joined by the $elemMatch over the shared prefix + // "a" can be combined. However, the predicate 'a.b'==1 cannot also be combined + // given that it is outside of the $elemMatch. + TEST_F(QueryPlannerTest, MultikeySharedPrefixNoIntersectOutsideElemMatch) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{'a.b': 1, a: {$elemMatch: {b: {$gt: 0}, c: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[0,Infinity,false,true]], " + " 'a.c': [[1,1,true,true]]}}}}}"); + } + + // Bounds for the predicates joined by the $elemMatch over the shared prefix + // "a" can be combined. However, the predicate outside the $elemMatch + // cannot also be combined. + TEST_F(QueryPlannerTest, MultikeySharedPrefixNoIntersectOutsideElemMatch2) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: 1, c: 1}}, 'a.b': 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], " + " 'a.c': [[1,1,true,true]]}}}}}"); + } + + // Bounds for the predicates joined by the $elemMatch over the shared prefix + // "a" can be combined. However, the predicate outside the $elemMatch + // cannot also be combined. + TEST_F(QueryPlannerTest, MultikeySharedPrefixNoIntersectOutsideElemMatch3) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1), true); + runQuery(fromjson("{'a.c': 2, a: {$elemMatch: {b: 1, c: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], " + " 'a.c': [[1,1,true,true]]}}}}}"); + } + + // There are two sets of fields that share a prefix: {'a.b', 'a.c'} and + // {'d.e', 'd.f'}. Since the index is multikey, we can only use the bounds from + // one member of each of these sets. + TEST_F(QueryPlannerTest, MultikeyTwoSharedPrefixesBasic) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1 << "d.e" << 1 << "d.f" << 1), true); + runQuery(fromjson("{'a.b': 1, 'a.c': 1, 'd.e': 1, 'd.f': 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1,'d.e':1,'d.f':1}," + "bounds: {'a.b':[[1,1,true,true]], " + " 'a.c':[['MinKey','MaxKey',true,true]], " + " 'd.e':[[1,1,true,true]], " + " 'd.f':[['MinKey','MaxKey',true,true]]}}}}}"); + } + + // All bounds can be combined. Although, 'a.b' and 'a.c' share prefix 'a', the + // relevant predicates are joined by an $elemMatch on 'a'. Similarly, predicates + // over 'd.e' and 'd.f' are joined by an $elemMatch on 'd'. + TEST_F(QueryPlannerTest, MultikeyTwoSharedPrefixesTwoElemMatch) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1 << "d.e" << 1 << "d.f" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: 1, c: 1}}, d: {$elemMatch: {e: 1, f: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and: [{a: {$elemMatch: {b: 1, c: 1}}}," + "{d: {$elemMatch: {e: 1, f: 1}}}]}," + "node: {ixscan: {pattern: {'a.b':1,'a.c':1,'d.e':1,'d.f':1}," + "bounds: {'a.b':[[1,1,true,true]], " + " 'a.c':[[1,1,true,true]], " + " 'd.e':[[1,1,true,true]], " + " 'd.f':[[1,1,true,true]]}}}}}"); + } + + // Bounds for 'a.b' and 'a.c' can be combined because of the $elemMatch on 'a'. + // Since predicates an 'd.e' and 'd.f' have no $elemMatch, we use the bounds + // for only one of the two. + TEST_F(QueryPlannerTest, MultikeyTwoSharedPrefixesOneElemMatch) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1 << "d.e" << 1 << "d.f" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: 1, c: 1}}, 'd.e': 1, 'd.f': 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and:[{a:{$elemMatch:{b:1,c:1}}}, {'d.f':1}]}," + "node: {ixscan: {pattern: {'a.b':1,'a.c':1,'d.e':1,'d.f':1}," + "bounds: {'a.b':[[1,1,true,true]], " + " 'a.c':[[1,1,true,true]], " + " 'd.e':[[1,1,true,true]], " + " 'd.f':[['MinKey','MaxKey',true,true]]}}}}}"); + } + + // Bounds for 'd.e' and 'd.f' can be combined because of the $elemMatch on 'd'. + // Since predicates an 'a.b' and 'a.c' have no $elemMatch, we use the bounds + // for only one of the two. + TEST_F(QueryPlannerTest, MultikeyTwoSharedPrefixesOneElemMatch2) { + // true means multikey + addIndex(BSON("a.b" << 1 << "a.c" << 1 << "d.e" << 1 << "d.f" << 1), true); + runQuery(fromjson("{'a.b': 1, 'a.c': 1, d: {$elemMatch: {e: 1, f: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {filter: {$and:[{d:{$elemMatch:{e:1,f:1}}}, {'a.c':1}]}," + "node: {ixscan: {pattern: {'a.b':1,'a.c':1,'d.e':1,'d.f':1}," + "bounds: {'a.b':[[1,1,true,true]], " + " 'a.c':[['MinKey','MaxKey',true,true]], " + " 'd.e':[[1,1,true,true]], " + " 'd.f':[[1,1,true,true]]}}}}}"); + } + + // The bounds cannot be compounded because 'a.b.x' and 'a.b.y' share prefix + // 'a.b' (and there is no $elemMatch). + TEST_F(QueryPlannerTest, MultikeyDoubleDottedNoElemMatch) { + // true means multikey + addIndex(BSON("a.b.x" << 1 << "a.b.y" << 1), true); + runQuery(fromjson("{'a.b.y': 1, 'a.b.x': 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.x':1,'a.b.y':1}, bounds: " + "{'a.b.x': [[1,1,true,true]], " + " 'a.b.y': [['MinKey','MaxKey',true,true]]}}}}}"); + } + + // The bounds can be compounded because the predicates are joined by an + // $elemMatch on the shared prefix "a.b". + TEST_F(QueryPlannerTest, MultikeyDoubleDottedElemMatch) { + // true means multikey + addIndex(BSON("a.b.x" << 1 << "a.b.y" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {x: 1, y: 1}}}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.x':1,'a.b.y':1}, bounds: " + "{'a.b.x': [[1,1,true,true]], " + " 'a.b.y': [[1,1,true,true]]}}}}}"); + } + + // The bounds cannot be compounded. Although there is an $elemMatch that appears + // to join the predicates, the path to which the $elemMatch is applied is "a". + // Therefore, the predicates contained in the $elemMatch are over "b.x" and "b.y". + // They cannot be compounded due to shared prefix "b". + TEST_F(QueryPlannerTest, MultikeyDoubleDottedUnhelpfulElemMatch) { + // true means multikey + addIndex(BSON("a.b.x" << 1 << "a.b.y" << 1), true); + runQuery(fromjson("{a: {$elemMatch: {'b.x': 1, 'b.y': 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.x':1,'a.b.y':1}, bounds: " + "{'a.b.x': [[1,1,true,true]], " + " 'a.b.y': [['MinKey','MaxKey',true,true]]}}}}}"); + } + + // The bounds can be compounded because the predicates are joined by an + // $elemMatch on the shared prefix "a.b". + TEST_F(QueryPlannerTest, MultikeyDoubleDottedElemMatchOnDotted) { + // true means multikey + addIndex(BSON("a.b.x" << 1 << "a.b.y" << 1), true); + runQuery(fromjson("{'a.b': {$elemMatch: {x: 1, y: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.x':1,'a.b.y':1}, bounds: " + "{'a.b.x': [[1,1,true,true]], " + " 'a.b.y': [[1,1,true,true]]}}}}}"); + } + + // This one is subtle. Say we compound the bounds for predicates over "a.b.c" and + // "a.b.d". This is okay because of the predicate over the shared prefix "a.b". + // It might seem like we can do the same for the $elemMatch over shared prefix "a.e", + // thus combining all bounds. But in fact, we can't combine any more bounds because + // we have already used prefix "a". In other words, this query is like having predicates + // over "a.b" and "a.e", so we can only use bounds from one of the two. + TEST_F(QueryPlannerTest, MultikeyComplexDoubleDotted) { + // true means multikey + addIndex(BSON("a.b.c" << 1 << "a.e.f" << 1 << "a.b.d" << 1 << "a.e.g" << 1), true); + runQuery(fromjson("{'a.b': {$elemMatch: {c: 1, d: 1}}, " + "'a.e': {$elemMatch: {f: 1, g: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c':1,'a.e.f':1,'a.b.d':1,'a.e.g':1}," + "bounds: {'a.b.c':[[1,1,true,true]], " + " 'a.e.f':[['MinKey','MaxKey',true,true]], " + " 'a.b.d':[[1,1,true,true]], " + " 'a.e.g':[['MinKey','MaxKey',true,true]]}}}}}"); + } + + // Similar to MultikeyComplexDoubleDotted above. + TEST_F(QueryPlannerTest, MultikeyComplexDoubleDotted2) { + // true means multikey + addIndex(BSON("a.b.c" << 1 << "a.e.c" << 1 << "a.b.d" << 1 << "a.e.d" << 1), true); + runQuery(fromjson("{'a.b': {$elemMatch: {c: 1, d: 1}}, " + "'a.e': {$elemMatch: {f: 1, g: 1}}}")); + + assertNumSolutions(2U); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c':1,'a.e.c':1,'a.b.d':1,'a.e.d':1}," + "bounds: {'a.b.c':[[1,1,true,true]], " + " 'a.e.c':[['MinKey','MaxKey',true,true]], " + " 'a.b.d':[[1,1,true,true]], " + " 'a.e.d':[['MinKey','MaxKey',true,true]]}}}}}"); + } + + // SERVER-13422: check that we plan $elemMatch object correctly with + // index intersection. + TEST_F(QueryPlannerTest, ElemMatchIndexIntersection) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("shortId" << 1)); + // true means multikey + addIndex(BSON("a.b.startDate" << 1), true); + addIndex(BSON("a.b.endDate" << 1), true); + + runQuery(fromjson("{shortId: 3, 'a.b': {$elemMatch: {startDate: {$lte: 3}," + "endDate: {$gt: 6}}}}")); + + assertNumSolutions(6U); + + // 3 single index solutions. + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {shortId: 1}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.startDate': 1}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.endDate': 1}}}}}"); + + // 3 index intersection solutions. The last one has to intersect two + // predicates within the $elemMatch object. + assertSolutionExists("{fetch: {node: {andHash: {nodes: [" + "{ixscan: {pattern: {shortId: 1}}}," + "{ixscan: {pattern: {'a.b.startDate': 1}}}]}}}}"); + assertSolutionExists("{fetch: {node: {andHash: {nodes: [" + "{ixscan: {pattern: {shortId: 1}}}," + "{ixscan: {pattern: {'a.b.endDate': 1}}}]}}}}"); + assertSolutionExists("{fetch: {node: {andHash: {nodes: [" + "{ixscan: {pattern: {'a.b.startDate': 1}}}," + "{ixscan: {pattern: {'a.b.endDate': 1}}}]}}}}"); + } + + // + // QueryPlannerParams option tests + // + + TEST_F(QueryPlannerTest, NoBlockingSortsAllowedTest) { + params.options = QueryPlannerParams::NO_BLOCKING_SORT; + runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); + assertNumSolutions(0U); + + addIndex(BSON("x" << 1)); + + runQuerySortProj(BSONObj(), BSON("x" << 1), BSONObj()); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NoTableScanBasic) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + runQuery(BSONObj()); + assertNumSolutions(0U); + + addIndex(BSON("x" << 1)); + + runQuery(BSONObj()); + assertNumSolutions(0U); + + runQuery(fromjson("{x: {$gte: 0}}")); + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, NoTableScanOrWithAndChild) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + runQuery(fromjson("{$or: [{a: 20}, {$and: [{a:1}, {b:7}]}]}")); + + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a: 1}}}, " + "{fetch: {filter: {b: 7}, node: {ixscan: " + "{filter: null, pattern: {a: 1}}}}}]}}}}"); + } + + // + // Index Intersection. + // + // We don't exhaustively check all plans here. Instead we check that there exists an + // intersection plan. The blending of >1 index plans and ==1 index plans is under development + // but we want to make sure that we create an >1 index plan when we should. + // + + TEST_F(QueryPlannerTest, IntersectBasicTwoPred) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuery(fromjson("{a:1, b:{$gt: 1}}")); + + assertSolutionExists("{fetch: {filter: null, node: {andHash: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}}}," + "{ixscan: {filter: null, pattern: {b:1}}}]}}}}"); + } + + TEST_F(QueryPlannerTest, IntersectBasicTwoPredCompound) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1 << "c" << 1)); + addIndex(BSON("b" << 1)); + runQuery(fromjson("{a:1, b:1, c:1}")); + + // There's an andSorted not andHash because the two seeks are point intervals. + assertSolutionExists("{fetch: {filter: null, node: {andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1, c:1}}}," + "{ixscan: {filter: null, pattern: {b:1}}}]}}}}"); + } + + // SERVER-12196 + TEST_F(QueryPlannerTest, IntersectBasicTwoPredCompoundMatchesIdxOrder1) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuery(fromjson("{a:1, b:1}")); + + assertNumSolutions(3U); + + assertSolutionExists("{fetch: {filter: {b:1}, node: " + "{ixscan: {filter: null, pattern: {a:1}}}}}"); + assertSolutionExists("{fetch: {filter: {a:1}, node: " + "{ixscan: {filter: null, pattern: {b:1}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}}}," + "{ixscan: {filter: null, pattern: {b:1}}}]}}}}"); + } + + // SERVER-12196 + TEST_F(QueryPlannerTest, IntersectBasicTwoPredCompoundMatchesIdxOrder2) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("b" << 1)); + addIndex(BSON("a" << 1)); + runQuery(fromjson("{a:1, b:1}")); + + assertNumSolutions(3U); + + assertSolutionExists("{fetch: {filter: {b:1}, node: " + "{ixscan: {filter: null, pattern: {a:1}}}}}"); + assertSolutionExists("{fetch: {filter: {a:1}, node: " + "{ixscan: {filter: null, pattern: {b:1}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: {andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}}}," + "{ixscan: {filter: null, pattern: {b:1}}}]}}}}"); + } + + TEST_F(QueryPlannerTest, IntersectManySelfIntersections) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + // True means multikey. + addIndex(BSON("a" << 1), true); + + // This one goes to 11. + runQuery(fromjson("{a:1, a:2, a:3, a:4, a:5, a:6, a:7, a:8, a:9, a:10, a:11}")); + + // But this one only goes to 10. + assertSolutionExists("{fetch: {filter: {a:11}, node: {andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}}}," // 1 + "{ixscan: {filter: null, pattern: {a:1}}}," // 2 + "{ixscan: {filter: null, pattern: {a:1}}}," // 3 + "{ixscan: {filter: null, pattern: {a:1}}}," // 4 + "{ixscan: {filter: null, pattern: {a:1}}}," // 5 + "{ixscan: {filter: null, pattern: {a:1}}}," // 6 + "{ixscan: {filter: null, pattern: {a:1}}}," // 7 + "{ixscan: {filter: null, pattern: {a:1}}}," // 8 + "{ixscan: {filter: null, pattern: {a:1}}}," // 9 + "{ixscan: {filter: null, pattern: {a:1}}}]}}}}"); // 10 + } + + TEST_F(QueryPlannerTest, IntersectSubtreeNodes) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + addIndex(BSON("d" << 1)); + + runQuery(fromjson("{$or: [{a: 1}, {b: 1}], $or: [{c:1}, {d:1}]}")); + assertSolutionExists("{fetch: {filter: null, node: {andHash: {nodes: [" + "{or: {nodes: [{ixscan:{filter:null, pattern:{a:1}}}," + "{ixscan:{filter:null, pattern:{b:1}}}]}}," + "{or: {nodes: [{ixscan:{filter:null, pattern:{c:1}}}," + "{ixscan:{filter:null, pattern:{d:1}}}]}}]}}}}"); + } + + TEST_F(QueryPlannerTest, IntersectSubtreeAndPred) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + runQuery(fromjson("{a: 1, $or: [{b:1}, {c:1}]}")); + + // This (can be) rewritten to $or:[ {a:1, b:1}, {c:1, d:1}]. We don't look for the various + // single $or solutions as that's tested elsewhere. We look for the intersect solution, + // where each AND inside of the root OR is an and_sorted. + size_t matches = 0; + matches += numSolutionMatches("{fetch: {filter: null, node: {or: {nodes: [" + "{andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {'a':1}}}," + "{ixscan: {filter: null, pattern: {'b':1}}}]}}," + "{andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {'a':1}}}," + "{ixscan: {filter: null, pattern: {'c':1}}}]}}]}}}}"); + matches += numSolutionMatches("{fetch: {filter: null, node: {andHash: {nodes:[" + "{or: {nodes: [{ixscan:{filter:null, pattern:{b:1}}}," + "{ixscan:{filter:null, pattern:{c:1}}}]}}," + "{ixscan:{filter: null, pattern:{a:1}}}]}}}}"); + ASSERT_GREATER_THAN_OR_EQUALS(matches, 1U); + } + + TEST_F(QueryPlannerTest, IntersectElemMatch) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a.b" << 1)); + addIndex(BSON("a.c" << 1)); + runQuery(fromjson("{a : {$elemMatch: {b:1, c:1}}}")); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:1, c:1}}}," + "node: {andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {'a.b':1}}}," + "{ixscan: {filter: null, pattern: {'a.c':1}}}]}}}}"); + } + + TEST_F(QueryPlannerTest, IntersectSortFromAndHash) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + runQuerySortProj(fromjson("{a: 1, b:{$gt: 1}}"), fromjson("{b:1}"), BSONObj()); + + // This provides the sort. + assertSolutionExists("{fetch: {filter: null, node: {andHash: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}}}," + "{ixscan: {filter: null, pattern: {b:1}}}]}}}}"); + + // Rearrange the preds, shouldn't matter. + runQuerySortProj(fromjson("{b: 1, a:{$lt: 7}}"), fromjson("{b:1}"), BSONObj()); + assertSolutionExists("{fetch: {filter: null, node: {andHash: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}}}," + "{ixscan: {filter: null, pattern: {b:1}}}]}}}}"); + } + + TEST_F(QueryPlannerTest, IntersectCanBeVeryBig) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + addIndex(BSON("d" << 1)); + runQuery(fromjson("{$or: [{ 'a' : null, 'b' : 94, 'c' : null, 'd' : null }," + "{ 'a' : null, 'b' : 98, 'c' : null, 'd' : null }," + "{ 'a' : null, 'b' : 1, 'c' : null, 'd' : null }," + "{ 'a' : null, 'b' : 2, 'c' : null, 'd' : null }," + "{ 'a' : null, 'b' : 7, 'c' : null, 'd' : null }," + "{ 'a' : null, 'b' : 9, 'c' : null, 'd' : null }," + "{ 'a' : null, 'b' : 16, 'c' : null, 'd' : null }]}")); + + assertNumSolutions(internalQueryEnumerationMaxOrSolutions); + } + + // Ensure that disabling AND_HASH intersection works properly. + TEST_F(QueryPlannerTest, IntersectDisableAndHash) { + bool oldEnableHashIntersection = internalQueryPlannerEnableHashIntersection; + + // Turn index intersection on but disable hash-based intersection. + internalQueryPlannerEnableHashIntersection = false; + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + + runQuery(fromjson("{a: {$gt: 1}, b: 1, c: 1}")); + + // We should do an AND_SORT intersection of {b: 1} and {c: 1}, but no AND_HASH plans. + assertNumSolutions(4U); + assertSolutionExists("{fetch: {filter: {b: 1, c: 1}, node: {ixscan: " + "{pattern: {a: 1}, bounds: {a: [[1,Infinity,false,true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {a:{$gt:1},c:1}, node: {ixscan: " + "{pattern: {b: 1}, bounds: {b: [[1,1,true,true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {a:{$gt:1},b:1}, node: {ixscan: " + "{pattern: {c: 1}, bounds: {c: [[1,1,true,true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {a:{$gt:1}}, node: {andSorted: {nodes: [" + "{ixscan: {filter: null, pattern: {b:1}}}," + "{ixscan: {filter: null, pattern: {c:1}}}]}}}}"); + + // Restore the old value of the has intersection switch. + internalQueryPlannerEnableHashIntersection = oldEnableHashIntersection; + } + + // + // Index intersection cases for SERVER-12825: make sure that + // we don't generate an ixisect plan if a compound index is + // available instead. + // + + // SERVER-12825 + TEST_F(QueryPlannerTest, IntersectCompoundInsteadBasic) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("a" << 1 << "b" << 1)); + runQuery(fromjson("{a: 1, b: 1}")); + + assertNumSolutions(3U); + assertSolutionExists("{fetch: {filter: {b:1}, node: " + "{ixscan: {filter: null, pattern: {a:1}}}}}"); + assertSolutionExists("{fetch: {filter: {a:1}, node: " + "{ixscan: {filter: null, pattern: {b:1}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a:1,b:1}}}}}"); + } + + // SERVER-12825 + TEST_F(QueryPlannerTest, IntersectCompoundInsteadThreeCompoundIndices) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1 << "b" << 1)); + addIndex(BSON("c" << 1 << "d" << 1)); + addIndex(BSON("a" << 1 << "c" << -1 << "b" << -1 << "d" << 1)); + runQuery(fromjson("{a: 1, b: 1, c: 1, d: 1}")); + + assertNumSolutions(3U); + assertSolutionExists("{fetch: {filter: {$and: [{c:1},{d:1}]}, node: " + "{ixscan: {filter: null, pattern: {a:1,b:1}}}}}"); + assertSolutionExists("{fetch: {filter: {$and:[{a:1},{b:1}]}, node: " + "{ixscan: {filter: null, pattern: {c:1,d:1}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a:1,c:-1,b:-1,d:1}}}}}"); + } + + // SERVER-12825 + TEST_F(QueryPlannerTest, IntersectCompoundInsteadUnusedField) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + runQuery(fromjson("{a: 1, b: 1}")); + + assertNumSolutions(3U); + assertSolutionExists("{fetch: {filter: {b:1}, node: " + "{ixscan: {filter: null, pattern: {a:1}}}}}"); + assertSolutionExists("{fetch: {filter: {a:1}, node: " + "{ixscan: {filter: null, pattern: {b:1}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a:1,b:1,c:1}}}}}"); + } + + // SERVER-12825 + TEST_F(QueryPlannerTest, IntersectCompoundInsteadUnusedField2) { + params.options = QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::INDEX_INTERSECTION; + addIndex(BSON("a" << 1 << "b" << 1)); + addIndex(BSON("c" << 1 << "d" << 1)); + addIndex(BSON("a" << 1 << "b" << 1 << "c" << 1)); + runQuery(fromjson("{a: 1, c: 1}")); + + assertNumSolutions(3U); + assertSolutionExists("{fetch: {filter: {c:1}, node: " + "{ixscan: {filter: null, pattern: {a:1,b:1}}}}}"); + assertSolutionExists("{fetch: {filter: {a:1}, node: " + "{ixscan: {filter: null, pattern: {c:1,d:1}}}}}"); + assertSolutionExists("{fetch: {filter: null, node: " + "{ixscan: {filter: null, pattern: {a:1,b:1,c:1}}}}}"); + } + + // + // 2dsphere V2 sparse indices, SERVER-9639 + // + + // Basic usage of a sparse 2dsphere index. V1 ignores the sparse field. We can use any prefix + // of the index as every document is indexed. + TEST_F(QueryPlannerTest, TwoDSphereSparseV1) { + // Create a V1 index. + addIndex(BSON("nonGeo" << 1 << "geo" << "2dsphere"), + BSON("2dsphereIndexVersion" << 1)); + + // Can use the index for this. + runQuery(fromjson("{nonGeo: 7}")); + assertNumSolutions(2); + assertSolutionExists("{cscan: {dir: 1}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {nonGeo: 1, geo: '2dsphere'}}}}}"); + } + + // V2 is "geo sparse" and removes the nonGeo assignment. + TEST_F(QueryPlannerTest, TwoDSphereSparseV2CantUse) { + // Create a V2 index. + addIndex(BSON("nonGeo" << 1 << "geo" << "2dsphere"), + BSON("2dsphereIndexVersion" << 2)); + + // Can't use the index prefix here as it's a V2 index and we have no geo pred. + runQuery(fromjson("{nonGeo: 7}")); + assertNumSolutions(1); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + TEST_F(QueryPlannerTest, TwoDSphereSparseOnePred) { + // Create a V2 index. + addIndex(BSON("geo" << "2dsphere"), + BSON("2dsphereIndexVersion" << 2)); + + // We can use the index here as we have a geo pred. + runQuery(fromjson("{geo : { $geoWithin : { $centerSphere : [[ 10, 20 ], 0.01 ] } }}}")); + assertNumSolutions(2); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // V2 is geo-sparse and the planner removes the nonGeo assignment when there's no geo pred + TEST_F(QueryPlannerTest, TwoDSphereSparseV2TwoPreds) { + addIndex(BSON("nonGeo" << 1 << "geo" << "2dsphere" << "geo2" << "2dsphere"), + BSON("2dsphereIndexVersion" << 2)); + + // Non-geo preds can only use a collscan. + runQuery(fromjson("{nonGeo: 7}")); + assertNumSolutions(1); + assertSolutionExists("{cscan: {dir: 1}}"); + + // One geo pred so we can use the index. + runQuery(fromjson("{nonGeo: 7, geo : { $geoWithin : { $centerSphere : [[ 10, 20 ], 0.01 ] } }}}")); + ASSERT_EQUALS(getNumSolutions(), 2U); + + // Two geo preds, so we can use the index still. + runQuery(fromjson("{nonGeo: 7, geo : { $geoWithin : { $centerSphere : [[ 10, 20 ], 0.01 ] }}," + " geo2 : { $geoWithin : { $centerSphere : [[ 10, 20 ], 0.01 ] }}}")); + ASSERT_EQUALS(getNumSolutions(), 2U); + } + + TEST_F(QueryPlannerTest, TwoDNearCompound) { + addIndex(BSON("geo" << "2dsphere" << "nongeo" << 1), + BSON("2dsphereIndexVersion" << 2)); + runQuery(fromjson("{geo: {$nearSphere: [-71.34895, 42.46037]}}")); + ASSERT_EQUALS(getNumSolutions(), 1U); + } + + TEST_F(QueryPlannerTest, TwoDSphereSparseV2BelowOr) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + addIndex(BSON("geo1" << "2dsphere" << "a" << 1 << "b" << 1), + BSON("2dsphereIndexVersion" << 2)); + addIndex(BSON("geo2" << "2dsphere" << "a" << 1 << "b" << 1), + BSON("2dsphereIndexVersion" << 2)); + + runQuery(fromjson("{a: 4, b: 5, $or: [" + "{geo1: {$geoWithin: {$centerSphere: [[10, 20], 0.01]}}}," + "{geo2: {$geoWithin: {$centerSphere: [[10, 20], 0.01]}}}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a: 4, b: 5}, node: {or: {nodes: [" + "{fetch: {node: {ixscan: {pattern: {geo1:'2dsphere',a:1,b:1}}}}}," + "{fetch: {node: {ixscan: {pattern: {geo2:'2dsphere',a:1,b:1}}}}}" + "]}}}}"); + } + + TEST_F(QueryPlannerTest, TwoDSphereSparseV2BelowElemMatch) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a.b" << "2dsphere" << "a.c" << 1), + BSON("2dsphereIndexVersion" << 2)); + + runQuery(fromjson("{a: {$elemMatch: {b: {$geoWithin: {$centerSphere: [[10,20], 0.01]}}," + "c: {$gt: 3}}}}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': '2dsphere', 'a.c': 1}}}}}"); + } + + // + // Test that we add a KeepMutations when we should and and we don't add one when we shouldn't. + // + + // Collection scan doesn't keep any state, so it can't produce flagged data. + TEST_F(QueryPlannerTest, NoMutationsForCollscan) { + params.options = QueryPlannerParams::KEEP_MUTATIONS; + runQuery(fromjson("")); + assertSolutionExists("{cscan: {dir: 1}}"); + } + + // Collscan + sort doesn't produce flagged data either. + TEST_F(QueryPlannerTest, NoMutationsForSort) { + params.options = QueryPlannerParams::KEEP_MUTATIONS; + runQuerySortProj(fromjson(""), fromjson("{a:1}"), BSONObj()); + assertSolutionExists("{sort: {pattern: {a: 1}, limit: 0, node: {cscan: {dir: 1}}}}"); + } + + // An index scan + fetch requires a keep node as it can flag data. Also make sure we put it in + // the right place, under the sort. + TEST_F(QueryPlannerTest, MutationsFromFetch) { + params.options = QueryPlannerParams::KEEP_MUTATIONS; + addIndex(BSON("a" << 1)); + runQuerySortProj(fromjson("{a: 5}"), fromjson("{b:1}"), BSONObj()); + assertSolutionExists("{sort: {pattern: {b:1}, limit: 0, node: {keep: {node: " + "{fetch: {node: {ixscan: {pattern: {a:1}}}}}}}}}"); + } + + // Index scan w/covering doesn't require a keep node as there's no fetch. + TEST_F(QueryPlannerTest, NoFetchNoKeep) { + params.options = QueryPlannerParams::KEEP_MUTATIONS; + addIndex(BSON("x" << 1)); + // query, sort, proj + runQuerySortProj(fromjson("{ x : {$gt: 1}}"), BSONObj(), fromjson("{_id: 0, x: 1}")); + + // cscan is a soln but we override the params that say to include it. + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{proj: {spec: {_id: 0, x: 1}, node: {ixscan: " + "{filter: null, pattern: {x: 1}}}}}"); + } + + // No keep with geoNear. + TEST_F(QueryPlannerTest, NoKeepWithGeoNear) { + params.options = QueryPlannerParams::KEEP_MUTATIONS; + addIndex(BSON("a" << "2d")); + runQuery(fromjson("{a: {$near: [0,0], $maxDistance:0.3 }}")); + ASSERT_EQUALS(getNumSolutions(), 1U); + assertSolutionExists("{geoNear2d: {a: '2d'}}"); + } + + // No keep when we have an indexed sort. + TEST_F(QueryPlannerTest, NoKeepWithIndexedSort) { + params.options = QueryPlannerParams::KEEP_MUTATIONS; + addIndex(BSON("a" << 1 << "b" << 1)); + runQuerySortProjSkipLimit(fromjson("{a: {$in: [1, 2]}}"), + BSON("b" << 1), BSONObj(), 0, 1); + + // cscan solution exists but we didn't turn on the "always include a collscan." + assertNumSolutions(1); + assertSolutionExists("{fetch: {node: {mergeSort: {nodes: " + "[{ixscan: {pattern: {a: 1, b: 1}}}, {ixscan: {pattern: {a: 1, b: 1}}}]}}}}"); + } + + // Make sure a top-level $or hits the limiting number + // of solutions that we are willing to consider. + TEST_F(QueryPlannerTest, OrEnumerationLimit) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + + // 6 $or clauses, each with 2 indexed predicates + // means 2^6 = 64 possibilities. We should hit the limit. + runQuery(fromjson("{$or: [{a: 1, b: 1}," + "{a: 2, b: 2}," + "{a: 3, b: 3}," + "{a: 4, b: 4}," + "{a: 5, b: 5}," + "{a: 6, b: 6}]}")); + + assertNumSolutions(internalQueryEnumerationMaxOrSolutions); + } + + TEST_F(QueryPlannerTest, OrEnumerationLimit2) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + addIndex(BSON("d" << 1)); + + // 3 $or clauses, and a few other preds. Each $or clause can + // generate up to the max number of allowed $or enumerations. + runQuery(fromjson("{$or: [{a: 1, b: 1, c: 1, d: 1}," + "{a: 2, b: 2, c: 2, d: 2}," + "{a: 3, b: 3, c: 3, d: 3}]}")); + + assertNumSolutions(internalQueryEnumerationMaxOrSolutions); + } + + // SERVER-13104: test that we properly enumerate all solutions for nested $or. + TEST_F(QueryPlannerTest, EnumerateNestedOr) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + + runQuery(fromjson("{d: 1, $or: [{a: 1, b: 1}, {c: 1}]}")); + + assertNumSolutions(2U); + assertSolutionExists("{fetch: {filter: {d: 1}, node: {or: {nodes: [" + "{fetch: {filter: {b: 1}, node: {ixscan: {pattern: {a: 1}}}}}," + "{ixscan: {pattern: {c: 1}}}]}}}}"); + assertSolutionExists("{fetch: {filter: {d: 1}, node: {or: {nodes: [" + "{fetch: {filter: {a: 1}, node: {ixscan: {pattern: {b: 1}}}}}," + "{ixscan: {pattern: {c: 1}}}]}}}}"); + } + + // SERVER-13104: test that we properly enumerate all solutions for nested $or. + TEST_F(QueryPlannerTest, EnumerateNestedOr2) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + addIndex(BSON("d" << 1)); + addIndex(BSON("e" << 1)); + addIndex(BSON("f" << 1)); + + runQuery(fromjson("{a: 1, b: 1, $or: [{c: 1, d: 1}, {e: 1, f: 1}]}")); + + assertNumSolutions(6U); + + // Four possibilities from indexing the $or. + assertSolutionExists("{fetch: {filter: {a: 1, b: 1}, node: {or: {nodes: [" + "{fetch: {filter: {d: 1}, node: {ixscan: {pattern: {c: 1}}}}}," + "{fetch: {filter: {f: 1}, node: {ixscan: {pattern: {e: 1}}}}}" + "]}}}}"); + assertSolutionExists("{fetch: {filter: {a: 1, b: 1}, node: {or: {nodes: [" + "{fetch: {filter: {c: 1}, node: {ixscan: {pattern: {d: 1}}}}}," + "{fetch: {filter: {f: 1}, node: {ixscan: {pattern: {e: 1}}}}}" + "]}}}}"); + assertSolutionExists("{fetch: {filter: {a: 1, b: 1}, node: {or: {nodes: [" + "{fetch: {filter: {d: 1}, node: {ixscan: {pattern: {c: 1}}}}}," + "{fetch: {filter: {e: 1}, node: {ixscan: {pattern: {f: 1}}}}}" + "]}}}}"); + assertSolutionExists("{fetch: {filter: {a: 1, b: 1}, node: {or: {nodes: [" + "{fetch: {filter: {c: 1}, node: {ixscan: {pattern: {d: 1}}}}}," + "{fetch: {filter: {e: 1}, node: {ixscan: {pattern: {f: 1}}}}}" + "]}}}}"); + + // Two possibilties from outside the $or. + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: 1}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {b: 1}}}}}"); + } + + // + // Test the "split limited sort stages" hack. + // + + TEST_F(QueryPlannerTest, SplitLimitedSort) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + params.options |= QueryPlannerParams::SPLIT_LIMITED_SORT; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + + runQuerySortProjSkipLimit(fromjson("{a: 1}"), fromjson("{b: 1}"), + BSONObj(), 0, 3); + + assertNumSolutions(2U); + // First solution has no blocking stage; no need to split. + assertSolutionExists("{fetch: {filter: {a:1}, node: " + "{ixscan: {filter: null, pattern: {b: 1}}}}}"); + // Second solution has a blocking sort with a limit: it gets split and + // joined with an OR stage. + assertSolutionExists("{or: {nodes: [" + "{sort: {pattern: {b: 1}, limit: 3, node: " + "{fetch: {node: {ixscan: {pattern: {a: 1}}}}}}}, " + "{sort: {pattern: {b: 1}, limit: 0, node: " + "{fetch: {node: {ixscan: {pattern: {a: 1}}}}}}}]}}"); + } + + // If a lookup against a unique index is available as a possible plan, then the planner + // should not generate other possibilities. + TEST_F(QueryPlannerTest, UniqueIndexLookup) { + params.options = QueryPlannerParams::INDEX_INTERSECTION; + params.options |= QueryPlannerParams::NO_TABLE_SCAN; + + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1), + false, // multikey + false, // sparse, + true); // unique + + runQuery(fromjson("{a: 1, b: 1}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a: 1}, node: " + "{ixscan: {filter: null, pattern: {b: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, HintOnNonUniqueIndex) { + params.options = QueryPlannerParams::INDEX_INTERSECTION; + + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1), + false, // multikey + false, // sparse, + true); // unique + + runQueryHint(fromjson("{a: 1, b: 1}"), BSON("a" << 1)); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {b: 1}, node: " + "{ixscan: {filter: null, pattern: {a: 1}}}}}"); + } + + TEST_F(QueryPlannerTest, UniqueIndexLookupBelowOr) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + addIndex(BSON("d" << 1), + false, // multikey + false, // sparse, + true); // unique + + runQuery(fromjson("{$or: [{a: 1, b: 1}, {c: 1, d: 1}]}")); + + // Only two plans because we throw out plans for the right branch of the $or that do not + // use equality over the unique index. + assertNumSolutions(2U); + assertSolutionExists("{or: {nodes: [" + "{fetch: {filter: {a: 1}, node: {ixscan: {pattern: {b: 1}}}}}," + "{fetch: {filter: {c: 1}, node: {ixscan: {pattern: {d: 1}}}}}]}}"); + assertSolutionExists("{or: {nodes: [" + "{fetch: {filter: {b: 1}, node: {ixscan: {pattern: {a: 1}}}}}," + "{fetch: {filter: {c: 1}, node: {ixscan: {pattern: {d: 1}}}}}]}}"); + } + + TEST_F(QueryPlannerTest, UniqueIndexLookupBelowOrBelowAnd) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("c" << 1)); + addIndex(BSON("d" << 1), + false, // multikey + false, // sparse, + true); // unique + + runQuery(fromjson("{e: 1, $or: [{a: 1, b: 1}, {c: 1, d: 1}]}")); + + // Only two plans because we throw out plans for the right branch of the $or that do not + // use equality over the unique index. + assertNumSolutions(2U); + assertSolutionExists("{fetch: {filter: {e: 1}, node: {or: {nodes: [" + "{fetch: {filter: {a: 1}, node: {ixscan: {pattern: {b: 1}}}}}," + "{fetch: {filter: {c: 1}, node: {ixscan: {pattern: {d: 1}}}}}" + "]}}}}"); + assertSolutionExists("{fetch: {filter: {e: 1}, node: {or: {nodes: [" + "{fetch: {filter: {b: 1}, node: {ixscan: {pattern: {a: 1}}}}}," + "{fetch: {filter: {c: 1}, node: {ixscan: {pattern: {d: 1}}}}}" + "]}}}}"); + } + + TEST_F(QueryPlannerTest, CoveredOrUniqueIndexLookup) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + addIndex(BSON("a" << 1 << "b" << 1)); + addIndex(BSON("a" << 1), + false, // multikey + false, // sparse, + true); // unique + + runQuerySortProj(fromjson("{a: 1, b: 1}"), BSONObj(), fromjson("{_id: 0, a: 1}")); + + assertNumSolutions(2U); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{fetch: {filter: {b: 1}, node: {ixscan: {pattern: {a: 1}}}}}}}"); + assertSolutionExists("{proj: {spec: {_id: 0, a: 1}, node: " + "{ixscan: {filter: null, pattern: {a: 1, b: 1}}}}}"); + } + + // + // Test bad input to query planner helpers. + // + + TEST(BadInputTest, CacheDataFromTaggedTree) { + PlanCacheIndexTree* indexTree; + + // Null match expression. + vector relevantIndices; + Status s = QueryPlanner::cacheDataFromTaggedTree(NULL, relevantIndices, &indexTree); + ASSERT_NOT_OK(s); + ASSERT(NULL == indexTree); + + // No relevant index matching the index tag. + relevantIndices.push_back(IndexEntry(BSON("a" << 1))); + + CanonicalQuery *cq; + Status cqStatus = CanonicalQuery::canonicalize(ns, BSON("a" << 3), &cq); + ASSERT_OK(cqStatus); + boost::scoped_ptr scopedCq(cq); + scopedCq->root()->setTag(new IndexTag(1)); + + s = QueryPlanner::cacheDataFromTaggedTree(scopedCq->root(), relevantIndices, &indexTree); + ASSERT_NOT_OK(s); + ASSERT(NULL == indexTree); + } + + TEST(BadInputTest, TagAccordingToCache) { + CanonicalQuery *cq; + Status cqStatus = CanonicalQuery::canonicalize(ns, BSON("a" << 3), &cq); + ASSERT_OK(cqStatus); + boost::scoped_ptr scopedCq(cq); + + PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); + indexTree->setIndexEntry(IndexEntry(BSON("a" << 1))); + + map indexMap; + + // Null filter. + Status s = QueryPlanner::tagAccordingToCache(NULL, indexTree, indexMap); + ASSERT_NOT_OK(s); + + // Null indexTree. + s = QueryPlanner::tagAccordingToCache(scopedCq->root(), NULL, indexMap); + ASSERT_NOT_OK(s); + + // Index not found. + s = QueryPlanner::tagAccordingToCache(scopedCq->root(), indexTree, indexMap); + ASSERT_NOT_OK(s); + + // Index found once added to the map. + indexMap[BSON("a" << 1)] = 0; + s = QueryPlanner::tagAccordingToCache(scopedCq->root(), indexTree, indexMap); + ASSERT_OK(s); + + // Regenerate canonical query in order to clear tags. + cqStatus = CanonicalQuery::canonicalize(ns, BSON("a" << 3), &cq); + ASSERT_OK(cqStatus); + scopedCq.reset(cq); + + // Mismatched tree topology. + PlanCacheIndexTree* child = new PlanCacheIndexTree(); + child->setIndexEntry(IndexEntry(BSON("a" << 1))); + indexTree->children.push_back(child); + s = QueryPlanner::tagAccordingToCache(scopedCq->root(), indexTree, indexMap); + ASSERT_NOT_OK(s); + } + +} // namespace diff --git a/src/mongo/db/query/query_planner_test_lib.cpp b/src/mongo/db/query/query_planner_test_lib.cpp new file mode 100644 index 00000000000..74b087a1406 --- /dev/null +++ b/src/mongo/db/query/query_planner_test_lib.cpp @@ -0,0 +1,483 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/query_planner.cpp + */ + +#include "mongo/db/query/query_planner_test_lib.h" + +#include +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +namespace { + + using namespace mongo; + + bool filterMatches(const BSONObj& testFilter, + const QuerySolutionNode* trueFilterNode) { + if (NULL == trueFilterNode->filter) { return false; } + StatusWithMatchExpression swme = MatchExpressionParser::parse(testFilter); + if (!swme.isOK()) { + return false; + } + const boost::scoped_ptr root(swme.getValue()); + CanonicalQuery::sortTree(root.get()); + const boost::scoped_ptr trueFilter(trueFilterNode->filter->shallowClone()); + CanonicalQuery::sortTree(trueFilter.get()); + return trueFilter->equivalent(root.get()); + } + + void appendIntervalBound(BSONObjBuilder& bob, BSONElement& el) { + if (el.type() == String) { + std::string data = el.String(); + if (data == "MaxKey") { + bob.appendMaxKey(""); + } + else if (data == "MinKey") { + bob.appendMinKey(""); + } + else { + bob.appendAs(el, ""); + } + } + else { + bob.appendAs(el, ""); + } + } + + bool intervalMatches(const BSONObj& testInt, const Interval trueInt) { + BSONObjIterator it(testInt); + if (!it.more()) { return false; } + BSONElement low = it.next(); + if (!it.more()) { return false; } + BSONElement high = it.next(); + if (!it.more()) { return false; } + bool startInclusive = it.next().Bool(); + if (!it.more()) { return false; } + bool endInclusive = it.next().Bool(); + if (it.more()) { return false; } + + BSONObjBuilder bob; + appendIntervalBound(bob, low); + appendIntervalBound(bob, high); + Interval toCompare(bob.obj(), startInclusive, endInclusive); + + return Interval::INTERVAL_EQUALS == trueInt.compare(toCompare); + } + + /** + * Returns whether the BSON representation of the index bounds in + * 'testBounds' matches 'trueBounds'. + * + * 'testBounds' should be of the following format: + * {: , : , ...} + * Each ordered interval list (e.g. ) is an array of arrays of + * the format: + * [[,,,], ...] + * + * For example, + * {a: [[1,2,true,false], [3,4,false,true]], b: [[-Infinity, Infinity]]} + * Means that the index bounds on field 'a' consist of the two intervals + * [1, 2) and (3, 4] and the index bounds on field 'b' are [-Infinity, Infinity]. + */ + bool boundsMatch(const BSONObj& testBounds, const IndexBounds trueBounds) { + // Iterate over the fields on which we have index bounds. + BSONObjIterator fieldIt(testBounds); + int fieldItCount = 0; + while (fieldIt.more()) { + BSONElement arrEl = fieldIt.next(); + if (arrEl.type() != Array) { + return false; + } + // Iterate over an ordered interval list for + // a particular field. + BSONObjIterator oilIt(arrEl.Obj()); + int oilItCount = 0; + while (oilIt.more()) { + BSONElement intervalEl = oilIt.next(); + if (intervalEl.type() != Array) { + return false; + } + Interval trueInt = trueBounds.getInterval(fieldItCount, oilItCount); + if (!intervalMatches(intervalEl.Obj(), trueInt)) { + return false; + } + ++oilItCount; + } + ++fieldItCount; + } + + return true; + } + +} // namespace + +namespace mongo { + + /** + * Looks in the children stored in the 'nodes' field of 'testSoln' + * to see if thet match the 'children' field of 'trueSoln'. + * + * This does an unordered comparison, i.e. childrenMatch returns + * true as long as the set of subtrees in testSoln's 'nodes' matches + * the set of subtrees in trueSoln's 'children' vector. + */ + static bool childrenMatch(const BSONObj& testSoln, const QuerySolutionNode* trueSoln) { + BSONElement children = testSoln["nodes"]; + if (children.eoo() || !children.isABSONObj()) { return false; } + + // The order of the children array in testSoln might not match + // the order in trueSoln, so we have to check all combos with + // these nested loops. + BSONObjIterator i(children.Obj()); + while (i.more()) { + BSONElement child = i.next(); + if (child.eoo() || !child.isABSONObj()) { return false; } + + // try to match against one of the QuerySolutionNode's children + bool found = false; + for (size_t j = 0; j < trueSoln->children.size(); ++j) { + if (QueryPlannerTestLib::solutionMatches(child.Obj(), trueSoln->children[j])) { + found = true; + break; + } + } + + // we couldn't match child + if (!found) { return false; } + } + + return true; + } + + // static + bool QueryPlannerTestLib::solutionMatches(const BSONObj& testSoln, + const QuerySolutionNode* trueSoln) { + // + // leaf nodes + // + if (STAGE_COLLSCAN == trueSoln->getType()) { + const CollectionScanNode* csn = static_cast(trueSoln); + BSONElement el = testSoln["cscan"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj csObj = el.Obj(); + + BSONElement dir = csObj["dir"]; + if (dir.eoo() || !dir.isNumber()) { return false; } + if (dir.numberInt() != csn->direction) { return false; } + + BSONElement filter = csObj["filter"]; + if (filter.eoo()) { + return true; + } + else if (filter.isNull()) { + return NULL == csn->filter; + } + else if (!filter.isABSONObj()) { + return false; + } + return filterMatches(filter.Obj(), trueSoln); + } + else if (STAGE_IXSCAN == trueSoln->getType()) { + const IndexScanNode* ixn = static_cast(trueSoln); + BSONElement el = testSoln["ixscan"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj ixscanObj = el.Obj(); + + BSONElement pattern = ixscanObj["pattern"]; + if (pattern.eoo() || !pattern.isABSONObj()) { return false; } + if (pattern.Obj() != ixn->indexKeyPattern) { return false; } + + BSONElement bounds = ixscanObj["bounds"]; + if (!bounds.eoo()) { + if (!bounds.isABSONObj()) { + return false; + } + else if (!boundsMatch(bounds.Obj(), ixn->bounds)) { + return false; + } + } + + BSONElement dir = ixscanObj["dir"]; + if (!dir.eoo() && NumberInt == dir.type()) { + if (dir.numberInt() != ixn->direction) { + return false; + } + } + + BSONElement filter = ixscanObj["filter"]; + if (filter.eoo()) { + return true; + } + else if (filter.isNull()) { + return NULL == ixn->filter; + } + else if (!filter.isABSONObj()) { + return false; + } + return filterMatches(filter.Obj(), trueSoln); + } + else if (STAGE_GEO_2D == trueSoln->getType()) { + const Geo2DNode* node = static_cast(trueSoln); + BSONElement el = testSoln["geo2d"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj geoObj = el.Obj(); + return geoObj == node->indexKeyPattern; + } + else if (STAGE_GEO_NEAR_2D == trueSoln->getType()) { + const GeoNear2DNode* node = static_cast(trueSoln); + BSONElement el = testSoln["geoNear2d"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj geoObj = el.Obj(); + return geoObj == node->indexKeyPattern; + } + else if (STAGE_GEO_NEAR_2DSPHERE == trueSoln->getType()) { + const GeoNear2DSphereNode* node = static_cast(trueSoln); + BSONElement el = testSoln["geoNear2dsphere"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj geoObj = el.Obj(); + return geoObj == node->indexKeyPattern; + } + else if (STAGE_TEXT == trueSoln->getType()) { + // {text: {search: "somestr", language: "something", filter: {blah: 1}}} + const TextNode* node = static_cast(trueSoln); + BSONElement el = testSoln["text"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj textObj = el.Obj(); + + BSONElement searchElt = textObj["search"]; + if (!searchElt.eoo()) { + if (searchElt.String() != node->query) { + return false; + } + } + + BSONElement languageElt = textObj["language"]; + if (!languageElt.eoo()) { + if (languageElt.String() != node->language) { + return false; + } + } + + BSONElement indexPrefix = textObj["prefix"]; + if (!indexPrefix.eoo()) { + if (!indexPrefix.isABSONObj()) { + return false; + } + + if (0 != indexPrefix.Obj().woCompare(node->indexPrefix)) { + return false; + } + } + + BSONElement filter = textObj["filter"]; + if (!filter.eoo()) { + if (filter.isNull()) { + if (NULL != node->filter) { return false; } + } + else if (!filter.isABSONObj()) { + return false; + } + else if (!filterMatches(filter.Obj(), trueSoln)) { + return false; + } + } + + return true; + } + + // + // internal nodes + // + if (STAGE_FETCH == trueSoln->getType()) { + const FetchNode* fn = static_cast(trueSoln); + + BSONElement el = testSoln["fetch"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj fetchObj = el.Obj(); + + BSONElement filter = fetchObj["filter"]; + if (!filter.eoo()) { + if (filter.isNull()) { + if (NULL != fn->filter) { return false; } + } + else if (!filter.isABSONObj()) { + return false; + } + else if (!filterMatches(filter.Obj(), trueSoln)) { + return false; + } + } + + BSONElement child = fetchObj["node"]; + if (child.eoo() || !child.isABSONObj()) { return false; } + return solutionMatches(child.Obj(), fn->children[0]); + } + else if (STAGE_OR == trueSoln->getType()) { + const OrNode * orn = static_cast(trueSoln); + BSONElement el = testSoln["or"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj orObj = el.Obj(); + return childrenMatch(orObj, orn); + } + else if (STAGE_AND_HASH == trueSoln->getType()) { + const AndHashNode* ahn = static_cast(trueSoln); + BSONElement el = testSoln["andHash"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj andHashObj = el.Obj(); + + BSONElement filter = andHashObj["filter"]; + if (!filter.eoo()) { + if (filter.isNull()) { + if (NULL != ahn->filter) { return false; } + } + else if (!filter.isABSONObj()) { + return false; + } + else if (!filterMatches(filter.Obj(), trueSoln)) { + return false; + } + } + + return childrenMatch(andHashObj, ahn); + } + else if (STAGE_AND_SORTED == trueSoln->getType()) { + const AndSortedNode* asn = static_cast(trueSoln); + BSONElement el = testSoln["andSorted"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj andSortedObj = el.Obj(); + + BSONElement filter = andSortedObj["filter"]; + if (!filter.eoo()) { + if (filter.isNull()) { + if (NULL != asn->filter) { return false; } + } + else if (!filter.isABSONObj()) { + return false; + } + else if (!filterMatches(filter.Obj(), trueSoln)) { + return false; + } + } + + return childrenMatch(andSortedObj, asn); + } + else if (STAGE_PROJECTION == trueSoln->getType()) { + const ProjectionNode* pn = static_cast(trueSoln); + + BSONElement el = testSoln["proj"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj projObj = el.Obj(); + + BSONElement spec = projObj["spec"]; + if (spec.eoo() || !spec.isABSONObj()) { return false; } + BSONElement child = projObj["node"]; + if (child.eoo() || !child.isABSONObj()) { return false; } + + return (spec.Obj() == pn->projection) + && solutionMatches(child.Obj(), pn->children[0]); + } + else if (STAGE_SORT == trueSoln->getType()) { + const SortNode* sn = static_cast(trueSoln); + BSONElement el = testSoln["sort"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj sortObj = el.Obj(); + + BSONElement patternEl = sortObj["pattern"]; + if (patternEl.eoo() || !patternEl.isABSONObj()) { return false; } + BSONElement limitEl = sortObj["limit"]; + if (!limitEl.isNumber()) { return false; } + BSONElement child = sortObj["node"]; + if (child.eoo() || !child.isABSONObj()) { return false; } + + size_t expectedLimit = limitEl.numberInt(); + return (patternEl.Obj() == sn->pattern) + && (expectedLimit == sn->limit) + && solutionMatches(child.Obj(), sn->children[0]); + } + else if (STAGE_SORT_MERGE == trueSoln->getType()) { + const MergeSortNode* msn = static_cast(trueSoln); + BSONElement el = testSoln["mergeSort"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj mergeSortObj = el.Obj(); + return childrenMatch(mergeSortObj, msn); + } + else if (STAGE_SKIP == trueSoln->getType()) { + const SkipNode* sn = static_cast(trueSoln); + BSONElement el = testSoln["skip"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj sortObj = el.Obj(); + + BSONElement skipEl = sortObj["n"]; + if (!skipEl.isNumber()) { return false; } + BSONElement child = sortObj["node"]; + if (child.eoo() || !child.isABSONObj()) { return false; } + + return (skipEl.numberInt() == sn->skip) + && solutionMatches(child.Obj(), sn->children[0]); + } + else if (STAGE_LIMIT == trueSoln->getType()) { + const LimitNode* ln = static_cast(trueSoln); + BSONElement el = testSoln["limit"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj sortObj = el.Obj(); + + BSONElement limitEl = sortObj["n"]; + if (!limitEl.isNumber()) { return false; } + BSONElement child = sortObj["node"]; + if (child.eoo() || !child.isABSONObj()) { return false; } + + return (limitEl.numberInt() == ln->limit) + && solutionMatches(child.Obj(), ln->children[0]); + } + else if (STAGE_KEEP_MUTATIONS == trueSoln->getType()) { + const KeepMutationsNode* kn = static_cast(trueSoln); + + BSONElement el = testSoln["keep"]; + if (el.eoo() || !el.isABSONObj()) { return false; } + BSONObj keepObj = el.Obj(); + + // Doesn't have any parameters really. + BSONElement child = keepObj["node"]; + if (child.eoo() || !child.isABSONObj()) { return false; } + + return solutionMatches(child.Obj(), kn->children[0]); + } + + return false; + } + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner_test_lib.h b/src/mongo/db/query/query_planner_test_lib.h new file mode 100644 index 00000000000..9c7d7eb9caf --- /dev/null +++ b/src/mongo/db/query/query_planner_test_lib.h @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/query_planner.cpp + */ + +#include +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + + class QueryPlannerTestLib { + public: + /** + * @param testSoln -- a BSON representation of a query solution + * @param trueSoln -- the root node of a query solution tree + * + * Returns true if the BSON representation matches the actual + * tree, otherwise returns false. + */ + static bool solutionMatches(const BSONObj& testSoln, const QuerySolutionNode* trueSoln); + + static bool solutionMatches(const string& testSoln, const QuerySolutionNode* trueSoln) { + return solutionMatches(fromjson(testSoln), trueSoln); + } + }; + +} // namespace mongo diff --git a/src/mongo/db/query/query_planner_text_test.cpp b/src/mongo/db/query/query_planner_text_test.cpp new file mode 100644 index 00000000000..d62e9638dcb --- /dev/null +++ b/src/mongo/db/query/query_planner_text_test.cpp @@ -0,0 +1,579 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for mongo/db/query/query_planner.cpp that involve $text. + */ + +#include "mongo/db/query/query_planner_test_lib.h" + +#include +#include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_solution.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +using namespace mongo; + +namespace { + + static const char* ns = "somebogusns"; + + class QueryPlannerTest : public mongo::unittest::Test { + protected: + void setUp() { + params.options = QueryPlannerParams::INCLUDE_COLLSCAN; + addIndex(BSON("_id" << 1)); + } + + void tearDown() { + delete cq; + + for (vector::iterator it = solns.begin(); it != solns.end(); ++it) { + delete *it; + } + } + + // + // Build up test. + // + + void addIndex(BSONObj keyPattern, bool multikey = false) { + // The first false means not multikey. + // The second false means not sparse. + params.indices.push_back(IndexEntry(keyPattern, + multikey, + false, + false, + "hari_king_of_the_stove", + BSONObj())); + } + + void addIndex(BSONObj keyPattern, bool multikey, bool sparse) { + params.indices.push_back(IndexEntry(keyPattern, + multikey, + sparse, + false, + "note_to_self_dont_break_build", + BSONObj())); + } + + // + // Execute planner. + // + + void runQuery(BSONObj query) { + runQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), 0, 0); + } + + void runQuerySortProj(const BSONObj& query, const BSONObj& sort, const BSONObj& proj) { + runQuerySortProjSkipLimit(query, sort, proj, 0, 0); + } + + void runQuerySkipLimit(const BSONObj& query, long long skip, long long limit) { + runQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), skip, limit); + } + + void runQueryHint(const BSONObj& query, const BSONObj& hint) { + runQuerySortProjSkipLimitHint(query, BSONObj(), BSONObj(), 0, 0, hint); + } + + void runQuerySortProjSkipLimit(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit) { + runQuerySortProjSkipLimitHint(query, sort, proj, skip, limit, BSONObj()); + } + + void runQuerySortHint(const BSONObj& query, const BSONObj& sort, const BSONObj& hint) { + runQuerySortProjSkipLimitHint(query, sort, BSONObj(), 0, 0, hint); + } + + void runQueryHintMinMax(const BSONObj& query, const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj) { + + runQueryFull(query, BSONObj(), BSONObj(), 0, 0, hint, minObj, maxObj, false); + } + + void runQuerySortProjSkipLimitHint(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint) { + runQueryFull(query, sort, proj, skip, limit, hint, BSONObj(), BSONObj(), false); + } + + void runQuerySnapshot(const BSONObj& query) { + runQueryFull(query, BSONObj(), BSONObj(), 0, 0, BSONObj(), BSONObj(), + BSONObj(), true); + } + + void runQueryFull(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + const BSONObj& minObj, + const BSONObj& maxObj, + bool snapshot) { + solns.clear(); + Status s = CanonicalQuery::canonicalize(ns, query, sort, proj, skip, limit, hint, + minObj, maxObj, snapshot, + false, // explain + &cq); + if (!s.isOK()) { cq = NULL; } + ASSERT_OK(s); + s = QueryPlanner::plan(*cq, params, &solns); + ASSERT_OK(s); + } + + /** + * Same as runQuery* functions except we expect a failed status from the planning stage. + */ + void runInvalidQuery(const BSONObj& query) { + runInvalidQuerySortProjSkipLimit(query, BSONObj(), BSONObj(), 0, 0); + } + + void runInvalidQuerySortProj(const BSONObj& query, const BSONObj& sort, + const BSONObj& proj) { + runInvalidQuerySortProjSkipLimit(query, sort, proj, 0, 0); + } + + void runInvalidQuerySortProjSkipLimit(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit) { + runInvalidQuerySortProjSkipLimitHint(query, sort, proj, skip, limit, BSONObj()); + } + + void runInvalidQueryHint(const BSONObj& query, const BSONObj& hint) { + runInvalidQuerySortProjSkipLimitHint(query, BSONObj(), BSONObj(), 0, 0, hint); + } + + void runInvalidQueryHintMinMax(const BSONObj& query, const BSONObj& hint, + const BSONObj& minObj, const BSONObj& maxObj) { + runInvalidQueryFull(query, BSONObj(), BSONObj(), 0, 0, hint, minObj, maxObj, false); + } + + void runInvalidQuerySortProjSkipLimitHint(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint) { + runInvalidQueryFull(query, sort, proj, skip, limit, hint, BSONObj(), BSONObj(), false); + } + + void runInvalidQueryFull(const BSONObj& query, + const BSONObj& sort, const BSONObj& proj, + long long skip, long long limit, + const BSONObj& hint, + const BSONObj& minObj, + const BSONObj& maxObj, + bool snapshot) { + solns.clear(); + Status s = CanonicalQuery::canonicalize(ns, query, sort, proj, skip, limit, hint, + minObj, maxObj, snapshot, + false, // explain + &cq); + if (!s.isOK()) { cq = NULL; } + ASSERT_OK(s); + s = QueryPlanner::plan(*cq, params, &solns); + ASSERT_NOT_OK(s); + } + + // + // Introspect solutions. + // + + size_t getNumSolutions() const { + return solns.size(); + } + + void dumpSolutions(mongoutils::str::stream& ost) const { + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + ost << (*it)->toString() << '\n'; + } + } + + void dumpSolutions() const { + mongoutils::str::stream ss; + dumpSolutions(ss); + cout << "Solutions: " << string(ss) << endl; + } + + /** + * Checks number solutions. Generates assertion message + * containing solution dump if applicable. + */ + void assertNumSolutions(size_t expectSolutions) const { + if (getNumSolutions() == expectSolutions) { + return; + } + mongoutils::str::stream ss; + ss << "expected " << expectSolutions << " solutions but got " << getNumSolutions() + << " instead. solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + } + + size_t numSolutionMatches(const string& solnJson) const { + BSONObj testSoln = fromjson(solnJson); + size_t matches = 0; + for (vector::const_iterator it = solns.begin(); + it != solns.end(); + ++it) { + QuerySolutionNode* root = (*it)->root.get(); + if (QueryPlannerTestLib::solutionMatches(testSoln, root)) { + ++matches; + } + } + return matches; + } + + /** + * Verifies that the solution tree represented in json by 'solnJson' is + * one of the solutions generated by QueryPlanner. + * + * The number of expected matches, 'numMatches', could be greater than + * 1 if solutions differ only by the pattern of index tags on a filter. + */ + void assertSolutionExists(const string& solnJson, size_t numMatches = 1) const { + size_t matches = numSolutionMatches(solnJson); + if (numMatches == matches) { + return; + } + mongoutils::str::stream ss; + ss << "expected " << numMatches << " matches for solution " << solnJson + << " but got " << matches + << " instead. all solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + } + + /** + * Given a vector of string-based solution tree representations 'solnStrs', + * verifies that the query planner generated exactly one of these solutions. + */ + void assertHasOneSolutionOf(const vector& solnStrs) const { + size_t matches = 0; + for (vector::const_iterator it = solnStrs.begin(); + it != solnStrs.end(); + ++it) { + if (1U == numSolutionMatches(*it)) { + ++matches; + } + } + if (1U == matches) { + return; + } + mongoutils::str::stream ss; + ss << "assertHasOneSolutionOf expected one matching solution" + << " but got " << matches + << " instead. all solutions generated: " << '\n'; + dumpSolutions(ss); + FAIL(ss); + } + + BSONObj queryObj; + CanonicalQuery* cq; + QueryPlannerParams params; + vector solns; + }; + + // + // Text + // Creating an FTS index {a:1, b:"text", c:1} actually + // creates an index with spec {a:1, _fts: "text", _ftsx: 1, c:1}. + // So, the latter is what we pass in to the planner. + // + // PS. You can also do {a:1, b:"text", d:"text", c:1} and it will create an index with the same + // key pattern. + // + + // Basic test that it works. + TEST_F(QueryPlannerTest, SimpleText) { + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$text: {$search: 'blah'}}")); + + assertNumSolutions(1); + assertSolutionExists("{text: {search: 'blah'}}"); + } + + // If you create an index {a:1, b: "text"} you can't use it for queries on just 'a'. + TEST_F(QueryPlannerTest, CantUseTextUnlessHaveTextPred) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{a:1}")); + + // No table scans allowed so there is no solution. + assertNumSolutions(0); + } + + // But if you create an index {a:1, b:"text"} you can use it if it has a pred on 'a' + // and a text query. + TEST_F(QueryPlannerTest, HaveOKPrefixOnTextIndex) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1)); + + runQuery(fromjson("{a:1, $text:{$search: 'blah'}}")); + assertNumSolutions(1); + assertSolutionExists("{text: {prefix: {a:1}, search: 'blah'}}}}"); + + // TODO: Do we want to $or a collection scan with a text search? + // runQuery(fromjson("{$or: [{b:1}, {a:1, $text: {$search: 'blah'}}]}")); + // assertNumSolutions(1); + + runQuery(fromjson("{$or: [{_id:1}, {a:1, $text: {$search: 'blah'}}]}")); + assertNumSolutions(1); + } + + // But the prefixes must be points. + TEST_F(QueryPlannerTest, HaveBadPrefixOnTextIndex) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1)); + runInvalidQuery(fromjson("{a:{$gt: 1}, $text:{$search: 'blah'}}")); + + runInvalidQuery(fromjson("{$text: {$search: 'blah'}}")); + + runInvalidQuery(fromjson("{$or: [{a:1}, {$text: {$search: 'blah'}}]}")); + } + + // There can be more than one prefix, but they all require points. + TEST_F(QueryPlannerTest, ManyPrefixTextIndex) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "b" << 1 << "_fts" << "text" << "_ftsx" << 1)); + + // Both points. + runQuery(fromjson("{a:1, b:1, $text:{$search: 'blah'}}")); + assertSolutionExists("{text: {prefix: {a:1, b:1}, search: 'blah'}}"); + assertNumSolutions(1); + + // Missing a. + runInvalidQuery(fromjson("{b:1, $text:{$search: 'blah'}}")); + + // Missing b. + runInvalidQuery(fromjson("{a:1, $text:{$search: 'blah'}}")); + + // a is not a point + runInvalidQuery(fromjson("{a:{$gt: 1}, b:1, $text:{$search: 'blah'}}")); + + // b is not a point + runInvalidQuery(fromjson("{a:1, b:{$gt: 1}, $text:{$search: 'blah'}}")); + } + + // And, suffixes. They're optional and don't need to be points. + TEST_F(QueryPlannerTest, SuffixOptional) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1 << "b" << 1)); + + runQuery(fromjson("{a:1, $text:{$search: 'blah'}}")); + assertNumSolutions(1); + assertSolutionExists("{text: {prefix: {a:1}, search: 'blah'}}}}"); + + runQuery(fromjson("{a:1, b:{$gt: 7}, $text:{$search: 'blah'}}")); + assertSolutionExists("{text: {prefix: {a:1}, filter: {b: {$gt: 7}}, search: 'blah'}}}}"); + assertNumSolutions(1); + } + + TEST_F(QueryPlannerTest, RemoveFromSubtree) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1 << "b" << 1)); + + runQuery(fromjson("{a:1, $or: [{a:1}, {b:7}], $text:{$search: 'blah'}}")); + assertNumSolutions(1); + + assertSolutionExists("{fetch: {filter: {$or:[{a:1},{b:7}]}," + "node: {text: {prefix: {a:1}, search: 'blah'}}}}"); + } + + // Text is quite often multikey. None of the prefixes can be arrays, and suffixes are indexed + // as-is, so we should compound even if it's multikey. + TEST_F(QueryPlannerTest, CompoundPrefixEvenIfMultikey) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "b" << 1 << "_fts" << "text" << "_ftsx" << 1), true); + + // Both points. + runQuery(fromjson("{a:1, b:1, $text:{$search: 'blah'}}")); + assertNumSolutions(1); + assertSolutionExists("{text: {prefix: {a:1, b:1}, search: 'blah'}}"); + } + + TEST_F(QueryPlannerTest, IndexOnOwnFieldButNotLeafPrefix) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1 << "b" << 1)); + + // 'a' is not an EQ so it doesn't compound w/the text pred. We also shouldn't use the text + // index to satisfy it w/o the text query. + runInvalidQuery(fromjson("{a:{$elemMatch:{$gt: 0, $lt: 2}}, $text:{$search: 'blah'}}")); + } + + TEST_F(QueryPlannerTest, IndexOnOwnFieldButNotLeafSuffixNoPrefix) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("_fts" << "text" << "_ftsx" << 1 << "b" << 1)); + + runQuery(fromjson("{b:{$elemMatch:{$gt: 0, $lt: 2}}, $text:{$search: 'blah'}}")); + assertNumSolutions(1); + } + + TEST_F(QueryPlannerTest, TextInsideAndWithCompoundIndex) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$and: [{a: 3}, {$text: {$search: 'foo'}}], a: 3}")); + + assertNumSolutions(1U); + assertSolutionExists("{text: {prefix: {a:3}, search: 'foo'}}"); + } + + // SERVER-15639: Test that predicates on index prefix fields which are not assigned to the index + // prefix are correctly included in the solution node filter. + TEST_F(QueryPlannerTest, TextInsideAndWithCompoundIndexAndMultiplePredsOnIndexPrefix) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1 << "_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$and: [{a: 1}, {a: 2}, {$text: {$search: 'foo'}}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{text: {prefix: {a: 1}, search: 'foo', filter: {a: 2}}}"); + } + + // SERVER-13039: Test that we don't generate invalid solutions when the TEXT node + // is buried beneath a logical node. + TEST_F(QueryPlannerTest, TextInsideOrBasic) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{a: 0, $or: [{_id: 1}, {$text: {$search: 'foo'}}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a:0}, node: {or: {nodes: [" + "{text: {search: 'foo'}}, " + "{ixscan: {filter: null, pattern: {_id: 1}}}]}}}}"); + } + + // SERVER-13039 + TEST_F(QueryPlannerTest, TextInsideOrWithAnotherOr) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$and: [{$or: [{a: 3}, {a: 4}]}, " + "{$or: [{$text: {$search: 'foo'}}, {a: 5}]}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {$or: [{a: 3}, {a: 4}]}, node: " + "{or: {nodes: [" + "{text: {search: 'foo'}}, " + "{ixscan: {filter: null, pattern: {a: 1}}}]}}}}"); + } + + // SERVER-13039 + TEST_F(QueryPlannerTest, TextInsideOrOfAnd) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$or: [{a: {$gt: 1, $gt: 2}}, " + "{a: {$gt: 3}, $text: {$search: 'foo'}}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: null, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}, bounds: " + "{a: [[2,Infinity,false,true]]}}}, " + "{fetch: {filter: {a:{$gt:3}}, node: " + "{text: {search: 'foo'}}}}]}}}}"); + } + + // SERVER-13039 + TEST_F(QueryPlannerTest, TextInsideAndOrAnd) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("b" << 1)); + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{a: 1, $or: [{a:2}, {b:2}, " + "{a: 1, $text: {$search: 'foo'}}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a:1}, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a:1}}}, " + "{fetch: {filter: {a:1}, node: {text: {search: 'foo'}}}}, " + "{ixscan: {filter: null, pattern: {b:1}}}]}}}}"); + } + + // SERVER-13039 + TEST_F(QueryPlannerTest, TextInsideAndOrAndOr) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$or: [{a: {$gt: 1, $gt: 2}}, " + "{a: {$gt: 3}, $or: [{$text: {$search: 'foo'}}, " + "{a: 6}]}], " + "a: 5}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {a:5}, node: {or: {nodes: [" + "{ixscan: {filter: null, pattern: {a: 1}}}, " + "{fetch: {filter: {a:{$gt:3}}, node: {or: {nodes: [" + "{text: {search: 'foo'}}, " + "{ixscan: {filter: null, pattern: {a: 1}}}]}}}}]}}}}"); + } + + // If only one branch of the $or can be indexed, then no indexed + // solutions are generated, even if one branch is $text. + TEST_F(QueryPlannerTest, TextInsideOrOneBranchNotIndexed) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{a: 1, $or: [{b: 2}, {$text: {$search: 'foo'}}]}")); + + assertNumSolutions(0); + } + + // If the unindexable $or is not the one containing the $text predicate, + // then we should still be able to generate an indexed solution. + TEST_F(QueryPlannerTest, TextInsideOrWithAnotherUnindexableOr) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + addIndex(BSON("a" << 1)); + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$and: [{$or: [{a: 1}, {b: 1}]}, " + "{$or: [{a: 2}, {$text: {$search: 'foo'}}]}]}")); + + assertNumSolutions(1U); + assertSolutionExists("{fetch: {filter: {$or:[{a:1},{b:1}]}, node: {or: {nodes: [" + "{text: {search: 'foo'}}, " + "{ixscan: {filter: null, pattern: {a:1}}}]}}}}"); + } + + TEST_F(QueryPlannerTest, AndTextWithGeoNonNear) { + addIndex(BSON("_fts" << "text" << "_ftsx" << 1)); + runQuery(fromjson("{$text: {$search: 'foo'}, a: {$geoIntersects: {$geometry: " + "{type: 'Point', coordinates: [3.0, 1.0]}}}}")); + + // Mandatory text index is used, and geo predicate becomes a filter. + assertNumSolutions(1U); + assertSolutionExists("{fetch: {node: {text: {search: 'foo'}}}}"); + } + +} // namespace diff --git a/src/mongo/db/query/query_settings.cpp b/src/mongo/db/query/query_settings.cpp new file mode 100644 index 00000000000..8de33cf4db8 --- /dev/null +++ b/src/mongo/db/query/query_settings.cpp @@ -0,0 +1,166 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_settings.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/plan_cache.h" + +namespace mongo { + + using std::vector; + + // + // HintOverride + // + + AllowedIndices::AllowedIndices(const std::vector& indexKeyPatterns) { + for (std::vector::const_iterator i = indexKeyPatterns.begin(); + i != indexKeyPatterns.end(); ++i) { + const BSONObj& indexKeyPattern = *i; + this->indexKeyPatterns.push_back(indexKeyPattern.getOwned()); + } + } + + AllowedIndices::~AllowedIndices() { } + + // + // AllowedIndexEntry + // + + AllowedIndexEntry::AllowedIndexEntry(const BSONObj& query, const BSONObj& sort, + const BSONObj& projection, + const std::vector& indexKeyPatterns) + : query(query.getOwned()), + sort(sort.getOwned()), + projection(projection.getOwned()) { + for (std::vector::const_iterator i = indexKeyPatterns.begin(); + i != indexKeyPatterns.end(); ++i) { + const BSONObj& indexKeyPattern = *i; + this->indexKeyPatterns.push_back(indexKeyPattern.getOwned()); + } + } + + AllowedIndexEntry::~AllowedIndexEntry() { } + + AllowedIndexEntry* AllowedIndexEntry::clone() const { + AllowedIndexEntry* entry = new AllowedIndexEntry(query, sort, projection, indexKeyPatterns); + return entry; + } + + // + // QuerySettings + // + + QuerySettings::QuerySettings() { } + + QuerySettings::~QuerySettings() { + _clear(); + } + + bool QuerySettings::getAllowedIndices(const CanonicalQuery& query, + AllowedIndices** allowedIndicesOut) const { + invariant(allowedIndicesOut); + + const PlanCacheKey& key = query.getPlanCacheKey(); + + boost::lock_guard cacheLock(_mutex); + AllowedIndexEntryMap::const_iterator cacheIter = _allowedIndexEntryMap.find(key); + + // Nothing to do if key does not exist in query settings. + if (cacheIter == _allowedIndexEntryMap.end()) { + *allowedIndicesOut = NULL; + return false; + } + + AllowedIndexEntry* entry = cacheIter->second; + + // Create a AllowedIndices from entry. + *allowedIndicesOut = new AllowedIndices(entry->indexKeyPatterns); + + return true; + } + + std::vector QuerySettings::getAllAllowedIndices() const { + boost::lock_guard cacheLock(_mutex); + vector entries; + for (AllowedIndexEntryMap::const_iterator i = _allowedIndexEntryMap.begin(); i != _allowedIndexEntryMap.end(); ++i) { + AllowedIndexEntry* entry = i->second; + entries.push_back(entry->clone()); + } + return entries; + } + + void QuerySettings::setAllowedIndices(const CanonicalQuery& canonicalQuery, + const std::vector& indexes) { + const LiteParsedQuery& lpq = canonicalQuery.getParsed(); + const BSONObj& query = lpq.getFilter(); + const BSONObj& sort = lpq.getSort(); + const BSONObj& projection = lpq.getProj(); + AllowedIndexEntry* entry = new AllowedIndexEntry(query, sort, projection, indexes); + + const PlanCacheKey& key = canonicalQuery.getPlanCacheKey(); + boost::lock_guard cacheLock(_mutex); + AllowedIndexEntryMap::iterator i = _allowedIndexEntryMap.find(key); + // Replace existing entry. + if (i != _allowedIndexEntryMap.end()) { + AllowedIndexEntry* entry = i->second; + delete entry; + } + _allowedIndexEntryMap[key] = entry; + } + + void QuerySettings::removeAllowedIndices(const CanonicalQuery& canonicalQuery) { + const PlanCacheKey& key = canonicalQuery.getPlanCacheKey(); + boost::lock_guard cacheLock(_mutex); + AllowedIndexEntryMap::iterator i = _allowedIndexEntryMap.find(key); + + // Nothing to do if key does not exist in query settings. + if (i == _allowedIndexEntryMap.end()) { + return; + } + + // Free up resources and delete entry. + AllowedIndexEntry* entry = i->second; + _allowedIndexEntryMap.erase(i); + delete entry; + } + + void QuerySettings::clearAllowedIndices() { + boost::lock_guard cacheLock(_mutex); + _clear(); + } + + void QuerySettings::_clear() { + for (AllowedIndexEntryMap::const_iterator i = _allowedIndexEntryMap.begin(); i != _allowedIndexEntryMap.end(); ++i) { + AllowedIndexEntry* entry = i->second; + delete entry; + } + _allowedIndexEntryMap.clear(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/query_settings.h b/src/mongo/db/query/query_settings.h new file mode 100644 index 00000000000..beb312daf91 --- /dev/null +++ b/src/mongo/db/query/query_settings.h @@ -0,0 +1,145 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include +#include "mongo/base/disallow_copying.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/index_entry.h" +#include "mongo/platform/unordered_map.h" + +namespace mongo { + + /** + * Holds allowed indices. + */ + class AllowedIndices { + private: + MONGO_DISALLOW_COPYING(AllowedIndices); + public: + AllowedIndices(const std::vector& indexKeyPatterns); + ~AllowedIndices(); + + // These are the index key patterns that + // we will use to override the indexes retrieved from + // the index catalog. + std::vector indexKeyPatterns; + }; + + /** + * Value type for query settings. + * Holds: + * query shape (query, sort, projection) + * vector of index specs + */ + class AllowedIndexEntry { + private: + MONGO_DISALLOW_COPYING(AllowedIndexEntry); + public: + AllowedIndexEntry(const BSONObj& query, const BSONObj& sort, + const BSONObj& projection, + const std::vector& indexKeyPatterns); + ~AllowedIndexEntry(); + AllowedIndexEntry* clone() const; + + // _query, _sort and _projection collectively + // represent the query shape that we are storing hint overrides for. + BSONObj query; + BSONObj sort; + BSONObj projection; + + // These are the index key patterns that + // we will use to override the indexes retrieved from + // the index catalog. + std::vector indexKeyPatterns; + }; + + /** + * Holds the index filters in a collection. + */ + class QuerySettings { + private: + MONGO_DISALLOW_COPYING(QuerySettings); + public: + QuerySettings(); + + ~QuerySettings(); + + /** + * Returns true and fills out allowedIndicesOut if a hint is set in the query settings + * for the query. + * Returns false and sets allowedIndicesOut to NULL otherwise. + * Caller owns AllowedIndices. + */ + bool getAllowedIndices(const CanonicalQuery& query, + AllowedIndices** allowedIndicesOut) const; + + /** + * Returns copies all overrides for the collection.. + * Caller owns overrides in vector. + */ + std::vector getAllAllowedIndices() const; + + /** + * Adds or replaces entry in query settings. + * If existing entry is found for the same key, + * frees resources for existing entry before replacing. + */ + void setAllowedIndices(const CanonicalQuery& canonicalQuery, + const std::vector& indexes); + + /** + * Removes single entry from query settings. No effect if query shape is not found. + */ + void removeAllowedIndices(const CanonicalQuery& canonicalQuery); + + /** + * Clears all allowed indices from query settings. + */ + void clearAllowedIndices(); + + private: + /** + * Clears entries without acquiring mutex. + */ + void _clear(); + + typedef unordered_map AllowedIndexEntryMap; + AllowedIndexEntryMap _allowedIndexEntryMap; + + /** + * Protects data in query settings. + */ + mutable boost::mutex _mutex; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/query_solution.cpp b/src/mongo/db/query/query_solution.cpp new file mode 100644 index 00000000000..6bc06047a1f --- /dev/null +++ b/src/mongo/db/query/query_solution.cpp @@ -0,0 +1,900 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_solution.h" + +#include "mongo/db/query/planner_analysis.h" +#include "mongo/db/query/query_planner_common.h" + +namespace mongo { + + string QuerySolutionNode::toString() const { + mongoutils::str::stream ss; + appendToString(&ss, 0); + return ss; + } + + // static + void QuerySolutionNode::addIndent(mongoutils::str::stream* ss, int level) { + for (int i = 0; i < level; ++i) { + *ss << "---"; + } + } + + void QuerySolutionNode::addCommon(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent + 1); + *ss << "fetched = " << fetched() << '\n'; + addIndent(ss, indent + 1); + *ss << "sortedByDiskLoc = " << sortedByDiskLoc() << '\n'; + addIndent(ss, indent + 1); + *ss << "getSort = ["; + for (BSONObjSet::const_iterator it = getSort().begin(); it != getSort().end(); it++) { + *ss << it->toString() << ", "; + } + *ss << "]" << '\n'; + } + + // + // TextNode + // + + void TextNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "TEXT\n"; + addIndent(ss, indent + 1); + *ss << "keyPattern = " << indexKeyPattern.toString() << '\n'; + addIndent(ss, indent + 1); + *ss << "query = " << query << '\n'; + addIndent(ss, indent + 1); + *ss << "language = " << language << '\n'; + addIndent(ss, indent + 1); + *ss << "indexPrefix = " << indexPrefix.toString() << '\n'; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << " filter = " << filter->toString(); + } + addCommon(ss, indent); + } + + QuerySolutionNode* TextNode::clone() const { + TextNode* copy = new TextNode(); + cloneBaseData(copy); + + copy->_sort = this->_sort; + copy->indexKeyPattern = this->indexKeyPattern; + copy->query = this->query; + copy->language = this->language; + copy->indexPrefix = this->indexPrefix; + + return copy; + } + + // + // CollectionScanNode + // + + CollectionScanNode::CollectionScanNode() : tailable(false), direction(1), maxScan(0) { } + + void CollectionScanNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "COLLSCAN\n"; + addIndent(ss, indent + 1); + *ss << "ns = " << name << '\n'; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << "filter = " << filter->toString(); + } + addCommon(ss, indent); + } + + QuerySolutionNode* CollectionScanNode::clone() const { + CollectionScanNode* copy = new CollectionScanNode(); + cloneBaseData(copy); + + copy->_sort = this->_sort; + copy->name = this->name; + copy->tailable = this->tailable; + copy->direction = this->direction; + copy->maxScan = this->maxScan; + + return copy; + } + + // + // AndHashNode + // + + AndHashNode::AndHashNode() { } + + AndHashNode::~AndHashNode() { } + + void AndHashNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "AND_HASH\n"; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << " filter = " << filter->toString() << '\n'; + } + addCommon(ss, indent); + for (size_t i = 0; i < children.size(); ++i) { + addIndent(ss, indent + 1); + *ss << "Child " << i << ":\n"; + children[i]->appendToString(ss, indent + 1); + } + } + + bool AndHashNode::fetched() const { + // Any WSM output from this stage came from all children stages. If any child provides + // fetched data, we merge that fetched data into the WSM we output. + for (size_t i = 0; i < children.size(); ++i) { + if (children[i]->fetched()) { + return true; + } + } + return false; + } + + bool AndHashNode::hasField(const string& field) const { + // Any WSM output from this stage came from all children stages. Therefore we have all + // fields covered in our children. + for (size_t i = 0; i < children.size(); ++i) { + if (children[i]->hasField(field)) { + return true; + } + } + return false; + } + + QuerySolutionNode* AndHashNode::clone() const { + AndHashNode* copy = new AndHashNode(); + cloneBaseData(copy); + + copy->_sort = this->_sort; + + return copy; + } + + // + // AndSortedNode + // + + AndSortedNode::AndSortedNode() { } + + AndSortedNode::~AndSortedNode() { } + + void AndSortedNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "AND_SORTED\n"; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << " filter = " << filter->toString() << '\n'; + } + addCommon(ss, indent); + for (size_t i = 0; i < children.size(); ++i) { + addIndent(ss, indent + 1); + *ss << "Child " << i << ":\n"; + children[i]->appendToString(ss, indent + 1); + } + } + + bool AndSortedNode::fetched() const { + // Any WSM output from this stage came from all children stages. If any child provides + // fetched data, we merge that fetched data into the WSM we output. + for (size_t i = 0; i < children.size(); ++i) { + if (children[i]->fetched()) { + return true; + } + } + return false; + } + + bool AndSortedNode::hasField(const string& field) const { + // Any WSM output from this stage came from all children stages. Therefore we have all + // fields covered in our children. + for (size_t i = 0; i < children.size(); ++i) { + if (children[i]->hasField(field)) { + return true; + } + } + return false; + } + + QuerySolutionNode* AndSortedNode::clone() const { + AndSortedNode* copy = new AndSortedNode(); + cloneBaseData(copy); + + copy->_sort = this->_sort; + + return copy; + } + + // + // OrNode + // + + OrNode::OrNode() : dedup(true) { } + + OrNode::~OrNode() { } + + void OrNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "OR\n"; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << " filter = " << filter->toString() << '\n'; + } + addCommon(ss, indent); + for (size_t i = 0; i < children.size(); ++i) { + addIndent(ss, indent + 1); + *ss << "Child " << i << ":\n"; + children[i]->appendToString(ss, indent + 2); + *ss << '\n'; + } + } + + bool OrNode::fetched() const { + // Any WSM output from this stage came exactly one child stage. Given that we don't know + // what child stage it came from, we require that all children provide fetched data in order + // to guarantee that our output is fetched. + for (size_t i = 0; i < children.size(); ++i) { + if (!children[i]->fetched()) { + return false; + } + } + return true; + } + + /** + * Any WSM output from this stage came from exactly one child stage. Therefore, if + * we want to guarantee that any output has a certain field, all of our children must + * have that field. + */ + bool OrNode::hasField(const string& field) const { + for (size_t i = 0; i < children.size(); ++i) { + if (!children[i]->hasField(field)) { + return false; + } + } + return true; + } + + QuerySolutionNode* OrNode::clone() const { + OrNode* copy = new OrNode(); + cloneBaseData(copy); + + copy->_sort = this->_sort; + copy->dedup = this->dedup; + + return copy; + } + + // + // MergeSortNode + // + + MergeSortNode::MergeSortNode() : dedup(true) { } + + MergeSortNode::~MergeSortNode() { } + + void MergeSortNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "MERGE_SORT\n"; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << " filter = " << filter->toString() << '\n'; + } + addCommon(ss, indent); + for (size_t i = 0; i < children.size(); ++i) { + addIndent(ss, indent + 1); + *ss << "Child " << i << ":\n"; + children[i]->appendToString(ss, indent + 2); + *ss << '\n'; + } + } + + bool MergeSortNode::fetched() const { + // Any WSM output from this stage came exactly one child stage. Given that we don't know + // what child stage it came from, we require that all children provide fetched data in order + // to guarantee that our output is fetched. + for (size_t i = 0; i < children.size(); ++i) { + if (!children[i]->fetched()) { + return false; + } + } + return true; + } + + /** + * Any WSM output from this stage came from exactly one child stage. Therefore, if + * we want to guarantee that any output has a certain field, all of our children must + * have that field. + */ + bool MergeSortNode::hasField(const string& field) const { + for (size_t i = 0; i < children.size(); ++i) { + if (!children[i]->hasField(field)) { + return false; + } + } + return true; + } + + QuerySolutionNode* MergeSortNode::clone() const { + MergeSortNode* copy = new MergeSortNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + copy->dedup = this->dedup; + copy->sort = this->sort; + + return copy; + } + + // + // FetchNode + // + + FetchNode::FetchNode() { } + + void FetchNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "FETCH\n"; + if (NULL != filter) { + addIndent(ss, indent + 1); + StringBuilder sb; + *ss << "filter:\n"; + filter->debugString(sb, indent + 2); + *ss << sb.str(); + } + addCommon(ss, indent); + addIndent(ss, indent + 1); + *ss << "Child:" << '\n'; + children[0]->appendToString(ss, indent + 2); + } + + QuerySolutionNode* FetchNode::clone() const { + FetchNode* copy = new FetchNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + + return copy; + } + + // + // IndexScanNode + // + + IndexScanNode::IndexScanNode() + : indexIsMultiKey(false), direction(1), maxScan(0), addKeyMetadata(false) { } + + void IndexScanNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "IXSCAN\n"; + addIndent(ss, indent + 1); + *ss << "keyPattern = " << indexKeyPattern << '\n'; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << "filter = " << filter->toString(); + } + addIndent(ss, indent + 1); + *ss << "direction = " << direction << '\n'; + addIndent(ss, indent + 1); + *ss << "bounds = " << bounds.toString() << '\n'; + addCommon(ss, indent); + } + + bool IndexScanNode::hasField(const string& field) const { + // There is no covering in a multikey index because you don't know whether or not the field + // in the key was extracted from an array in the original document. + if (indexIsMultiKey) { return false; } + + BSONObjIterator it(indexKeyPattern); + while (it.more()) { + if (field == it.next().fieldName()) { + return true; + } + } + return false; + } + + bool IndexScanNode::sortedByDiskLoc() const { + // Indices use DiskLoc as an additional key after the actual index key. + // Therefore, if we're only examining one index key, the output is sorted + // by DiskLoc. + + // If it's a simple range query, it's easy to determine if the range is a point. + if (bounds.isSimpleRange) { + return 0 == bounds.startKey.woCompare(bounds.endKey, indexKeyPattern); + } + + // If it's a more complex bounds query, we make sure that each field is a point. + for (size_t i = 0; i < bounds.fields.size(); ++i) { + const OrderedIntervalList& oil = bounds.fields[i]; + if (1 != oil.intervals.size()) { + return false; + } + const Interval& interval = oil.intervals[0]; + if (0 != interval.start.woCompare(interval.end, false)) { + return false; + } + } + + return true; + } + + void IndexScanNode::computeProperties() { + _sorts.clear(); + + BSONObj sortPattern = QueryPlannerAnalysis::getSortPattern(indexKeyPattern); + if (direction == -1) { + sortPattern = QueryPlannerCommon::reverseSortObj(sortPattern); + } + + _sorts.insert(sortPattern); + + const int nFields = sortPattern.nFields(); + if (nFields > 1) { + // We're sorted not only by sortPattern but also by all prefixes of it. + for (int i = 0; i < nFields; ++i) { + // Make obj out of fields [0,i] + BSONObjIterator it(sortPattern); + BSONObjBuilder prefixBob; + for (int j = 0; j <= i; ++j) { + prefixBob.append(it.next()); + } + _sorts.insert(prefixBob.obj()); + } + } + + // If we are using the index {a:1, b:1} to answer the predicate {a: 10}, it's sorted + // both by the index key pattern and by the pattern {b: 1}. + + // See if there are any fields with equalities for bounds. We can drop these + // from any sort orders created. + set equalityFields; + if (!bounds.isSimpleRange) { + // Figure out how many fields are point intervals. + for (size_t i = 0; i < bounds.fields.size(); ++i) { + const OrderedIntervalList& oil = bounds.fields[i]; + if (oil.intervals.size() != 1) { + continue; + } + const Interval& ival = oil.intervals[0]; + if (!ival.isPoint()) { + continue; + } + equalityFields.insert(oil.name); + } + } + + if (equalityFields.empty()) { + return; + } + + // TODO: Each field in equalityFields could be dropped from the sort order since it is + // a point interval. The full set of sort orders is as follows: + // For each sort in _sorts: + // For each drop in powerset(equalityFields): + // Remove fields in 'drop' from 'sort' and add resulting sort to output. + // + // Since this involves a powerset, we don't generate the full set of possibilities. + // Instead, we generate sort orders by removing possible contiguous prefixes of equality + // predicates. For example, if the key pattern is {a: 1, b: 1, c: 1, d: 1, e: 1} + // and and there are equality predicates on 'a', 'b', and 'c', then here we add the sort + // orders {b: 1, c: 1, d: 1, e: 1} and {c: 1, d: 1, e: 1}. (We also end up adding + // {d: 1, e: 1} and {d: 1}, but this is done later on.) + BSONObjIterator it(sortPattern); + BSONObjBuilder suffixBob; + while (it.more()) { + BSONElement elt = it.next(); + // TODO: string slowness. fix when bounds are stringdata not string. + if (equalityFields.end() == equalityFields.find(string(elt.fieldName()))) { + suffixBob.append(elt); + // This field isn't a point interval, can't drop. + break; + } + + // We add the sort obtained by dropping 'elt' and all preceding elements from the index + // key pattern. + BSONObjIterator droppedPrefixIt = it; + BSONObjBuilder droppedPrefixBob; + while (droppedPrefixIt.more()) { + droppedPrefixBob.append(droppedPrefixIt.next()); + } + _sorts.insert(droppedPrefixBob.obj()); + } + + while (it.more()) { + suffixBob.append(it.next()); + } + + // We've found the suffix following the contiguous prefix of equality fields. + // Ex. For index {a: 1, b: 1, c: 1, d: 1} and query {a: 3, b: 5}, this suffix + // of the key pattern is {c: 1, d: 1}. + // + // Now we have to add all prefixes of this suffix as possible sort orders. + // Ex. Continuing the example from above, we have to include sort orders + // {c: 1} and {c: 1, d: 1}. + BSONObj filterPointsObj = suffixBob.obj(); + for (int i = 0; i < filterPointsObj.nFields(); ++i) { + // Make obj out of fields [0,i] + BSONObjIterator it(filterPointsObj); + BSONObjBuilder prefixBob; + for (int j = 0; j <= i; ++j) { + prefixBob.append(it.next()); + } + _sorts.insert(prefixBob.obj()); + } + } + + QuerySolutionNode* IndexScanNode::clone() const { + IndexScanNode* copy = new IndexScanNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + copy->indexKeyPattern = this->indexKeyPattern; + copy->indexIsMultiKey = this->indexIsMultiKey; + copy->direction = this->direction; + copy->maxScan = this->maxScan; + copy->addKeyMetadata = this->addKeyMetadata; + copy->bounds = this->bounds; + + return copy; + } + + // + // ProjectionNode + // + + void ProjectionNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "PROJ\n"; + addIndent(ss, indent + 1); + *ss << "proj = " << projection.toString() << '\n'; + addIndent(ss, indent + 1); + if (DEFAULT == projType) { + *ss << "type = DEFAULT\n"; + } + else if (COVERED_ONE_INDEX == projType) { + *ss << "type = COVERED_ONE_INDEX\n"; + } + else { + invariant(SIMPLE_DOC == projType); + *ss << "type = SIMPLE_DOC\n"; + } + addCommon(ss, indent); + addIndent(ss, indent + 1); + *ss << "Child:" << '\n'; + children[0]->appendToString(ss, indent + 2); + } + + QuerySolutionNode* ProjectionNode::clone() const { + ProjectionNode* copy = new ProjectionNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + copy->fullExpression = this->fullExpression; + + // This MatchExpression* is owned by the canonical query, not by the + // ProjectionNode. Just copying the pointer is fine. + copy->projection = this->projection; + + return copy; + } + + // + // SortNode + // + + void SortNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "SORT\n"; + addIndent(ss, indent + 1); + *ss << "pattern = " << pattern.toString() << '\n'; + addIndent(ss, indent + 1); + *ss << "query for bounds = " << query.toString() << '\n'; + addIndent(ss, indent + 1); + *ss << "limit = " << limit << '\n'; + addCommon(ss, indent); + addIndent(ss, indent + 1); + *ss << "Child:" << '\n'; + children[0]->appendToString(ss, indent + 2); + } + + QuerySolutionNode* SortNode::clone() const { + SortNode* copy = new SortNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + copy->pattern = this->pattern; + copy->query = this->query; + copy->limit = this->limit; + + return copy; + } + + // + // LimitNode + // + + + void LimitNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "LIMIT\n"; + addIndent(ss, indent + 1); + *ss << "limit = " << limit << '\n'; + addIndent(ss, indent + 1); + addCommon(ss, indent); + addIndent(ss, indent + 1); + *ss << "Child:" << '\n'; + children[0]->appendToString(ss, indent + 2); + } + + QuerySolutionNode* LimitNode::clone() const { + LimitNode* copy = new LimitNode(); + cloneBaseData(copy); + + copy->limit = this->limit; + + return copy; + } + + // + // SkipNode + // + + void SkipNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "SKIP\n"; + addIndent(ss, indent + 1); + *ss << "skip= " << skip << '\n'; + addCommon(ss, indent); + addIndent(ss, indent + 1); + *ss << "Child:" << '\n'; + children[0]->appendToString(ss, indent + 2); + } + + QuerySolutionNode* SkipNode::clone() const { + SkipNode* copy = new SkipNode(); + cloneBaseData(copy); + + copy->skip = this->skip; + + return copy; + } + + // + // GeoNear2DNode + // + + void GeoNear2DNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "GEO_NEAR_2D\n"; + addIndent(ss, indent + 1); + *ss << "keyPattern = " << indexKeyPattern.toString() << '\n'; + addCommon(ss, indent); + *ss << "nearQuery = " << nq.toString() << '\n'; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << " filter = " << filter->toString(); + } + } + + QuerySolutionNode* GeoNear2DNode::clone() const { + GeoNear2DNode* copy = new GeoNear2DNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + copy->nq = this->nq; + copy->numWanted = this->numWanted; + copy->indexKeyPattern = this->indexKeyPattern; + copy->addPointMeta = this->addPointMeta; + copy->addDistMeta = this->addDistMeta; + + return copy; + } + + // + // GeoNear2DSphereNode + // + + void GeoNear2DSphereNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "GEO_NEAR_2DSPHERE\n"; + addIndent(ss, indent + 1); + *ss << "keyPattern = " << indexKeyPattern.toString() << '\n'; + addCommon(ss, indent); + *ss << "baseBounds = " << baseBounds.toString() << '\n'; + addIndent(ss, indent + 1); + *ss << "nearQuery = " << nq.toString() << '\n'; + if (NULL != filter) { + addIndent(ss, indent + 1); + *ss << " filter = " << filter->toString(); + } + } + + QuerySolutionNode* GeoNear2DSphereNode::clone() const { + GeoNear2DSphereNode* copy = new GeoNear2DSphereNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + copy->nq = this->nq; + copy->baseBounds = this->baseBounds; + copy->indexKeyPattern = this->indexKeyPattern; + copy->addPointMeta = this->addPointMeta; + copy->addDistMeta = this->addDistMeta; + + return copy; + } + + // + // Geo2DNode + // + + void Geo2DNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "GEO_2D\n"; + addIndent(ss, indent + 1); + *ss << "keyPattern = " << indexKeyPattern.toString() << '\n'; + addCommon(ss, indent); + } + + bool Geo2DNode::hasField(const string& field) const { + BSONObjIterator it(indexKeyPattern); + while (it.more()) { + if (field == it.next().fieldName()) { + return true; + } + } + return false; + } + + QuerySolutionNode* Geo2DNode::clone() const { + Geo2DNode* copy = new Geo2DNode(); + cloneBaseData(copy); + + copy->_sorts = this->_sorts; + copy->indexKeyPattern = this->indexKeyPattern; + copy->gq = this->gq; + + return copy; + } + + // + // ShardingFilterNode + // + + void ShardingFilterNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "SHARDING_FILTER\n"; + if (NULL != filter) { + addIndent(ss, indent + 1); + StringBuilder sb; + *ss << "filter:\n"; + filter->debugString(sb, indent + 2); + *ss << sb.str(); + } + addCommon(ss, indent); + addIndent(ss, indent + 1); + *ss << "Child:" << '\n'; + children[0]->appendToString(ss, indent + 2); + } + + QuerySolutionNode* ShardingFilterNode::clone() const { + ShardingFilterNode* copy = new ShardingFilterNode(); + cloneBaseData(copy); + return copy; + } + + // + // KeepMutationsNode + // + + void KeepMutationsNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "KEEP_MUTATIONS\n"; + if (NULL != filter) { + addIndent(ss, indent + 1); + StringBuilder sb; + *ss << "filter:\n"; + filter->debugString(sb, indent + 2); + *ss << sb.str(); + } + addCommon(ss, indent); + addIndent(ss, indent + 1); + *ss << "Child:" << '\n'; + children[0]->appendToString(ss, indent + 2); + } + + QuerySolutionNode* KeepMutationsNode::clone() const { + KeepMutationsNode* copy = new KeepMutationsNode(); + cloneBaseData(copy); + + copy->sorts = this->sorts; + + return copy; + } + + // + // DistinctNode + // + + void DistinctNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "DISTINCT\n"; + addIndent(ss, indent + 1); + *ss << "keyPattern = " << indexKeyPattern << '\n'; + addIndent(ss, indent + 1); + *ss << "direction = " << direction << '\n'; + addIndent(ss, indent + 1); + *ss << "bounds = " << bounds.toString() << '\n'; + } + + QuerySolutionNode* DistinctNode::clone() const { + DistinctNode* copy = new DistinctNode(); + cloneBaseData(copy); + + copy->sorts = this->sorts; + copy->indexKeyPattern = this->indexKeyPattern; + copy->direction = this->direction; + copy->bounds = this->bounds; + copy->fieldNo = this->fieldNo; + + return copy; + } + + // + // CountNode + // + + void CountNode::appendToString(mongoutils::str::stream* ss, int indent) const { + addIndent(ss, indent); + *ss << "COUNT\n"; + addIndent(ss, indent + 1); + *ss << "keyPattern = " << indexKeyPattern << '\n'; + addIndent(ss, indent + 1); + *ss << "startKey = " << startKey << '\n'; + addIndent(ss, indent + 1); + *ss << "endKey = " << endKey << '\n'; + } + + QuerySolutionNode* CountNode::clone() const { + CountNode* copy = new CountNode(); + cloneBaseData(copy); + + copy->sorts = this->sorts; + copy->indexKeyPattern = this->indexKeyPattern; + copy->startKey = this->startKey; + copy->startKeyInclusive = this->startKeyInclusive; + copy->endKey = this->endKey; + copy->endKeyInclusive = this->endKeyInclusive; + + return copy; + } + +} // namespace mongo diff --git a/src/mongo/db/query/query_solution.h b/src/mongo/db/query/query_solution.h new file mode 100644 index 00000000000..7721e718f83 --- /dev/null +++ b/src/mongo/db/query/query_solution.h @@ -0,0 +1,751 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression.h" +#include "mongo/db/geo/geoquery.h" +#include "mongo/db/fts/fts_query.h" +#include "mongo/db/query/index_bounds.h" +#include "mongo/db/query/plan_cache.h" +#include "mongo/db/query/stage_types.h" + +namespace mongo { + + using mongo::fts::FTSQuery; + + /** + * This is an abstract representation of a query plan. It can be transcribed into a tree of + * PlanStages, which can then be handed to a PlanRunner for execution. + */ + struct QuerySolutionNode { + QuerySolutionNode() { } + virtual ~QuerySolutionNode() { + for (size_t i = 0; i < children.size(); ++i) { + delete children[i]; + } + } + + /** + * Return a string representation of this node and any children. + */ + string toString() const; + + /** + * What stage should this be transcribed to? See stage_types.h. + */ + virtual StageType getType() const = 0; + + /** + * Internal function called by toString() + * + * TODO: Consider outputting into a BSONObj or builder thereof. + */ + virtual void appendToString(mongoutils::str::stream* ss, int indent) const = 0; + + // + // Computed properties + // + + /** + * Must be called before any properties are examined. + */ + virtual void computeProperties() { + for (size_t i = 0; i < children.size(); ++i) { + children[i]->computeProperties(); + } + } + + /** + * If true, one of these are true: + * 1. All outputs are already fetched, or + * 2. There is a projection in place and a fetch is not required. + * + * If false, a fetch needs to be placed above the root in order to provide results. + * + * Usage: To determine if every possible result that might reach the root + * will be fully-fetched or not. We don't want any surplus fetches. + */ + virtual bool fetched() const = 0; + + /** + * Returns true if the tree rooted at this node provides data with the field name 'field'. + * This data can come from any of the types of the WSM. + * + * Usage: If an index-only plan has all the fields we're interested in, we don't + * have to fetch to show results with those fields. + * + * TODO: 'field' is probably more appropriate as a FieldRef or string. + */ + virtual bool hasField(const string& field) const = 0; + + /** + * Returns true if the tree rooted at this node provides data that is sorted by the + * its location on disk. + * + * Usage: If all the children of an STAGE_AND_HASH have this property, we can compute the + * AND faster by replacing the STAGE_AND_HASH with STAGE_AND_SORTED. + */ + virtual bool sortedByDiskLoc() const = 0; + + /** + * Return a BSONObjSet representing the possible sort orders of the data stream from this + * node. If the data is not sorted in any particular fashion, returns an empty set. + * + * Usage: + * 1. If our plan gives us a sort order, we don't have to add a sort stage. + * 2. If all the children of an OR have the same sort order, we can maintain that + * sort order with a STAGE_SORT_MERGE instead of STAGE_OR. + */ + virtual const BSONObjSet& getSort() const = 0; + + /** + * Make a deep copy. + */ + virtual QuerySolutionNode* clone() const = 0; + + /** + * Copy base query solution data from 'this' to 'other'. + */ + void cloneBaseData(QuerySolutionNode* other) const { + for (size_t i = 0; i < this->children.size(); i++) { + other->children.push_back(this->children[i]->clone()); + } + if (NULL != this->filter) { + other->filter.reset(this->filter->shallowClone()); + } + } + + // These are owned here. + vector children; + + // If a stage has a non-NULL filter all values outputted from that stage must pass that + // filter. + scoped_ptr filter; + + protected: + /** + * Formatting helper used by toString(). + */ + static void addIndent(mongoutils::str::stream* ss, int level); + + /** + * Every solution node has properties and this adds the debug info for the + * properties. + */ + void addCommon(mongoutils::str::stream* ss, int indent) const; + + private: + MONGO_DISALLOW_COPYING(QuerySolutionNode); + }; + + /** + * A QuerySolution must be entirely self-contained and own everything inside of it. + * + * A tree of stages may be built from a QuerySolution. The QuerySolution must outlive the tree + * of stages. + */ + struct QuerySolution { + QuerySolution() : hasBlockingStage(false), indexFilterApplied(false) { } + + // Owned here. + scoped_ptr root; + + // Any filters in root or below point into this object. Must be owned. + BSONObj filterData; + + string ns; + + // There are two known scenarios in which a query solution might potentially block: + // + // Sort stage: + // If the solution has a sort stage, the sort wasn't provided by an index, so we might want + // to scan an index to provide that sort in a non-blocking fashion. + // + // Hashed AND stage: + // The hashed AND stage buffers data from multiple index scans and could block. In that case, + // we would want to fall back on an alternate non-blocking solution. + bool hasBlockingStage; + + // Runner executing this solution might be interested in knowing + // if the planning process for this solution was based on filtered indices. + bool indexFilterApplied; + + // Owned here. Used by the plan cache. + boost::scoped_ptr cacheData; + + /** + * Output a human-readable string representing the plan. + */ + string toString() { + if (NULL == root) { + return "empty query solution"; + } + + mongoutils::str::stream ss; + root->appendToString(&ss, 0); + return ss; + } + private: + MONGO_DISALLOW_COPYING(QuerySolution); + }; + + struct TextNode : public QuerySolutionNode { + TextNode() { } + virtual ~TextNode() { } + + virtual StageType getType() const { return STAGE_TEXT; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + // text's return is LOC_AND_UNOWNED_OBJ so it's fetched and has all fields. + bool fetched() const { return true; } + bool hasField(const string& field) const { return true; } + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return _sort; } + + QuerySolutionNode* clone() const; + + BSONObjSet _sort; + + BSONObj indexKeyPattern; + std::string query; + std::string language; + + // "Prefix" fields of a text index can handle equality predicates. We group them with the + // text node while creating the text leaf node and convert them into a BSONObj index prefix + // when we finish the text leaf node. + BSONObj indexPrefix; + }; + + struct CollectionScanNode : public QuerySolutionNode { + CollectionScanNode(); + virtual ~CollectionScanNode() { } + + virtual StageType getType() const { return STAGE_COLLSCAN; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return true; } + bool hasField(const string& field) const { return true; } + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return _sort; } + + QuerySolutionNode* clone() const; + + BSONObjSet _sort; + + // Name of the namespace. + string name; + + // Should we make a tailable cursor? + bool tailable; + + int direction; + + // maxScan option to .find() limits how many docs we look at. + int maxScan; + }; + + struct AndHashNode : public QuerySolutionNode { + AndHashNode(); + virtual ~AndHashNode(); + + virtual StageType getType() const { return STAGE_AND_HASH; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const; + bool hasField(const string& field) const; + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return children.back()->getSort(); } + + QuerySolutionNode* clone() const; + + BSONObjSet _sort; + }; + + struct AndSortedNode : public QuerySolutionNode { + AndSortedNode(); + virtual ~AndSortedNode(); + + virtual StageType getType() const { return STAGE_AND_SORTED; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const; + bool hasField(const string& field) const; + bool sortedByDiskLoc() const { return true; } + const BSONObjSet& getSort() const { return _sort; } + + QuerySolutionNode* clone() const; + + BSONObjSet _sort; + }; + + struct OrNode : public QuerySolutionNode { + OrNode(); + virtual ~OrNode(); + + virtual StageType getType() const { return STAGE_OR; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const; + bool hasField(const string& field) const; + bool sortedByDiskLoc() const { + // Even if our children are sorted by their diskloc or other fields, we don't maintain + // any order on the output. + return false; + } + const BSONObjSet& getSort() const { return _sort; } + + QuerySolutionNode* clone() const; + + BSONObjSet _sort; + + bool dedup; + }; + + struct MergeSortNode : public QuerySolutionNode { + MergeSortNode(); + virtual ~MergeSortNode(); + + virtual StageType getType() const { return STAGE_SORT_MERGE; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const; + bool hasField(const string& field) const; + bool sortedByDiskLoc() const { return false; } + + const BSONObjSet& getSort() const { return _sorts; } + + QuerySolutionNode* clone() const; + + virtual void computeProperties() { + for (size_t i = 0; i < children.size(); ++i) { + children[i]->computeProperties(); + } + _sorts.clear(); + _sorts.insert(sort); + } + + BSONObjSet _sorts; + + BSONObj sort; + bool dedup; + }; + + struct FetchNode : public QuerySolutionNode { + FetchNode(); + virtual ~FetchNode() { } + + virtual StageType getType() const { return STAGE_FETCH; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return true; } + bool hasField(const string& field) const { return true; } + bool sortedByDiskLoc() const { return children[0]->sortedByDiskLoc(); } + const BSONObjSet& getSort() const { return children[0]->getSort(); } + + QuerySolutionNode* clone() const; + + BSONObjSet _sorts; + }; + + struct IndexScanNode : public QuerySolutionNode { + IndexScanNode(); + virtual ~IndexScanNode() { } + + virtual void computeProperties(); + + virtual StageType getType() const { return STAGE_IXSCAN; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return false; } + bool hasField(const string& field) const; + bool sortedByDiskLoc() const; + const BSONObjSet& getSort() const { return _sorts; } + + QuerySolutionNode* clone() const; + + BSONObjSet _sorts; + + BSONObj indexKeyPattern; + bool indexIsMultiKey; + + int direction; + + // maxScan option to .find() limits how many docs we look at. + int maxScan; + + // If there's a 'returnKey' projection we add key metadata. + bool addKeyMetadata; + + // BIG NOTE: + // If you use simple bounds, we'll use whatever index access method the keypattern implies. + // If you use the complex bounds, we force Btree access. + // The complex bounds require Btree access. + IndexBounds bounds; + }; + + struct ProjectionNode : public QuerySolutionNode { + /** + * We have a few implementations of the projection functionality. The most general + * implementation 'DEFAULT' is much slower than the fast-path implementations + * below. We only really have all the information available to choose a projection + * implementation at planning time. + */ + enum ProjectionType { + // This is the most general implementation of the projection functionality. It handles + // every case. + DEFAULT, + + // This is a fast-path for when the projection is fully covered by one index. + COVERED_ONE_INDEX, + + // This is a fast-path for when the projection only has inclusions on non-dotted fields. + SIMPLE_DOC, + }; + + ProjectionNode() : fullExpression(NULL), projType(DEFAULT) { } + + virtual ~ProjectionNode() { } + + virtual StageType getType() const { return STAGE_PROJECTION; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + /** + * This node changes the type to OWNED_OBJ. There's no fetching possible after this. + */ + bool fetched() const { return true; } + + bool hasField(const string& field) const { + // TODO: Returning false isn't always the right answer -- we may either be including + // certain fields, or we may be dropping fields (in which case hasField returns true). + // + // Given that projection sits on top of everything else in .find() it doesn't matter + // what we do here. + return false; + } + + bool sortedByDiskLoc() const { + // Projections destroy the DiskLoc. By returning true here, this kind of implies that a + // fetch could still be done upstream. + // + // Perhaps this should be false to not imply that there *is* a DiskLoc? Kind of a + // corner case. + return children[0]->sortedByDiskLoc(); + } + + const BSONObjSet& getSort() const { + // TODO: If we're applying a projection that maintains sort order, the prefix of the + // sort order we project is the sort order. + return _sorts; + } + + QuerySolutionNode* clone() const; + + BSONObjSet _sorts; + + // The full query tree. Needed when we have positional operators. + // Owned in the CanonicalQuery, not here. + MatchExpression* fullExpression; + + // Given that we don't yet have a MatchExpression analogue for the expression language, we + // use a BSONObj. + BSONObj projection; + + // What implementation of the projection algorithm should we use? + ProjectionType projType; + + // Only meaningful if projType == COVERED_ONE_INDEX. This is the key pattern of the index + // supplying our covered data. We can pre-compute which fields to include and cache that + // data for later if we know we only have one index. + BSONObj coveredKeyObj; + }; + + struct SortNode : public QuerySolutionNode { + SortNode() : limit(0) { } + virtual ~SortNode() { } + + virtual StageType getType() const { return STAGE_SORT; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return children[0]->fetched(); } + bool hasField(const string& field) const { return children[0]->hasField(field); } + bool sortedByDiskLoc() const { return false; } + + const BSONObjSet& getSort() const { return _sorts; } + + QuerySolutionNode* clone() const; + + virtual void computeProperties() { + for (size_t i = 0; i < children.size(); ++i) { + children[i]->computeProperties(); + } + _sorts.clear(); + _sorts.insert(pattern); + } + + BSONObjSet _sorts; + + BSONObj pattern; + + BSONObj query; + + // Sum of both limit and skip count in the parsed query. + size_t limit; + }; + + struct LimitNode : public QuerySolutionNode { + LimitNode() { } + virtual ~LimitNode() { } + + virtual StageType getType() const { return STAGE_LIMIT; } + + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return children[0]->fetched(); } + bool hasField(const string& field) const { return children[0]->hasField(field); } + bool sortedByDiskLoc() const { return children[0]->sortedByDiskLoc(); } + const BSONObjSet& getSort() const { return children[0]->getSort(); } + + QuerySolutionNode* clone() const; + + int limit; + }; + + struct SkipNode : public QuerySolutionNode { + SkipNode() { } + virtual ~SkipNode() { } + + virtual StageType getType() const { return STAGE_SKIP; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return children[0]->fetched(); } + bool hasField(const string& field) const { return children[0]->hasField(field); } + bool sortedByDiskLoc() const { return children[0]->sortedByDiskLoc(); } + const BSONObjSet& getSort() const { return children[0]->getSort(); } + + QuerySolutionNode* clone() const; + + int skip; + }; + + // + // Geo nodes. A thin wrapper above an IXSCAN until we can yank functionality out of + // the IXSCAN layer into the stage layer. + // + + // TODO: This is probably an expression index. + struct Geo2DNode : public QuerySolutionNode { + Geo2DNode() { } + virtual ~Geo2DNode() { } + + virtual StageType getType() const { return STAGE_GEO_2D; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return false; } + bool hasField(const string& field) const; + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return _sorts; } + BSONObjSet _sorts; + + QuerySolutionNode* clone() const; + + BSONObj indexKeyPattern; + GeoQuery gq; + }; + + // This is a standalone stage. + struct GeoNear2DNode : public QuerySolutionNode { + GeoNear2DNode() : numWanted(100), addPointMeta(false), addDistMeta(false) { } + virtual ~GeoNear2DNode() { } + + virtual StageType getType() const { return STAGE_GEO_NEAR_2D; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return true; } + bool hasField(const string& field) const { return true; } + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return _sorts; } + + QuerySolutionNode* clone() const; + + BSONObjSet _sorts; + + NearQuery nq; + int numWanted; + BSONObj indexKeyPattern; + bool addPointMeta; + bool addDistMeta; + }; + + // This is actually its own standalone stage. + struct GeoNear2DSphereNode : public QuerySolutionNode { + GeoNear2DSphereNode() : addPointMeta(false), addDistMeta(false) { } + virtual ~GeoNear2DSphereNode() { } + + virtual StageType getType() const { return STAGE_GEO_NEAR_2DSPHERE; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return true; } + bool hasField(const string& field) const { return true; } + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return _sorts; } + + QuerySolutionNode* clone() const; + + BSONObjSet _sorts; + + NearQuery nq; + IndexBounds baseBounds; + + BSONObj indexKeyPattern; + bool addPointMeta; + bool addDistMeta; + }; + + // + // Internal nodes used to provide functionality + // + + /** + * If we're answering a query on a sharded cluster, docs must be checked against the shard key + * to ensure that we don't return data that shouldn't be there. This must be done prior to + * projection, and in fact should be done as early as possible to avoid propagating stale data + * through the pipeline. + */ + struct ShardingFilterNode : public QuerySolutionNode { + ShardingFilterNode() { } + virtual ~ShardingFilterNode() { } + + virtual StageType getType() const { return STAGE_SHARDING_FILTER; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return children[0]->fetched(); } + bool hasField(const string& field) const { return children[0]->hasField(field); } + bool sortedByDiskLoc() const { return children[0]->sortedByDiskLoc(); } + const BSONObjSet& getSort() const { return children[0]->getSort(); } + + QuerySolutionNode* clone() const; + }; + + /** + * If documents mutate or are deleted during a query, we can (in some cases) fetch them + * and still return them. This stage merges documents that have been mutated or deleted + * into the query result stream. + */ + struct KeepMutationsNode : public QuerySolutionNode { + KeepMutationsNode() { } + virtual ~KeepMutationsNode() { } + + virtual StageType getType() const { return STAGE_KEEP_MUTATIONS; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + // Any flagged results are OWNED_OBJ and therefore we're covered if our child is. + bool fetched() const { return children[0]->fetched(); } + + // Any flagged results are OWNED_OBJ and as such they'll have any field we need. + bool hasField(const string& field) const { return children[0]->hasField(field); } + + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return sorts; } + + QuerySolutionNode* clone() const; + + // Since we merge in flagged results we have no sort order. + BSONObjSet sorts; + }; + + /** + * Distinct queries only want one value for a given field. We run an index scan but + * *always* skip over the current key to the next key. + */ + struct DistinctNode : public QuerySolutionNode { + DistinctNode() { } + virtual ~DistinctNode() { } + + virtual StageType getType() const { return STAGE_DISTINCT; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + // This stage is created "on top" of normal planning and as such the properties + // below don't really matter. + bool fetched() const { return false; } + bool hasField(const string& field) const { return !indexKeyPattern[field].eoo(); } + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return sorts; } + + QuerySolutionNode* clone() const; + + BSONObjSet sorts; + + BSONObj indexKeyPattern; + int direction; + IndexBounds bounds; + // We are distinct-ing over the 'fieldNo'-th field of 'indexKeyPattern'. + int fieldNo; + }; + + /** + * Some count queries reduce to counting how many keys are between two entries in a + * Btree. + */ + struct CountNode : public QuerySolutionNode { + CountNode() { } + virtual ~CountNode() { } + + virtual StageType getType() const { return STAGE_COUNT; } + virtual void appendToString(mongoutils::str::stream* ss, int indent) const; + + bool fetched() const { return true; } + bool hasField(const string& field) const { return true; } + bool sortedByDiskLoc() const { return false; } + const BSONObjSet& getSort() const { return sorts; } + + QuerySolutionNode* clone() const; + + BSONObjSet sorts; + + BSONObj indexKeyPattern; + + BSONObj startKey; + bool startKeyInclusive; + + BSONObj endKey; + bool endKeyInclusive; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/runner.h b/src/mongo/db/query/runner.h new file mode 100644 index 00000000000..4a43fd9a5d0 --- /dev/null +++ b/src/mongo/db/query/runner.h @@ -0,0 +1,223 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/invalidation_type.h" + +namespace mongo { + + class Collection; + class DiskLoc; + class TypeExplain; + struct PlanInfo; + + /** + * A runner runs a query. + */ + class Runner { + public: + virtual ~Runner() { } + + enum RunnerState { + // We successfully populated the out parameter. + RUNNER_ADVANCED, + + // We're EOF. We won't return any more results (edge case exception: capped+tailable). + RUNNER_EOF, + + // We were killed or had an error. + RUNNER_DEAD, + + // getNext was asked for data it cannot provide, or the underlying PlanStage had an + // unrecoverable error. + // If the underlying PlanStage has any information on the error, it will be available in + // the objOut parameter. Call WorkingSetCommon::toStatusString() to retrieve the error + // details from the output BSON object. + RUNNER_ERROR, + }; + + static string statestr(RunnerState s) { + if (RUNNER_ADVANCED == s) { + return "RUNNER_ADVANCED"; + } + else if (RUNNER_EOF == s) { + return "RUNNER_EOF"; + } + else if (RUNNER_DEAD == s) { + return "RUNNER_DEAD"; + } + else { + verify(RUNNER_ERROR == s); + return "RUNNER_ERROR"; + } + } + + /** + * The yielding policy of the runner. By default, a runner does not yield itself + * (YIELD_MANUAL). + */ + enum YieldPolicy { + // Any call to getNext() may yield. In particular, the runner may be killed during any + // call to getNext(). If this occurs, getNext() will return RUNNER_DEAD. + // + // If you are enabling autoyield, you must register the Runner with ClientCursor via + // ClientCursor::registerRunner and deregister via ClientCursor::deregisterRunnerwhen + // done. Registered runners are informed about DiskLoc deletions and Namespace + // invalidations and other important events. + // + // Exception: This is not required if the Runner is cached inside of a ClientCursor. + // This is only done if the Runner is cached and can be referred to by a cursor id. + // This is not a popular thing to do. + YIELD_AUTO, + + // Owner must yield manually if yields are requested. How to yield yourself: + // + // 0. Let's say you have Runner* runner. + // + // 1. Register your runner with ClientCursor. Registered runners are informed about + // DiskLoc deletions and Namespace invalidation and other important events. Do this by + // calling ClientCursor::registerRunner(runner). This could be done once when you get + // your runner, or per-yield. + // + // 2. Call runner->saveState() before you yield. + // + // 3. Call RunnerYieldPolicy::staticYield(runner->ns(), NULL) to yield. Any state that + // may change between yields must be checked by you. (For example, DiskLocs may not be + // valid across yielding, indices may be dropped, etc.) + // + // 4. Call runner->restoreState() before using the runner again. + // + // 5. Your runner's next call to getNext may return RUNNER_DEAD. + // + // 6. When you're done with your runner, deregister it from ClientCursor via + // ClientCursor::deregister(runner). + YIELD_MANUAL, + }; + + /** + * Set the yielding policy of the underlying runner. See the RunnerYieldPolicy enum above. + */ + virtual void setYieldPolicy(YieldPolicy policy) = 0; + + /** + * Get the next result from the query. + * + * If objOut is not NULL, only results that have a BSONObj are returned. The BSONObj may + * point to on-disk data (isOwned will be false) and must be copied by the caller before + * yielding. + * + * If dlOut is not NULL, only results that have a valid DiskLoc are returned. + * + * If both objOut and dlOut are not NULL, only results with both a valid BSONObj and DiskLoc + * will be returned. The BSONObj is the object located at the DiskLoc provided. + * + * If the underlying query machinery produces a result that does not have the data requested + * by the user, it will be silently dropped. + * + * If the caller is running a query, they probably only care about the object. + * If the caller is an internal client, they may only care about DiskLocs (index scan), or + * about object + DiskLocs (collection scan). + * + * Some notes on objOut and ownership: + * + * objOut may be an owned object in certain cases: invalidation of the underlying DiskLoc, + * the object is created from covered index key data, the object is projected or otherwise + * the result of a computation. + * + * objOut will also be owned when the underlying PlanStage has provided error details in the + * event of a RUNNER_ERROR. Call WorkingSetCommon::toStatusString() to convert the object + * to a loggable format. + * + * objOut will be unowned if it's the result of a fetch or a collection scan. + */ + virtual RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut) = 0; + + /** + * Will the next call to getNext() return EOF? It's useful to know if the runner is done + * without having to take responsibility for a result. + */ + virtual bool isEOF() = 0; + + /** + * Inform the runner about changes to DiskLoc(s) that occur while the runner is yielded. + * The runner must take any actions required to continue operating correctly, including + * broadcasting the invalidation request to the PlanStage tree being run. + * + * Called from CollectionCursorCache::invalidateDocument. + * + * See db/invalidation_type.h for InvalidationType. + */ + virtual void invalidate(const DiskLoc& dl, InvalidationType type) = 0; + + /** + * Mark the Runner as no longer valid. Can happen when a runner yields and the underlying + * database is dropped/indexes removed/etc. All future to calls to getNext return + * RUNNER_DEAD. Every other call is a NOOP. + * + * The runner must guarantee as a postcondition that future calls to collection() will + * return NULL. + */ + virtual void kill() = 0; + + /** + * Save any state required to yield. + */ + virtual void saveState() = 0; + + /** + * Restore saved state, possibly after a yield. Return true if the runner is OK, false if + * it was killed. + */ + virtual bool restoreState() = 0; + + /** + * Return the NS that the query is running over. + */ + virtual const string& ns() = 0; + + /** + * Return the Collection that the query is running over. + */ + virtual const Collection* collection() = 0; + + /** + * Returns OK, allocating and filling '*explain' or '*planInfo' with a description of the + * chosen plan, depending on which is non-NULL (one of the two should be NULL). Caller + * takes onwership of either '*explain' and '*planInfo'. Otherwise, returns false + * a detailed error status. + * + * If 'explain' is NULL, then this out-parameter is ignored. Similarly, if 'staticInfo' + * is NULL, then no static debug information is produced. + */ + virtual Status getInfo(TypeExplain** explain, PlanInfo** planInfo) const = 0; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/runner_yield_policy.h b/src/mongo/db/query/runner_yield_policy.h new file mode 100644 index 00000000000..b2113243fa9 --- /dev/null +++ b/src/mongo/db/query/runner_yield_policy.h @@ -0,0 +1,130 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/clientcursor.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/util/elapsed_tracker.h" + +namespace mongo { + + class RunnerYieldPolicy { + public: + RunnerYieldPolicy() : _elapsedTracker(128, 10), _runnerYielding(NULL) { } + + ~RunnerYieldPolicy() { + if (NULL != _runnerYielding) { + // We were destructed mid-yield. Since we're being used to yield a runner, we have + // to deregister the runner. + if ( _runnerYielding->collection() ) { + _runnerYielding->collection()->cursorCache()->deregisterRunner(_runnerYielding); + } + } + } + + bool shouldYield() { + return _elapsedTracker.intervalHasElapsed(); + } + + /** + * Yield the provided runner, registering and deregistering it appropriately. Deal with + * deletion during a yield by setting _runnerYielding to ensure deregistration. + * + * Provided runner MUST be YIELD_MANUAL. + */ + bool yieldAndCheckIfOK(Runner* runner, Record* record = NULL) { + invariant(runner); + invariant(runner->collection()); + + int micros = ClientCursor::suggestYieldMicros(); + + // If micros is not positive, no point in yielding, nobody waiting. + // + // TODO: Do we want to yield anyway if record is not NULL? + // + // TODO: Track how many times we actually yield, how many times micros is <0, etc. + if (micros <= 0) { return true; } + + // If micros > 0, we should yield. + runner->saveState(); + _runnerYielding = runner; + + runner->collection()->cursorCache()->registerRunner( _runnerYielding ); + + // Note that this call checks for interrupt, and thus can throw if interrupt flag is set + staticYield(micros, record); + + // staticYield does not check for interrupt after regaining the lock, but the + // bg index building interrupt code is depending on this behavior. Otherwise, we may + // return an unexpected error when restoreState(), below, notices its cursor has been + // invalidated. + killCurrentOp.checkForInterrupt(); + + if ( runner->collection() ) { + // if the runner was killed, runner->collection() will return NULL + // so we don't deregister as it was done when killed + runner->collection()->cursorCache()->deregisterRunner( _runnerYielding ); + } + _runnerYielding = NULL; + _elapsedTracker.resetLastTime(); + return runner->restoreState(); + } + + /** + * Yield, possibly fetching the provided record. Caller is in charge of all runner + * registration. + * + * Used for YIELD_AUTO runners. + */ + void yield(Record* rec = NULL) { + int micros = ClientCursor::suggestYieldMicros(); + + // If there is anyone waiting on us or if there's a record to page-in, yield. TODO: Do + // we want to page in the record in the lock even if nobody is waiting for the lock? + if (micros > 0 || (NULL != rec)) { + staticYield(micros, rec); + // TODO: When do we really want to reset this? Currently we reset it when we + // actually yield. As such we'll keep on trying to yield once the tracker has + // elapsed. If we reset it even if we don't yield, we'll wait until the time + // interval elapses again to try yielding. + _elapsedTracker.resetLastTime(); + } + } + + static void staticYield(int micros, const Record* rec = NULL) { + ClientCursor::staticYield(micros, "", rec); + } + + private: + ElapsedTracker _elapsedTracker; + Runner* _runnerYielding; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/single_solution_runner.cpp b/src/mongo/db/query/single_solution_runner.cpp new file mode 100644 index 00000000000..65c54e757a2 --- /dev/null +++ b/src/mongo/db/query/single_solution_runner.cpp @@ -0,0 +1,133 @@ +/** + * Copyright 2013 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/single_solution_runner.h" + +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/explain_plan.h" +#include "mongo/db/query/type_explain.h" +#include "mongo/db/query/plan_executor.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + SingleSolutionRunner::SingleSolutionRunner(const Collection* collection, + CanonicalQuery* canonicalQuery, + QuerySolution* soln, + PlanStage* root, + WorkingSet* ws) + : _collection( collection ), + _canonicalQuery(canonicalQuery), + _solution(soln), + _exec(new PlanExecutor(ws, root)) { } + + SingleSolutionRunner::~SingleSolutionRunner() { } + + Runner::RunnerState SingleSolutionRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { + return _exec->getNext(objOut, dlOut); + // TODO: I'm not convinced we want to cache this run. What if it's a collscan solution + // and the user adds an index later? We don't want to reach for this. But if solving + // the query is v. hard, we do want to cache it. Maybe we can remove single solution + // cache entries when we build an index? + } + + bool SingleSolutionRunner::isEOF() { + return _exec->isEOF(); + } + + void SingleSolutionRunner::saveState() { + _exec->saveState(); + } + + bool SingleSolutionRunner::restoreState() { + return _exec->restoreState(); + } + + void SingleSolutionRunner::setYieldPolicy(Runner::YieldPolicy policy) { + _exec->setYieldPolicy(policy); + } + + void SingleSolutionRunner::invalidate(const DiskLoc& dl, InvalidationType type) { + _exec->invalidate(dl, type); + } + + const std::string& SingleSolutionRunner::ns() { + return _canonicalQuery->getParsed().ns(); + } + + void SingleSolutionRunner::kill() { + _exec->kill(); + _collection = NULL; + } + + Status SingleSolutionRunner::getInfo(TypeExplain** explain, + PlanInfo** planInfo) const { + if (NULL != explain) { + verify(_exec.get()); + + scoped_ptr stats(_exec->getStats()); + if (NULL == stats.get()) { + return Status(ErrorCodes::InternalError, "no stats available to explain plan"); + } + + Status status = explainPlan(*stats, explain, true /* full details */); + if (!status.isOK()) { + return status; + } + + // Fill in explain fields that are accounted by on the runner level. + TypeExplain* chosenPlan = NULL; + explainPlan(*stats, &chosenPlan, false /* no full details */); + if (chosenPlan) { + (*explain)->addToAllPlans(chosenPlan); + } + (*explain)->setNScannedObjectsAllPlans((*explain)->getNScannedObjects()); + (*explain)->setNScannedAllPlans((*explain)->getNScanned()); + + // _solution could be NULL in certain cases such as when QueryOption_OplogReplay + // is enabled in the query flags. + if (_solution) { + (*explain)->setIndexFilterApplied(_solution->indexFilterApplied); + } + } + else if (NULL != planInfo) { + if (NULL == _solution.get()) { + return Status(ErrorCodes::InternalError, + "no solution available for plan info"); + } + getPlanInfo(*_solution, planInfo); + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/db/query/single_solution_runner.h b/src/mongo/db/query/single_solution_runner.h new file mode 100644 index 00000000000..99e573a4846 --- /dev/null +++ b/src/mongo/db/query/single_solution_runner.h @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/query/runner.h" + +namespace mongo { + + class BSONObj; + class CanonicalQuery; + class DiskLoc; + class PlanExecutor; + class PlanStage; + struct QuerySolution; + class TypeExplain; + class WorkingSet; + + /** + * SingleSolutionRunner runs a plan that was the only possible solution to a query. It exists + * only to dump stats into the cache after running. + */ + class SingleSolutionRunner : public Runner { + public: + + /** Takes ownership of all the arguments except collection */ + SingleSolutionRunner(const Collection* collection, + CanonicalQuery* canonicalQuery, QuerySolution* soln, + PlanStage* root, WorkingSet* ws); + + virtual ~SingleSolutionRunner(); + + Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + virtual bool isEOF(); + + virtual void saveState(); + + virtual bool restoreState(); + + virtual void setYieldPolicy(Runner::YieldPolicy policy); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual const std::string& ns(); + + virtual void kill(); + + virtual const Collection* collection() { return _collection; } + /** + * Returns OK, allocating and filling in '*explain' and '*staticInfo' with the details + * of the plan used by this runner. Caller takes ownership of '*explain' and + * '*staticInfo'. Otherwise, return a status describing the error. + */ + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const; + + private: + const Collection* _collection; + + boost::scoped_ptr _canonicalQuery; + boost::scoped_ptr _solution; + boost::scoped_ptr _exec; + }; + +} // namespace mongo + diff --git a/src/mongo/db/query/stage_builder.cpp b/src/mongo/db/query/stage_builder.cpp new file mode 100644 index 00000000000..56bb5bb4862 --- /dev/null +++ b/src/mongo/db/query/stage_builder.cpp @@ -0,0 +1,343 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/stage_builder.h" + +#include "mongo/db/exec/2d.h" +#include "mongo/db/exec/2dnear.h" +#include "mongo/db/exec/and_hash.h" +#include "mongo/db/exec/and_sorted.h" +#include "mongo/db/exec/collection_scan.h" +#include "mongo/db/exec/count.h" +#include "mongo/db/exec/distinct_scan.h" +#include "mongo/db/exec/fetch.h" +#include "mongo/db/exec/index_scan.h" +#include "mongo/db/exec/keep_mutations.h" +#include "mongo/db/exec/limit.h" +#include "mongo/db/exec/merge_sort.h" +#include "mongo/db/exec/or.h" +#include "mongo/db/exec/projection.h" +#include "mongo/db/exec/s2near.h" +#include "mongo/db/exec/shard_filter.h" +#include "mongo/db/exec/sort.h" +#include "mongo/db/exec/skip.h" +#include "mongo/db/exec/text.h" +#include "mongo/db/index/fts_access_method.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/catalog/collection.h" + +namespace mongo { + + PlanStage* buildStages(const QuerySolution& qsol, const QuerySolutionNode* root, WorkingSet* ws) { + if (STAGE_COLLSCAN == root->getType()) { + const CollectionScanNode* csn = static_cast(root); + CollectionScanParams params; + params.ns = csn->name; + params.tailable = csn->tailable; + params.direction = (csn->direction == 1) ? CollectionScanParams::FORWARD + : CollectionScanParams::BACKWARD; + params.maxScan = csn->maxScan; + return new CollectionScan(params, ws, csn->filter.get()); + } + else if (STAGE_IXSCAN == root->getType()) { + const IndexScanNode* ixn = static_cast(root); + + Database* db = cc().database(); + Collection* collection = db ? db->getCollection(qsol.ns) : NULL; + if (NULL == collection) { + warning() << "Can't ixscan null namespace " << qsol.ns << endl; + return NULL; + } + + IndexScanParams params; + + params.descriptor = collection->getIndexCatalog()->findIndexByKeyPattern( ixn->indexKeyPattern ); + if ( params.descriptor == NULL ) { + warning() << "Can't find index " << ixn->indexKeyPattern.toString() + << "in namespace " << qsol.ns << endl; + return NULL; + } + + params.bounds = ixn->bounds; + params.direction = ixn->direction; + params.maxScan = ixn->maxScan; + params.addKeyMetadata = ixn->addKeyMetadata; + return new IndexScan(params, ws, ixn->filter.get()); + } + else if (STAGE_FETCH == root->getType()) { + const FetchNode* fn = static_cast(root); + PlanStage* childStage = buildStages(qsol, fn->children[0], ws); + if (NULL == childStage) { return NULL; } + return new FetchStage(ws, childStage, fn->filter.get()); + } + else if (STAGE_SORT == root->getType()) { + const SortNode* sn = static_cast(root); + PlanStage* childStage = buildStages(qsol, sn->children[0], ws); + if (NULL == childStage) { return NULL; } + SortStageParams params; + params.pattern = sn->pattern; + params.query = sn->query; + params.limit = sn->limit; + return new SortStage(params, ws, childStage); + } + else if (STAGE_PROJECTION == root->getType()) { + const ProjectionNode* pn = static_cast(root); + PlanStage* childStage = buildStages(qsol, pn->children[0], ws); + if (NULL == childStage) { return NULL; } + ProjectionStageParams params; + params.projObj = pn->projection; + + // Stuff the right data into the params depending on what proj impl we use. + if (ProjectionNode::DEFAULT == pn->projType) { + params.fullExpression = pn->fullExpression; + params.projImpl = ProjectionStageParams::NO_FAST_PATH; + } + else if (ProjectionNode::COVERED_ONE_INDEX == pn->projType) { + params.projImpl = ProjectionStageParams::COVERED_ONE_INDEX; + params.coveredKeyObj = pn->coveredKeyObj; + invariant(!pn->coveredKeyObj.isEmpty()); + } + else { + invariant(ProjectionNode::SIMPLE_DOC == pn->projType); + params.projImpl = ProjectionStageParams::SIMPLE_DOC; + } + + return new ProjectionStage(params, ws, childStage); + } + else if (STAGE_LIMIT == root->getType()) { + const LimitNode* ln = static_cast(root); + PlanStage* childStage = buildStages(qsol, ln->children[0], ws); + if (NULL == childStage) { return NULL; } + return new LimitStage(ln->limit, ws, childStage); + } + else if (STAGE_SKIP == root->getType()) { + const SkipNode* sn = static_cast(root); + PlanStage* childStage = buildStages(qsol, sn->children[0], ws); + if (NULL == childStage) { return NULL; } + return new SkipStage(sn->skip, ws, childStage); + } + else if (STAGE_AND_HASH == root->getType()) { + const AndHashNode* ahn = static_cast(root); + auto_ptr ret(new AndHashStage(ws, ahn->filter.get())); + for (size_t i = 0; i < ahn->children.size(); ++i) { + PlanStage* childStage = buildStages(qsol, ahn->children[i], ws); + if (NULL == childStage) { return NULL; } + ret->addChild(childStage); + } + return ret.release(); + } + else if (STAGE_OR == root->getType()) { + const OrNode * orn = static_cast(root); + auto_ptr ret(new OrStage(ws, orn->dedup, orn->filter.get())); + for (size_t i = 0; i < orn->children.size(); ++i) { + PlanStage* childStage = buildStages(qsol, orn->children[i], ws); + if (NULL == childStage) { return NULL; } + ret->addChild(childStage); + } + return ret.release(); + } + else if (STAGE_AND_SORTED == root->getType()) { + const AndSortedNode* asn = static_cast(root); + auto_ptr ret(new AndSortedStage(ws, asn->filter.get())); + for (size_t i = 0; i < asn->children.size(); ++i) { + PlanStage* childStage = buildStages(qsol, asn->children[i], ws); + if (NULL == childStage) { return NULL; } + ret->addChild(childStage); + } + return ret.release(); + } + else if (STAGE_SORT_MERGE == root->getType()) { + const MergeSortNode* msn = static_cast(root); + MergeSortStageParams params; + params.dedup = msn->dedup; + params.pattern = msn->sort; + auto_ptr ret(new MergeSortStage(params, ws)); + for (size_t i = 0; i < msn->children.size(); ++i) { + PlanStage* childStage = buildStages(qsol, msn->children[i], ws); + if (NULL == childStage) { return NULL; } + ret->addChild(childStage); + } + return ret.release(); + } + else if (STAGE_GEO_2D == root->getType()) { + const Geo2DNode* node = static_cast(root); + TwoDParams params; + params.gq = node->gq; + params.filter = node->filter.get(); + params.indexKeyPattern = node->indexKeyPattern; + params.ns = qsol.ns; + return new TwoD(params, ws); + } + else if (STAGE_GEO_NEAR_2D == root->getType()) { + const GeoNear2DNode* node = static_cast(root); + TwoDNearParams params; + params.nearQuery = node->nq; + params.ns = qsol.ns; + params.indexKeyPattern = node->indexKeyPattern; + params.filter = node->filter.get(); + params.numWanted = node->numWanted; + params.addPointMeta = node->addPointMeta; + params.addDistMeta = node->addDistMeta; + return new TwoDNear(params, ws); + } + else if (STAGE_GEO_NEAR_2DSPHERE == root->getType()) { + const GeoNear2DSphereNode* node = static_cast(root); + S2NearParams params; + params.ns = qsol.ns; + params.indexKeyPattern = node->indexKeyPattern; + params.nearQuery = node->nq; + params.baseBounds = node->baseBounds; + params.filter = node->filter.get(); + params.addPointMeta = node->addPointMeta; + params.addDistMeta = node->addDistMeta; + return new S2NearStage(params, ws); + } + else if (STAGE_TEXT == root->getType()) { + const TextNode* node = static_cast(root); + + Database* db = cc().database(); + Collection* collection = db ? db->getCollection(qsol.ns) : NULL; + if (NULL == collection) { + warning() << "Null collection for text"; + return NULL; + } + vector idxMatches; + collection->getIndexCatalog()->findIndexByType("text", idxMatches); + if (1 != idxMatches.size()) { + warning() << "No text index, or more than one text index"; + return NULL; + } + IndexDescriptor* index = idxMatches[0]; + const FTSAccessMethod* fam = + static_cast( collection->getIndexCatalog()->getIndex( index ) ); + TextStageParams params(fam->getSpec()); + + params.ns = qsol.ns; + params.index = index; + params.spec = fam->getSpec(); + params.indexPrefix = node->indexPrefix; + + const std::string& language = ("" == node->language + ? fam->getSpec().defaultLanguage().str() + : node->language); + + Status parseStatus = params.query.parse(node->query, language, + fam->getSpec().getTextIndexVersion()); + if (!parseStatus.isOK()) { + warning() << "Can't parse text search query"; + return NULL; + } + + return new TextStage(params, ws, node->filter.get()); + } + else if (STAGE_SHARDING_FILTER == root->getType()) { + const ShardingFilterNode* fn = static_cast(root); + PlanStage* childStage = buildStages(qsol, fn->children[0], ws); + if (NULL == childStage) { return NULL; } + return new ShardFilterStage(shardingState.getCollectionMetadata(qsol.ns), ws, childStage); + } + else if (STAGE_KEEP_MUTATIONS == root->getType()) { + const KeepMutationsNode* km = static_cast(root); + PlanStage* childStage = buildStages(qsol, km->children[0], ws); + if (NULL == childStage) { return NULL; } + return new KeepMutationsStage(km->filter.get(), ws, childStage); + } + else if (STAGE_DISTINCT == root->getType()) { + const DistinctNode* dn = static_cast(root); + + Database* db = cc().database(); + Collection* collection = db ? db->getCollection(qsol.ns) : NULL; + if (NULL == collection) { + warning() << "Can't distinct-scan null namespace " << qsol.ns << endl; + return NULL; + } + + DistinctParams params; + + params.descriptor = + collection->getIndexCatalog()->findIndexByKeyPattern(dn->indexKeyPattern); + params.direction = dn->direction; + params.bounds = dn->bounds; + params.fieldNo = dn->fieldNo; + return new DistinctScan(params, ws); + } + else if (STAGE_COUNT == root->getType()) { + const CountNode* cn = static_cast(root); + + Database* db = cc().database(); + if (NULL == db) { + warning() << "Can't fast-count null namespace (database null)" << qsol.ns << endl; + return NULL; + } + + Collection* collection = db ? db->getCollection(qsol.ns) : NULL; + if (NULL == collection) { + warning() << "Can't fast-count null namespace (collection null)" << qsol.ns << endl; + return NULL; + } + + CountParams params; + + params.descriptor = + collection->getIndexCatalog()->findIndexByKeyPattern(cn->indexKeyPattern); + params.startKey = cn->startKey; + params.startKeyInclusive = cn->startKeyInclusive; + params.endKey = cn->endKey; + params.endKeyInclusive = cn->endKeyInclusive; + + return new Count(params, ws); + } + else { + mongoutils::str::stream ss; + root->appendToString(&ss, 0); + string nodeStr(ss); + warning() << "Can't build exec tree for node " << nodeStr << endl; + return NULL; + } + } + + // static + bool StageBuilder::build(const QuerySolution& solution, PlanStage** rootOut, + WorkingSet** wsOut) { + QuerySolutionNode* root = solution.root.get(); + if (NULL == root) { return false; } + + auto_ptr ws(new WorkingSet()); + PlanStage* stageRoot = buildStages(solution, root, ws.get()); + + if (NULL != stageRoot) { + *rootOut = stageRoot; + *wsOut = ws.release(); + return true; + } + else { + return false; + } + } + +} // namespace mongo diff --git a/src/mongo/db/query/stage_builder.h b/src/mongo/db/query/stage_builder.h new file mode 100644 index 00000000000..c352fd1b9b6 --- /dev/null +++ b/src/mongo/db/query/stage_builder.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/working_set.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + /** + * The StageBuilder converts a QuerySolution to an executable tree of PlanStage(s). + */ + class StageBuilder { + public: + /** + * Turns 'solution' into an executable tree of PlanStage(s). This function accesses cc() + * and catalog information and as such the caller must have a lock. + * + * Returns true if the PlanStage tree was built successfully. The root of the tree is in + * *rootOut and the WorkingSet that the tree uses is in *wsOut. + * + * Returns false otherwise. *rootOut and *wsOut are invalid. + */ + static bool build(const QuerySolution& solution, PlanStage** rootOut, WorkingSet** wsOut); + }; + +} // namespace mongo diff --git a/src/mongo/db/query/stage_types.h b/src/mongo/db/query/stage_types.h new file mode 100644 index 00000000000..cb1ffb391d1 --- /dev/null +++ b/src/mongo/db/query/stage_types.h @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { + + /** + * These map to implementations of the PlanStage interface, all of which live in db/exec/ + */ + enum StageType { + STAGE_AND_HASH, + STAGE_AND_SORTED, + STAGE_COLLSCAN, + + // If we're running a .count(), the query is fully covered by one ixscan, and the ixscan is + // from one key to another, we can just skip through the keys without bothering to examine + // them. + STAGE_COUNT, + + // If we're running a distinct, we only care about one value for each key. The distinct + // stage is an ixscan with some key-skipping behvaior that only distinct uses. + STAGE_DISTINCT, + + // This is more of an "internal-only" stage where we try to keep docs that were mutated + // during query execution. + STAGE_KEEP_MUTATIONS, + + STAGE_FETCH, + + // TODO: This is secretly an expression index but we need geometry -> covering for our + // geohash. + STAGE_GEO_2D, + + // The two $geoNear impls imply a fetch+sort and must be stages. + STAGE_GEO_NEAR_2D, + STAGE_GEO_NEAR_2DSPHERE, + + STAGE_IXSCAN, + STAGE_LIMIT, + STAGE_OR, + STAGE_PROJECTION, + STAGE_SHARDING_FILTER, + STAGE_SKIP, + STAGE_SORT, + STAGE_SORT_MERGE, + STAGE_TEXT, + STAGE_UNKNOWN, + }; + +} // namespace mongo diff --git a/src/mongo/db/query/subplan_runner.cpp b/src/mongo/db/query/subplan_runner.cpp new file mode 100644 index 00000000000..2fb7ca30073 --- /dev/null +++ b/src/mongo/db/query/subplan_runner.cpp @@ -0,0 +1,501 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/subplan_runner.h" + +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/diskloc.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/get_runner.h" +#include "mongo/db/query/multi_plan_runner.h" +#include "mongo/db/query/planner_analysis.h" +#include "mongo/db/query/planner_access.h" +#include "mongo/db/query/qlog.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/stage_builder.h" +#include "mongo/db/query/type_explain.h" + +namespace mongo { + + // static + bool SubplanRunner::canUseSubplanRunner(const CanonicalQuery& query) { + const LiteParsedQuery& lpq = query.getParsed(); + const MatchExpression* expr = query.root(); + + // Only rooted ORs work with the subplan scheme. + if (MatchExpression::OR != expr->matchType()) { + return false; + } + + // Collection scan + // No sort order requested + if (lpq.getSort().isEmpty() && + expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) { + return false; + } + + // Hint provided + if (!lpq.getHint().isEmpty()) { + return false; + } + + // Min provided + // Min queries are a special case of hinted queries. + if (!lpq.getMin().isEmpty()) { + return false; + } + + // Max provided + // Similar to min, max queries are a special case of hinted queries. + if (!lpq.getMax().isEmpty()) { + return false; + } + + // Tailable cursors won't get cached, just turn into collscans. + if (query.getParsed().hasOption(QueryOption_CursorTailable)) { + return false; + } + + // Snapshot is really a hint. + if (query.getParsed().isSnapshot()) { + return false; + } + + return true; + } + + // static + Status SubplanRunner::make(Collection* collection, + const QueryPlannerParams& params, + CanonicalQuery* cq, + SubplanRunner** out) { + auto_ptr autoRunner(new SubplanRunner(collection, params, cq)); + Status planningStatus = autoRunner->planSubqueries(); + if (!planningStatus.isOK()) { + return planningStatus; + } + + *out = autoRunner.release(); + return Status::OK(); + } + + SubplanRunner::SubplanRunner(Collection* collection, + const QueryPlannerParams& params, + CanonicalQuery* cq) + : _state(SubplanRunner::PLANNING), + _collection(collection), + _plannerParams(params), + _query(cq), + _killed(false), + _policy(Runner::YIELD_MANUAL), + _ns(cq->getParsed().ns()) { } + + SubplanRunner::~SubplanRunner() { + while (!_solutions.empty()) { + vector solns = _solutions.front(); + for (size_t i = 0; i < solns.size(); i++) { + delete solns[i]; + } + _solutions.pop(); + } + + while (!_cqs.empty()) { + delete _cqs.front(); + _cqs.pop(); + } + } + + Runner::RunnerState SubplanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) { + if (_killed) { + return Runner::RUNNER_DEAD; + } + + if (isEOF()) { return Runner::RUNNER_EOF; } + + if (SubplanRunner::PLANNING == _state) { + // Try to run as sub-plans. + if (runSubplans()) { + // If runSubplans returns true we expect something here. + invariant(_underlyingRunner.get()); + } + else if (!_killed) { + // Couldn't run as subplans so we'll just call normal getRunner. + + Runner* runner; + Status status = getRunnerAlwaysPlan( + _collection, _query.release(), _plannerParams, &runner); + + if (!status.isOK()) { + // We utterly failed. + _killed = true; + + // Propagate the error to the user wrapped in a BSONObj + if (NULL != objOut) { + BSONObjBuilder bob; + bob.append("ok", status.isOK() ? 1.0 : 0.0); + bob.append("code", status.code()); + bob.append("errmsg", status.reason()); + *objOut = bob.obj(); + } + return Runner::RUNNER_ERROR; + } + else { + _underlyingRunner.reset(runner); + _underlyingRunner->setYieldPolicy(_policy); + } + } + + // We can change state when we're either killed or we have an underlying runner. + invariant(_killed || NULL != _underlyingRunner.get()); + _state = SubplanRunner::RUNNING; + } + + if (_killed) { + return Runner::RUNNER_DEAD; + } + + if (isEOF()) { + return Runner::RUNNER_EOF; + } + + // If we're here we should have planned already. + invariant(SubplanRunner::RUNNING == _state); + invariant(_underlyingRunner.get()); + return _underlyingRunner->getNext(objOut, dlOut); + } + + Status SubplanRunner::planSubqueries() { + MatchExpression* theOr = _query->root(); + + for (size_t i = 0; i < _plannerParams.indices.size(); ++i) { + const IndexEntry& ie = _plannerParams.indices[i]; + _indexMap[ie.keyPattern] = i; + QLOG() << "Subplanner: index " << i << " is " << ie.toString() << endl; + } + + for (size_t i = 0; i < theOr->numChildren(); ++i) { + // Turn the i-th child into its own query. + MatchExpression* orChild = theOr->getChild(i); + CanonicalQuery* orChildCQ; + Status childCQStatus = CanonicalQuery::canonicalize(*_query, + orChild, + &orChildCQ); + if (!childCQStatus.isOK()) { + mongoutils::str::stream ss; + ss << "Subplanner: Can't canonicalize subchild " << orChild->toString() + << " " << childCQStatus.reason(); + return Status(ErrorCodes::BadValue, ss); + } + + // Make sure it gets cleaned up. + auto_ptr safeOrChildCQ(orChildCQ); + + // Plan the i-th child. + vector solutions; + + // We don't set NO_TABLE_SCAN because peeking at the cache data will keep us from + // considering any plan that's a collscan. + QLOG() << "Subplanner: planning child " << i << " of " << theOr->numChildren(); + Status status = QueryPlanner::plan(*safeOrChildCQ, _plannerParams, &solutions); + + if (!status.isOK()) { + mongoutils::str::stream ss; + ss << "Subplanner: Can't plan for subchild " << orChildCQ->toString() + << " " << status.reason(); + return Status(ErrorCodes::BadValue, ss); + } + QLOG() << "Subplanner: got " << solutions.size() << " solutions"; + + if (0 == solutions.size()) { + // If one child doesn't have an indexed solution, bail out. + mongoutils::str::stream ss; + ss << "Subplanner: No solutions for subchild " << orChildCQ->toString(); + return Status(ErrorCodes::BadValue, ss); + } + + // Hang onto the canonicalized subqueries and the corresponding query solutions + // so that they can be used in subplan running later on. + _cqs.push(safeOrChildCQ.release()); + _solutions.push(solutions); + } + + return Status::OK(); + } + + bool SubplanRunner::runSubplans() { + // This is what we annotate with the index selections and then turn into a solution. + auto_ptr theOr( + static_cast(_query->root()->shallowClone())); + + // This is the skeleton of index selections that is inserted into the cache. + auto_ptr cacheData(new PlanCacheIndexTree()); + + for (size_t i = 0; i < theOr->numChildren(); ++i) { + MatchExpression* orChild = theOr->getChild(i); + + auto_ptr orChildCQ(_cqs.front()); + _cqs.pop(); + + // 'solutions' is owned by the SubplanRunner instance until + // it is popped from the queue. + vector solutions = _solutions.front(); + + // We already checked for zero solutions in planSubqueries(...). + invariant(!solutions.empty()); + + if (1 == solutions.size()) { + // There is only one solution. Transfer ownership to an auto_ptr. + _solutions.pop(); + auto_ptr autoSoln(solutions[0]); + + // We want a well-formed *indexed* solution. + if (NULL == autoSoln->cacheData.get()) { + // For example, we don't cache things for 2d indices. + QLOG() << "Subplanner: No cache data for subchild " << orChild->toString(); + return false; + } + + if (SolutionCacheData::USE_INDEX_TAGS_SOLN != autoSoln->cacheData->solnType) { + QLOG() << "Subplanner: No indexed cache data for subchild " + << orChild->toString(); + return false; + } + + // Add the index assignments to our original query. + Status tagStatus = QueryPlanner::tagAccordingToCache( + orChild, autoSoln->cacheData->tree.get(), _indexMap); + + if (!tagStatus.isOK()) { + QLOG() << "Subplanner: Failed to extract indices from subchild " + << orChild->toString(); + return false; + } + + // Add the child's cache data to the cache data we're creating for the main query. + cacheData->children.push_back(autoSoln->cacheData->tree->clone()); + } + else { + // N solutions, rank them. Takes ownership of safeOrChildCQ. + MultiPlanRunner* mpr = new MultiPlanRunner(_collection, orChildCQ.release()); + + // Dump all the solutions into the MPR. The MPR takes ownership of + // each solution. + _solutions.pop(); + for (size_t i = 0; i < solutions.size(); ++i) { + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*solutions[i], &root, &ws)); + // Takes ownership of all arguments. + mpr->addPlan(solutions[i], root, ws); + } + + // If we're allowed to yield, let the MPR know. + mpr->setYieldPolicy(_policy); + + // Calling pickBestPlan can yield so we must propagate events down to the MPR. + _underlyingRunner.reset(mpr); + + // Pull out the best plan. + size_t bestPlan; + BSONObj errorObj; + if (!mpr->pickBestPlan(&bestPlan, &errorObj)) { + QLOG() << "Subplanner: Failed to pick best plan for subchild " + << orChild->toString() + << " error obj is " << errorObj.toString(); + return false; + } + + // pickBestPlan can yield. Make sure we're not dead any which way. + if (_killed) { + QLOG() << "Subplanner: Killed while picking best plan for subchild " + << orChild->toString(); + return false; + } + + QuerySolution* bestSoln = solutions[bestPlan]; + + // Check that we have good cache data. For example, we don't cache things for 2d + // indices. + if (NULL == bestSoln->cacheData.get()) { + QLOG() << "Subplanner: No cache data for subchild: " << orChild->toString(); + return false; + } + + if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) { + QLOG() << "Subplanner: No indexed cache data for subchild " + << orChild->toString(); + return false; + } + + // Add the index assignments to our original query. + Status tagStatus = QueryPlanner::tagAccordingToCache( + orChild, bestSoln->cacheData->tree.get(), _indexMap); + + if (!tagStatus.isOK()) { + QLOG() << "Subplanner: Failed to extract indices from subchild " + << orChild->toString(); + return false; + } + + cacheData->children.push_back(solutions[bestPlan]->cacheData->tree->clone()); + } + } + + // Must do this before using the planner functionality. + sortUsingTags(theOr.get()); + + // Use the cached index assignments to build solnRoot. Takes ownership of 'theOr' + QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess( + *_query, theOr.release(), false, _plannerParams.indices); + + if (NULL == solnRoot) { + QLOG() << "Subplanner: Failed to build indexed data path for subplanned query\n"; + return false; + } + + QLOG() << "Subplanner: fully tagged tree is " << solnRoot->toString(); + + // Takes ownership of 'solnRoot' + QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*_query, + _plannerParams, + solnRoot); + + if (NULL == soln) { + QLOG() << "Subplanner: Failed to analyze subplanned query"; + return false; + } + + // We want our franken-solution to be cached. + SolutionCacheData* scd = new SolutionCacheData(); + scd->tree.reset(cacheData.release()); + soln->cacheData.reset(scd); + + QLOG() << "Subplanner: Composite solution is " << soln->toString() << endl; + + // We use one of these even if there is one plan. We do this so that the entry is cached + // with stats obtained in the same fashion as a competitive ranking would have obtained + // them. + MultiPlanRunner* mpr = new MultiPlanRunner(_collection, _query.release()); + WorkingSet* ws; + PlanStage* root; + verify(StageBuilder::build(*soln, &root, &ws)); + // Takes ownership of all arguments. + mpr->addPlan(soln, root, ws); + + mpr->setYieldPolicy(_policy); + _underlyingRunner.reset(mpr); + + return true; + } + + bool SubplanRunner::isEOF() { + if (_killed) { + return true; + } + + // If we're still planning we're not done yet. + if (SubplanRunner::PLANNING == _state) { + return false; + } + + // If we're running we best have a runner. + invariant(_underlyingRunner.get()); + return _underlyingRunner->isEOF(); + } + + void SubplanRunner::saveState() { + if (_killed) { + return; + } + + // We're ranking a sub-plan via an MPR or we're streaming results from this Runner. Either + // way, pass on the request. + if (NULL != _underlyingRunner.get()) { + _underlyingRunner->saveState(); + } + } + + bool SubplanRunner::restoreState() { + if (_killed) { + return false; + } + + // We're ranking a sub-plan via an MPR or we're streaming results from this Runner. Either + // way, pass on the request. + if (NULL != _underlyingRunner.get()) { + return _underlyingRunner->restoreState(); + } + + return true; + } + + void SubplanRunner::setYieldPolicy(Runner::YieldPolicy policy) { + if (_killed) { return; } + + // If somebody sets this before calling work() we need to know how to set it in our subquery + // runners. + _policy = policy; + + if (NULL != _underlyingRunner.get()) { + _underlyingRunner->setYieldPolicy(policy); + } + } + + void SubplanRunner::invalidate(const DiskLoc& dl, InvalidationType type) { + if (_killed) { return; } + + if (NULL != _underlyingRunner.get()) { + _underlyingRunner->invalidate(dl, type); + } + } + + const std::string& SubplanRunner::ns() { + return _ns; + } + + void SubplanRunner::kill() { + _killed = true; + _collection = NULL; + + if (NULL != _underlyingRunner.get()) { + _underlyingRunner->kill(); + } + } + + Status SubplanRunner::getInfo(TypeExplain** explain, PlanInfo** planInfo) const { + if (SubplanRunner::RUNNING == _state) { + invariant(_underlyingRunner.get()); + return _underlyingRunner->getInfo(explain, planInfo); + } + else { + return Status(ErrorCodes::BadValue, "no sub-plan to defer getInfo to"); + } + } + +} // namespace mongo diff --git a/src/mongo/db/query/subplan_runner.h b/src/mongo/db/query/subplan_runner.h new file mode 100644 index 00000000000..7415b50c934 --- /dev/null +++ b/src/mongo/db/query/subplan_runner.h @@ -0,0 +1,140 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/db/query/runner.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/db/query/query_solution.h" + +namespace mongo { + + class BSONObj; + class CanonicalQuery; + class DiskLoc; + class TypeExplain; + struct PlanInfo; + + class SubplanRunner : public Runner { + public: + /** + * Used to create SubplanRunner instances. The caller owns the instance + * returned through 'out'. + * + * 'out' is valid only if an OK status is returned. + */ + static Status make(Collection* collection, + const QueryPlannerParams& params, + CanonicalQuery* cq, + SubplanRunner** out); + + static bool canUseSubplanRunner(const CanonicalQuery& query); + + virtual ~SubplanRunner(); + + virtual Runner::RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut); + + virtual bool isEOF(); + + virtual void saveState(); + + virtual bool restoreState(); + + virtual void setYieldPolicy(Runner::YieldPolicy policy); + + virtual void invalidate(const DiskLoc& dl, InvalidationType type); + + virtual const std::string& ns(); + + virtual void kill(); + + virtual const Collection* collection() { + return _collection; + } + + virtual Status getInfo(TypeExplain** explain, + PlanInfo** planInfo) const; + + /** + * Plan each branch of the $or independently, and store the resulting + * lists of query solutions in '_solutions'. + * + * Called from SubplanRunner::make so that getRunner can fail if + * subquery planning fails, rather than returning a runner and failing + * through getNext(...). + */ + Status planSubqueries(); + + private: + SubplanRunner(Collection* collection, + const QueryPlannerParams& params, + CanonicalQuery* cq); + + bool runSubplans(); + + enum SubplanRunnerState { + PLANNING, + RUNNING, + }; + + SubplanRunnerState _state; + + Collection* _collection; + + QueryPlannerParams _plannerParams; + + std::auto_ptr _query; + + bool _killed; + + Runner::YieldPolicy _policy; + + boost::scoped_ptr _underlyingRunner; + + std::string _ns; + + // We do the subquery planning up front, and keep the resulting + // query solutions here. Lists of query solutions are dequeued + // and ownership is transferred to the underlying runners one + // at a time. + std::queue< std::vector > _solutions; + + // Holds the canonicalized subqueries. Ownership is transferred + // to the underlying runners one at a time. + std::queue _cqs; + + // We need this to extract cache-friendly index data from the index assignments. + map _indexMap; + }; + +} // namespace mongo diff --git a/src/mongo/db/query/type_explain.cpp b/src/mongo/db/query/type_explain.cpp new file mode 100644 index 00000000000..b6415b4db41 --- /dev/null +++ b/src/mongo/db/query/type_explain.cpp @@ -0,0 +1,805 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/query/type_explain.h" + +#include "mongo/db/field_parser.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + + // TODO: This doesn't need to be so complicated or serializable. Let's throw this out when we + // move to explain V2 + + using mongoutils::str::stream; + + const BSONField > TypeExplain::clauses("clauses"); + const BSONField TypeExplain::cursor("cursor"); + const BSONField TypeExplain::isMultiKey("isMultiKey"); + const BSONField TypeExplain::n("n", 0); + const BSONField TypeExplain::nScannedObjects("nscannedObjects", 0); + const BSONField TypeExplain::nScanned("nscanned", 0); + const BSONField TypeExplain::nScannedObjectsAllPlans("nscannedObjectsAllPlans"); + const BSONField TypeExplain::nScannedAllPlans("nscannedAllPlans"); + const BSONField TypeExplain::scanAndOrder("scanAndOrder"); + const BSONField TypeExplain::indexOnly("indexOnly"); + const BSONField TypeExplain::nYields("nYields"); + const BSONField TypeExplain::nChunkSkips("nChunkSkips"); + const BSONField TypeExplain::millis("millis"); + const BSONField TypeExplain::indexBounds("indexBounds"); + const BSONField > TypeExplain::allPlans("allPlans"); + const BSONField TypeExplain::oldPlan("oldPlan"); + const BSONField TypeExplain::indexFilterApplied("filterSet"); + const BSONField TypeExplain::server("server"); + + TypeExplain::TypeExplain() { + clear(); + } + + TypeExplain::~TypeExplain() { + unsetClauses(); + unsetAllPlans(); + } + + bool TypeExplain::isValid(std::string* errMsg) const { + std::string dummy; + if (errMsg == NULL) { + errMsg = &dummy; + } + + // All the mandatory fields must be present. + if (!_isNSet) { + *errMsg = stream() << "missing " << n.name() << " field"; + return false; + } + + if (!_isNScannedObjectsSet) { + *errMsg = stream() << "missing " << nScannedObjects.name() << " field"; + return false; + } + + if (!_isNScannedSet) { + *errMsg = stream() << "missing " << nScanned.name() << " field"; + return false; + } + + return true; + } + + BSONObj TypeExplain::toBSON() const { + BSONObjBuilder builder; + + if (_clauses.get()) { + BSONArrayBuilder clausesBuilder(builder.subarrayStart(clauses())); + for (std::vector::const_iterator it = _clauses->begin(); + it != _clauses->end(); + ++it) { + BSONObj clausesDocument = (*it)->toBSON(); + clausesBuilder.append(clausesDocument); + } + clausesBuilder.done(); + } + + if (_isCursorSet) builder.append(cursor(), _cursor); + + if (_isIsMultiKeySet) builder.append(isMultiKey(), _isMultiKey); + + if (_isNSet) { + builder.appendNumber(n(), _n); + } + else { + builder.appendNumber(n(), n.getDefault()); + } + + if (_isNScannedObjectsSet) { + builder.appendNumber(nScannedObjects(), _nScannedObjects); + } + else { + builder.appendNumber(nScannedObjects(), nScannedObjects.getDefault()); + } + + if (_isNScannedSet) { + builder.appendNumber(nScanned(), _nScanned); + } + else { + builder.appendNumber(nScanned(), nScanned.getDefault()); + } + + if (_isNScannedObjectsAllPlansSet) + builder.appendNumber(nScannedObjectsAllPlans(), _nScannedObjectsAllPlans); + + if (_isNScannedAllPlansSet) builder.appendNumber(nScannedAllPlans(), _nScannedAllPlans); + + if (_isScanAndOrderSet) builder.append(scanAndOrder(), _scanAndOrder); + + if (_isIndexOnlySet) builder.append(indexOnly(), _indexOnly); + + if (_isNYieldsSet) builder.appendNumber(nYields(), _nYields); + + if (_isNChunkSkipsSet) builder.appendNumber(nChunkSkips(), _nChunkSkips); + + if (_isMillisSet) builder.appendNumber(millis(), _millis); + + if (_isIndexBoundsSet) builder.append(indexBounds(), _indexBounds); + + if (_allPlans.get()) { + BSONArrayBuilder allPlansBuilder(builder.subarrayStart(allPlans())); + for (std::vector::const_iterator it = _allPlans->begin(); + it != _allPlans->end(); + ++it) { + BSONObj allPlansObject = (*it)->toBSON(); + allPlansBuilder.append(allPlansObject); + } + allPlansBuilder.done(); + } + + if (_oldPlan.get()) builder.append(oldPlan(), _oldPlan->toBSON()); + + if (_isServerSet) builder.append(server(), _server); + + if (_isIndexFilterAppliedSet) builder.append(indexFilterApplied(), _indexFilterApplied); + + // Add this at the end as it can be huge + if (!stats.isEmpty()) { + builder.append("stats", stats); + } + + return builder.obj(); + } + + bool TypeExplain::parseBSON(const BSONObj& source, string* errMsg) { + clear(); + + std::string dummy; + if (!errMsg) errMsg = &dummy; + + FieldParser::FieldState fieldState; + + std::vector* bareClauses = NULL; + fieldState = FieldParser::extract(source, clauses, &bareClauses, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + if (fieldState == FieldParser::FIELD_SET) _clauses.reset(bareClauses); + + fieldState = FieldParser::extract(source, cursor, &_cursor, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isCursorSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, isMultiKey, &_isMultiKey, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isIsMultiKeySet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, n, &_n, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isNSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, nScannedObjects, &_nScannedObjects, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isNScannedObjectsSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, nScanned, &_nScanned, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isNScannedSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, + nScannedObjectsAllPlans, + &_nScannedObjectsAllPlans, + errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isNScannedObjectsAllPlansSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, nScannedAllPlans, &_nScannedAllPlans, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isNScannedAllPlansSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, scanAndOrder, &_scanAndOrder, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isScanAndOrderSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, indexOnly, &_indexOnly, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isIndexOnlySet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, nYields, &_nYields, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isNYieldsSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, nChunkSkips, &_nChunkSkips, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isNChunkSkipsSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, millis, &_millis, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isMillisSet = fieldState == FieldParser::FIELD_SET; + + fieldState = FieldParser::extract(source, indexBounds, &_indexBounds, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isIndexBoundsSet = fieldState == FieldParser::FIELD_SET; + + std::vector* bareAllPlans = NULL; + fieldState = FieldParser::extract(source, allPlans, &bareAllPlans, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + if (fieldState == FieldParser::FIELD_SET) _allPlans.reset(bareAllPlans); + + TypeExplain* bareOldPlan = NULL; + fieldState = FieldParser::extract(source, oldPlan, &bareOldPlan, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + if (fieldState == FieldParser::FIELD_SET) _oldPlan.reset(bareOldPlan); + + fieldState = FieldParser::extract(source, server, &_server, errMsg); + if (fieldState == FieldParser::FIELD_INVALID) return false; + _isServerSet = fieldState == FieldParser::FIELD_SET; + + return true; + } + + void TypeExplain::clear() { + unsetClauses(); + + _cursor.clear(); + _isCursorSet = false; + + _isMultiKey = false; + _isIsMultiKeySet = false; + + _n = 0; + _isNSet = false; + + _nScannedObjects = 0; + _isNScannedObjectsSet = false; + + _nScanned = 0; + _isNScannedSet = false; + + _nScannedObjectsAllPlans = 0; + _isNScannedObjectsAllPlansSet = false; + + _nScannedAllPlans = 0; + _isNScannedAllPlansSet = false; + + _scanAndOrder = false; + _isScanAndOrderSet = false; + + _indexOnly = false; + _isIndexOnlySet = false; + + _idHack = false; + _isIDHackSet = false; + + _indexFilterApplied = false; + _isIndexFilterAppliedSet = false; + + _nYields = 0; + _isNYieldsSet = false; + + _nChunkSkips = 0; + _isNChunkSkipsSet = false; + + _millis = 0; + _isMillisSet = false; + + _indexBounds = BSONObj(); + _isIndexBoundsSet = false; + + unsetAllPlans(); + + unsetOldPlan(); + + _server.clear(); + _isServerSet = false; + + } + + void TypeExplain::cloneTo(TypeExplain* other) const { + other->clear(); + + other->unsetClauses(); + if (_clauses.get()) { + for(std::vector::const_iterator it = _clauses->begin(); + it != _clauses->end(); + ++it) { + TypeExplain* clausesItem = new TypeExplain; + (*it)->cloneTo(clausesItem); + other->addToClauses(clausesItem); + } + } + + other->_cursor = _cursor; + other->_isCursorSet = _isCursorSet; + + other->_isMultiKey = _isMultiKey; + other->_isIsMultiKeySet = _isIsMultiKeySet; + + other->_n = _n; + other->_isNSet = _isNSet; + + other->_nScannedObjects = _nScannedObjects; + other->_isNScannedObjectsSet = _isNScannedObjectsSet; + + other->_nScanned = _nScanned; + other->_isNScannedSet = _isNScannedSet; + + other->_nScannedObjectsAllPlans = _nScannedObjectsAllPlans; + other->_isNScannedObjectsAllPlansSet = _isNScannedObjectsAllPlansSet; + + other->_nScannedAllPlans = _nScannedAllPlans; + other->_isNScannedAllPlansSet = _isNScannedAllPlansSet; + + other->_scanAndOrder = _scanAndOrder; + other->_isScanAndOrderSet = _isScanAndOrderSet; + + other->_indexOnly = _indexOnly; + other->_isIndexOnlySet = _isIndexOnlySet; + + other->_idHack = _idHack; + other->_isIDHackSet = _isIDHackSet; + + other->_indexFilterApplied = _indexFilterApplied; + other->_isIndexFilterAppliedSet = _isIndexFilterAppliedSet; + + other->_nYields = _nYields; + other->_isNYieldsSet = _isNYieldsSet; + + other->_nChunkSkips = _nChunkSkips; + other->_isNChunkSkipsSet = _isNChunkSkipsSet; + + other->_millis = _millis; + other->_isMillisSet = _isMillisSet; + + other->_indexBounds = _indexBounds; + other->_isIndexBoundsSet = _isIndexBoundsSet; + + other->unsetAllPlans(); + if (_allPlans.get()) { + for(std::vector::const_iterator it = _allPlans->begin(); + it != _allPlans->end(); + ++it) { + TypeExplain* allPlansItem = new TypeExplain; + (*it)->cloneTo(allPlansItem); + other->addToAllPlans(allPlansItem); + } + } + + other->unsetOldPlan(); + if (_oldPlan.get()) { + TypeExplain* oldPlanCopy = new TypeExplain; + _oldPlan->cloneTo(oldPlanCopy); + other->setOldPlan(oldPlanCopy); + } + + other->_server = _server; + other->_isServerSet = _isServerSet; + } + + std::string TypeExplain::toString() const { + return toBSON().toString(); + } + + void TypeExplain::setClauses(const std::vector& clauses) { + unsetClauses(); + for(std::vector::const_iterator it = clauses.begin(); + it != clauses.end(); + ++it) { + TypeExplain* clausesItem = new TypeExplain; + (*it)->cloneTo(clausesItem); + addToClauses(clausesItem); + } + } + + void TypeExplain::addToClauses(TypeExplain* clauses) { + if (_clauses.get() == NULL) { + _clauses.reset(new std::vector); + } + _clauses->push_back(clauses); + } + + void TypeExplain::unsetClauses() { + if (_clauses.get()) { + for(std::vector::const_iterator it = _clauses->begin(); + it != _clauses->end(); + ++it) { + delete *it; + } + } + _clauses.reset(); + } + + bool TypeExplain::isClausesSet() const { + return _clauses.get() != NULL; + } + + size_t TypeExplain::sizeClauses() const { + verify(_clauses.get()); + return _clauses->size(); + } + + const std::vector& TypeExplain::getClauses() const { + verify(_clauses.get()); + return *_clauses; + } + + const TypeExplain* TypeExplain::getClausesAt(size_t pos) const { + verify(_clauses.get()); + verify(_clauses->size() > pos); + return _clauses->at(pos); + } + + void TypeExplain::setCursor(const StringData& cursor) { + _cursor = cursor.toString(); + _isCursorSet = true; + } + + void TypeExplain::unsetCursor() { + _isCursorSet = false; + } + + bool TypeExplain::isCursorSet() const { + return _isCursorSet; + } + + const std::string& TypeExplain::getCursor() const { + verify(_isCursorSet); + return _cursor; + } + + void TypeExplain::setIsMultiKey(bool isMultiKey) { + _isMultiKey = isMultiKey; + _isIsMultiKeySet = true; + } + + void TypeExplain::unsetIsMultiKey() { + _isIsMultiKeySet = false; + } + + bool TypeExplain::isIsMultiKeySet() const { + return _isIsMultiKeySet; + } + + bool TypeExplain::getIsMultiKey() const { + verify(_isIsMultiKeySet); + return _isMultiKey; + } + + void TypeExplain::setN(long long n) { + _n = n; + _isNSet = true; + } + + void TypeExplain::unsetN() { + _isNSet = false; + } + + bool TypeExplain::isNSet() const { + return _isNSet; + } + + long long TypeExplain::getN() const { + verify(_isNSet); + return _n; + } + + void TypeExplain::setNScannedObjects(long long nScannedObjects) { + _nScannedObjects = nScannedObjects; + _isNScannedObjectsSet = true; + } + + void TypeExplain::unsetNScannedObjects() { + _isNScannedObjectsSet = false; + } + + bool TypeExplain::isNScannedObjectsSet() const { + return _isNScannedObjectsSet; + } + + long long TypeExplain::getNScannedObjects() const { + verify(_isNScannedObjectsSet); + return _nScannedObjects; + } + + void TypeExplain::setNScanned(long long nScanned) { + _nScanned = nScanned; + _isNScannedSet = true; + } + + void TypeExplain::unsetNScanned() { + _isNScannedSet = false; + } + + bool TypeExplain::isNScannedSet() const { + return _isNScannedSet; + } + + long long TypeExplain::getNScanned() const { + verify(_isNScannedSet); + return _nScanned; + } + + void TypeExplain::setNScannedObjectsAllPlans(long long nScannedObjectsAllPlans) { + _nScannedObjectsAllPlans = nScannedObjectsAllPlans; + _isNScannedObjectsAllPlansSet = true; + } + + void TypeExplain::unsetNScannedObjectsAllPlans() { + _isNScannedObjectsAllPlansSet = false; + } + + bool TypeExplain::isNScannedObjectsAllPlansSet() const { + return _isNScannedObjectsAllPlansSet; + } + + long long TypeExplain::getNScannedObjectsAllPlans() const { + verify(_isNScannedObjectsAllPlansSet); + return _nScannedObjectsAllPlans; + } + + void TypeExplain::setNScannedAllPlans(long long nScannedAllPlans) { + _nScannedAllPlans = nScannedAllPlans; + _isNScannedAllPlansSet = true; + } + + void TypeExplain::unsetNScannedAllPlans() { + _isNScannedAllPlansSet = false; + } + + bool TypeExplain::isNScannedAllPlansSet() const { + return _isNScannedAllPlansSet; + } + + long long TypeExplain::getNScannedAllPlans() const { + verify(_isNScannedAllPlansSet); + return _nScannedAllPlans; + } + + void TypeExplain::setScanAndOrder(bool scanAndOrder) { + _scanAndOrder = scanAndOrder; + _isScanAndOrderSet = true; + } + + void TypeExplain::unsetScanAndOrder() { + _isScanAndOrderSet = false; + } + + bool TypeExplain::isScanAndOrderSet() const { + return _isScanAndOrderSet; + } + + bool TypeExplain::getScanAndOrder() const { + verify(_isScanAndOrderSet); + return _scanAndOrder; + } + + void TypeExplain::setIndexOnly(bool indexOnly) { + _indexOnly = indexOnly; + _isIndexOnlySet = true; + } + + void TypeExplain::unsetIndexOnly() { + _isIndexOnlySet = false; + } + + bool TypeExplain::isIndexOnlySet() const { + return _isIndexOnlySet; + } + + bool TypeExplain::getIndexOnly() const { + verify(_isIndexOnlySet); + return _indexOnly; + } + + void TypeExplain::setIDHack(bool idhack) { + _idHack = idhack; + _isIDHackSet = true; + } + + void TypeExplain::unsetIDHack() { + _isIDHackSet = false; + } + + bool TypeExplain::isIDHackSet() const { + return _isIDHackSet; + } + + bool TypeExplain::getIDHack() const { + verify(_isIDHackSet); + return _idHack; + } + + void TypeExplain::setIndexFilterApplied(bool indexFilterApplied) { + _indexFilterApplied = indexFilterApplied; + _isIndexFilterAppliedSet = true; + } + + void TypeExplain::unsetIndexFilterApplied() { + _isIndexFilterAppliedSet = false; + } + + bool TypeExplain::isIndexFilterAppliedSet() const { + return _isIndexFilterAppliedSet; + } + + bool TypeExplain::getIndexFilterApplied() const { + verify(_isIndexFilterAppliedSet); + return _indexFilterApplied; + } + + void TypeExplain::setNYields(long long nYields) { + _nYields = nYields; + _isNYieldsSet = true; + } + + void TypeExplain::unsetNYields() { + _isNYieldsSet = false; + } + + bool TypeExplain::isNYieldsSet() const { + return _isNYieldsSet; + } + + long long TypeExplain::getNYields() const { + verify(_isNYieldsSet); + return _nYields; + } + + void TypeExplain::setNChunkSkips(long long nChunkSkips) { + _nChunkSkips = nChunkSkips; + _isNChunkSkipsSet = true; + } + + void TypeExplain::unsetNChunkSkips() { + _isNChunkSkipsSet = false; + } + + bool TypeExplain::isNChunkSkipsSet() const { + return _isNChunkSkipsSet; + } + + long long TypeExplain::getNChunkSkips() const { + verify(_isNChunkSkipsSet); + return _nChunkSkips; + } + + void TypeExplain::setMillis(long long millis) { + _millis = millis; + _isMillisSet = true; + } + + void TypeExplain::unsetMillis() { + _isMillisSet = false; + } + + bool TypeExplain::isMillisSet() const { + return _isMillisSet; + } + + long long TypeExplain::getMillis() const { + verify(_isMillisSet); + return _millis; + } + + void TypeExplain::setIndexBounds(const BSONObj& indexBounds) { + _indexBounds = indexBounds.getOwned(); + _isIndexBoundsSet = true; + } + + void TypeExplain::unsetIndexBounds() { + _isIndexBoundsSet = false; + } + + bool TypeExplain::isIndexBoundsSet() const { + return _isIndexBoundsSet; + } + + const BSONObj& TypeExplain::getIndexBounds() const { + verify(_isIndexBoundsSet); + return _indexBounds; + } + + void TypeExplain::setAllPlans(const std::vector& allPlans) { + unsetAllPlans(); + for (std::vector::const_iterator it = allPlans.begin(); + it != allPlans.end(); + ++it) { + TypeExplain* allPlansItem = new TypeExplain; + (*it)->cloneTo(allPlansItem); + addToClauses(allPlansItem); + } + } + + void TypeExplain::addToAllPlans(TypeExplain* allPlans) { + if (_allPlans.get() == NULL) { + _allPlans.reset(new std::vector); + } + _allPlans->push_back(allPlans); + } + + void TypeExplain::unsetAllPlans() { + if (_allPlans.get()) { + for (std::vector::const_iterator it = _allPlans->begin(); + it != _allPlans->end(); + ++it) { + delete *it; + } + _allPlans.reset(); + } + } + + bool TypeExplain::isAllPlansSet() const { + return _allPlans.get() != NULL; + } + + size_t TypeExplain::sizeAllPlans() const { + verify(_allPlans.get()); + return _allPlans->size(); + } + + const std::vector& TypeExplain::getAllPlans() const { + verify(_allPlans.get()); + return *_allPlans; + } + + const TypeExplain* TypeExplain::getAllPlansAt(size_t pos) const { + verify(_allPlans.get()); + verify(_allPlans->size() > pos); + return _allPlans->at(pos); + } + + void TypeExplain::setOldPlan(TypeExplain* oldPlan) { + _oldPlan.reset(oldPlan); + } + + void TypeExplain::unsetOldPlan() { + _oldPlan.reset(); + } + + bool TypeExplain::isOldPlanSet() const { + return _oldPlan.get() != NULL; + } + + const TypeExplain* TypeExplain::getOldPlan() const { + verify(_oldPlan.get()); + return _oldPlan.get(); + } + + void TypeExplain::setServer(const StringData& server) { + _server = server.toString(); + _isServerSet = true; + } + + void TypeExplain::unsetServer() { + _isServerSet = false; + } + + bool TypeExplain::isServerSet() const { + return _isServerSet; + } + + const std::string& TypeExplain::getServer() const { + verify(_isServerSet); + return _server; + } + +} // namespace mongo diff --git a/src/mongo/db/query/type_explain.h b/src/mongo/db/query/type_explain.h new file mode 100644 index 00000000000..1048e591c85 --- /dev/null +++ b/src/mongo/db/query/type_explain.h @@ -0,0 +1,289 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include + +#include "mongo/base/string_data.h" +#include "mongo/s/bson_serializable.h" + +namespace mongo { + + /** + * Contains query debug information that describes the + * query plan. Generally this information depends only on + * the planning process that happens without running the + * query. The exception is the multi plan runner, in which + * case plan selection depends on actually running the query. + * + * Currently, just a summary string describing the plan + * used to run the query. + */ + struct PlanInfo { + PlanInfo() : planSummary("") { } + std::string planSummary; + }; + + /** + * This class represents the layout and content of a TypeExplain runCommand, + * the response side. + */ + class TypeExplain : public BSONSerializable { + MONGO_DISALLOW_COPYING(TypeExplain); + public: + + // + // schema declarations + // + + static const BSONField > clauses; + static const BSONField cursor; + static const BSONField isMultiKey; + static const BSONField n; + static const BSONField nScannedObjects; + static const BSONField nScanned; + static const BSONField nScannedObjectsAllPlans; + static const BSONField nScannedAllPlans; + static const BSONField scanAndOrder; + static const BSONField indexOnly; + static const BSONField nYields; + static const BSONField nChunkSkips; + static const BSONField millis; + static const BSONField indexBounds; + static const BSONField > allPlans; + static const BSONField oldPlan; + static const BSONField indexFilterApplied; + static const BSONField server; + + // + // construction / destruction + // + + TypeExplain(); + virtual ~TypeExplain(); + + /** Copies all the fields present in 'this' to 'other'. */ + void cloneTo(TypeExplain* other) const; + + // + // bson serializable interface implementation + // + + virtual bool isValid(std::string* errMsg) const; + virtual BSONObj toBSON() const; + virtual bool parseBSON(const BSONObj& source, std::string* errMsg); + virtual void clear(); + virtual std::string toString() const; + + // + // individual field accessors + // + + void setClauses(const std::vector& clauses); + void addToClauses(TypeExplain* clauses); + void unsetClauses(); + bool isClausesSet() const; + size_t sizeClauses() const; + const std::vector& getClauses() const; + const TypeExplain* getClausesAt(size_t pos) const; + + void setCursor(const StringData& cursor); + void unsetCursor(); + bool isCursorSet() const; + const std::string& getCursor() const; + + void setIsMultiKey(bool isMultiKey); + void unsetIsMultiKey(); + bool isIsMultiKeySet() const; + bool getIsMultiKey() const; + + void setN(long long n); + void unsetN(); + bool isNSet() const; + long long getN() const; + + void setNScannedObjects(long long nScannedObjects); + void unsetNScannedObjects(); + bool isNScannedObjectsSet() const; + long long getNScannedObjects() const; + + void setNScanned(long long nScanned); + void unsetNScanned(); + bool isNScannedSet() const; + long long getNScanned() const; + + void setNScannedObjectsAllPlans(long long nScannedObjectsAllPlans); + void unsetNScannedObjectsAllPlans(); + bool isNScannedObjectsAllPlansSet() const; + long long getNScannedObjectsAllPlans() const; + + void setNScannedAllPlans(long long nScannedAllPlans); + void unsetNScannedAllPlans(); + bool isNScannedAllPlansSet() const; + long long getNScannedAllPlans() const; + + void setScanAndOrder(bool scanAndOrder); + void unsetScanAndOrder(); + bool isScanAndOrderSet() const; + bool getScanAndOrder() const; + + void setIndexOnly(bool indexOnly); + void unsetIndexOnly(); + bool isIndexOnlySet() const; + bool getIndexOnly() const; + + void setIDHack(bool idhack); + void unsetIDHack(); + bool isIDHackSet() const; + bool getIDHack() const; + + void setIndexFilterApplied(bool indexFilterApplied); + void unsetIndexFilterApplied(); + bool isIndexFilterAppliedSet() const; + bool getIndexFilterApplied() const; + + void setNYields(long long nYields); + void unsetNYields(); + bool isNYieldsSet() const; + long long getNYields() const; + + void setNChunkSkips(long long nChunkSkips); + void unsetNChunkSkips(); + bool isNChunkSkipsSet() const; + long long getNChunkSkips() const; + + void setMillis(long long millis); + void unsetMillis(); + bool isMillisSet() const; + long long getMillis() const; + + void setIndexBounds(const BSONObj& indexBounds); + void unsetIndexBounds(); + bool isIndexBoundsSet() const; + const BSONObj& getIndexBounds() const; + + void setAllPlans(const std::vector& allPlans); + void addToAllPlans(TypeExplain* allPlans); + void unsetAllPlans(); + bool isAllPlansSet() const; + size_t sizeAllPlans() const; + const std::vector& getAllPlans() const; + const TypeExplain* getAllPlansAt(size_t pos) const; + + void setOldPlan(TypeExplain* oldPlan); + void unsetOldPlan(); + bool isOldPlanSet() const; + const TypeExplain* getOldPlan() const; + + void setServer(const StringData& server); + void unsetServer(); + bool isServerSet() const; + const std::string& getServer() const; + + // Opaque stats object + BSONObj stats; + + private: + // Convention: (M)andatory, (O)ptional + + // (O) explain for branches on a $or query + boost::scoped_ptr >_clauses; + + // (O) type and name of the cursor used on the leaf stage + std::string _cursor; + bool _isCursorSet; + + // (O) type and name of the cursor used on the leaf stage + bool _isMultiKey; + bool _isIsMultiKeySet; + + // (M) number of documents returned by the query + long long _n; + bool _isNSet; + + // (M) number of documents fetched entirely from the disk + long long _nScannedObjects; + bool _isNScannedObjectsSet; + + // (M) number of entries retrieved either from an index or collection + long long _nScanned; + bool _isNScannedSet; + + // (O) number of documents fetched entirely from the disk across all plans + long long _nScannedObjectsAllPlans; + bool _isNScannedObjectsAllPlansSet; + + // (O) number of entries retrieved either from an index or collection across all plans + long long _nScannedAllPlans; + bool _isNScannedAllPlansSet; + + // (O) whether this plan involved sorting + bool _scanAndOrder; + bool _isScanAndOrderSet; + + // (O) number of entries retrieved either from an index or collection across all plans + bool _indexOnly; + bool _isIndexOnlySet; + + // (O) whether the idhack was used to answer this query + bool _idHack; + bool _isIDHackSet; + + // (O) whether index filters were used in planning this query + bool _indexFilterApplied; + bool _isIndexFilterAppliedSet; + + // (O) number times this plan released and reacquired its lock + long long _nYields; + bool _isNYieldsSet; + + // (O) number times this plan skipped over migrated data + long long _nChunkSkips; + bool _isNChunkSkipsSet; + + // (O) elapsed time this plan took running, in milliseconds + long long _millis; + bool _isMillisSet; + + // (O) keys used to seek in and out of an index + BSONObj _indexBounds; + bool _isIndexBoundsSet; + + // (O) alternative plans considered + boost::scoped_ptr > _allPlans; + + // (O) cached plan for this query + boost::scoped_ptr _oldPlan; + + // (O) server's host:port against which the query ran + std::string _server; + bool _isServerSet; + }; + +} // namespace mongo diff --git a/src/mongo/db/queryoptimizer.cpp b/src/mongo/db/queryoptimizer.cpp deleted file mode 100644 index 469d7bf1691..00000000000 --- a/src/mongo/db/queryoptimizer.cpp +++ /dev/null @@ -1,2038 +0,0 @@ -// @file queryoptimizer.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "mongo/pch.h" - -#include "mongo/db/queryoptimizer.h" - -#include "mongo/client/dbclientinterface.h" -#include "mongo/db/btreecursor.h" -#include "mongo/db/cmdline.h" -#include "mongo/db/db.h" -#include "mongo/db/intervalbtreecursor.h" -#include "mongo/db/pagefault.h" -#include "mongo/server.h" - -//#define DEBUGQO(x) cout << x << endl; -#define DEBUGQO(x) - -namespace mongo { - - QueryPlanSummary QueryPlan::summary() const { return QueryPlanSummary( *this ); } - - double elementDirection( const BSONElement &e ) { - if ( e.isNumber() ) - return e.number(); - return 1; - } - - // returns an IndexDetails * for a hint, 0 if hint is $natural. - // hint must not be eoo() - IndexDetails *parseHint( const BSONElement &hint, NamespaceDetails *d ) { - massert( 13292, "hint eoo", !hint.eoo() ); - if( hint.type() == String ) { - string hintstr = hint.valuestr(); - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - IndexDetails& ii = i.next(); - if ( ii.indexName() == hintstr ) { - return ⅈ - } - } - } - else if( hint.type() == Object ) { - BSONObj hintobj = hint.embeddedObject(); - uassert( 10112 , "bad hint", !hintobj.isEmpty() ); - if ( !strcmp( hintobj.firstElementFieldName(), "$natural" ) ) { - return 0; - } - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - IndexDetails& ii = i.next(); - if( ii.keyPattern().woCompare(hintobj) == 0 ) { - return ⅈ - } - } - } - uassert( 10113 , "bad hint", false ); - return 0; - } - - QueryPlan *QueryPlan::make( NamespaceDetails *d, - int idxNo, - const FieldRangeSetPair &frsp, - const FieldRangeSetPair *originalFrsp, - const BSONObj &originalQuery, - const BSONObj &order, - const shared_ptr &parsedQuery, - const BSONObj &startKey, - const BSONObj &endKey, - const std::string& special ) { - auto_ptr ret( new QueryPlan( d, idxNo, frsp, originalQuery, order, parsedQuery, - special ) ); - ret->init( originalFrsp, startKey, endKey ); - return ret.release(); - } - - QueryPlan::QueryPlan( NamespaceDetails *d, - int idxNo, - const FieldRangeSetPair &frsp, - const BSONObj &originalQuery, - const BSONObj &order, - const shared_ptr &parsedQuery, - const std::string& special ) : - _d(d), - _idxNo(idxNo), - _frs( frsp.frsForIndex( _d, _idxNo ) ), - _frsMulti( frsp.frsForIndex( _d, -1 ) ), - _originalQuery( originalQuery ), - _order( order ), - _parsedQuery( parsedQuery ), - _index( 0 ), - _scanAndOrderRequired( true ), - _matcherNecessary( true ), - _direction( 0 ), - _endKeyInclusive(), - _utility( Helpful ), - _special( special ), - _type(0), - _startOrEndSpec() { - } - - void QueryPlan::init( const FieldRangeSetPair *originalFrsp, - const BSONObj &startKey, - const BSONObj &endKey ) { - _endKeyInclusive = endKey.isEmpty(); - _startOrEndSpec = !startKey.isEmpty() || !endKey.isEmpty(); - - BSONObj idxKey = _idxNo < 0 ? BSONObj() : _d->idx( _idxNo ).keyPattern(); - - if ( !_frs.matchPossibleForIndex( idxKey ) ) { - _utility = Impossible; - _scanAndOrderRequired = false; - return; - } - - if ( willScanTable() ) { - if ( _order.isEmpty() || !strcmp( _order.firstElementFieldName(), "$natural" ) ) - _scanAndOrderRequired = false; - return; - } - - _index = &_d->idx(_idxNo); - - // If the parsing or index indicates this is a special query, don't continue the processing - if (!_special.empty() || - ( _index->getSpec().getType() && - _index->getSpec().getType()->suitability( _frs, _order ) != USELESS ) ) { - - _type = _index->getSpec().getType(); - if (_special.empty()) _special = _index->getSpec().getType()->getPlugin()->getName(); - - massert( 13040 , (string)"no type for special: " + _special , _type ); - // hopefully safe to use original query in these contexts; - // don't think we can mix special with $or clause separation yet - _scanAndOrderRequired = _type->scanAndOrderRequired( _originalQuery , _order ); - return; - } - - const IndexSpec &idxSpec = _index->getSpec(); - BSONObjIterator o( _order ); - BSONObjIterator k( idxKey ); - if ( !o.moreWithEOO() ) - _scanAndOrderRequired = false; - while( o.moreWithEOO() ) { - BSONElement oe = o.next(); - if ( oe.eoo() ) { - _scanAndOrderRequired = false; - break; - } - if ( !k.moreWithEOO() ) - break; - BSONElement ke; - while( 1 ) { - ke = k.next(); - if ( ke.eoo() ) - goto doneCheckOrder; - if ( strcmp( oe.fieldName(), ke.fieldName() ) == 0 ) - break; - if ( !_frs.range( ke.fieldName() ).equality() ) - goto doneCheckOrder; - } - int d = elementDirection( oe ) == elementDirection( ke ) ? 1 : -1; - if ( _direction == 0 ) - _direction = d; - else if ( _direction != d ) - break; - } -doneCheckOrder: - if ( _scanAndOrderRequired ) - _direction = 0; - BSONObjIterator i( idxKey ); - int exactIndexedQueryCount = 0; - int optimalIndexedQueryCount = 0; - bool awaitingLastOptimalField = true; - set orderFieldsUnindexed; - _order.getFieldNames( orderFieldsUnindexed ); - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - const FieldRange &fr = _frs.range( e.fieldName() ); - if ( awaitingLastOptimalField ) { - if ( !fr.universal() ) - ++optimalIndexedQueryCount; - if ( !fr.equality() ) - awaitingLastOptimalField = false; - } - else { - if ( !fr.universal() ) - optimalIndexedQueryCount = -1; - } - if ( fr.equality() ) { - BSONElement e = fr.max(); - if ( !e.isNumber() && !e.mayEncapsulate() && e.type() != RegEx ) - ++exactIndexedQueryCount; - } - orderFieldsUnindexed.erase( e.fieldName() ); - } - if ( !_scanAndOrderRequired && - ( optimalIndexedQueryCount == _frs.numNonUniversalRanges() ) ) - _utility = Optimal; - _frv.reset( new FieldRangeVector( _frs, idxSpec, _direction ) ); - - if ( // If all field range constraints are on indexed fields and ... - _utility == Optimal && - // ... the field ranges exactly represent the query and ... - _frs.mustBeExactMatchRepresentation() && - // ... all indexed ranges are represented in the field range vector ... - _frv->hasAllIndexedRanges() ) { - - // ... then the field range vector is sufficient to perform query matching against index - // keys. No matcher is required. - _matcherNecessary = false; - } - - if ( originalFrsp ) { - _originalFrv.reset( new FieldRangeVector( originalFrsp->frsForIndex( _d, _idxNo ), - idxSpec, _direction ) ); - } - else { - _originalFrv = _frv; - } - if ( _startOrEndSpec ) { - BSONObj newStart, newEnd; - if ( !startKey.isEmpty() ) - _startKey = startKey; - else - _startKey = _frv->startKey(); - if ( !endKey.isEmpty() ) - _endKey = endKey; - else - _endKey = _frv->endKey(); - } - - if ( ( _scanAndOrderRequired || _order.isEmpty() ) && - _frs.range( idxKey.firstElementFieldName() ).universal() ) { // NOTE SERVER-2140 - _utility = Unhelpful; - } - - if ( idxSpec.isSparse() && hasPossibleExistsFalsePredicate() ) { - _utility = Disallowed; - } - - if ( _parsedQuery && _parsedQuery->getFields() && !_d->isMultikey( _idxNo ) ) { // Does not check modifiedKeys() - _keyFieldsOnly.reset( _parsedQuery->getFields()->checkKey( _index->keyPattern() ) ); - } - } - - shared_ptr QueryPlan::newCursor( const DiskLoc& startLoc, - bool requestIntervalCursor ) const { - - if ( _type ) { - // hopefully safe to use original query in these contexts - don't think we can mix type with $or clause separation yet - int numWanted = 0; - if ( _parsedQuery ) { - // SERVER-5390 - numWanted = _parsedQuery->getSkip() + _parsedQuery->getNumToReturn(); - } - return _type->newCursor( _originalQuery , _order , numWanted ); - } - - if ( _utility == Impossible ) { - // Dummy table scan cursor returning no results. Allowed in --notablescan mode. - return shared_ptr( new BasicCursor( DiskLoc() ) ); - } - - if ( willScanTable() ) { - checkTableScanAllowed(); - return findTableScan( _frs.ns(), _order, startLoc ); - } - - massert( 10363 , "newCursor() with start location not implemented for indexed plans", startLoc.isNull() ); - - if ( _startOrEndSpec ) { - // we are sure to spec _endKeyInclusive - return shared_ptr( BtreeCursor::make( _d, - *_index, - _startKey, - _endKey, - _endKeyInclusive, - _direction >= 0 ? 1 : -1 ) ); - } - - if ( _index->getSpec().getType() ) { - return shared_ptr( BtreeCursor::make( _d, - *_index, - _frv->startKey(), - _frv->endKey(), - true, - _direction >= 0 ? 1 : -1 ) ); - } - - // An IntervalBtreeCursor is returned if explicitly requested AND _frv is exactly - // represented by a single interval within the btree. - if ( // If an interval cursor is requested and ... - requestIntervalCursor && - // ... equalities come before ranges (a requirement of Optimal) and ... - _utility == Optimal && - // ... the field range vector exactly represents a single interval ... - _frv->isSingleInterval() ) { - // ... and an interval cursor can be created ... - shared_ptr ret( IntervalBtreeCursor::make( _d, - *_index, - _frv->startKey(), - _frv->startKeyInclusive(), - _frv->endKey(), - _frv->endKeyInclusive() ) ); - if ( ret ) { - // ... then return the interval cursor. - return ret; - } - } - - return shared_ptr( BtreeCursor::make( _d, - *_index, - _frv, - independentRangesSingleIntervalLimit(), - _direction >= 0 ? 1 : -1 ) ); - } - - shared_ptr QueryPlan::newReverseCursor() const { - if ( willScanTable() ) { - int orderSpec = _order.getIntField( "$natural" ); - if ( orderSpec == INT_MIN ) - orderSpec = 1; - return findTableScan( _frs.ns(), BSON( "$natural" << -orderSpec ) ); - } - massert( 10364 , "newReverseCursor() not implemented for indexed plans", false ); - return shared_ptr(); - } - - BSONObj QueryPlan::indexKey() const { - if ( !_index ) - return BSON( "$natural" << 1 ); - return _index->keyPattern(); - } - - void QueryPlan::registerSelf( long long nScanned, - CandidatePlanCharacter candidatePlans ) const { - // Impossible query constraints can be detected before scanning and historically could not - // generate a QueryPattern. - if ( _utility == Impossible ) { - return; - } - - SimpleMutex::scoped_lock lk(NamespaceDetailsTransient::_qcMutex); - QueryPattern queryPattern = _frs.pattern( _order ); - CachedQueryPlan queryPlanToCache( indexKey(), nScanned, candidatePlans ); - NamespaceDetailsTransient &nsdt = NamespaceDetailsTransient::get_inlock( ns() ); - nsdt.registerCachedQueryPlanForPattern( queryPattern, queryPlanToCache ); - } - - void QueryPlan::checkTableScanAllowed() const { - if ( likely( !cmdLine.noTableScan ) ) - return; - - // TODO - is this desirable? See SERVER-2222. - if ( _frs.numNonUniversalRanges() == 0 ) - return; - - if ( strstr( ns() , ".system." ) ) - return; - - if( str::startsWith(ns(), "local.") ) - return; - - if ( ! nsdetails( ns() ) ) - return; - - uassert( 10111 , (string)"table scans not allowed:" + ns() , ! cmdLine.noTableScan ); - } - - int QueryPlan::independentRangesSingleIntervalLimit() const { - if ( _scanAndOrderRequired && - _parsedQuery && - !_parsedQuery->wantMore() && - !isMultiKey() && - queryBoundsExactOrderSuffix() ) { - verify( _direction == 0 ); - // Limit the results for each compound interval. SERVER-5063 - return _parsedQuery->getSkip() + _parsedQuery->getNumToReturn(); - } - return 0; - } - - /** - * Detects $exists:false predicates in a matcher. All $exists:false predicates will be - * detected. Some $exists:true predicates may be incorrectly reported as $exists:false due to - * the approximate nature of the implementation. - */ - class ExistsFalseDetector : public MatcherVisitor { - public: - ExistsFalseDetector( const Matcher& originalMatcher ); - bool hasFoundExistsFalse() const { return _foundExistsFalse; } - void visitMatcher( const Matcher& matcher ) { _currentMatcher = &matcher; } - void visitElementMatcher( const ElementMatcher& elementMatcher ); - private: - const Matcher* _originalMatcher; - const Matcher* _currentMatcher; - bool _foundExistsFalse; - }; - - ExistsFalseDetector::ExistsFalseDetector( const Matcher& originalMatcher ) : - _originalMatcher( &originalMatcher ), - _currentMatcher( 0 ), - _foundExistsFalse() { - } - - /** Matches $exists:false and $not:{$exists:true} exactly. */ - static bool isExistsFalsePredicate( const ElementMatcher& elementMatcher ) { - bool hasTrueValue = elementMatcher._toMatch.trueValue(); - bool hasNotModifier = elementMatcher._isNot; - return hasNotModifier ? hasTrueValue : !hasTrueValue; - } - - void ExistsFalseDetector::visitElementMatcher( const ElementMatcher& elementMatcher ) { - if ( elementMatcher._compareOp != BSONObj::opEXISTS ) { - // Only consider $exists predicates. - return; - } - if ( _currentMatcher != _originalMatcher ) { - // Treat all $exists predicates nested below the original matcher as $exists:false. - // This approximation is used because a nesting operator may change the matching - // semantics of $exists:true. - _foundExistsFalse = true; - return; - } - if ( isExistsFalsePredicate( elementMatcher ) ) { - // Top level $exists operators are matched exactly. - _foundExistsFalse = true; - } - } - - bool QueryPlan::hasPossibleExistsFalsePredicate() const { - ExistsFalseDetector detector( matcher()->docMatcher() ); - matcher()->docMatcher().visit( detector ); - return detector.hasFoundExistsFalse(); - } - - bool QueryPlan::queryBoundsExactOrderSuffix() const { - if ( !indexed() || - !_frs.matchPossible() || - !_frs.mustBeExactMatchRepresentation() ) { - return false; - } - BSONObj idxKey = indexKey(); - BSONObjIterator index( idxKey ); - BSONObjIterator order( _order ); - int coveredNonUniversalRanges = 0; - while( index.more() ) { - const FieldRange& indexFieldRange = _frs.range( (*index).fieldName() ); - if ( !indexFieldRange.isPointIntervalSet() ) { - if ( !indexFieldRange.universal() ) { - // The last indexed range may be a non point set containing a single interval. - // SERVER-5777 - if ( indexFieldRange.intervals().size() > 1 ) { - return false; - } - ++coveredNonUniversalRanges; - } - break; - } - ++coveredNonUniversalRanges; - if ( order.more() && str::equals( (*index).fieldName(), (*order).fieldName() ) ) { - ++order; - } - ++index; - } - if ( coveredNonUniversalRanges != _frs.numNonUniversalRanges() ) { - return false; - } - while( index.more() && order.more() ) { - if ( !str::equals( (*index).fieldName(), (*order).fieldName() ) ) { - return false; - } - if ( ( elementDirection( *index ) < 0 ) != ( elementDirection( *order ) < 0 ) ) { - return false; - } - ++order; - ++index; - } - return !order.more(); - } - - string QueryPlan::toString() const { - return BSON( - "index" << indexKey() << - "frv" << ( _frv ? _frv->toString() : "" ) << - "order" << _order - ).jsonString(); - } - - shared_ptr QueryPlan::matcher() const { - if ( !_matcher ) { - _matcher.reset( new CoveredIndexMatcher( originalQuery(), indexKey() ) ); - } - return _matcher; - } - - bool QueryPlan::isMultiKey() const { - if ( _idxNo < 0 ) - return false; - return _d->isMultikey( _idxNo ); - } - - std::ostream &operator<< ( std::ostream &out, const QueryPlan::Utility &utility ) { - out << "QueryPlan::"; - switch( utility ) { - case QueryPlan::Impossible: return out << "Impossible"; - case QueryPlan::Optimal: return out << "Optimal"; - case QueryPlan::Helpful: return out << "Helpful"; - case QueryPlan::Unhelpful: return out << "Unhelpful"; - case QueryPlan::Disallowed: return out << "Disallowed"; - default: - return out << "UNKNOWN(" << utility << ")"; - } - } - - CachedMatchCounter::CachedMatchCounter( long long& aggregateNscanned, - int cumulativeCount ) : - _aggregateNscanned( aggregateNscanned ), - _nscanned(), - _cumulativeCount( cumulativeCount ), - _count(), - _checkDups(), - _match( Unknown ), - _counted() { - } - - void CachedMatchCounter::resetMatch() { - _match = Unknown; - _counted = false; - } - - bool CachedMatchCounter::setMatch( bool match ) { - MatchState oldMatch = _match; - _match = match ? True : False; - return _match == True && oldMatch != True; - } - - void CachedMatchCounter::incMatch( const DiskLoc& loc ) { - if ( !_counted && _match == True && !getsetdup( loc ) ) { - ++_cumulativeCount; - ++_count; - _counted = true; - } - } - - bool CachedMatchCounter::wouldIncMatch( const DiskLoc& loc ) const { - return !_counted && _match == True && !getdup( loc ); - } - - bool CachedMatchCounter::enoughCumulativeMatchesToChooseAPlan() const { - // This is equivalent to the default condition for switching from - // a query to a getMore, which was the historical default match count for - // choosing a plan. - return _cumulativeCount >= 101; - } - - bool CachedMatchCounter::enoughMatchesToRecordPlan() const { - // Recording after 50 matches is a historical default (101 default limit / 2). - return _count > 50; - } - - void CachedMatchCounter::updateNscanned( long long nscanned ) { - _aggregateNscanned += ( nscanned - _nscanned ); - _nscanned = nscanned; - } - - bool CachedMatchCounter::getsetdup( const DiskLoc& loc ) { - if ( !_checkDups ) { - return false; - } - pair::iterator, bool> p = _dups.insert( loc ); - return !p.second; - } - - bool CachedMatchCounter::getdup( const DiskLoc& loc ) const { - if ( !_checkDups ) { - return false; - } - return _dups.find( loc ) != _dups.end(); - } - - QueryPlanRunner::QueryPlanRunner( long long& aggregateNscanned, - const QueryPlanSelectionPolicy& selectionPolicy, - const bool& requireOrder, - bool alwaysCountMatches, - int cumulativeCount ) : - _complete(), - _stopRequested(), - _queryPlan(), - _error(), - _matchCounter( aggregateNscanned, cumulativeCount ), - _countingMatches(), - _mustAdvance(), - _capped(), - _selectionPolicy( selectionPolicy ), - _requireOrder( requireOrder ), - _alwaysCountMatches( alwaysCountMatches ) { - } - - void QueryPlanRunner::next() { - checkCursorOrdering(); - - mayAdvance(); - - if ( countMatches() && _matchCounter.enoughCumulativeMatchesToChooseAPlan() ) { - setStop(); - if ( _explainPlanInfo ) _explainPlanInfo->notePicked(); - return; - } - if ( !_c || !_c->ok() ) { - if ( _explainPlanInfo && _c ) _explainPlanInfo->noteDone( *_c ); - setComplete(); - return; - } - - _mustAdvance = true; - } - - long long QueryPlanRunner::nscanned() const { - return _c ? _c->nscanned() : _matchCounter.nscanned(); - } - - void QueryPlanRunner::prepareToYield() { - if ( _c && !_cc ) { - _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout, _c, queryPlan().ns() ) ); - // Set 'doing deletes' as deletes may occur; if there are no deletes this has no - // effect. - _cc->setDoingDeletes( true ); - } - if ( _cc ) { - recordCursorLocation(); - _cc->prepareToYield( _yieldData ); - } - } - - void QueryPlanRunner::recoverFromYield() { - if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) { - // !!! The collection may be gone, and any namespace or index specific memory may - // have become invalid. - _c.reset(); - _cc.reset(); - - if ( _capped ) { - msgassertedNoTrace( 13338, - str::stream() << "capped cursor overrun: " - << queryPlan().ns() ); - } - msgassertedNoTrace( 15892, - str::stream() << - "QueryPlanRunner::recoverFromYield() failed to recover" ); - } - else { - checkCursorAdvanced(); - } - } - - void QueryPlanRunner::prepareToTouchEarlierIterate() { - recordCursorLocation(); - if ( _c ) { - _c->prepareToTouchEarlierIterate(); - } - } - - void QueryPlanRunner::recoverFromTouchingEarlierIterate() { - if ( _c ) { - _c->recoverFromTouchingEarlierIterate(); - } - checkCursorAdvanced(); - } - - bool QueryPlanRunner::currentMatches( MatchDetails* details ) { - if ( !_c || !_c->ok() ) { - _matchCounter.setMatch( false ); - return false; - } - - MatchDetails myDetails; - if ( !details && _explainPlanInfo ) { - details = &myDetails; - } - - bool match = queryPlan().matcher()->matchesCurrent( _c.get(), details ); - // Cache the match, so we can count it in mayAdvance(). - bool newMatch = _matchCounter.setMatch( match ); - - if ( _explainPlanInfo ) { - // Note iterate results as if this is the only query plan running. But do not account - // for query parameters that may be appled to the whole result set (results from - // interleaved plans), for example the 'skip' parameter. - bool countableMatch = newMatch && _matchCounter.wouldIncMatch( _c->currLoc() ); - bool matchWouldBeLoadedForReturn = countableMatch && hasDocumentLoadingQueryPlan(); - _explainPlanInfo->noteIterate( countableMatch, - details->hasLoadedRecord() || - matchWouldBeLoadedForReturn, - *_c ); - } - - return match; - } - - bool QueryPlanRunner::mayRecordPlan() const { - return complete() && ( !stopRequested() || _matchCounter.enoughMatchesToRecordPlan() ); - } - - QueryPlanRunner* QueryPlanRunner::createChild() const { - return new QueryPlanRunner( _matchCounter.aggregateNscanned(), - _selectionPolicy, - _requireOrder, - _alwaysCountMatches, - _matchCounter.cumulativeCount() ); - } - - void QueryPlanRunner::setQueryPlan( const QueryPlan* queryPlan ) { - _queryPlan = queryPlan; - verify( _queryPlan != NULL ); - } - - void QueryPlanRunner::init() { - checkCursorOrdering(); - if ( !_selectionPolicy.permitPlan( queryPlan() ) ) { - throw MsgAssertionException( 9011, - str::stream() - << "Plan not permitted by query plan selection policy '" - << _selectionPolicy.name() - << "'" ); - } - - _c = queryPlan().newCursor(); - // The basic and btree cursors used by this implementation do not supply their own - // matchers, and a matcher from a query plan will be used instead. - verify( !_c->matcher() ); - // Such cursors all support deduplication. - verify( _c->autoDedup() ); - - // The query plan must have a matcher. The matcher's constructor performs some aspects - // of query validation that should occur as part of this class's init() if not handled - // already. - fassert( 16249, queryPlan().matcher() ); - - // All candidate cursors must support yields for QueryOptimizerCursorImpl's - // prepareToYield() and prepareToTouchEarlierIterate() to work. - verify( _c->supportYields() ); - _capped = _c->capped(); - - // TODO This violates the current Cursor interface abstraction, but for now it's simpler to keep our own set of - // dups rather than avoid poisoning the cursor's dup set with unreturned documents. Deduping documents - // matched in this QueryOptimizerCursorOp will run against the takeover cursor. - _matchCounter.setCheckDups( countMatches() && _c->isMultiKey() ); - // TODO ok if cursor becomes multikey later? - - _matchCounter.updateNscanned( _c->nscanned() ); - } - - void QueryPlanRunner::setException( const DBException &e ) { - _error = true; - _exception = e.getInfo(); - } - - shared_ptr QueryPlanRunner::generateExplainInfo() { - if ( !_c ) { - return shared_ptr( new ExplainPlanInfo() ); - } - _explainPlanInfo.reset( new ExplainPlanInfo() ); - _explainPlanInfo->notePlan( *_c, queryPlan().scanAndOrderRequired(), - queryPlan().keyFieldsOnly() ); - return _explainPlanInfo; - } - - void QueryPlanRunner::mayAdvance() { - if ( !_c ) { - return; - } - if ( countingMatches() ) { - // Check match if not yet known. - if ( !_matchCounter.knowMatch() ) { - currentMatches( 0 ); - } - _matchCounter.incMatch( currLoc() ); - } - if ( _mustAdvance ) { - _c->advance(); - handleCursorAdvanced(); - } - _matchCounter.updateNscanned( _c->nscanned() ); - } - - bool QueryPlanRunner::countingMatches() { - if ( _countingMatches ) { - return true; - } - if ( countMatches() ) { - // Only count matches after the first call to next(), which occurs before the first - // result is returned. - _countingMatches = true; - } - return false; - } - - bool QueryPlanRunner::countMatches() const { - return _alwaysCountMatches || !queryPlan().scanAndOrderRequired(); - } - - bool QueryPlanRunner::hasDocumentLoadingQueryPlan() const { - if ( queryPlan().parsedQuery() && queryPlan().parsedQuery()->returnKey() ) { - // Index keys will be returned using $returnKey. - return false; - } - if ( queryPlan().scanAndOrderRequired() ) { - // The in memory sort implementation operates on full documents. - return true; - } - if ( keyFieldsOnly() ) { - // A covered index projection will be used. - return false; - } - // Documents will be loaded for a standard query. - return true; - } - - void QueryPlanRunner::recordCursorLocation() { - _posBeforeYield = currLoc(); - } - - void QueryPlanRunner::checkCursorAdvanced() { - // This check will not correctly determine if we are looking at a different document in - // all cases, but it is adequate for updating the query plan's match count (just used to pick - // plans, not returned to the client) and adjust iteration via _mustAdvance. - if ( _posBeforeYield != currLoc() ) { - // If the yield advanced our position, the next next() will be a no op. - handleCursorAdvanced(); - } - } - - void QueryPlanRunner::handleCursorAdvanced() { - _mustAdvance = false; - _matchCounter.resetMatch(); - } - - void QueryPlanRunner::checkCursorOrdering() { - if ( _requireOrder && queryPlan().scanAndOrderRequired() ) { - throw MsgAssertionException( OutOfOrderDocumentsAssertionCode, "order spec cannot be satisfied with index" ); - } - } - - QueryPlanGenerator::QueryPlanGenerator( QueryPlanSet &qps, - auto_ptr originalFrsp, - const shared_ptr &parsedQuery, - const BSONObj &hint, - RecordedPlanPolicy recordedPlanPolicy, - const BSONObj &min, - const BSONObj &max, - bool allowSpecial ) : - _qps( qps ), - _originalFrsp( originalFrsp ), - _parsedQuery( parsedQuery ), - _hint( hint.getOwned() ), - _recordedPlanPolicy( recordedPlanPolicy ), - _min( min.getOwned() ), - _max( max.getOwned() ), - _allowSpecial( allowSpecial ) { - } - - void QueryPlanGenerator::addInitialPlans() { - const char *ns = _qps.frsp().ns(); - NamespaceDetails *d = nsdetails( ns ); - - if ( addShortCircuitPlan( d ) ) { - return; - } - - addStandardPlans( d ); - warnOnCappedIdTableScan(); - } - - void QueryPlanGenerator::addFallbackPlans() { - const char *ns = _qps.frsp().ns(); - NamespaceDetails *d = nsdetails( ns ); - verify( d ); - - vector > plans; - shared_ptr optimalPlan; - shared_ptr specialPlan; - for( int i = 0; i < d->nIndexes; ++i ) { - - if ( !QueryUtilIndexed::indexUseful( _qps.frsp(), d, i, _qps.order() ) ) { - continue; - } - - shared_ptr p = newPlan( d, i ); - switch( p->utility() ) { - case QueryPlan::Impossible: - _qps.setSinglePlan( p ); - return; - case QueryPlan::Optimal: - if ( !optimalPlan ) { - optimalPlan = p; - } - break; - case QueryPlan::Helpful: - if ( p->special().empty() ) { - // Not a 'special' plan. - plans.push_back( p ); - } - else if ( _allowSpecial ) { - specialPlan = p; - } - break; - default: - break; - } - } - - if ( optimalPlan ) { - _qps.setSinglePlan( optimalPlan ); - // Record an optimal plan in the query cache immediately, with a small nscanned value - // that will be ignored. - optimalPlan->registerSelf - ( 0, CandidatePlanCharacter( !optimalPlan->scanAndOrderRequired(), - optimalPlan->scanAndOrderRequired() ) ); - return; - } - - // Only add a special plan if no standard btree plans have been added. SERVER-4531 - if ( plans.empty() && specialPlan ) { - _qps.setSinglePlan( specialPlan ); - return; - } - - for( vector >::const_iterator i = plans.begin(); i != plans.end(); - ++i ) { - _qps.addCandidatePlan( *i ); - } - - _qps.addCandidatePlan( newPlan( d, -1 ) ); - } - - bool QueryPlanGenerator::addShortCircuitPlan( NamespaceDetails *d ) { - return - // The collection is missing. - setUnindexedPlanIf( !d, d ) || - // No match is possible. - setUnindexedPlanIf( !_qps.frsp().matchPossible(), d ) || - // The hint, min, or max parameters are specified. - addHintPlan( d ) || - // A special index operation is requested. - addSpecialPlan( d ) || - // No indexable ranges or ordering are specified. - setUnindexedPlanIf( _qps.frsp().noNonUniversalRanges() && _qps.order().isEmpty(), d ) || - // $natural sort is requested. - setUnindexedPlanIf( !_qps.order().isEmpty() && - str::equals( _qps.order().firstElementFieldName(), "$natural" ), d ); - } - - bool QueryPlanGenerator::addHintPlan( NamespaceDetails *d ) { - BSONElement hint = _hint.firstElement(); - if ( !hint.eoo() ) { - IndexDetails *id = parseHint( hint, d ); - if ( id ) { - setHintedPlanForIndex( *id ); - } - else { - uassert( 10366, "natural order cannot be specified with $min/$max", - _min.isEmpty() && _max.isEmpty() ); - setSingleUnindexedPlan( d ); - } - return true; - } - - if ( !_min.isEmpty() || !_max.isEmpty() ) { - string errmsg; - BSONObj keyPattern; - IndexDetails *idx = indexDetailsForRange( _qps.frsp().ns(), errmsg, _min, _max, - keyPattern ); - uassert( 10367 , errmsg, idx ); - validateAndSetHintedPlan( newPlan( d, d->idxNo( *idx ), _min, _max ) ); - return true; - } - - return false; - } - - bool QueryPlanGenerator::addSpecialPlan( NamespaceDetails *d ) { - DEBUGQO( "\t special : " << _qps.frsp().getSpecial().toString() ); - SpecialIndices special = _qps.frsp().getSpecial(); - if (!special.empty()) { - // Try to handle the special part of the query with an index - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - int j = i.pos(); - IndexDetails& ii = i.next(); - const IndexSpec& spec = ii.getSpec(); - if (special.has(spec.getTypeName()) && - spec.suitability( _qps.frsp().frsForIndex(d, j), _qps.order() ) != USELESS ) { - uassert( 16330, "'special' query operator not allowed", _allowSpecial ); - _qps.setSinglePlan( newPlan( d, j, BSONObj(), BSONObj(), spec.getTypeName())); - return true; - } - } - // If all possible special indices require an index and we don't have one, - // error. - if (special.allRequireIndex()) { - uassert(13038, "can't find any special indices: " + special.toString() - + " for: " + _qps.originalQuery().toString(), false ); - } - // Otherwise, we can get the same functionality from the matcher. - } - return false; - } - - void QueryPlanGenerator::addStandardPlans( NamespaceDetails *d ) { - if ( !addCachedPlan( d ) ) { - addFallbackPlans(); - } - } - - bool QueryPlanGenerator::addCachedPlan( NamespaceDetails *d ) { - if ( _recordedPlanPolicy == Ignore ) { - return false; - } - - CachedQueryPlan best = QueryUtilIndexed::bestIndexForPatterns( _qps.frsp(), _qps.order() ); - BSONObj bestIndex = best.indexKey(); - if ( bestIndex.isEmpty() ) { - return false; - } - - shared_ptr p; - if ( str::equals( bestIndex.firstElementFieldName(), "$natural" ) ) { - p = newPlan( d, -1 ); - } - - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - int j = i.pos(); - IndexDetails& ii = i.next(); - if( ii.keyPattern().woCompare(bestIndex) == 0 ) { - p = newPlan( d, j ); - } - } - - massert( 10368 , "Unable to locate previously recorded index", p ); - - if ( p->utility() == QueryPlan::Unhelpful || - p->utility() == QueryPlan::Disallowed ) { - return false; - } - - if ( _recordedPlanPolicy == UseIfInOrder && p->scanAndOrderRequired() ) { - return false; - } - - if ( !_allowSpecial && !p->special().empty() ) { - return false; - } - - _qps.setCachedPlan( p, best ); - return true; - } - - shared_ptr QueryPlanGenerator::newPlan( NamespaceDetails *d, - int idxNo, - const BSONObj &min, - const BSONObj &max, - const string &special ) const { - shared_ptr ret( QueryPlan::make( d, idxNo, _qps.frsp(), _originalFrsp.get(), - _qps.originalQuery(), _qps.order(), _parsedQuery, - min, max, special ) ); - return ret; - } - - bool QueryPlanGenerator::setUnindexedPlanIf( bool set, NamespaceDetails *d ) { - if ( set ) { - setSingleUnindexedPlan( d ); - } - return set; - } - - void QueryPlanGenerator::setSingleUnindexedPlan( NamespaceDetails *d ) { - _qps.setSinglePlan( newPlan( d, -1 ) ); - } - - void QueryPlanGenerator::setHintedPlanForIndex( IndexDetails& id ) { - if ( !_min.isEmpty() || !_max.isEmpty() ) { - string errmsg; - BSONObj keyPattern = id.keyPattern(); - // This reformats _min and _max to be used for index lookup. - massert( 10365 , errmsg, indexDetailsForRange( _qps.frsp().ns(), errmsg, _min, _max, - keyPattern ) ); - } - NamespaceDetails *d = nsdetails( _qps.frsp().ns() ); - validateAndSetHintedPlan( newPlan( d, d->idxNo( id ), _min, _max ) ); - } - - void QueryPlanGenerator::validateAndSetHintedPlan( const shared_ptr& plan ) { - uassert( 16331, "'special' plan hint not allowed", - _allowSpecial || plan->special().empty() ); - _qps.setSinglePlan( plan ); - } - - void QueryPlanGenerator::warnOnCappedIdTableScan() const { - // if we are doing a table scan on _id - // and it's a capped collection - // we warn as it's a common user error - // .system. and local collections are exempt - const char *ns = _qps.frsp().ns(); - NamespaceDetails *d = nsdetails( ns ); - if ( d && - d->isCapped() && - _qps.nPlans() == 1 && - ( _qps.firstPlan()->utility() != QueryPlan::Impossible ) && - !_qps.firstPlan()->indexed() && - !_qps.firstPlan()->multikeyFrs().range( "_id" ).universal() ) { - if (!str::contains( ns , ".system." ) && !str::startsWith( ns , "local." )) { - warning() << "unindexed _id query on capped collection, " - << "performance will be poor collection: " << ns << endl; - } - } - } - - QueryPlanSet* QueryPlanSet::make( const char* ns, - auto_ptr frsp, - auto_ptr originalFrsp, - const BSONObj& originalQuery, - const BSONObj& order, - const shared_ptr& parsedQuery, - const BSONObj& hint, - QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy, - const BSONObj& min, - const BSONObj& max, - bool allowSpecial ) { - auto_ptr ret( new QueryPlanSet( ns, frsp, originalFrsp, originalQuery, order, - parsedQuery, hint, recordedPlanPolicy, min, - max, allowSpecial ) ); - ret->init(); - return ret.release(); - } - - - QueryPlanSet::QueryPlanSet( const char *ns, - auto_ptr frsp, - auto_ptr originalFrsp, - const BSONObj &originalQuery, - const BSONObj &order, - const shared_ptr &parsedQuery, - const BSONObj &hint, - QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy, - const BSONObj &min, - const BSONObj &max, - bool allowSpecial ) : - _generator( *this, originalFrsp, parsedQuery, hint, recordedPlanPolicy, min, max, - allowSpecial ), - _originalQuery( originalQuery ), - _frsp( frsp ), - _mayRecordPlan(), - _usingCachedPlan(), - _order( order.getOwned() ), - _oldNScanned( 0 ), - _yieldSometimesTracker( 256, 20 ), - _allowSpecial( allowSpecial ) { - } - - bool QueryPlanSet::hasMultiKey() const { - for( PlanVector::const_iterator i = _plans.begin(); i != _plans.end(); ++i ) - if ( (*i)->isMultiKey() ) - return true; - return false; - } - - void QueryPlanSet::init() { - DEBUGQO( "QueryPlanSet::init " << ns << "\t" << _originalQuery ); - _plans.clear(); - _usingCachedPlan = false; - - _generator.addInitialPlans(); - } - - void QueryPlanSet::setSinglePlan( const QueryPlanPtr &plan ) { - if ( nPlans() == 0 ) { - pushPlan( plan ); - } - } - - void QueryPlanSet::setCachedPlan( const QueryPlanPtr &plan, - const CachedQueryPlan &cachedPlan ) { - verify( nPlans() == 0 ); - _usingCachedPlan = true; - _oldNScanned = cachedPlan.nScanned(); - _cachedPlanCharacter = cachedPlan.planCharacter(); - pushPlan( plan ); - } - - void QueryPlanSet::addCandidatePlan( const QueryPlanPtr &plan ) { - // If _plans is nonempty, the new plan may be supplementing a recorded plan at the first - // position of _plans. It must not duplicate the first plan. - if ( nPlans() > 0 && plan->indexKey() == firstPlan()->indexKey() ) { - return; - } - pushPlan( plan ); - _mayRecordPlan = true; - } - - void QueryPlanSet::addFallbackPlans() { - _generator.addFallbackPlans(); - _mayRecordPlan = true; - } - - void QueryPlanSet::pushPlan( const QueryPlanSet::QueryPlanPtr& plan ) { - verify( _allowSpecial || plan->special().empty() ); - _plans.push_back( plan ); - } - - bool QueryPlanSet::hasPossiblyExcludedPlans() const { - return - _usingCachedPlan && - ( nPlans() == 1 ) && - ( firstPlan()->utility() != QueryPlan::Optimal ); - } - - QueryPlanSet::QueryPlanPtr QueryPlanSet::getBestGuess() const { - verify( _plans.size() ); - if ( _plans[ 0 ]->scanAndOrderRequired() ) { - for ( unsigned i=1; i<_plans.size(); i++ ) { - if ( ! _plans[i]->scanAndOrderRequired() ) - return _plans[i]; - } - - warning() << "best guess query plan requested, but scan and order are required for all plans " - << " query: " << _originalQuery - << " order: " << _order - << " choices: "; - - for ( unsigned i=0; i<_plans.size(); i++ ) - warning() << _plans[i]->indexKey() << " "; - warning() << endl; - - return QueryPlanPtr(); - } - return _plans[0]; - } - - bool QueryPlanSet::haveInOrderPlan() const { - for( PlanVector::const_iterator i = _plans.begin(); i != _plans.end(); ++i ) { - if ( !(*i)->scanAndOrderRequired() ) { - return true; - } - } - return false; - } - - bool QueryPlanSet::possibleInOrderPlan() const { - if ( haveInOrderPlan() ) { - return true; - } - return _cachedPlanCharacter.mayRunInOrderPlan(); - } - - bool QueryPlanSet::possibleOutOfOrderPlan() const { - for( PlanVector::const_iterator i = _plans.begin(); i != _plans.end(); ++i ) { - if ( (*i)->scanAndOrderRequired() ) { - return true; - } - } - return _cachedPlanCharacter.mayRunOutOfOrderPlan(); - } - - CandidatePlanCharacter QueryPlanSet::characterizeCandidatePlans() const { - return CandidatePlanCharacter( possibleInOrderPlan(), possibleOutOfOrderPlan() ); - } - - bool QueryPlanSet::prepareToRetryQuery() { - if ( !hasPossiblyExcludedPlans() || _plans.size() > 1 ) { - return false; - } - - // A cached plan was used, so clear the plan for this query pattern so the query may be - // retried without a cached plan. - QueryUtilIndexed::clearIndexesForPatterns( *_frsp, _order ); - init(); - return true; - } - - string QueryPlanSet::toString() const { - BSONArrayBuilder bab; - for( PlanVector::const_iterator i = _plans.begin(); i != _plans.end(); ++i ) { - bab << (*i)->toString(); - } - return bab.arr().jsonString(); - } - - MultiPlanScanner *MultiPlanScanner::make( const StringData& ns, - const BSONObj &query, - const BSONObj &order, - const shared_ptr &parsedQuery, - const BSONObj &hint, - QueryPlanGenerator::RecordedPlanPolicy - recordedPlanPolicy, - const BSONObj &min, - const BSONObj &max ) { - auto_ptr ret( new MultiPlanScanner( ns, query, parsedQuery, hint, - recordedPlanPolicy ) ); - ret->init( order, min, max ); - return ret.release(); - } - - shared_ptr MultiPlanScanner::iterateRunnerQueue - ( QueryPlanRunner& originalRunner, bool retried ) { - - if ( _runnerQueue ) { - return _runnerQueue->next(); - } - - _runnerQueue.reset( new QueryPlanRunnerQueue( *_currentQps, originalRunner ) ); - shared_ptr explainClause; - if ( _explainQueryInfo ) { - explainClause = _runnerQueue->generateExplainInfo(); - } - - shared_ptr runner = _runnerQueue->next(); - if ( runner->error() && - _currentQps->prepareToRetryQuery() ) { - - // Avoid an infinite loop here - this should never occur. - verify( !retried ); - _runnerQueue.reset(); - return iterateRunnerQueue( originalRunner, true ); - } - - if ( _explainQueryInfo ) { - _explainQueryInfo->addClauseInfo( explainClause ); - } - return runner; - } - - void MultiPlanScanner::updateCurrentQps( QueryPlanSet *qps ) { - _currentQps.reset( qps ); - _runnerQueue.reset(); - } - - QueryPlanRunnerQueue::QueryPlanRunnerQueue( QueryPlanSet& plans, - const QueryPlanRunner& prototypeRunner ) : - _prototypeRunner( prototypeRunner ), - _plans( plans ), - _done() { - } - - void QueryPlanRunnerQueue::prepareToYield() { - for( vector >::const_iterator i = _runners.begin(); - i != _runners.end(); ++i ) { - prepareToYieldRunner( **i ); - } - } - - void QueryPlanRunnerQueue::recoverFromYield() { - for( vector >::const_iterator i = _runners.begin(); - i != _runners.end(); ++i ) { - recoverFromYieldRunner( **i ); - } - } - - shared_ptr QueryPlanRunnerQueue::init() { - massert( 10369 , "no plans", _plans.plans().size() > 0 ); - - if ( _plans.plans().size() > 1 ) - LOG(1) << " running multiple plans" << endl; - for( QueryPlanSet::PlanVector::const_iterator i = _plans.plans().begin(); - i != _plans.plans().end(); ++i ) { - shared_ptr runner( _prototypeRunner.createChild() ); - runner->setQueryPlan( i->get() ); - _runners.push_back( runner ); - } - - // Initialize runners. - for( vector >::iterator i = _runners.begin(); - i != _runners.end(); ++i ) { - initRunner( **i ); - if ( _explainClauseInfo ) { - _explainClauseInfo->addPlanInfo( (*i)->generateExplainInfo() ); - } - } - - // See if an op has completed. - for( vector >::iterator i = _runners.begin(); - i != _runners.end(); ++i ) { - if ( (*i)->complete() ) { - return *i; - } - } - - // Put runnable ops in the priority queue. - for( vector >::iterator i = _runners.begin(); - i != _runners.end(); ++i ) { - if ( !(*i)->error() ) { - _queue.push( *i ); - } - } - - if ( _queue.empty() ) { - return _runners.front(); - } - - return shared_ptr(); - } - - shared_ptr QueryPlanRunnerQueue::next() { - verify( !done() ); - - if ( _runners.empty() ) { - shared_ptr initialRet = init(); - if ( initialRet ) { - _done = true; - return initialRet; - } - } - - shared_ptr ret; - do { - ret = _next(); - } while( ret->error() && !_queue.empty() ); - - if ( _queue.empty() ) { - _done = true; - } - - return ret; - } - - shared_ptr QueryPlanRunnerQueue::_next() { - verify( !_queue.empty() ); - RunnerHolder holder = _queue.pop(); - QueryPlanRunner& runner = *holder._runner; - nextRunner( runner ); - if ( runner.complete() ) { - if ( _plans.mayRecordPlan() && runner.mayRecordPlan() ) { - runner.queryPlan().registerSelf( runner.nscanned(), - _plans.characterizeCandidatePlans() ); - } - _done = true; - return holder._runner; - } - if ( runner.error() ) { - return holder._runner; - } - if ( _plans.hasPossiblyExcludedPlans() && - runner.nscanned() > _plans.oldNScanned() * 10 ) { - verify( _plans.nPlans() == 1 && _plans.firstPlan()->special().empty() ); - holder._offset = -runner.nscanned(); - _plans.addFallbackPlans(); - QueryPlanSet::PlanVector::const_iterator i = _plans.plans().begin(); - ++i; - for( ; i != _plans.plans().end(); ++i ) { - shared_ptr runner( _prototypeRunner.createChild() ); - runner->setQueryPlan( i->get() ); - _runners.push_back( runner ); - initRunner( *runner ); - if ( runner->complete() ) - return runner; - _queue.push( runner ); - } - _plans.setUsingCachedPlan( false ); - } - _queue.push( holder ); - return holder._runner; - } - -#define GUARD_RUNNER_EXCEPTION( runner, expression ) \ - try { \ - expression; \ - } \ - catch ( DBException& e ) { \ - runner.setException( e.getInfo() ); \ - } \ - catch ( const std::exception &e ) { \ - runner.setException( ExceptionInfo( e.what(), 0 ) ); \ - } \ - catch ( PageFaultException& pfe ) { \ - throw pfe; \ - } \ - catch ( ... ) { \ - runner.setException( ExceptionInfo( "Caught unknown exception", 0 ) ); \ - } - - - void QueryPlanRunnerQueue::initRunner( QueryPlanRunner &runner ) { - GUARD_RUNNER_EXCEPTION( runner, runner.init() ); - } - - void QueryPlanRunnerQueue::nextRunner( QueryPlanRunner& runner ) { - GUARD_RUNNER_EXCEPTION( runner, if ( !runner.error() ) { runner.next(); } ); - } - - void QueryPlanRunnerQueue::prepareToYieldRunner( QueryPlanRunner& runner ) { - GUARD_RUNNER_EXCEPTION( runner, if ( !runner.error() ) { runner.prepareToYield(); } ); - } - - void QueryPlanRunnerQueue::recoverFromYieldRunner( QueryPlanRunner& runner ) { - GUARD_RUNNER_EXCEPTION( runner, if ( !runner.error() ) { runner.recoverFromYield(); } ); - } - - /** - * NOTE on our $or implementation: In our current qo implementation we don't - * keep statistics on our data, but we can conceptualize the problem of - * selecting an index when statistics exist for all index ranges. The - * d-hitting set problem on k sets and n elements can be reduced to the - * problem of index selection on k $or clauses and n index ranges (where - * d is the max number of indexes, and the number of ranges n is unbounded). - * In light of the fact that d-hitting set is np complete, and we don't even - * track statistics (so cost calculations are expensive) our first - * implementation uses the following greedy approach: We take one $or clause - * at a time and treat each as a separate query for index selection purposes. - * But if an index range is scanned for a particular $or clause, we eliminate - * that range from all subsequent clauses. One could imagine an opposite - * implementation where we select indexes based on the union of index ranges - * for all $or clauses, but this can have much poorer worst case behavior. - * (An index range that suits one $or clause may not suit another, and this - * is worse than the typical case of index range choice staleness because - * with $or the clauses may likely be logically distinct.) The greedy - * implementation won't do any worse than all the $or clauses individually, - * and it can often do better. In the first cut we are intentionally using - * QueryPattern tracking to record successful plans on $or clauses for use by - * subsequent $or clauses, even though there may be a significant aggregate - * $nor component that would not be represented in QueryPattern. - */ - - MultiPlanScanner::MultiPlanScanner( const StringData& ns, - const BSONObj &query, - const shared_ptr &parsedQuery, - const BSONObj &hint, - QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy ) : - _ns( ns.toString() ), - _or( !query.getField( "$or" ).eoo() ), - _query( query.getOwned() ), - _parsedQuery( parsedQuery ), - _i(), - _recordedPlanPolicy( recordedPlanPolicy ), - _hint( hint.getOwned() ), - _tableScanned(), - _doneRunners() { - } - - void MultiPlanScanner::init( const BSONObj &order, const BSONObj &min, const BSONObj &max ) { - if ( !order.isEmpty() || !min.isEmpty() || !max.isEmpty() ) { - _or = false; - } - if ( _or ) { - // Only construct an OrRangeGenerator if we may handle $or clauses. - _org.reset( new OrRangeGenerator( _ns.c_str(), _query ) ); - if ( !_org->getSpecial().empty() ) { - _or = false; - } - else if ( haveUselessOr() ) { - _or = false; - } - } - - // if _or == false, don't use or clauses for index selection - if ( !_or ) { - ++_i; - auto_ptr frsp( new FieldRangeSetPair( _ns.c_str(), _query, true ) ); - updateCurrentQps( QueryPlanSet::make( _ns.c_str(), frsp, auto_ptr(), - _query, order, _parsedQuery, _hint, - _recordedPlanPolicy, - min, max, true ) ); - } - else { - BSONElement e = _query.getField( "$or" ); - massert( 13268, "invalid $or spec", - e.type() == Array && e.embeddedObject().nFields() > 0 ); - handleBeginningOfClause(); - } - } - - void MultiPlanScanner::handleEndOfClause( const QueryPlan &clausePlan ) { - if ( clausePlan.willScanTable() ) { - _tableScanned = true; - } else { - _org->popOrClause( clausePlan.nsd(), clausePlan.idxNo(), - clausePlan.indexed() ? clausePlan.indexKey() : BSONObj() ); - } - } - - void MultiPlanScanner::handleBeginningOfClause() { - assertHasMoreClauses(); - ++_i; - auto_ptr frsp( _org->topFrsp() ); - auto_ptr originalFrsp( _org->topFrspOriginal() ); - updateCurrentQps( QueryPlanSet::make( _ns.c_str(), frsp, originalFrsp, _query, - BSONObj(), _parsedQuery, _hint, _recordedPlanPolicy, - BSONObj(), BSONObj(), - // 'Special' plans are not supported within $or. - false ) ); - } - - bool MultiPlanScanner::mayHandleBeginningOfClause() { - if ( hasMoreClauses() ) { - handleBeginningOfClause(); - return true; - } - return false; - } - - shared_ptr MultiPlanScanner::nextRunner() { - verify( !doneRunners() ); - shared_ptr ret = _or ? nextRunnerOr() : nextRunnerSimple(); - if ( ret->error() || ret->complete() ) { - _doneRunners = true; - } - return ret; - } - - shared_ptr MultiPlanScanner::nextRunnerSimple() { - return iterateRunnerQueue( *_baseRunner ); - } - - shared_ptr MultiPlanScanner::nextRunnerOr() { - shared_ptr runner; - do { - runner = nextRunnerSimple(); - if ( !runner->completeWithoutStop() ) { - return runner; - } - handleEndOfClause( runner->queryPlan() ); - _baseRunner = runner; - } while( mayHandleBeginningOfClause() ); - return runner; - } - - const QueryPlan *MultiPlanScanner::nextClauseBestGuessPlan( const QueryPlan ¤tPlan ) { - assertHasMoreClauses(); - handleEndOfClause( currentPlan ); - if ( !hasMoreClauses() ) { - return 0; - } - handleBeginningOfClause(); - shared_ptr bestGuess = _currentQps->getBestGuess(); - verify( bestGuess ); - return bestGuess.get(); - } - - void MultiPlanScanner::prepareToYield() { - if ( _runnerQueue ) { - _runnerQueue->prepareToYield(); - } - } - - void MultiPlanScanner::recoverFromYield() { - if ( _runnerQueue ) { - _runnerQueue->recoverFromYield(); - } - } - - void MultiPlanScanner::clearRunnerQueue() { - if ( _runnerQueue ) { - _runnerQueue.reset(); - } - } - - int MultiPlanScanner::currentNPlans() const { - return _currentQps->nPlans(); - } - - const QueryPlan *MultiPlanScanner::singlePlan() const { - if ( _or || - _currentQps->nPlans() != 1 || - _currentQps->hasPossiblyExcludedPlans() ) { - return 0; - } - return _currentQps->firstPlan().get(); - } - - bool MultiPlanScanner::haveUselessOr() const { - NamespaceDetails *nsd = nsdetails( _ns ); - if ( !nsd ) { - return true; - } - BSONElement hintElt = _hint.firstElement(); - if ( !hintElt.eoo() ) { - IndexDetails *id = parseHint( hintElt, nsd ); - if ( !id ) { - return true; - } - return QueryUtilIndexed::uselessOr( *_org, nsd, nsd->idxNo( *id ) ); - } - return QueryUtilIndexed::uselessOr( *_org, nsd, -1 ); - } - - BSONObj MultiPlanScanner::cachedPlanExplainSummary() const { - if ( _or || !_currentQps->usingCachedPlan() ) { - return BSONObj(); - } - QueryPlanSet::QueryPlanPtr plan = _currentQps->firstPlan(); - shared_ptr cursor = plan->newCursor(); - return BSON( "cursor" << cursor->toString() - << "indexBounds" << cursor->prettyIndexBounds() ); - } - - void MultiPlanScanner::clearIndexesForPatterns() const { - QueryUtilIndexed::clearIndexesForPatterns( _currentQps->frsp(), _currentQps->order() ); - } - - bool MultiPlanScanner::haveInOrderPlan() const { - return _or ? true : _currentQps->haveInOrderPlan(); - } - - bool MultiPlanScanner::possibleInOrderPlan() const { - return _or ? true : _currentQps->possibleInOrderPlan(); - } - - bool MultiPlanScanner::possibleOutOfOrderPlan() const { - return _or ? false : _currentQps->possibleOutOfOrderPlan(); - } - - string MultiPlanScanner::toString() const { - return BSON( - "or" << _or << - "currentQps" << _currentQps->toString() - ).jsonString(); - } - - MultiCursor::MultiCursor( auto_ptr mps, const shared_ptr &c, - const shared_ptr &matcher, - const shared_ptr &explainPlanInfo, - const QueryPlanRunner& runner, long long nscanned ) : - _mps( mps ), - _c( c ), - _matcher( matcher ), - _queryPlan( &runner.queryPlan() ), - _nscanned( nscanned ), - _explainPlanInfo( explainPlanInfo ) { - _mps->clearRunnerQueue(); - _mps->setRecordedPlanPolicy( QueryPlanGenerator::UseIfInOrder ); - if ( !ok() ) { - // If the supplied cursor is exhausted, try to advance it. - advance(); - } - } - - bool MultiCursor::advance() { - _c->advance(); - advanceExhaustedClauses(); - return ok(); - } - - void MultiCursor::recoverFromYield() { - Cursor::recoverFromYield(); - advanceExhaustedClauses(); - } - - void MultiCursor::advanceClause() { - _nscanned += _c->nscanned(); - if ( _explainPlanInfo ) _explainPlanInfo->noteDone( *_c ); - shared_ptr oldClauseFrv = _queryPlan->originalFrv(); - _queryPlan = _mps->nextClauseBestGuessPlan( *_queryPlan ); - if ( _queryPlan ) { - _matcher.reset( _matcher->nextClauseMatcher( oldClauseFrv, _queryPlan->indexKey() ) ); - _c = _queryPlan->newCursor(); - // The basic and btree cursors used by this implementation support deduplication. - verify( _c->autoDedup() ); - // All sub cursors must support yields. - verify( _c->supportYields() ); - if ( _explainPlanInfo ) { - _explainPlanInfo.reset( new ExplainPlanInfo() ); - _explainPlanInfo->notePlan( *_c, _queryPlan->scanAndOrderRequired(), - _queryPlan->keyFieldsOnly() ); - shared_ptr clauseInfo( new ExplainClauseInfo() ); - clauseInfo->addPlanInfo( _explainPlanInfo ); - _mps->addClauseInfo( clauseInfo ); - } - } - } - - void MultiCursor::advanceExhaustedClauses() { - while( !ok() && _mps->hasMoreClauses() ) { - advanceClause(); - } - } - - void MultiCursor::noteIterate( bool match, bool loadedRecord ) { - if ( _explainPlanInfo ) _explainPlanInfo->noteIterate( match, loadedRecord, *_c ); - } - - bool indexWorks( const BSONObj &idxPattern, const BSONObj &sampleKey, int direction, int firstSignificantField ) { - BSONObjIterator p( idxPattern ); - BSONObjIterator k( sampleKey ); - int i = 0; - while( 1 ) { - BSONElement pe = p.next(); - BSONElement ke = k.next(); - if ( pe.eoo() && ke.eoo() ) - return true; - if ( pe.eoo() || ke.eoo() ) - return false; - if ( strcmp( pe.fieldName(), ke.fieldName() ) != 0 ) - return false; - if ( ( i == firstSignificantField ) && !( ( direction > 0 ) == ( pe.number() > 0 ) ) ) - return false; - ++i; - } - return false; - } - - BSONObj extremeKeyForIndex( const BSONObj &idxPattern, int baseDirection ) { - BSONObjIterator i( idxPattern ); - BSONObjBuilder b; - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - int idxDirection = e.number() >= 0 ? 1 : -1; - int direction = idxDirection * baseDirection; - switch( direction ) { - case 1: - b.appendMaxKey( e.fieldName() ); - break; - case -1: - b.appendMinKey( e.fieldName() ); - break; - default: - verify( false ); - } - } - return b.obj(); - } - - pair keyAudit( const BSONObj &min, const BSONObj &max ) { - int direction = 0; - int firstSignificantField = 0; - BSONObjIterator i( min ); - BSONObjIterator a( max ); - while( 1 ) { - BSONElement ie = i.next(); - BSONElement ae = a.next(); - if ( ie.eoo() && ae.eoo() ) - break; - if ( ie.eoo() || ae.eoo() || strcmp( ie.fieldName(), ae.fieldName() ) != 0 ) { - return make_pair( -1, -1 ); - } - int cmp = ie.woCompare( ae ); - if ( cmp < 0 ) - direction = 1; - if ( cmp > 0 ) - direction = -1; - if ( direction != 0 ) - break; - ++firstSignificantField; - } - return make_pair( direction, firstSignificantField ); - } - - pair flexibleKeyAudit( const BSONObj &min, const BSONObj &max ) { - if ( min.isEmpty() || max.isEmpty() ) { - return make_pair( 1, -1 ); - } - else { - return keyAudit( min, max ); - } - } - - // NOTE min, max, and keyPattern will be updated to be consistent with the selected index. - IndexDetails *indexDetailsForRange( const char *ns, string &errmsg, BSONObj &min, BSONObj &max, BSONObj &keyPattern ) { - if ( min.isEmpty() && max.isEmpty() ) { - errmsg = "one of min or max must be specified"; - return 0; - } - - Client::Context ctx( ns ); - IndexDetails *id = 0; - NamespaceDetails *d = nsdetails( ns ); - if ( !d ) { - errmsg = "ns not found"; - return 0; - } - - pair ret = flexibleKeyAudit( min, max ); - if ( ret == make_pair( -1, -1 ) ) { - errmsg = "min and max keys do not share pattern"; - return 0; - } - if ( keyPattern.isEmpty() ) { - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - IndexDetails& ii = i.next(); - if ( indexWorks( ii.keyPattern(), min.isEmpty() ? max : min, ret.first, ret.second ) ) { - if ( ii.getSpec().getType() == 0 ) { - id = ⅈ - keyPattern = ii.keyPattern(); - break; - } - } - } - - } - else { - if ( !indexWorks( keyPattern, min.isEmpty() ? max : min, ret.first, ret.second ) ) { - errmsg = "requested keyPattern does not match specified keys"; - return 0; - } - NamespaceDetails::IndexIterator i = d->ii(); - while( i.more() ) { - IndexDetails& ii = i.next(); - if( ii.keyPattern().woCompare(keyPattern) == 0 ) { - id = ⅈ - break; - } - if ( keyPattern.nFields() == 1 && ii.keyPattern().nFields() == 1 && - IndexDetails::isIdIndexPattern( keyPattern ) && - ii.isIdIndex() ) { - id = ⅈ - break; - } - - } - } - - if ( min.isEmpty() ) { - min = extremeKeyForIndex( keyPattern, -1 ); - } - else if ( max.isEmpty() ) { - max = extremeKeyForIndex( keyPattern, 1 ); - } - - if ( !id ) { - errmsg = str::stream() << "no index found for specified keyPattern: " << keyPattern.toString() - << " min: " << min << " max: " << max; - return 0; - } - - min = min.extractFieldsUnDotted( keyPattern ); - max = max.extractFieldsUnDotted( keyPattern ); - - return id; - } - - shared_ptr NamespaceDetailsTransient::bestGuessCursor( const char *ns, - const BSONObj &query, - const BSONObj &sort ) { - auto_ptr frsp( new FieldRangeSetPair( ns, query, true ) ); - auto_ptr origFrsp( new FieldRangeSetPair( *frsp ) ); - - scoped_ptr qps( QueryPlanSet::make( ns, frsp, origFrsp, query, sort, - shared_ptr(), BSONObj(), - QueryPlanGenerator::UseIfInOrder, - BSONObj(), BSONObj(), true ) ); - QueryPlanSet::QueryPlanPtr qpp = qps->getBestGuess(); - if( ! qpp.get() ) return shared_ptr(); - - shared_ptr ret = qpp->newCursor(); - - // If we don't already have a matcher, supply one. - if ( !query.isEmpty() && ! ret->matcher() ) { - ret->setMatcher( qpp->matcher() ); - } - return ret; - } - - bool QueryUtilIndexed::indexUseful( const FieldRangeSetPair &frsp, NamespaceDetails *d, int idxNo, const BSONObj &order ) { - DEV frsp.assertValidIndex( d, idxNo ); - BSONObj keyPattern = d->idx( idxNo ).keyPattern(); - if ( !frsp.matchPossibleForIndex( d, idxNo, keyPattern ) ) { - // No matches are possible in the index so the index may be useful. - return true; - } - - return d->idx( idxNo ).getSpec().suitability( frsp.frsForIndex( d , idxNo ) , order ) - != USELESS; - } - - void QueryUtilIndexed::clearIndexesForPatterns( const FieldRangeSetPair &frsp, const BSONObj &order ) { - SimpleMutex::scoped_lock lk(NamespaceDetailsTransient::_qcMutex); - NamespaceDetailsTransient &nsdt = NamespaceDetailsTransient::get_inlock( frsp.ns() ); - CachedQueryPlan noCachedPlan; - nsdt.registerCachedQueryPlanForPattern( frsp._singleKey.pattern( order ), noCachedPlan ); - nsdt.registerCachedQueryPlanForPattern( frsp._multiKey.pattern( order ), noCachedPlan ); - } - - CachedQueryPlan QueryUtilIndexed::bestIndexForPatterns( const FieldRangeSetPair &frsp, const BSONObj &order ) { - SimpleMutex::scoped_lock lk(NamespaceDetailsTransient::_qcMutex); - NamespaceDetailsTransient &nsdt = NamespaceDetailsTransient::get_inlock( frsp.ns() ); - // TODO Maybe it would make sense to return the index with the lowest - // nscanned if there are two possibilities. - { - QueryPattern pattern = frsp._singleKey.pattern( order ); - CachedQueryPlan cachedQueryPlan = nsdt.cachedQueryPlanForPattern( pattern ); - if ( !cachedQueryPlan.indexKey().isEmpty() ) { - return cachedQueryPlan; - } - } - { - QueryPattern pattern = frsp._multiKey.pattern( order ); - CachedQueryPlan cachedQueryPlan = nsdt.cachedQueryPlanForPattern( pattern ); - if ( !cachedQueryPlan.indexKey().isEmpty() ) { - return cachedQueryPlan; - } - } - return CachedQueryPlan(); - } - - bool QueryUtilIndexed::uselessOr( const OrRangeGenerator &org, NamespaceDetails *d, int hintIdx ) { - for( list::const_iterator i = org._originalOrSets.begin(); i != org._originalOrSets.end(); ++i ) { - if ( hintIdx != -1 ) { - if ( !indexUseful( *i, d, hintIdx, BSONObj() ) ) { - return true; - } - } - else { - bool useful = false; - for( int j = 0; j < d->nIndexes; ++j ) { - if ( indexUseful( *i, d, j, BSONObj() ) ) { - useful = true; - break; - } - } - if ( !useful ) { - return true; - } - } - } - return false; - } - -} // namespace mongo diff --git a/src/mongo/db/queryoptimizer.h b/src/mongo/db/queryoptimizer.h deleted file mode 100644 index fe82801f417..00000000000 --- a/src/mongo/db/queryoptimizer.h +++ /dev/null @@ -1,870 +0,0 @@ -// @file queryoptimizer.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/base/disallow_copying.h" -#include "mongo/db/clientcursor.h" -#include "mongo/db/cursor.h" -#include "mongo/db/explain.h" -#include "mongo/db/matcher.h" -#include "mongo/db/queryoptimizercursor.h" -#include "mongo/db/querypattern.h" -#include "mongo/db/queryutil.h" -#include "mongo/util/elapsed_tracker.h" - -namespace mongo { - - static const int OutOfOrderDocumentsAssertionCode = 14810; - - class IndexDetails; - class IndexType; - class QueryPlanSummary; - - /** - * A plan for executing a query using the given index spec and FieldRangeSet. An object of this - * class may only be used by one thread at a time. - */ - class QueryPlan : boost::noncopyable { - public: - - /** - * @param originalFrsp - original constraints for this query clause. If null, frsp will be used instead. - */ - static QueryPlan *make( NamespaceDetails *d, - int idxNo, // -1 = no index - const FieldRangeSetPair &frsp, - const FieldRangeSetPair *originalFrsp, - const BSONObj &originalQuery, - const BSONObj &order, - const shared_ptr &parsedQuery = - shared_ptr(), - const BSONObj &startKey = BSONObj(), - const BSONObj &endKey = BSONObj(), - const std::string& special="" ); - - /** Categorical classification of a QueryPlan's utility. */ - enum Utility { - Impossible, // Cannot produce any matches, so the query must have an empty result set. - // No other plans need to be considered. - Optimal, // Should run as the only candidate plan in the absence of an Impossible - // plan. - Helpful, // Should be considered. - Unhelpful, // Should not be considered. - Disallowed // Must not be considered unless explicitly hinted. May produce a - // semantically incorrect result set. - }; - - Utility utility() const { return _utility; } - - /** @return true if ScanAndOrder processing will be required for result set. */ - bool scanAndOrderRequired() const { return _scanAndOrderRequired; } - /** - * @return false if document matching can be determined entirely using index keys and the - * FieldRangeSetPair generated for the query, without using a Matcher. This function may - * return false positives but not false negatives. For example, if the field range set's - * mustBeExactMatchRepresentation() returns a false negative, this function will return a - * false positive. - */ - bool mayBeMatcherNecessary() const { return _matcherNecessary; } - /** @return true if this QueryPlan would perform an unindexed scan. */ - bool willScanTable() const { return _idxNo < 0 && ( _utility != Impossible ); } - /** @return 'special' attribute of the plan, which was either set explicitly or generated from the index. */ - const string &special() const { return _special; } - - /** @return a new cursor based on this QueryPlan's index and FieldRangeSet. */ - shared_ptr newCursor( const DiskLoc& startLoc = DiskLoc(), - bool requestIntervalCursor = false ) const; - /** @return a new reverse cursor if this is an unindexed plan. */ - shared_ptr newReverseCursor() const; - /** Register this plan as a winner for its QueryPattern, with specified 'nscanned'. */ - void registerSelf( long long nScanned, CandidatePlanCharacter candidatePlans ) const; - - int direction() const { return _direction; } - BSONObj indexKey() const; - bool indexed() const { return _index != 0; } - const IndexDetails *index() const { return _index; } - int idxNo() const { return _idxNo; } - const char *ns() const { return _frs.ns(); } - NamespaceDetails *nsd() const { return _d; } - BSONObj originalQuery() const { return _originalQuery; } - shared_ptr originalFrv() const { return _originalFrv; } - - const FieldRangeSet &multikeyFrs() const { return _frsMulti; } - - shared_ptr keyFieldsOnly() const { return _keyFieldsOnly; } - const ParsedQuery* parsedQuery() const { return _parsedQuery.get(); } - - /** @return a shared, lazily initialized matcher for the query plan. */ - shared_ptr matcher() const; - - QueryPlanSummary summary() const; - - /** The following member functions are for testing, or public for testing. */ - - shared_ptr frv() const { return _frv; } - bool isMultiKey() const; - string toString() const; - bool queryBoundsExactOrderSuffix() const; - - private: - - QueryPlan(NamespaceDetails *d, - int idxNo, - const FieldRangeSetPair &frsp, - const BSONObj &originalQuery, - const BSONObj &order, - const shared_ptr &parsedQuery, - const std::string& special ); - void init( const FieldRangeSetPair *originalFrsp, - const BSONObj &startKey, - const BSONObj &endKey ); - - void checkTableScanAllowed() const; - int independentRangesSingleIntervalLimit() const; - /** @return true when the plan's query may contains an $exists:false predicate. */ - bool hasPossibleExistsFalsePredicate() const; - - NamespaceDetails * _d; - int _idxNo; - const FieldRangeSet &_frs; - const FieldRangeSet &_frsMulti; - const BSONObj _originalQuery; - const BSONObj _order; - shared_ptr _parsedQuery; - const IndexDetails * _index; - bool _scanAndOrderRequired; - bool _matcherNecessary; - int _direction; - shared_ptr _frv; - shared_ptr _originalFrv; - BSONObj _startKey; - BSONObj _endKey; - bool _endKeyInclusive; - Utility _utility; - string _special; - IndexType * _type; - bool _startOrEndSpec; - shared_ptr _keyFieldsOnly; - mutable shared_ptr _matcher; // Lazy initialization. - }; - - std::ostream &operator<< ( std::ostream &out, const QueryPlan::Utility &utility ); - - /** - * A QueryPlanSummary owns its own attributes and may be shared. Currently a QueryPlan - * should only be owned by a QueryPlanSet. - */ - class QueryPlanSummary { - public: - QueryPlanSummary() : - _scanAndOrderRequired() { - } - QueryPlanSummary( const QueryPlan &queryPlan ) : - _fieldRangeSetMulti( new FieldRangeSet( queryPlan.multikeyFrs() ) ), - _keyFieldsOnly( queryPlan.keyFieldsOnly() ), - _scanAndOrderRequired( queryPlan.scanAndOrderRequired() ) { - } - bool valid() const { return _fieldRangeSetMulti; } - shared_ptr _fieldRangeSetMulti; - shared_ptr _keyFieldsOnly; - bool _scanAndOrderRequired; - }; - - /** - * Helper class for a QueryPlanRunner to cache and count matches. One object of this type is - * used per candidate QueryPlan (as there is one QueryPlanRunner per QueryPlan). - * - * Typical usage: - * 1) resetMatch() - reset stored match value to Unkonwn. - * 2) setMatch() - set match value to a definite true/false value. - * 3) knowMatch() - check if setMatch() has been called. - * 4) incMatch() - increment count if match is true. - */ - class CachedMatchCounter { - public: - /** - * @param aggregateNscanned - shared count of nscanned for this and other plans. - * @param cumulativeCount - starting point for accumulated count over a series of plans. - */ - CachedMatchCounter( long long& aggregateNscanned, int cumulativeCount ); - - /** Set whether dup checking is enabled when counting. */ - void setCheckDups( bool checkDups ) { _checkDups = checkDups; } - - void resetMatch(); - - /** @return true if the match was not previously recorded. */ - bool setMatch( bool match ); - - bool knowMatch() const { return _match != Unknown; } - - void incMatch( const DiskLoc& loc ); - - bool wouldIncMatch( const DiskLoc& loc ) const; - - bool enoughCumulativeMatchesToChooseAPlan() const; - - bool enoughMatchesToRecordPlan() const; - - int cumulativeCount() const { return _cumulativeCount; } - int count() const { return _count; } - - /** Update local and aggregate nscanned counts. */ - void updateNscanned( long long nscanned ); - - long long nscanned() const { return _nscanned; } - long long& aggregateNscanned() const { return _aggregateNscanned; } - - private: - bool getsetdup( const DiskLoc& loc ); - bool getdup( const DiskLoc& loc ) const; - - long long& _aggregateNscanned; - long long _nscanned; - int _cumulativeCount; - int _count; - bool _checkDups; - enum MatchState { Unknown, False, True }; - MatchState _match; - bool _counted; - set _dups; - }; - - /** - * Iterates through a QueryPlan's candidate matches, keeping track of accumulated nscanned. - * Generally used along with runners for other QueryPlans in a QueryPlanRunnerQueue priority - * queue. Eg if there are three candidate QueryPlans evaluated in parallel, there are three - * QueryPlanRunners, one checking for matches on each query. - * - * Typical usage: - * 1) A new QueryPlanRunner is generated using createChild(). - * 2) A QueryPlan is assigned using setQueryPlan(). - * 3) init() is called to initialize the runner. - * 4) next() is called repeatedly, with nscanned() checked after each call. - * 5) In one of these calls to next(), setComplete() is called internally. - * 6) The QueryPattern for the QueryPlan may be recorded as a winning plan. - */ - class QueryPlanRunner { - MONGO_DISALLOW_COPYING( QueryPlanRunner ); - public: - /** - * @param aggregateNscanned Shared long long counting total nscanned for runners for all - * cursors. - * @param selectionPolicy Characterizes the set of QueryPlans allowed for this operation. - * See queryoptimizercursor.h for more information. - * @param requireOrder Whether only ordered plans are allowed. - * @param alwaysCountMatches Whether matches are to be counted regardless of ordering. - * @param cumulativeCount Total count. - */ - QueryPlanRunner( long long& aggregateNscanned, - const QueryPlanSelectionPolicy& selectionPolicy, - const bool& requireOrder, - bool alwaysCountMatches, - int cumulativeCount = 0 ); - - /** @return QueryPlan assigned to this runner by the query optimizer. */ - const QueryPlan& queryPlan() const { return *_queryPlan; } - - /** Advance to the next potential matching document (eg using a cursor). */ - void next(); - - /** - * @return current 'nscanned' metric for this runner. Used to compare cost to other - * runners. - */ - long long nscanned() const; - - /** Take any steps necessary before the db mutex is yielded. */ - void prepareToYield(); - - /** Recover once the db mutex is regained. */ - void recoverFromYield(); - - /** Take any steps necessary before an earlier iterate of the cursor is modified. */ - void prepareToTouchEarlierIterate(); - - /** Recover after the earlier iterate is modified. */ - void recoverFromTouchingEarlierIterate(); - - DiskLoc currLoc() const { return _c ? _c->currLoc() : DiskLoc(); } - BSONObj currKey() const { return _c ? _c->currKey() : BSONObj(); } - bool currentMatches( MatchDetails* details ); - - /** - * @return true iff the QueryPlan for this runner may be registered - * as a winning plan. - */ - bool mayRecordPlan() const; - - shared_ptr cursor() const { return _c; } - - /** @return true iff the implementation called setComplete() or setStop(). */ - bool complete() const { return _complete; } - /** @return true iff the implementation called setStop(). */ - bool stopRequested() const { return _stopRequested; } - bool completeWithoutStop() const { return complete() && !stopRequested(); } - /** @return true iff the implementation errored out. */ - bool error() const { return _error; } - /** @return the error information. */ - ExceptionInfo exception() const { return _exception; } - - /** To be called by QueryPlanSet::Runner only. */ - - /** - * @return a copy of the inheriting class, which will be run with its own query plan. The - * child runner will assume its parent runner has completed execution. - */ - QueryPlanRunner* createChild() const; - - void setQueryPlan( const QueryPlan* queryPlan ); - - /** Handle initialization after a QueryPlan has been set. */ - void init(); - - void setException( const DBException& e ); - - /** @return an ExplainPlanInfo object that will be updated as the query runs. */ - shared_ptr generateExplainInfo(); - shared_ptr explainInfo() const { return _explainPlanInfo; } - - const Projection::KeyOnly* keyFieldsOnly() const { - return queryPlan().keyFieldsOnly().get(); - } - - private: - /** Call if all results have been found. */ - void setComplete() { _complete = true; } - /** Call if the scan is complete even if not all results have been found. */ - void setStop() { setComplete(); _stopRequested = true; } - - void mayAdvance(); - bool countingMatches(); - bool countMatches() const; - /** - * @return true if the results generated by this query plan will be loaded from the record - * store (not built from an index entry). - */ - bool hasDocumentLoadingQueryPlan() const; - - void recordCursorLocation(); - void checkCursorAdvanced(); - void handleCursorAdvanced(); - void checkCursorOrdering(); - - bool _complete; - bool _stopRequested; - ExceptionInfo _exception; - const QueryPlan* _queryPlan; - bool _error; - CachedMatchCounter _matchCounter; - bool _countingMatches; - bool _mustAdvance; - bool _capped; - shared_ptr _c; - ClientCursor::Holder _cc; - DiskLoc _posBeforeYield; - ClientCursor::YieldData _yieldData; - const QueryPlanSelectionPolicy& _selectionPolicy; - const bool& _requireOrder; // TODO don't use a ref for this, but signal change explicitly - shared_ptr _explainPlanInfo; - bool _alwaysCountMatches; - }; - - /** - * This class works if T::operator< is variant unlike a regular stl priority queue, but it's - * very slow. However if _vec.size() is always very small, it would be fine, maybe even faster - * than a smart impl that does more memory allocations. - * TODO Clean up this temporary code. - */ - template - class PriorityQueue { - MONGO_DISALLOW_COPYING( PriorityQueue ); - public: - PriorityQueue() { - _vec.reserve(4); - } - int size() const { return _vec.size(); } - bool empty() const { return _vec.empty(); } - void push(const T & x) { - _vec.push_back(x); - } - T pop() { - size_t t = 0; - for( size_t i = 1; i < _vec.size(); i++ ) { - if( _vec[t] < _vec[i] ) - t = i; - } - T ret = _vec[t]; - _vec.erase(_vec.begin()+t); - return ret; - } - private: - vector _vec; - }; - - class QueryPlanSet; - - /** Populates a provided QueryPlanSet with candidate query plans, when requested. */ - class QueryPlanGenerator { - public: - - /** Policies for utilizing recorded plans. */ - typedef enum { - Ignore, // Ignore the recorded plan and try all candidate plans. - UseIfInOrder, // Use the recorded plan if it is properly ordered. - Use // Always use the recorded plan. - } RecordedPlanPolicy; - - /** @param qps The QueryPlanSet to which plans will be provided. */ - QueryPlanGenerator( QueryPlanSet &qps, - auto_ptr originalFrsp, - const shared_ptr &parsedQuery, - const BSONObj &hint, - RecordedPlanPolicy recordedPlanPolicy, - const BSONObj &min, - const BSONObj &max, - bool allowSpecial ); - /** Populate the provided QueryPlanSet with an initial set of plans. */ - void addInitialPlans(); - /** Supplement a cached plan provided earlier by adding additional query plans. */ - void addFallbackPlans(); - - private: - - bool addShortCircuitPlan( NamespaceDetails *d ); - bool addHintPlan( NamespaceDetails *d ); - bool addSpecialPlan( NamespaceDetails *d ); - void addStandardPlans( NamespaceDetails *d ); - bool addCachedPlan( NamespaceDetails *d ); - shared_ptr newPlan( NamespaceDetails *d, - int idxNo, - const BSONObj &min = BSONObj(), - const BSONObj &max = BSONObj(), - const string &special = "" ) const; - bool setUnindexedPlanIf( bool set, NamespaceDetails *d ); - void setSingleUnindexedPlan( NamespaceDetails *d ); - void setHintedPlanForIndex( IndexDetails& id ); - void validateAndSetHintedPlan( const shared_ptr& plan ); - void warnOnCappedIdTableScan() const; - QueryPlanSet &_qps; - auto_ptr _originalFrsp; - shared_ptr _parsedQuery; - BSONObj _hint; - RecordedPlanPolicy _recordedPlanPolicy; - BSONObj _min; - BSONObj _max; - bool _allowSpecial; - }; - - /** A set of candidate query plans for a query. */ - class QueryPlanSet { - public: - typedef boost::shared_ptr QueryPlanPtr; - typedef vector PlanVector; - - /** - * @param originalFrsp - original constraints for this query clause; if null, frsp will be - * used. - */ - static QueryPlanSet* make( const char* ns, - auto_ptr frsp, - auto_ptr originalFrsp, - const BSONObj& originalQuery, - const BSONObj& order, - const shared_ptr& parsedQuery, - const BSONObj& hint, - QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy, - const BSONObj& min, - const BSONObj& max, - bool allowSpecial ); - - /** @return number of candidate plans. */ - int nPlans() const { return _plans.size(); } - - QueryPlanPtr firstPlan() const { return _plans[ 0 ]; } - - /** @return true if a plan is selected based on previous success of this plan. */ - bool usingCachedPlan() const { return _usingCachedPlan; } - /** @return true if some candidate plans may have been excluded due to plan caching. */ - bool hasPossiblyExcludedPlans() const; - /** @return a single plan that may work well for the specified query. */ - QueryPlanPtr getBestGuess() const; - - const FieldRangeSetPair &frsp() const { return *_frsp; } - BSONObj originalQuery() const { return _originalQuery; } - BSONObj order() const { return _order; } - - /** @return true if an active plan is in order. */ - bool haveInOrderPlan() const; - /** @return true if an active or fallback plan is in order. */ - bool possibleInOrderPlan() const; - /** @return true if an active or fallback plan is out of order. */ - bool possibleOutOfOrderPlan() const; - - CandidatePlanCharacter characterizeCandidatePlans() const; - - bool prepareToRetryQuery(); - - string toString() const; - - /** Configure a single query plan if one has not already been provided. */ - void setSinglePlan( const QueryPlanPtr &plan ); - /** Configure a query plan from the plan cache. */ - void setCachedPlan( const QueryPlanPtr &plan, const CachedQueryPlan &cachedPlan ); - /** Add a candidate query plan, potentially one of many. */ - void addCandidatePlan( const QueryPlanPtr &plan ); - - const PlanVector& plans() const { return _plans; } - bool mayRecordPlan() const { return _mayRecordPlan; } - int oldNScanned() const { return _oldNScanned; } - void addFallbackPlans(); - void setUsingCachedPlan( bool usingCachedPlan ) { _usingCachedPlan = usingCachedPlan; } - - //for testing - bool modifiedKeys() const; - bool hasMultiKey() const; - - private: - - QueryPlanSet( const char *ns, - auto_ptr frsp, - auto_ptr originalFrsp, - const BSONObj &originalQuery, - const BSONObj &order, - const shared_ptr &parsedQuery, - const BSONObj &hint, - QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy, - const BSONObj &min, - const BSONObj &max, - bool allowSpecial ); - void init(); - - void pushPlan( const QueryPlanPtr& plan ); - - QueryPlanGenerator _generator; - BSONObj _originalQuery; - auto_ptr _frsp; - PlanVector _plans; - bool _mayRecordPlan; - bool _usingCachedPlan; - CandidatePlanCharacter _cachedPlanCharacter; - BSONObj _order; - long long _oldNScanned; - ElapsedTracker _yieldSometimesTracker; - bool _allowSpecial; - }; - - /** - * A priority queue of QueryPlanRunners ordered by their nscanned values. The QueryPlanRunners - * are iterated sequentially and reinserted into the queue until one runner completes or all - * runners error out. - */ - class QueryPlanRunnerQueue { - public: - QueryPlanRunnerQueue( QueryPlanSet& plans, const QueryPlanRunner& prototypeRunner ); - - /** - * Pull a runner from the priority queue, advance it if possible, re-insert it into the - * queue if it is not done, and return it. But if this runner errors out, retry with - * another runner until a non error runner is found or all runners have errored out. - * @return the next non error runner if there is one, otherwise an error runner. - * If the returned runner is complete() or error(), this queue becomes done(). - */ - shared_ptr next(); - /** @return true if done iterating. */ - bool done() const { return _done; } - - /** Prepare all runners for a database mutex yield. */ - void prepareToYield(); - /** Restore all runners after a database mutex yield. */ - void recoverFromYield(); - - /** @return an ExplainClauseInfo object that will be updated as the query runs. */ - shared_ptr generateExplainInfo() { - _explainClauseInfo.reset( new ExplainClauseInfo() ); - return _explainClauseInfo; - } - - private: - const QueryPlanRunner& _prototypeRunner; - QueryPlanSet& _plans; - static void initRunner( QueryPlanRunner& runner ); - static void nextRunner( QueryPlanRunner& runner ); - static void prepareToYieldRunner( QueryPlanRunner& runner ); - static void recoverFromYieldRunner( QueryPlanRunner& runner ); - - /** Initialize the Runner. */ - shared_ptr init(); - /** Move the Runner forward one iteration, and @return the plan for the iteration. */ - shared_ptr _next(); - - vector > _runners; - struct RunnerHolder { - RunnerHolder( const shared_ptr& runner ) : - _runner( runner ), - _offset() { - } - shared_ptr _runner; - long long _offset; - bool operator<( const RunnerHolder& other ) const { - return _runner->nscanned() + _offset > other._runner->nscanned() + other._offset; - } - }; - PriorityQueue _queue; - shared_ptr _explainClauseInfo; - bool _done; - }; - - /** Handles $or type queries by generating a QueryPlanSet for each $or clause. */ - class MultiPlanScanner { - public: - - static MultiPlanScanner *make( const StringData& ns, - const BSONObj &query, - const BSONObj &order, - const shared_ptr &parsedQuery = - shared_ptr(), - const BSONObj &hint = BSONObj(), - QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy = - QueryPlanGenerator::Use, - const BSONObj &min = BSONObj(), - const BSONObj &max = BSONObj() ); - - /** Set the originalRunner for QueryPlanSet iteration. */ - void initialRunner( const shared_ptr& originalRunner ) { - _baseRunner = originalRunner; - } - /** - * Advance to the next runner, if not doneRunners(). - * @return the next non error runner if there is one, otherwise an error runner. - * If the returned runner is complete() or error(), the MultiPlanScanner becomes - * doneRunners() and no further runner iteration is possible. - */ - shared_ptr nextRunner(); - /** @return true if done with runner iteration. */ - bool doneRunners() const { return _doneRunners; } - - /** - * Advance to the next $or clause; hasMoreClauses() must be true. - * @param currentPlan QueryPlan of the current $or clause - * @return best guess query plan of the next $or clause, 0 if there is no such plan. - */ - const QueryPlan *nextClauseBestGuessPlan( const QueryPlan ¤tPlan ); - - /** Add explain information for a new clause. */ - void addClauseInfo( const shared_ptr &clauseInfo ) { - verify( _explainQueryInfo ); - _explainQueryInfo->addClauseInfo( clauseInfo ); - } - - /** @return an ExplainQueryInfo object that will be updated as the query runs. */ - shared_ptr generateExplainInfo() { - _explainQueryInfo.reset( new ExplainQueryInfo() ); - return _explainQueryInfo; - } - - /** Yield the runner member. */ - - void prepareToYield(); - void recoverFromYield(); - - /** Clear the runner member. */ - void clearRunnerQueue(); - - void setRecordedPlanPolicy( QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy ) { - _recordedPlanPolicy = recordedPlanPolicy; - } - - int currentNPlans() const; - - /** - * @return the query plan that would be used if the scanner would run a single - * cursor for this query, otherwise 0. The returned plan is invalid if this - * MultiPlanScanner is destroyed, hence we return a raw pointer. - */ - const QueryPlan *singlePlan() const; - - /** @return true if more $or clauses need to be scanned. */ - bool hasMoreClauses() const { - return _or ? ( !_tableScanned && !_org->orRangesExhausted() ) : _i == 0; - } - /** - * @return plan information if there is a cached plan for a non $or query, otherwise an - * empty object. - */ - BSONObj cachedPlanExplainSummary() const; - /** - * @return true if this is not a $or query and some candidate plans may have been excluded - * due to plan caching. - */ - bool hasPossiblyExcludedPlans() const { - return !_or && _currentQps->hasPossiblyExcludedPlans(); - } - bool hasMultiKey() const { return _currentQps->hasMultiKey(); } - - /** Clear recorded indexes for the current QueryPlanSet's patterns. */ - void clearIndexesForPatterns() const; - - /** @return true if an active plan of _currentQps is in order. */ - bool haveInOrderPlan() const; - /** @return true if an active or fallback plan of _currentQps is in order. */ - bool possibleInOrderPlan() const; - /** @return true if an active or fallback plan of _currentQps is out of order. */ - bool possibleOutOfOrderPlan() const; - - int i() const { return _i; } - - string toString() const; - - private: - - MultiPlanScanner( const StringData& ns, - const BSONObj &query, - const shared_ptr &parsedQuery, - const BSONObj &hint, - QueryPlanGenerator::RecordedPlanPolicy recordedPlanPolicy ); - void init( const BSONObj &order, - const BSONObj &min, - const BSONObj &max ); - - /** Initialize or iterate a runner generated from @param originalOp. */ - shared_ptr iterateRunnerQueue( QueryPlanRunner& originalRunner, - bool retried = false ); - - shared_ptr nextRunnerSimple(); - shared_ptr nextRunnerOr(); - - void updateCurrentQps( QueryPlanSet *qps ); - - void assertNotOr() const { - massert( 13266, "not implemented for $or query", !_or ); - } - void assertHasMoreClauses() const { - massert( 13271, "no more clauses", hasMoreClauses() ); - } - - void handleEndOfClause( const QueryPlan &clausePlan ); - void handleBeginningOfClause(); - bool mayHandleBeginningOfClause(); - - bool haveUselessOr() const; - - const string _ns; - bool _or; - BSONObj _query; - shared_ptr _parsedQuery; - scoped_ptr _org; // May be null in certain non $or query cases. - scoped_ptr _currentQps; - int _i; - QueryPlanGenerator::RecordedPlanPolicy _recordedPlanPolicy; - BSONObj _hint; - bool _tableScanned; - shared_ptr _baseRunner; - scoped_ptr _runnerQueue; - shared_ptr _explainQueryInfo; - bool _doneRunners; - }; - - /** - * Provides a cursor interface for serial single Cursor iteration using a MultiPlanScanner. - * Currently used internally by a QueryOptimizerCursor. - * - * A MultiCursor is backed by one BasicCursor or BtreeCursor at a time and forwards calls for - * ensuring a consistent state after a write to its backing Cursor. - */ - class MultiCursor : public Cursor { - public: - /** @param nscanned is the initial nscanned value. */ - MultiCursor( auto_ptr mps, const shared_ptr &c, - const shared_ptr &matcher, - const shared_ptr &explainPlanInfo, - const QueryPlanRunner& runner, long long nscanned ); - - virtual bool ok() { return _c->ok(); } - virtual Record* _current() { return _c->_current(); } - virtual BSONObj current() { return _c->current(); } - virtual DiskLoc currLoc() { return _c->currLoc(); } - virtual bool advance(); - virtual BSONObj currKey() const { return _c->currKey(); } - virtual DiskLoc refLoc() { return _c->refLoc(); } - virtual void noteLocation() { _c->noteLocation(); } - virtual void checkLocation() { _c->checkLocation(); } - virtual void recoverFromYield(); - virtual bool supportGetMore() { return true; } - virtual bool supportYields() { return true; } - virtual BSONObj indexKeyPattern() { return _c->indexKeyPattern(); } - - /** Deduping documents from a prior cursor is handled by the matcher. */ - virtual bool getsetdup(DiskLoc loc) { return _c->getsetdup( loc ); } - - virtual bool modifiedKeys() const { return true; } - - virtual bool isMultiKey() const { return _mps->hasMultiKey(); } - - virtual CoveredIndexMatcher* matcher() const { return _matcher.get(); } - - virtual bool capped() const { return _c->capped(); } - - virtual long long nscanned() { return _nscanned + _c->nscanned(); } - - void noteIterate( bool match, bool loadedRecord ); - - const QueryPlan &queryPlan() const { - verify( _c->ok() && _queryPlan ); - return *_queryPlan; - } - - const Projection::KeyOnly *keyFieldsOnly() const { - verify( _c->ok() && _queryPlan ); - return _queryPlan->keyFieldsOnly().get(); - } - private: - void advanceClause(); - void advanceExhaustedClauses(); - auto_ptr _mps; - shared_ptr _c; - shared_ptr _matcher; - const QueryPlan *_queryPlan; - long long _nscanned; - shared_ptr _explainPlanInfo; - }; - - /** NOTE min, max, and keyPattern will be updated to be consistent with the selected index. */ - IndexDetails *indexDetailsForRange( const char *ns, string &errmsg, BSONObj &min, BSONObj &max, BSONObj &keyPattern ); - - class CachedQueryPlan; - - /** - * Add-on functionality for queryutil classes requiring access to indexing - * functionality not currently linked to mongos. - * TODO Clean this up a bit, possibly with separate sharded and non sharded - * implementations for the appropriate queryutil classes or by pulling index - * related functionality into separate wrapper classes. - */ - struct QueryUtilIndexed { - /** @return true if the index may be useful according to its KeySpec. */ - static bool indexUseful( const FieldRangeSetPair &frsp, NamespaceDetails *d, int idxNo, const BSONObj &order ); - /** Clear any indexes recorded as the best for either the single or multi key pattern. */ - static void clearIndexesForPatterns( const FieldRangeSetPair &frsp, const BSONObj &order ); - /** Return a recorded best index for the single or multi key pattern. */ - static CachedQueryPlan bestIndexForPatterns( const FieldRangeSetPair &frsp, const BSONObj &order ); - static bool uselessOr( const OrRangeGenerator& org, NamespaceDetails *d, int hintIdx ); - }; - -} // namespace mongo diff --git a/src/mongo/db/queryoptimizercursor.h b/src/mongo/db/queryoptimizercursor.h deleted file mode 100644 index 5736667d008..00000000000 --- a/src/mongo/db/queryoptimizercursor.h +++ /dev/null @@ -1,147 +0,0 @@ -// @file queryoptimizercursor.h - Interface for a cursor interleaving multiple candidate cursors. - -/** - * Copyright (C) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "cursor.h" -#include "diskloc.h" - -namespace mongo { - - class QueryPlan; - class CandidatePlanCharacter; - - /** - * An interface for policies overriding the query optimizer's default behavior for selecting - * query plans and creating cursors. - */ - class QueryPlanSelectionPolicy { - public: - virtual ~QueryPlanSelectionPolicy() {} - virtual string name() const = 0; - virtual bool permitOptimalNaturalPlan() const { return true; } - virtual bool permitOptimalIdPlan() const { return true; } - virtual bool permitPlan( const QueryPlan &plan ) const { return true; } - virtual BSONObj planHint( const StringData& ns ) const { return BSONObj(); } - - /** - * @return true to request that a created Cursor provide a matcher(). If false, the - * Cursor's matcher() may be NULL if the Cursor can perform accurate query matching - * internally using a non Matcher mechanism. One case where a Matcher might be requested - * even though not strictly necessary to select matching documents is if metadata about - * matches may be requested using MatchDetails. NOTE This is a hint that the Cursor use a - * Matcher, but the hint may be ignored. In some cases the Cursor may not provide - * a Matcher even if 'requestMatcher' is true. - */ - virtual bool requestMatcher() const { return true; } - - /** - * @return true to request creating an IntervalBtreeCursor rather than a BtreeCursor when - * possible. An IntervalBtreeCursor is optimized for counting the number of documents - * between two endpoints in a btree. NOTE This is a hint to create an interval cursor, but - * the hint may be ignored. In some cases a different cursor type may be created even if - * 'requestIntervalCursor' is true. - */ - virtual bool requestIntervalCursor() const { return false; } - - /** Allow any query plan selection, permitting the query optimizer's default behavior. */ - static const QueryPlanSelectionPolicy &any(); - - /** Prevent unindexed collection scans. */ - static const QueryPlanSelectionPolicy &indexOnly(); - - /** - * Generally hints to use the _id plan, falling back to the $natural plan. However, the - * $natural plan will always be used if optimal for the query. - */ - static const QueryPlanSelectionPolicy &idElseNatural(); - - private: - class Any; - static Any __any; - class IndexOnly; - static IndexOnly __indexOnly; - class IdElseNatural; - static IdElseNatural __idElseNatural; - }; - - class QueryPlanSelectionPolicy::Any : public QueryPlanSelectionPolicy { - public: - virtual string name() const { return "any"; } - }; - - class QueryPlanSelectionPolicy::IndexOnly : public QueryPlanSelectionPolicy { - public: - virtual string name() const { return "indexOnly"; } - virtual bool permitOptimalNaturalPlan() const { return false; } - virtual bool permitPlan( const QueryPlan &plan ) const; - }; - - class QueryPlanSelectionPolicy::IdElseNatural : public QueryPlanSelectionPolicy { - public: - virtual string name() const { return "idElseNatural"; } - virtual bool permitPlan( const QueryPlan &plan ) const; - virtual BSONObj planHint( const StringData& ns ) const; - }; - - class FieldRangeSet; - class ExplainQueryInfo; - - /** - * Adds functionality to Cursor for running multiple plans, running out of order plans, - * utilizing covered indexes, and generating explain output. - */ - class QueryOptimizerCursor : public Cursor { - public: - - /** Candidate plans for the query before it begins running. */ - virtual CandidatePlanCharacter initialCandidatePlans() const = 0; - /** FieldRangeSet for the query before it begins running. */ - virtual const FieldRangeSet *initialFieldRangeSet() const = 0; - - /** @return true if the plan for the current iterate is out of order. */ - virtual bool currentPlanScanAndOrderRequired() const = 0; - - /** @return true when there may be multiple plans running and some are in order. */ - virtual bool runningInitialInOrderPlan() const = 0; - /** - * @return true when some query plans may have been excluded due to plan caching, for a - * non-$or query. - */ - virtual bool hasPossiblyExcludedPlans() const = 0; - - /** - * @return true when both in order and out of order candidate plans were available, and - * an out of order candidate plan completed iteration. - */ - virtual bool completePlanOfHybridSetScanAndOrderRequired() const = 0; - - /** Clear recorded indexes for the current clause's query patterns. */ - virtual void clearIndexesForPatterns() = 0; - /** Stop returning results from out of order plans and do not allow them to complete. */ - virtual void abortOutOfOrderPlans() = 0; - - /** Note match information for the current iterate, to generate explain output. */ - virtual void noteIterate( bool match, bool loadedDocument, bool chunkSkip ) = 0; - /** Note a lock yield for explain output reporting. */ - virtual void noteYield() = 0; - /** @return explain output for the query run by this cursor. */ - virtual shared_ptr explainQueryInfo() const = 0; - }; - -} // namespace mongo diff --git a/src/mongo/db/queryoptimizercursorimpl.cpp b/src/mongo/db/queryoptimizercursorimpl.cpp deleted file mode 100644 index 2c4d05c590d..00000000000 --- a/src/mongo/db/queryoptimizercursorimpl.cpp +++ /dev/null @@ -1,542 +0,0 @@ -// @file queryoptimizercursorimpl.cpp - A cursor interleaving multiple candidate cursors. - -/** - * Copyright (C) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - - -#include "pch.h" - -#include "mongo/db/queryoptimizercursorimpl.h" - -#include "mongo/db/btreecursor.h" -#include "mongo/db/queryoptimizer.h" - -namespace mongo { - - extern bool useHints; - - QueryPlanSelectionPolicy::Any QueryPlanSelectionPolicy::__any; - const QueryPlanSelectionPolicy &QueryPlanSelectionPolicy::any() { return __any; } - - bool QueryPlanSelectionPolicy::IndexOnly::permitPlan( const QueryPlan &plan ) const { - return !plan.willScanTable(); - } - QueryPlanSelectionPolicy::IndexOnly QueryPlanSelectionPolicy::__indexOnly; - const QueryPlanSelectionPolicy &QueryPlanSelectionPolicy::indexOnly() { return __indexOnly; } - - bool QueryPlanSelectionPolicy::IdElseNatural::permitPlan( const QueryPlan &plan ) const { - return !plan.indexed() || plan.index()->isIdIndex(); - } - BSONObj QueryPlanSelectionPolicy::IdElseNatural::planHint( const StringData& ns ) const { - NamespaceDetails *nsd = nsdetails( ns ); - if ( !nsd || !nsd->haveIdIndex() ) { - return BSON( "$hint" << BSON( "$natural" << 1 ) ); - } - return BSON( "$hint" << nsd->idx( nsd->findIdIndex() ).indexName() ); - } - QueryPlanSelectionPolicy::IdElseNatural QueryPlanSelectionPolicy::__idElseNatural; - const QueryPlanSelectionPolicy &QueryPlanSelectionPolicy::idElseNatural() { - return __idElseNatural; - } - - QueryOptimizerCursorImpl* QueryOptimizerCursorImpl::make - ( auto_ptr& mps, - const QueryPlanSelectionPolicy& planPolicy, - bool requireOrder, - bool explain ) { - auto_ptr ret( new QueryOptimizerCursorImpl( mps, planPolicy, - requireOrder ) ); - ret->init( explain ); - return ret.release(); - } - - bool QueryOptimizerCursorImpl::ok() { - return _takeover ? _takeover->ok() : !currLoc().isNull(); - } - - Record* QueryOptimizerCursorImpl::_current() { - if ( _takeover ) { - return _takeover->_current(); - } - assertOk(); - return currLoc().rec(); - } - - BSONObj QueryOptimizerCursorImpl::current() { - if ( _takeover ) { - return _takeover->current(); - } - assertOk(); - return currLoc().obj(); - } - - DiskLoc QueryOptimizerCursorImpl::currLoc() { - return _takeover ? _takeover->currLoc() : _currLoc(); - } - - DiskLoc QueryOptimizerCursorImpl::_currLoc() const { - dassert( !_takeover ); - return _currRunner ? _currRunner->currLoc() : DiskLoc(); - } - - bool QueryOptimizerCursorImpl::advance() { - return _advance( false ); - } - - BSONObj QueryOptimizerCursorImpl::currKey() const { - if ( _takeover ) { - return _takeover->currKey(); - } - assertOk(); - return _currRunner->currKey(); - } - - DiskLoc QueryOptimizerCursorImpl::refLoc() { - return _takeover ? _takeover->refLoc() : DiskLoc(); - } - - BSONObj QueryOptimizerCursorImpl::indexKeyPattern() { - if ( _takeover ) { - return _takeover->indexKeyPattern(); - } - assertOk(); - return _currRunner->cursor()->indexKeyPattern(); - } - - void QueryOptimizerCursorImpl::prepareToTouchEarlierIterate() { - if ( _takeover ) { - _takeover->prepareToTouchEarlierIterate(); - } - else if ( _currRunner ) { - if ( _mps->currentNPlans() == 1 ) { - // This single plan version is a bit more performant, so we use it when possible. - _currRunner->prepareToTouchEarlierIterate(); - } - else { - // With multiple plans, the 'earlier iterate' could be the current iterate of one of - // the component plans. We do a full yield of all plans, using ClientCursors. - _mps->prepareToYield(); - } - } - } - - void QueryOptimizerCursorImpl::recoverFromTouchingEarlierIterate() { - if ( _takeover ) { - _takeover->recoverFromTouchingEarlierIterate(); - } - else if ( _currRunner ) { - if ( _mps->currentNPlans() == 1 ) { - _currRunner->recoverFromTouchingEarlierIterate(); - } - else { - recoverFromYield(); - } - } - } - - void QueryOptimizerCursorImpl::prepareToYield() { - if ( _takeover ) { - _takeover->prepareToYield(); - } - else if ( _currRunner ) { - _mps->prepareToYield(); - } - } - - void QueryOptimizerCursorImpl::recoverFromYield() { - if ( _takeover ) { - _takeover->recoverFromYield(); - return; - } - if ( _currRunner ) { - _mps->recoverFromYield(); - if ( _currRunner->error() || !ok() ) { - // Advance to a non error op if one of the ops errored out. - // Advance to a following $or clause if the $or clause returned all results. - verify( !_mps->doneRunners() ); - _advance( true ); - } - } - } - - bool QueryOptimizerCursorImpl::getsetdup(DiskLoc loc) { - if ( _takeover ) { - if ( getdupInternal( loc ) ) { - return true; - } - return _takeover->getsetdup( loc ); - } - assertOk(); - return getsetdupInternal( loc ); - } - - bool QueryOptimizerCursorImpl::isMultiKey() const { - if ( _takeover ) { - return _takeover->isMultiKey(); - } - assertOk(); - return _currRunner->cursor()->isMultiKey(); - } - - bool QueryOptimizerCursorImpl::capped() const { - // Initial capped wrapping cases (before takeover) are handled internally by a component - // ClientCursor. - return _takeover ? _takeover->capped() : false; - } - - long long QueryOptimizerCursorImpl::nscanned() { - return _takeover ? _takeover->nscanned() : _nscanned; - } - - CoveredIndexMatcher* QueryOptimizerCursorImpl::matcher() const { - if ( _takeover ) { - return _takeover->matcher(); - } - assertOk(); - return _currRunner->queryPlan().matcher().get(); - } - - bool QueryOptimizerCursorImpl::currentMatches( MatchDetails* details ) { - if ( _takeover ) { - return _takeover->currentMatches( details ); - } - assertOk(); - return _currRunner->currentMatches( details ); - } - - const FieldRangeSet* QueryOptimizerCursorImpl::initialFieldRangeSet() const { - if ( _takeover ) { - return 0; - } - assertOk(); - return &_currRunner->queryPlan().multikeyFrs(); - } - - bool QueryOptimizerCursorImpl::currentPlanScanAndOrderRequired() const { - if ( _takeover ) { - return _takeover->queryPlan().scanAndOrderRequired(); - } - assertOk(); - return _currRunner->queryPlan().scanAndOrderRequired(); - } - - const Projection::KeyOnly* QueryOptimizerCursorImpl::keyFieldsOnly() const { - if ( _takeover ) { - return _takeover->keyFieldsOnly(); - } - assertOk(); - return _currRunner->keyFieldsOnly(); - } - - bool QueryOptimizerCursorImpl::runningInitialInOrderPlan() const { - if ( _takeover ) { - return false; - } - assertOk(); - return _mps->haveInOrderPlan(); - } - - bool QueryOptimizerCursorImpl::hasPossiblyExcludedPlans() const { - if ( _takeover ) { - return false; - } - assertOk(); - return _mps->hasPossiblyExcludedPlans(); - } - - void QueryOptimizerCursorImpl::clearIndexesForPatterns() { - if ( !_takeover ) { - _mps->clearIndexesForPatterns(); - } - } - - void QueryOptimizerCursorImpl::abortOutOfOrderPlans() { - _requireOrder = true; - } - - void QueryOptimizerCursorImpl::noteIterate( bool match, bool loadedDocument, bool chunkSkip ) { - if ( _explainQueryInfo ) { - _explainQueryInfo->noteIterate( match, loadedDocument, chunkSkip ); - } - if ( _takeover ) { - _takeover->noteIterate( match, loadedDocument ); - } - } - - void QueryOptimizerCursorImpl::noteYield() { - if ( _explainQueryInfo ) { - _explainQueryInfo->noteYield(); - } - } - - QueryOptimizerCursorImpl::QueryOptimizerCursorImpl( auto_ptr& mps, - const QueryPlanSelectionPolicy& planPolicy, - bool requireOrder ) : - _requireOrder( requireOrder ), - _mps( mps ), - _initialCandidatePlans( _mps->possibleInOrderPlan(), _mps->possibleOutOfOrderPlan() ), - _originalRunner( new QueryPlanRunner( _nscanned, - planPolicy, - _requireOrder, - !_initialCandidatePlans.hybridPlanSet() ) ), - _currRunner(), - _completePlanOfHybridSetScanAndOrderRequired(), - _nscanned() { - } - - void QueryOptimizerCursorImpl::init( bool explain ) { - _mps->initialRunner( _originalRunner ); - if ( explain ) { - _explainQueryInfo = _mps->generateExplainInfo(); - } - shared_ptr runner = _mps->nextRunner(); - rethrowOnError( runner ); - if ( !runner->complete() ) { - _currRunner = runner.get(); - } - } - - bool QueryOptimizerCursorImpl::_advance( bool force ) { - if ( _takeover ) { - return _takeover->advance(); - } - - if ( !force && !ok() ) { - return false; - } - - _currRunner = 0; - shared_ptr runner = _mps->nextRunner(); - rethrowOnError( runner ); - - if ( !runner->complete() ) { - // The 'runner' will be valid until we call _mps->nextOp() again. We return 'current' - // values from this op. - _currRunner = runner.get(); - } - else if ( runner->stopRequested() ) { - if ( runner->cursor() ) { - _takeover.reset( new MultiCursor( _mps, - runner->cursor(), - runner->queryPlan().matcher(), - runner->explainInfo(), - *runner, - _nscanned - runner->cursor()->nscanned() ) ); - } - } - else { - if ( _initialCandidatePlans.hybridPlanSet() ) { - _completePlanOfHybridSetScanAndOrderRequired = - runner->queryPlan().scanAndOrderRequired(); - } - } - - return ok(); - } - - /** Forward an exception when the runner errs out. */ - void QueryOptimizerCursorImpl::rethrowOnError( const shared_ptr< QueryPlanRunner > &runner ) { - if ( runner->error() ) { - throw MsgAssertionException( runner->exception() ); - } - } - - bool QueryOptimizerCursorImpl::getsetdupInternal(const DiskLoc &loc) { - return _dups.getsetdup( loc ); - } - - bool QueryOptimizerCursorImpl::getdupInternal(const DiskLoc &loc) { - dassert( _takeover ); - return _dups.getdup( loc ); - } - - shared_ptr newQueryOptimizerCursor( auto_ptr mps, - const QueryPlanSelectionPolicy &planPolicy, - bool requireOrder, bool explain ) { - try { - shared_ptr ret - ( QueryOptimizerCursorImpl::make( mps, planPolicy, requireOrder, explain ) ); - return ret; - } catch( const AssertionException &e ) { - if ( e.getCode() == OutOfOrderDocumentsAssertionCode ) { - // If no indexes follow the requested sort order, return an - // empty pointer. This is legacy behavior based on bestGuessCursor(). - return shared_ptr(); - } - throw; - } - return shared_ptr(); - } - - shared_ptr - NamespaceDetailsTransient::getCursor( const StringData &ns, - const BSONObj &query, - const BSONObj &order, - const QueryPlanSelectionPolicy &planPolicy, - const shared_ptr &parsedQuery, - bool requireOrder, - QueryPlanSummary *singlePlanSummary ) { - - CursorGenerator generator( ns, - query, - order, - planPolicy, - parsedQuery, - requireOrder, - singlePlanSummary ); - return generator.generate(); - } - - CursorGenerator::CursorGenerator( const StringData &ns, - const BSONObj &query, - const BSONObj &order, - const QueryPlanSelectionPolicy &planPolicy, - const shared_ptr &parsedQuery, - bool requireOrder, - QueryPlanSummary *singlePlanSummary ) : - _ns( ns ), - _query( query ), - _order( order ), - _planPolicy( planPolicy ), - _parsedQuery( parsedQuery ), - _requireOrder( requireOrder ), - _singlePlanSummary( singlePlanSummary ) { - // Initialize optional return variables. - if ( _singlePlanSummary ) { - *_singlePlanSummary = QueryPlanSummary(); - } - } - - void CursorGenerator::setArgumentsHint() { - if ( useHints && _parsedQuery ) { - _argumentsHint = _parsedQuery->getHint(); - } - - if ( snapshot() ) { - NamespaceDetails *d = nsdetails( _ns ); - if ( d ) { - int i = d->findIdIndex(); - if( i < 0 ) { - if ( _ns.find( ".system." ) == string::npos ) - log() << "warning: no _id index on $snapshot query, ns:" << _ns << endl; - } - else { - /* [dm] the name of an _id index tends to vary, so we build the hint the hard - way here. probably need a better way to specify "use the _id index" as a hint. - if someone is in the query optimizer please fix this then! - */ - _argumentsHint = BSON( "$hint" << d->idx(i).indexName() ); - } - } - } - } - - shared_ptr CursorGenerator::shortcutCursor() const { - if ( !mayShortcutQueryOptimizer() ) { - return shared_ptr(); - } - - if ( _planPolicy.permitOptimalNaturalPlan() && _query.isEmpty() && _order.isEmpty() ) { - return theDataFileMgr.findAll( _ns ); - } - if ( _planPolicy.permitOptimalIdPlan() && isSimpleIdQuery( _query ) ) { - Database *database = cc().database(); - verify( database ); - NamespaceDetails *d = database->namespaceIndex.details( _ns ); - if ( d ) { - int idxNo = d->findIdIndex(); - if ( idxNo >= 0 ) { - IndexDetails& i = d->idx( idxNo ); - BSONObj key = i.getKeyFromQuery( _query ); - return shared_ptr( BtreeCursor::make( d, i, key, key, true, 1 ) ); - } - } - } - - return shared_ptr(); - } - - void CursorGenerator::setMultiPlanScanner() { - _mps.reset( MultiPlanScanner::make( _ns, _query, _order, _parsedQuery, hint(), - explain() ? QueryPlanGenerator::Ignore : - QueryPlanGenerator::Use, - min(), max() ) ); - } - - shared_ptr CursorGenerator::singlePlanCursor() { - const QueryPlan *singlePlan = _mps->singlePlan(); - if ( !singlePlan || ( isOrderRequired() && singlePlan->scanAndOrderRequired() ) ) { - return shared_ptr(); - } - if ( !_planPolicy.permitPlan( *singlePlan ) ) { - return shared_ptr(); - } - - if ( _singlePlanSummary ) { - *_singlePlanSummary = singlePlan->summary(); - } - shared_ptr single = singlePlan->newCursor( DiskLoc(), - _planPolicy.requestIntervalCursor() ); - if ( !_query.isEmpty() && !single->matcher() ) { - - // The query plan must have a matcher. The matcher's constructor performs some aspects - // of query validation that should occur before a cursor is returned. - fassert( 16449, singlePlan->matcher() ); - - if ( // If a matcher is requested or ... - _planPolicy.requestMatcher() || - // ... the index ranges do not exactly match the query or ... - singlePlan->mayBeMatcherNecessary() || - // ... the matcher must look at the full record ... - singlePlan->matcher()->needRecord() ) { - - // ... then set the cursor's matcher to the query plan's matcher. - single->setMatcher( singlePlan->matcher() ); - } - } - if ( singlePlan->keyFieldsOnly() ) { - single->setKeyFieldsOnly( singlePlan->keyFieldsOnly() ); - } - return single; - } - - shared_ptr CursorGenerator::generate() { - - setArgumentsHint(); - shared_ptr cursor = shortcutCursor(); - if ( cursor ) { - return cursor; - } - - setMultiPlanScanner(); - cursor = singlePlanCursor(); - if ( cursor ) { - return cursor; - } - - return newQueryOptimizerCursor( _mps, _planPolicy, isOrderRequired(), explain() ); - } - - /** This interface is just available for testing. */ - shared_ptr newQueryOptimizerCursor - ( const char *ns, const BSONObj &query, const BSONObj &order, - const QueryPlanSelectionPolicy &planPolicy, bool requireOrder, - const shared_ptr &parsedQuery ) { - auto_ptr mps( MultiPlanScanner::make( ns, query, order, parsedQuery ) ); - return newQueryOptimizerCursor( mps, planPolicy, requireOrder, false ); - } - -} // namespace mongo; diff --git a/src/mongo/db/queryoptimizercursorimpl.h b/src/mongo/db/queryoptimizercursorimpl.h deleted file mode 100644 index 3f3a97d3ce0..00000000000 --- a/src/mongo/db/queryoptimizercursorimpl.h +++ /dev/null @@ -1,298 +0,0 @@ -// @file queryoptimizercursorimpl.h - A cursor interleaving multiple candidate cursors. - -/** - * Copyright (C) 2011 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "mongo/db/queryutil.h" -#include "mongo/db/queryoptimizercursor.h" -#include "mongo/db/querypattern.h" - -namespace mongo { - - class MultiCursor; - class MultiPlanScanner; - class QueryPlanRunner; - class QueryPlanSummary; - - /** Dup tracking class, optimizing one common case with small set and few initial reads. */ - class SmallDupSet { - public: - SmallDupSet() : _accesses() { - _vec.reserve( 250 ); - } - /** @return true if @param 'loc' already added to the set, false if adding to the set in this call. */ - bool getsetdup( const DiskLoc &loc ) { - access(); - return vec() ? getsetdupVec( loc ) : getsetdupSet( loc ); - } - /** @return true when @param loc in the set. */ - bool getdup( const DiskLoc &loc ) { - access(); - return vec() ? getdupVec( loc ) : getdupSet( loc ); - } - private: - void access() { - ++_accesses; - mayUpgrade(); - } - void mayUpgrade() { - if ( vec() && _accesses > 500 ) { - _set.insert( _vec.begin(), _vec.end() ); - } - } - bool vec() const { - return _set.size() == 0; - } - bool getsetdupVec( const DiskLoc &loc ) { - if ( getdupVec( loc ) ) { - return true; - } - _vec.push_back( loc ); - return false; - } - bool getdupVec( const DiskLoc &loc ) const { - for( vector::const_iterator i = _vec.begin(); i != _vec.end(); ++i ) { - if ( *i == loc ) { - return true; - } - } - return false; - } - bool getsetdupSet( const DiskLoc &loc ) { - pair::iterator, bool> p = _set.insert(loc); - return !p.second; - } - bool getdupSet( const DiskLoc &loc ) { - return _set.count( loc ) > 0; - } - vector _vec; - set _set; - long long _accesses; - }; - - /** - * This cursor runs a MultiPlanScanner iteratively and returns results from - * the scanner's cursors as they become available. Once the scanner chooses - * a single plan, this cursor becomes a simple wrapper around that single - * plan's cursor (called the 'takeover' cursor). - * - * A QueryOptimizerCursor employs a delegation strategy to ensure consistency after writes - * during its initial phase when multiple delegate Cursors may be active (before _takeover is - * set). - * - * Before takeover, the return value of refLoc() will be isNull(), causing ClientCursor to - * ignore a QueryOptimizerCursor (though not its delegate Cursors) when a delete occurs. - * Requests to prepareToYield() or recoverFromYield() will be forwarded to - * prepareToYield()/recoverFromYield() on ClientCursors of delegate Cursors. If a delegate - * Cursor becomes eof() or invalid after a yield recovery, - * QueryOptimizerCursor::recoverFromYield() may advance _currRunner to another delegate Cursor. - * - * Requests to prepareToTouchEarlierIterate() or recoverFromTouchingEarlierIterate() are - * forwarded as prepareToTouchEarlierIterate()/recoverFromTouchingEarlierIterate() to the - * delegate Cursor when a single delegate Cursor is active. If multiple delegate Cursors are - * active, the advance() call preceeding prepareToTouchEarlierIterate() may not properly advance - * all delegate Cursors, so the calls are forwarded as prepareToYield()/recoverFromYield() to a - * ClientCursor for each delegate Cursor. - * - * After _takeover is set, consistency after writes is ensured by delegation to the _takeover - * MultiCursor. - */ - class QueryOptimizerCursorImpl : public QueryOptimizerCursor { - public: - static QueryOptimizerCursorImpl* make( auto_ptr& mps, - const QueryPlanSelectionPolicy& planPolicy, - bool requireOrder, - bool explain ); - - virtual bool ok(); - - virtual Record* _current(); - - virtual BSONObj current(); - - virtual DiskLoc currLoc(); - - DiskLoc _currLoc() const; - - virtual bool advance(); - - virtual BSONObj currKey() const; - - /** - * When return value isNull(), our cursor will be ignored for deletions by the ClientCursor - * implementation. In such cases, internal ClientCursors will update the positions of - * component Cursors when necessary. - * !!! Use care if changing this behavior, as some ClientCursor functionality may not work - * recursively. - */ - virtual DiskLoc refLoc(); - - virtual BSONObj indexKeyPattern(); - - virtual bool supportGetMore() { return true; } - - virtual bool supportYields() { return true; } - - virtual void prepareToTouchEarlierIterate(); - - virtual void recoverFromTouchingEarlierIterate(); - - virtual void prepareToYield(); - - virtual void recoverFromYield(); - - virtual string toString() { return "QueryOptimizerCursor"; } - - virtual bool getsetdup(DiskLoc loc); - - /** Matcher needs to know if the the cursor being forwarded to is multikey. */ - virtual bool isMultiKey() const; - - // TODO fix - virtual bool modifiedKeys() const { return true; } - - virtual bool capped() const; - - virtual long long nscanned(); - - virtual CoveredIndexMatcher *matcher() const; - - virtual bool currentMatches( MatchDetails* details = 0 ); - - virtual CandidatePlanCharacter initialCandidatePlans() const { - return _initialCandidatePlans; - } - - virtual const FieldRangeSet* initialFieldRangeSet() const; - - virtual bool currentPlanScanAndOrderRequired() const; - - virtual const Projection::KeyOnly* keyFieldsOnly() const; - - virtual bool runningInitialInOrderPlan() const; - - virtual bool hasPossiblyExcludedPlans() const; - - virtual bool completePlanOfHybridSetScanAndOrderRequired() const { - return _completePlanOfHybridSetScanAndOrderRequired; - } - - virtual void clearIndexesForPatterns(); - - virtual void abortOutOfOrderPlans(); - - virtual void noteIterate( bool match, bool loadedDocument, bool chunkSkip ); - - virtual void noteYield(); - - virtual shared_ptr explainQueryInfo() const { - return _explainQueryInfo; - } - - private: - - QueryOptimizerCursorImpl( auto_ptr& mps, - const QueryPlanSelectionPolicy& planPolicy, - bool requireOrder ); - - void init( bool explain ); - - /** - * Advances the QueryPlanSet::Runner. - * @param force - advance even if the current query op is not valid. The 'force' param should only be specified - * when there are plans left in the runner. - */ - bool _advance( bool force ); - - /** Forward an exception when the runner errs out. */ - void rethrowOnError( const shared_ptr< QueryPlanRunner >& runner ); - - void assertOk() const { - massert( 14809, "Invalid access for cursor that is not ok()", !_currLoc().isNull() ); - } - - /** Insert and check for dups before takeover occurs */ - bool getsetdupInternal(const DiskLoc& loc); - - /** Just check for dups - after takeover occurs */ - bool getdupInternal(const DiskLoc& loc); - - bool _requireOrder; - auto_ptr _mps; - CandidatePlanCharacter _initialCandidatePlans; - shared_ptr _originalRunner; - QueryPlanRunner* _currRunner; - bool _completePlanOfHybridSetScanAndOrderRequired; - shared_ptr _takeover; - long long _nscanned; - // Using a SmallDupSet seems a bit hokey, but I've measured a 5% performance improvement - // with ~100 document non multi key scans. - SmallDupSet _dups; - shared_ptr _explainQueryInfo; - }; - - /** - * Helper class for generating a simple Cursor or QueryOptimizerCursor from a set of query - * parameters. This class was refactored from a single function call and is not expected to - * outlive its constructor arguments. - */ - class CursorGenerator { - public: - CursorGenerator( const StringData& ns, - const BSONObj &query, - const BSONObj &order, - const QueryPlanSelectionPolicy &planPolicy, - const shared_ptr &parsedQuery, - bool requireOrder, - QueryPlanSummary *singlePlanSummary ); - - shared_ptr generate(); - - private: - bool snapshot() const { return _parsedQuery && _parsedQuery->isSnapshot(); } - bool explain() const { return _parsedQuery && _parsedQuery->isExplain(); } - BSONObj min() const { return _parsedQuery ? _parsedQuery->getMin() : BSONObj(); } - BSONObj max() const { return _parsedQuery ? _parsedQuery->getMax() : BSONObj(); } - bool hasFields() const { return _parsedQuery && _parsedQuery->getFieldPtr(); } - - bool isOrderRequired() const { return _requireOrder; } - bool mayShortcutQueryOptimizer() const { - return min().isEmpty() && max().isEmpty() && !hasFields() && _argumentsHint.isEmpty(); - } - BSONObj hint() const { - return _argumentsHint.isEmpty() ? _planPolicy.planHint( _ns ) : _argumentsHint; - } - - void setArgumentsHint(); - shared_ptr shortcutCursor() const; - void setMultiPlanScanner(); - shared_ptr singlePlanCursor(); - - const StringData _ns; - BSONObj _query; - BSONObj _order; - const QueryPlanSelectionPolicy &_planPolicy; - shared_ptr _parsedQuery; - bool _requireOrder; - QueryPlanSummary *_singlePlanSummary; - - BSONObj _argumentsHint; - auto_ptr _mps; - }; - -} // namespace mongo diff --git a/src/mongo/db/querypattern.cpp b/src/mongo/db/querypattern.cpp deleted file mode 100644 index 8abdc8c281a..00000000000 --- a/src/mongo/db/querypattern.cpp +++ /dev/null @@ -1,117 +0,0 @@ -// @file querypattern.cpp - Query pattern matching for selecting similar plans given similar queries. - -/* Copyright 2011 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "querypattern.h" -#include "mongo/db/queryutil.h" - -namespace mongo { - - QueryPattern::QueryPattern( const FieldRangeSet &frs, const BSONObj &sort ) { - for( map::const_iterator i = frs.ranges().begin(); i != frs.ranges().end(); ++i ) { - if ( i->second.equality() ) { - _fieldTypes[ i->first ] = QueryPattern::Equality; - } - else if ( i->second.empty() ) { - _fieldTypes[ i->first ] = QueryPattern::Empty; - } - else if ( !i->second.universal() ) { - bool upper = i->second.max().type() != MaxKey; - bool lower = i->second.min().type() != MinKey; - if ( upper && lower ) { - _fieldTypes[ i->first ] = QueryPattern::UpperAndLowerBound; - } - else if ( upper ) { - _fieldTypes[ i->first ] = QueryPattern::UpperBound; - } - else if ( lower ) { - _fieldTypes[ i->first ] = QueryPattern::LowerBound; - } - else { - _fieldTypes[ i->first ] = QueryPattern::ConstraintPresent; - } - } - } - setSort( sort ); - } - - /** for testing only - speed unimportant */ - bool QueryPattern::operator==( const QueryPattern &other ) const { - bool less = operator<( other ); - bool more = other.operator<( *this ); - verify( !( less && more ) ); - return !( less || more ); - } - - /** for testing only - speed unimportant */ - bool QueryPattern::operator!=( const QueryPattern &other ) const { - return !operator==( other ); - } - - string typeToString( enum QueryPattern::Type t ) { - switch (t) { - case QueryPattern::Empty: - return "Empty"; - case QueryPattern::Equality: - return "Equality"; - case QueryPattern::LowerBound: - return "LowerBound"; - case QueryPattern::UpperBound: - return "UpperBound"; - case QueryPattern::UpperAndLowerBound: - return "UpperAndLowerBound"; - case QueryPattern::ConstraintPresent: - return "ConstraintPresent"; - } - return ""; - } - - string QueryPattern::toString() const { - BSONObjBuilder b; - for( map::const_iterator i = _fieldTypes.begin(); i != _fieldTypes.end(); ++i ) { - b << i->first << typeToString( i->second ); - } - return BSON( "query" << b.done() << "sort" << _sort ).toString(); - } - - void QueryPattern::setSort( const BSONObj sort ) { - _sort = normalizeSort( sort ); - } - - BSONObj QueryPattern::normalizeSort( const BSONObj &spec ) { - if ( spec.isEmpty() ) - return spec; - int direction = ( spec.firstElement().number() >= 0 ) ? 1 : -1; - BSONObjIterator i( spec ); - BSONObjBuilder b; - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - b.append( e.fieldName(), direction * ( ( e.number() >= 0 ) ? -1 : 1 ) ); - } - return b.obj(); - } - - CachedQueryPlan::CachedQueryPlan( const BSONObj &indexKey, long long nScanned, - CandidatePlanCharacter planCharacter ) : - _indexKey( indexKey ), - _nScanned( nScanned ), - _planCharacter( planCharacter ) { - } - - -} // namespace mongo diff --git a/src/mongo/db/querypattern.h b/src/mongo/db/querypattern.h deleted file mode 100644 index 3c7ad07fa9b..00000000000 --- a/src/mongo/db/querypattern.h +++ /dev/null @@ -1,118 +0,0 @@ -// @file querypattern.h - Query pattern matching for selecting similar plans given similar queries. - -/* Copyright 2011 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "jsobj.h" - -namespace mongo { - - class FieldRangeSet; - - /** - * Implements query pattern matching, used to determine if a query is - * similar to an earlier query and should use the same plan. - * - * Two queries will generate the same QueryPattern, and therefore match each - * other, if their fields have the same Types and they have the same sort - * spec. - */ - class QueryPattern { - public: - QueryPattern( const FieldRangeSet &frs, const BSONObj &sort ); - enum Type { - Empty, - Equality, - LowerBound, - UpperBound, - UpperAndLowerBound, - ConstraintPresent - }; - bool operator<( const QueryPattern &other ) const; - /** for testing only */ - bool operator==( const QueryPattern &other ) const; - /** for testing only */ - bool operator!=( const QueryPattern &other ) const; - /** for development / debugging */ - string toString() const; - private: - void setSort( const BSONObj sort ); - static BSONObj normalizeSort( const BSONObj &spec ); - map _fieldTypes; - BSONObj _sort; - }; - - /** Summarizes the candidate plans that may run for a query. */ - class CandidatePlanCharacter { - public: - CandidatePlanCharacter( bool mayRunInOrderPlan, bool mayRunOutOfOrderPlan ) : - _mayRunInOrderPlan( mayRunInOrderPlan ), - _mayRunOutOfOrderPlan( mayRunOutOfOrderPlan ) { - } - CandidatePlanCharacter() : - _mayRunInOrderPlan(), - _mayRunOutOfOrderPlan() { - } - bool mayRunInOrderPlan() const { return _mayRunInOrderPlan; } - bool mayRunOutOfOrderPlan() const { return _mayRunOutOfOrderPlan; } - bool valid() const { return mayRunInOrderPlan() || mayRunOutOfOrderPlan(); } - bool hybridPlanSet() const { return mayRunInOrderPlan() && mayRunOutOfOrderPlan(); } - private: - bool _mayRunInOrderPlan; - bool _mayRunOutOfOrderPlan; - }; - - /** Information about a query plan that ran successfully for a QueryPattern. */ - class CachedQueryPlan { - public: - CachedQueryPlan() : - _nScanned() { - } - CachedQueryPlan( const BSONObj &indexKey, long long nScanned, - CandidatePlanCharacter planCharacter ); - BSONObj indexKey() const { return _indexKey; } - long long nScanned() const { return _nScanned; } - CandidatePlanCharacter planCharacter() const { return _planCharacter; } - private: - BSONObj _indexKey; - long long _nScanned; - CandidatePlanCharacter _planCharacter; - }; - - inline bool QueryPattern::operator<( const QueryPattern &other ) const { - map::const_iterator i = _fieldTypes.begin(); - map::const_iterator j = other._fieldTypes.begin(); - while( i != _fieldTypes.end() ) { - if ( j == other._fieldTypes.end() ) - return false; - if ( i->first < j->first ) - return true; - else if ( i->first > j->first ) - return false; - if ( i->second < j->second ) - return true; - else if ( i->second > j->second ) - return false; - ++i; - ++j; - } - if ( j != other._fieldTypes.end() ) - return true; - return _sort.woCompare( other._sort ) < 0; - } - -} // namespace mongo diff --git a/src/mongo/db/queryutil.cpp b/src/mongo/db/queryutil.cpp index dd8a16060f7..773833d78ae 100644 --- a/src/mongo/db/queryutil.cpp +++ b/src/mongo/db/queryutil.cpp @@ -1,5 +1,3 @@ -// @file queryutil.cpp - /* Copyright 2009 10gen Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,22 +13,17 @@ * limitations under the License. */ -#include "pch.h" +#include "mongo/pch.h" #include "mongo/db/queryutil.h" -#include "pdfile.h" -#include "../util/startup_test.h" -#include "dbmessage.h" -#include "../util/mongoutils/str.h" + +#include "mongo/db/index_names.h" +#include "mongo/db/matcher.h" +#include "mongo/db/pdfile.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { - ParsedQuery::ParsedQuery( QueryMessage& qm ) - : _ns( qm.ns ) , _ntoskip( qm.ntoskip ) , _ntoreturn( qm.ntoreturn ) , _options( qm.queryOptions ) { - init( qm.query ); - initFields( qm.fields ); - } - extern BSONObj staticNull; extern BSONObj staticUndefined; @@ -446,15 +439,15 @@ namespace mongo { break; } case BSONObj::opWITHIN: - _special.add("2d", SpecialIndices::NO_INDEX_REQUIRED); - _special.add("2dsphere", SpecialIndices::NO_INDEX_REQUIRED); + _special.add(IndexNames::GEO_2D, SpecialIndices::NO_INDEX_REQUIRED); + _special.add(IndexNames::GEO_2DSPHERE, SpecialIndices::NO_INDEX_REQUIRED); break; case BSONObj::opNEAR: - _special.add("2d", SpecialIndices::INDEX_REQUIRED); - _special.add("2dsphere", SpecialIndices::INDEX_REQUIRED); + _special.add(IndexNames::GEO_2D, SpecialIndices::INDEX_REQUIRED); + _special.add(IndexNames::GEO_2DSPHERE, SpecialIndices::INDEX_REQUIRED); break; case BSONObj::opGEO_INTERSECTS: - _special.add("2dsphere", SpecialIndices::NO_INDEX_REQUIRED); + _special.add(IndexNames::GEO_2DSPHERE, SpecialIndices::NO_INDEX_REQUIRED); break; case BSONObj::opEXISTS: { if ( !existsSpec ) { @@ -1060,7 +1053,8 @@ namespace mongo { return; } - if ( str::equals( matchFieldName, "$atomic" ) ) { + if ( str::equals( matchFieldName, "$atomic" ) || + str::equals( matchFieldName, "$isolated" ) ) { return; } } @@ -1167,15 +1161,26 @@ namespace mongo { return true; } - FieldRangeVector::FieldRangeVector( const FieldRangeSet &frs, const IndexSpec &indexSpec, + FieldRangeVector::FieldRangeVector( const FieldRangeSet &frs, BSONObj keyPattern, int direction ) : - _indexSpec( indexSpec ), + _keyPattern(keyPattern), _direction( direction >= 0 ? 1 : -1 ), _hasAllIndexedRanges( true ) { - verify( frs.matchPossibleForIndex( _indexSpec.keyPattern ) ); + verify( frs.matchPossibleForIndex( keyPattern)); _queries = frs._queries; - BSONObjIterator i( _indexSpec.keyPattern ); + + // For key generation + BSONObjIterator it(_keyPattern); + while (it.more()) { + BSONElement elt = it.next(); + _fieldNames.push_back(elt.fieldName()); + _fixed.push_back(BSONElement()); + } + + _keyGenerator.reset(new BtreeKeyGeneratorV1(_fieldNames, _fixed, false)); + map topFieldElemMatchContexts; + BSONObjIterator i(keyPattern); while( i.more() ) { BSONElement e = i.next(); const FieldRange *range = &frs.range( e.fieldName() ); @@ -1269,7 +1274,7 @@ namespace mongo { BSONObj FieldRangeVector::startKey() const { BSONObjBuilder b; - BSONObjIterator keys( _indexSpec.keyPattern ); + BSONObjIterator keys(_keyPattern); vector::const_iterator i = _ranges.begin(); for( ; i != _ranges.end(); ++i, ++keys ) { // Append lower bounds until an exclusive bound is found. @@ -1307,7 +1312,7 @@ namespace mongo { BSONObj FieldRangeVector::endKey() const { BSONObjBuilder b; - BSONObjIterator keys( _indexSpec.keyPattern ); + BSONObjIterator keys(_keyPattern); vector::const_iterator i = _ranges.begin(); for( ; i != _ranges.end(); ++i, ++keys ) { // Append upper bounds until an exclusive bound is found. @@ -1345,7 +1350,7 @@ namespace mongo { BSONObj FieldRangeVector::obj() const { BSONObjBuilder b; - BSONObjIterator k( _indexSpec.keyPattern ); + BSONObjIterator k(_keyPattern); for( int i = 0; i < (int)_ranges.size(); ++i ) { BSONArrayBuilder a( b.subarrayStart( k.next().fieldName() ) ); for( vector::const_iterator j = _ranges[ i ].intervals().begin(); @@ -1384,10 +1389,6 @@ namespace mongo { return true; } - QueryPattern FieldRangeSet::pattern( const BSONObj &sort ) const { - return QueryPattern( *this, sort ); - } - int FieldRangeSet::numNonUniversalRanges() const { int count = 0; for( map::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i ) { @@ -1451,11 +1452,13 @@ namespace mongo { "multiKey" << _multiKey.toString() ).jsonString(); } - + void FieldRangeSetPair::assertValidIndex( const NamespaceDetails *d, int idxNo ) const { - massert( 14048, "FieldRangeSetPair invalid index specified", idxNo >= 0 && idxNo < d->nIndexes ); + massert( 14048, + "FieldRangeSetPair invalid index specified", + idxNo >= 0 && idxNo < d->getCompletedIndexCount() ); } - + const FieldRangeSet &FieldRangeSetPair::frsForIndex( const NamespaceDetails* nsd, int idxNo ) const { assertValidIndexOrNoIndex( nsd, idxNo ); if ( idxNo < 0 ) { @@ -1521,7 +1524,7 @@ namespace mongo { bool FieldRangeVector::matchesKey( const BSONObj &key ) const { BSONObjIterator j( key ); - BSONObjIterator k( _indexSpec.keyPattern ); + BSONObjIterator k(_keyPattern); for( int l = 0; l < (int)_ranges.size(); ++l ) { int number = (int) k.next().number(); bool forward = ( number >= 0 ? 1 : -1 ) * ( _direction >= 0 ? 1 : -1 ) > 0; @@ -1533,16 +1536,22 @@ namespace mongo { } bool FieldRangeVector::matches( const BSONObj &obj ) const { - bool ok = false; + BSONObjSet keys; + + /** + * Key generation by design is behind the index interface. There is an exception here + * because $or uses key generation to dedup its results. When $or is fixed to not require + * this, key generation will be removed from here. + */ + _keyGenerator->getKeys(obj, &keys); + // TODO The representation of matching keys could potentially be optimized // more for the case at hand. (For example, we can potentially consider // fields individually instead of constructing several bson objects using // multikey arrays.) But getKeys() canonically defines the key set for a // given object and for now we are using it as is. - BSONObjSet keys; - _indexSpec.getKeys( obj, keys ); for( BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i ) { if ( matchesKey( *i ) ) { ok = true; @@ -1558,8 +1567,10 @@ namespace mongo { BSONObj FieldRangeVector::firstMatch( const BSONObj &obj ) const { // NOTE Only works in forward direction. verify( _direction >= 0 ); - BSONObjSet keys( BSONObjCmp( _indexSpec.keyPattern ) ); - _indexSpec.getKeys( obj, keys ); + BSONObjCmp oc(_keyPattern); + BSONObjSet keys(oc); + // See FieldRangeVector::matches for comment on key generation. + _keyGenerator->getKeys(obj, &keys); for( BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i ) { if ( matchesKey( *i ) ) { return *i; @@ -1570,7 +1581,7 @@ namespace mongo { string FieldRangeVector::toString() const { BSONObjBuilder bob; - BSONObjIterator i( _indexSpec.keyPattern ); + BSONObjIterator i(_keyPattern); for( vector::const_iterator r = _ranges.begin(); r != _ranges.end() && i.more(); ++r ) { BSONElement e = i.next(); @@ -1592,7 +1603,7 @@ namespace mongo { // TODO optimize more SERVER-5450. int FieldRangeVectorIterator::advance( const BSONObj &curr ) { BSONObjIterator j( curr ); - BSONObjIterator o( _v._indexSpec.keyPattern ); + BSONObjIterator o( _v._keyPattern); // track first field for which we are not at the end of the valid values, // since we may need to advance from the key prefix ending with this field int latestNonEndpoint = -1; @@ -1865,76 +1876,6 @@ namespace mongo { _orSets.pop_front(); _originalOrSets.pop_front(); } - - struct SimpleRegexUnitTest : StartupTest { - void run() { - { - BSONObjBuilder b; - b.appendRegex("r", "^foo"); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "foo" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "^f?oo"); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "^fz?oo"); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "f" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "^f", ""); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "f" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "\\Af", ""); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "f" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "^f", "m"); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "\\Af", "m"); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "f" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "\\Af", "mi"); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "" ); - } - { - BSONObjBuilder b; - b.appendRegex("r", "\\Af \t\vo\n\ro \\ \\# #comment", "mx"); - BSONObj o = b.done(); - verify( simpleRegex(o.firstElement()) == "foo #" ); - } - { - verify( simpleRegex("^\\Qasdf\\E", "", NULL) == "asdf" ); - verify( simpleRegex("^\\Qasdf\\E.*", "", NULL) == "asdf" ); - verify( simpleRegex("^\\Qasdf", "", NULL) == "asdf" ); // PCRE supports this - verify( simpleRegex("^\\Qasdf\\\\E", "", NULL) == "asdf\\" ); - verify( simpleRegex("^\\Qas.*df\\E", "", NULL) == "as.*df" ); - verify( simpleRegex("^\\Qas\\Q[df\\E", "", NULL) == "as\\Q[df" ); - verify( simpleRegex("^\\Qas\\E\\\\E\\Q$df\\E", "", NULL) == "as\\E$df" ); // quoted string containing \E - } - - } - } simple_regex_unittest; - long long applySkipLimit( long long num , const BSONObj& cmd ) { BSONElement s = cmd["skip"]; diff --git a/src/mongo/db/queryutil.h b/src/mongo/db/queryutil.h index 6e5c86976b3..9dfc7a0cf3d 100644 --- a/src/mongo/db/queryutil.h +++ b/src/mongo/db/queryutil.h @@ -1,5 +1,3 @@ -// @file queryutil.h - Utility classes representing ranges of valid BSONElement values for a query. - /* Copyright 2009 10gen Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,234 +16,13 @@ #pragma once #include "jsobj.h" -#include "indexkey.h" -#include "projection.h" -#include "mongo/client/dbclientinterface.h" +#include "mongo/db/index/btree_key_generator.h" namespace mongo { - - extern const int MaxBytesToReturnToClientAtOnce; - + //maximum number of intervals produced by $in queries. static const unsigned MAX_IN_COMBINATIONS = 4000000; - /* This is for languages whose "objects" are not well ordered (JSON is well ordered). - [ { a : ... } , { b : ... } ] -> { a : ..., b : ... } - */ - inline BSONObj transformOrderFromArrayFormat(BSONObj order) { - /* note: this is slow, but that is ok as order will have very few pieces */ - BSONObjBuilder b; - char p[2] = "0"; - - while ( 1 ) { - BSONObj j = order.getObjectField(p); - if ( j.isEmpty() ) - break; - BSONElement e = j.firstElement(); - uassert( 10102 , "bad order array", !e.eoo()); - uassert( 10103 , "bad order array [2]", e.isNumber()); - b.append(e); - (*p)++; - uassert( 10104 , "too many ordering elements", *p <= '9'); - } - - return b.obj(); - } - - class QueryMessage; - - /** - * this represents a total user query - * includes fields from the query message, both possible query levels - * parses everything up front - */ - class ParsedQuery : boost::noncopyable { - public: - ParsedQuery( QueryMessage& qm ); - ParsedQuery( const char* ns , int ntoskip , int ntoreturn , int queryoptions , const BSONObj& query , const BSONObj& fields ) - : _ns( ns ) , _ntoskip( ntoskip ) , _ntoreturn( ntoreturn ) , _options( queryoptions ) { - init( query ); - initFields( fields ); - } - - const char * ns() const { return _ns; } - bool isLocalDB() const { return strncmp(_ns, "local.", 6) == 0; } - - const BSONObj& getFilter() const { return _filter; } - Projection* getFields() const { return _fields.get(); } - shared_ptr getFieldPtr() const { return _fields; } - - int getSkip() const { return _ntoskip; } - int getNumToReturn() const { return _ntoreturn; } - bool wantMore() const { return _wantMore; } - int getOptions() const { return _options; } - bool hasOption( int x ) const { return ( x & _options ) != 0; } - bool hasReadPref() const { return _hasReadPref; } - - bool isExplain() const { return _explain; } - bool isSnapshot() const { return _snapshot; } - bool returnKey() const { return _returnKey; } - bool showDiskLoc() const { return _showDiskLoc; } - - const BSONObj& getMin() const { return _min; } - const BSONObj& getMax() const { return _max; } - const BSONObj& getOrder() const { return _order; } - const BSONObj& getHint() const { return _hint; } - int getMaxScan() const { return _maxScan; } - - bool couldBeCommand() const { - /* we assume you are using findOne() for running a cmd... */ - return _ntoreturn == 1 && strstr( _ns , ".$cmd" ); - } - - bool hasIndexSpecifier() const { - return ! _hint.isEmpty() || ! _min.isEmpty() || ! _max.isEmpty(); - } - - /* if ntoreturn is zero, we return up to 101 objects. on the subsequent getmore, there - is only a size limit. The idea is that on a find() where one doesn't use much results, - we don't return much, but once getmore kicks in, we start pushing significant quantities. - - The n limit (vs. size) is important when someone fetches only one small field from big - objects, which causes massive scanning server-side. - */ - bool enoughForFirstBatch( int n , int len ) const { - if ( _ntoreturn == 0 ) - return ( len > 1024 * 1024 ) || n >= 101; - return n >= _ntoreturn || len > MaxBytesToReturnToClientAtOnce; - } - - bool enough( int n ) const { - if ( _ntoreturn == 0 ) - return false; - return n >= _ntoreturn; - } - - bool enoughForExplain( long long n ) const { - if ( _wantMore || _ntoreturn == 0 ) { - return false; - } - return n >= _ntoreturn; - } - - private: - void init( const BSONObj& q ) { - _reset(); - uassert( 10105 , "bad skip value in query", _ntoskip >= 0); - - if ( _ntoreturn < 0 ) { - /* _ntoreturn greater than zero is simply a hint on how many objects to send back per - "cursor batch". - A negative number indicates a hard limit. - */ - _wantMore = false; - _ntoreturn = -_ntoreturn; - } - - - BSONElement e = q["query"]; - if ( ! e.isABSONObj() ) - e = q["$query"]; - - if ( e.isABSONObj() ) { - _filter = e.embeddedObject(); - _initTop( q ); - } - else { - _filter = q; - } - - _filter = _filter.getOwned(); - - _hasReadPref = q.hasField(Query::ReadPrefField.name()); - } - - void _reset() { - _wantMore = true; - _explain = false; - _snapshot = false; - _returnKey = false; - _showDiskLoc = false; - _maxScan = 0; - } - - void _initTop( const BSONObj& top ) { - BSONObjIterator i( top ); - while ( i.more() ) { - BSONElement e = i.next(); - const char * name = e.fieldName(); - - if ( strcmp( "$orderby" , name ) == 0 || - strcmp( "orderby" , name ) == 0 ) { - if ( e.type() == Object ) { - _order = e.embeddedObject(); - } - else if ( e.type() == Array ) { - _order = transformOrderFromArrayFormat( _order ); - } - else { - uasserted(13513, "sort must be an object or array"); - } - continue; - } - - if( *name == '$' ) { - name++; - if ( strcmp( "explain" , name ) == 0 ) - _explain = e.trueValue(); - else if ( strcmp( "snapshot" , name ) == 0 ) - _snapshot = e.trueValue(); - else if ( strcmp( "min" , name ) == 0 ) - _min = e.embeddedObject(); - else if ( strcmp( "max" , name ) == 0 ) - _max = e.embeddedObject(); - else if ( strcmp( "hint" , name ) == 0 ) - _hint = e.wrap(); - else if ( strcmp( "returnKey" , name ) == 0 ) - _returnKey = e.trueValue(); - else if ( strcmp( "maxScan" , name ) == 0 ) - _maxScan = e.numberInt(); - else if ( strcmp( "showDiskLoc" , name ) == 0 ) - _showDiskLoc = e.trueValue(); - else if ( strcmp( "comment" , name ) == 0 ) { - ; // no-op - } - } - } - - if ( _snapshot ) { - uassert( 12001 , "E12001 can't sort with $snapshot", _order.isEmpty() ); - uassert( 12002 , "E12002 can't use hint with $snapshot", _hint.isEmpty() ); - } - - } - - void initFields( const BSONObj& fields ) { - if ( fields.isEmpty() ) - return; - _fields.reset( new Projection() ); - _fields->init( fields.getOwned() ); - } - - const char * const _ns; - const int _ntoskip; - int _ntoreturn; - BSONObj _filter; - BSONObj _order; - const int _options; - shared_ptr< Projection > _fields; - bool _wantMore; - bool _explain; - bool _snapshot; - bool _returnKey; - bool _showDiskLoc; - bool _hasReadPref; - BSONObj _min; - BSONObj _max; - BSONObj _hint; - int _maxScan; - }; - /** * One side of an interval of BSONElements, defined by a value and a boolean indicating if the * interval includes the value. @@ -435,8 +212,6 @@ namespace mongo { // element having field name '$elemMatch'. }; - class QueryPattern; - /** * A set of FieldRanges determined from constraints on the fields of a query, * that may be used to determine index bounds. @@ -502,7 +277,7 @@ namespace mongo { const char *ns() const { return _ns.c_str(); } - QueryPattern pattern( const BSONObj &sort = BSONObj() ) const; + // QueryPattern pattern( const BSONObj &sort = BSONObj() ) const; SpecialIndices getSpecial() const; /** @@ -644,8 +419,6 @@ namespace mongo { friend struct QueryUtilIndexed; }; - class IndexSpec; - /** * An ordered list of fields and their FieldRanges, corresponding to valid * index keys for a given index spec. @@ -658,7 +431,7 @@ namespace mongo { * @param indexSpec The index spec (key pattern and info) * @param direction The direction of index traversal */ - FieldRangeVector( const FieldRangeSet &frs, const IndexSpec &indexSpec, int direction ); + FieldRangeVector( const FieldRangeSet &frs, BSONObj keyPattern, int direction ); /** * Methods for identifying compound start and end btree bounds describing this field range @@ -724,8 +497,6 @@ namespace mongo { /** @return a client readable representation of 'this' */ BSONObj obj() const; - const IndexSpec& getSpec(){ return _indexSpec; } - /** * @return true iff the provided document matches valid ranges on all * of this FieldRangeVector's fields, which is the case iff this document @@ -766,11 +537,16 @@ namespace mongo { int matchingLowElement( const BSONElement &e, int i, bool direction, bool &lowEquality ) const; bool matchesElement( const BSONElement &e, int i, bool direction ) const; vector _ranges; - const IndexSpec _indexSpec; + BSONObj _keyPattern; int _direction; vector _queries; // make sure mem owned bool _hasAllIndexedRanges; friend class FieldRangeVectorIterator; + + vector _fieldNames; + vector _fixed; + // See FieldRangeVector::matches for comment on key generation. + scoped_ptr _keyGenerator; }; /** diff --git a/src/mongo/db/range_deleter.cpp b/src/mongo/db/range_deleter.cpp new file mode 100644 index 00000000000..e534068e217 --- /dev/null +++ b/src/mongo/db/range_deleter.cpp @@ -0,0 +1,623 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/range_deleter.h" + +#include +#include + +#include "mongo/s/range_arithmetic.h" +#include "mongo/db/range_deleter_stats.h" +#include "mongo/util/concurrency/synchronization.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/time_support.h" + +using std::auto_ptr; +using std::set; +using std::pair; +using std::string; + +using mongoutils::str::stream; + +namespace { + + const long int NotEmptyTimeoutMillis = 200; + const long long int MaxCurorCheckIntervalMillis = 500; + + /** + * Removes an element from the container that holds a pointer type, and deletes the + * pointer as well. Returns true if the element was found. + */ + template + bool deletePtrElement(ContainerType* container, ContainerElementType elem) { + typename ContainerType::iterator iter = container->find(elem); + + if (iter == container->end()) { + return false; + } + + delete *iter; + container->erase(iter); + return true; + } +} + +namespace mongo { + + namespace duration = boost::posix_time; + + struct RangeDeleter::RangeDeleteEntry { + RangeDeleteEntry(): + secondaryThrottle(true), + queueStartTS(0), + lastLoggedTS(0), + notifyDone(NULL) { + } + + std::string ns; + + // Inclusive lower range. + BSONObj min; + + // Exclusive upper range. + BSONObj max; + + // The key pattern of the index the range refers to. + // This is relevant especially with special indexes types + // like hash indexes. + BSONObj shardKeyPattern; + + bool secondaryThrottle; + + // Sets of cursors to wait to close until this can be ready + // for deletion. + std::set cursorsToWait; + + // Time we started waiting for cursors to disappear + Date_t queueStartTS; + // Time we last reported the cursors + Date_t lastLoggedTS; + + // Not owned here. + // Important invariant: Can only be set and used by one thread. + Notification* notifyDone; + + // For debugging only + BSONObj toBSON() const { + return BSON("ns" << ns + << "min" << min + << "max" << max + << "notifyDoneAddr" << reinterpret_cast(notifyDone)); + } + }; + + struct RangeDeleter::NSMinMax { + NSMinMax(std::string ns, const BSONObj min, const BSONObj max): + ns(ns), min(min), max(max) { + } + + std::string ns; + + // Inclusive lower range. + BSONObj min; + + // Exclusive upper range. + BSONObj max; + }; + + bool RangeDeleter::NSMinMaxCmp::operator()( + const NSMinMax* lhs, const NSMinMax* rhs) const { + const int nsComp = lhs->ns.compare(rhs->ns); + + if (nsComp < 0) { + return true; + } + + if (nsComp > 0) { + return false; + } + + return compareRanges(lhs->min, lhs->max, rhs->min, rhs->max) < 0; + } + + RangeDeleter::RangeDeleter(RangeDeleterEnv* env): + _env(env), // ownership xfer + _stopMutex("stopRangeDeleter"), + _stopRequested(false), + _queueMutex("RangeDeleter"), + _stats(new RangeDeleterStats(&_queueMutex)) { + } + + RangeDeleter::~RangeDeleter() { + for(TaskList::iterator it = _notReadyQueue.begin(); + it != _notReadyQueue.end(); + ++it) { + delete (*it); + } + + for(TaskList::iterator it = _taskQueue.begin(); + it != _taskQueue.end(); + ++it) { + delete (*it); + } + + for(NSMinMaxSet::iterator it = _deleteSet.begin(); + it != _deleteSet.end(); + ++it) { + delete (*it); + } + + for(NSMinMaxSet::iterator it = _blackList.begin(); + it != _blackList.end(); + ++it) { + delete (*it); + } + } + + void RangeDeleter::startWorkers() { + if (!_worker) { + _worker.reset(new boost::thread(boost::bind(&RangeDeleter::doWork, this))); + } + } + + void RangeDeleter::stopWorkers() { + { + scoped_lock sl(_stopMutex); + _stopRequested = true; + } + + if (_worker) { + _worker->join(); + } + + scoped_lock sl(_queueMutex); + while (_stats->hasInProgress_inlock()) { + _nothingInProgressCV.wait(sl.boost()); + } + } + + /** + * Logs a periodic "waiting for cursors" message after a predefined time threshold. + * Returns the new time of the last log message. + */ + static Date_t logCursorsWaiting(const std::string& ns, + const mongo::BSONObj& min, + const mongo::BSONObj& max, + const std::set& cursorsToWait, + Date_t queueStartTS, + Date_t lastLoggedTS) { + + // We always log the first cursors waiting message (so we have cursor ids in the logs). + // After 15 minutes (the cursor timeout period), we start logging additional messages at + // a 1 minute interval. + static const long long kLogCursorsThresholdMillis = 15 * 60 * 1000; + static const long long kLogCursorsIntervalMillis = 1 * 60 * 1000; + + if (cursorsToWait.empty()) + return false; + + Date_t currentTime = jsTime(); + long long elapsedMillisSinceQueued = 0; + + // We always log the first message when lastLoggedTS == 0 + if (lastLoggedTS != 0) { + + if (currentTime > queueStartTS) + elapsedMillisSinceQueued = currentTime - queueStartTS; + + if (elapsedMillisSinceQueued < kLogCursorsThresholdMillis) + return lastLoggedTS; + + long long elapsedMillisSinceLog = 0; + if (currentTime > lastLoggedTS) + elapsedMillisSinceLog = currentTime - lastLoggedTS; + + if (elapsedMillisSinceLog < kLogCursorsIntervalMillis) + return lastLoggedTS; + } + + mongo::StringBuilder cursorList; + for (std::set::const_iterator it = cursorsToWait.begin(); + it != cursorsToWait.end(); ++it) { + cursorList << *it << " "; + } + + mongo::log() << "rangeDeleter waiting for open cursors in: " << ns + << ", min: " << min << ", max: " << max + << (lastLoggedTS == 0 ? string("") : + string(stream() << ", elapsedSecs: " << elapsedMillisSinceQueued / 1000)) + << ", cursors: [ " << cursorList.str() << "]"; + + return currentTime; + } + + bool RangeDeleter::queueDelete(const std::string& ns, + const BSONObj& min, + const BSONObj& max, + const BSONObj& shardKeyPattern, + bool secondaryThrottle, + Notification* notifyDone, + std::string* errMsg) { + string dummy; + if (errMsg == NULL) errMsg = &dummy; + + auto_ptr toDelete(new RangeDeleteEntry); + toDelete->ns = ns; + toDelete->min = min.getOwned(); + toDelete->max = max.getOwned(); + toDelete->shardKeyPattern = shardKeyPattern.getOwned(); + toDelete->secondaryThrottle = secondaryThrottle; + toDelete->notifyDone = notifyDone; + + { + scoped_lock sl(_queueMutex); + if (_stopRequested) { + *errMsg = "deleter is already stopped."; + return false; + } + + if (!canEnqueue_inlock(ns, min, max, errMsg)) { + return false; + } + + _deleteSet.insert(new NSMinMax(ns, min, max)); + _stats->incTotalDeletes_inlock(); + _stats->incPendingDeletes_inlock(); + } + + _env->getCursorIds(ns, &toDelete->cursorsToWait); + + // Log first waiting message (if we have cursors to wait for) + toDelete->lastLoggedTS = logCursorsWaiting(toDelete->ns, toDelete->min, toDelete->max, + toDelete->cursorsToWait, 0, 0); + toDelete->queueStartTS = toDelete->lastLoggedTS; + + { + scoped_lock sl(_queueMutex); + + if (toDelete->cursorsToWait.empty()) { + _taskQueue.push_back(toDelete.release()); + _taskQueueNotEmptyCV.notify_one(); + } + else { + _notReadyQueue.push_back(toDelete.release()); + } + } + + return true; + } + + bool RangeDeleter::deleteNow(const std::string& ns, + const BSONObj& min, + const BSONObj& max, + const BSONObj& shardKeyPattern, + bool secondaryThrottle, + string* errMsg) { + if (stopRequested()) { + *errMsg = "deleter is already stopped."; + return false; + } + + string dummy; + if (errMsg == NULL) errMsg = &dummy; + + NSMinMax deleteRange(ns, min, max); + { + scoped_lock sl(_queueMutex); + if (!canEnqueue_inlock(ns, min, max, errMsg)) { + return false; + } + + _deleteSet.insert(&deleteRange); + _stats->incTotalDeletes_inlock(); + + // Note: count for pending deletes is an integral part of the shutdown story. + // Therefore, to simplify things, there is no "pending" state for deletes in + // deleteNow, the state transition is simply inProgress -> done. + _stats->incInProgressDeletes_inlock(); + } + + set cursorsToWait; + _env->getCursorIds(ns, &cursorsToWait); + + long long checkIntervalMillis = 5; + + // Log first waiting message (if we have cursors to wait for) + Date_t lastLoggedTS = logCursorsWaiting(ns, min, max, cursorsToWait, 0, 0); + Date_t queueStartTS = lastLoggedTS; + + while (!cursorsToWait.empty()) { + + // Log waiting for cursor messages + lastLoggedTS = logCursorsWaiting(ns, min, max, cursorsToWait, + queueStartTS, lastLoggedTS); + + set cursorsNow; + _env->getCursorIds(ns, &cursorsNow); + + set cursorsLeft; + std::set_intersection(cursorsToWait.begin(), + cursorsToWait.end(), + cursorsNow.begin(), + cursorsNow.end(), + std::inserter(cursorsLeft, cursorsLeft.end())); + + cursorsToWait.swap(cursorsLeft); + + if (stopRequested()) { + *errMsg = "deleter was stopped."; + + scoped_lock sl(_queueMutex); + _deleteSet.erase(&deleteRange); + + _stats->decInProgressDeletes_inlock(); + _stats->decTotalDeletes_inlock(); + + if (!_stats->hasInProgress_inlock()) { + _nothingInProgressCV.notify_one(); + } + + return false; + } + + if (checkIntervalMillis < MaxCurorCheckIntervalMillis) { + checkIntervalMillis *= 2; + } + + sleepmillis(checkIntervalMillis); + } + + bool result = _env->deleteRange(ns, min, max, shardKeyPattern, + secondaryThrottle, errMsg); + + { + scoped_lock sl(_queueMutex); + _deleteSet.erase(&deleteRange); + + _stats->decInProgressDeletes_inlock(); + _stats->decTotalDeletes_inlock(); + + if (!_stats->hasInProgress_inlock()) { + _nothingInProgressCV.notify_one(); + } + } + + return result; + } + + bool RangeDeleter::addToBlackList(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + std::string* errMsg) { + string dummy; + if (errMsg == NULL) errMsg = &dummy; + + scoped_lock sl(_queueMutex); + + if (isBlacklisted_inlock(ns, min, max, errMsg)) { + return false; + } + + for (NSMinMaxSet::const_iterator iter = _deleteSet.begin(); + iter != _deleteSet.end(); ++iter) { + const NSMinMax* const entry = *iter; + if (entry->ns == ns && rangeOverlaps(entry->min, entry->max, min, max)) { + *errMsg = stream() << "Cannot black list ns: " << ns + << ", min: " << min + << ", max: " << max + << " since it is already queued for deletion."; + return false; + } + } + + _blackList.insert(new NSMinMax(ns.toString(), min, max)); + return true; + } + + bool RangeDeleter::removeFromBlackList(const StringData& ns, + const BSONObj& min, + const BSONObj& max) { + scoped_lock sl(_queueMutex); + NSMinMax entry(ns.toString(), min, max); + return deletePtrElement(&_blackList, &entry); + } + + const RangeDeleterStats* RangeDeleter::getStats() const { + return _stats.get(); + } + + BSONObj RangeDeleter::toBSON() const { + scoped_lock sl(_queueMutex); + + BSONObjBuilder builder; + + BSONArrayBuilder notReadyBuilder(builder.subarrayStart("notReady")); + for (TaskList::const_iterator iter = _notReadyQueue.begin(); + iter != _notReadyQueue.end(); ++iter) { + notReadyBuilder.append((*iter)->toBSON()); + } + notReadyBuilder.doneFast(); + + BSONArrayBuilder readyBuilder(builder.subarrayStart("ready")); + for (TaskList::const_iterator iter = _taskQueue.begin(); + iter != _taskQueue.end(); ++iter) { + readyBuilder.append((*iter)->toBSON()); + } + readyBuilder.doneFast(); + + return builder.obj(); + } + + void RangeDeleter::doWork() { + _env->initThread(); + + while (!inShutdown() && !stopRequested()) { + string errMsg; + + RangeDeleteEntry* nextTask = NULL; + + { + scoped_lock sl(_queueMutex); + while (_taskQueue.empty()) { + _taskQueueNotEmptyCV.timed_wait( + sl.boost(), duration::milliseconds(NotEmptyTimeoutMillis)); + + if (stopRequested()) { + log() << "stopping range deleter worker" << endl; + return; + } + + if (_taskQueue.empty()) { + // Try to check if some deletes are ready and move them to the + // ready queue. + + TaskList::iterator iter = _notReadyQueue.begin(); + while (iter != _notReadyQueue.end()) { + RangeDeleteEntry* entry = *iter; + + set cursorsNow; + _env->getCursorIds(entry->ns, &cursorsNow); + + set cursorsLeft; + std::set_intersection(entry->cursorsToWait.begin(), + entry->cursorsToWait.end(), + cursorsNow.begin(), + cursorsNow.end(), + std::inserter(cursorsLeft, + cursorsLeft.end())); + + entry->cursorsToWait.swap(cursorsLeft); + + if (entry->cursorsToWait.empty()) { + _taskQueue.push_back(*iter); + _taskQueueNotEmptyCV.notify_one(); + iter = _notReadyQueue.erase(iter); + } + else { + // Log waiting for cursor messages + entry->lastLoggedTS = logCursorsWaiting(entry->ns, + entry->min, + entry->max, + entry->cursorsToWait, + entry->queueStartTS, + entry->lastLoggedTS); + ++iter; + } + } + } + } + + if (stopRequested()) { + log() << "stopping range deleter worker" << endl; + return; + } + + nextTask = _taskQueue.front(); + _taskQueue.pop_front(); + + _stats->decPendingDeletes_inlock(); + _stats->incInProgressDeletes_inlock(); + } + + if (!_env->deleteRange(nextTask->ns, + nextTask->min, + nextTask->max, + nextTask->shardKeyPattern, + nextTask->secondaryThrottle, + &errMsg)) { + warning() << "Error encountered while trying to delete range: " + << errMsg << endl; + } + + { + scoped_lock sl(_queueMutex); + + NSMinMax setEntry(nextTask->ns, nextTask->min, nextTask->max); + deletePtrElement(&_deleteSet, &setEntry); + _stats->decInProgressDeletes_inlock(); + _stats->decTotalDeletes_inlock(); + + if (nextTask->notifyDone) { + nextTask->notifyDone->notifyOne(); + } + + delete nextTask; + nextTask = NULL; + } + } + } + + bool RangeDeleter::isBlacklisted_inlock(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + std::string* errMsg) const { + for (NSMinMaxSet::const_iterator iter = _blackList.begin(); + iter != _blackList.end(); ++iter) { + const NSMinMax* const entry = *iter; + if (ns != entry->ns) continue; + + if (rangeOverlaps(min, max, entry->min, entry->max)) { + *errMsg = stream() << "ns: " << ns + << ", min: " << min + << ", max: " << max + << " intersects with black list" + << " min: " << entry->min + << ", max: " << entry->max; + return true; + } + } + + return false; + } + + bool RangeDeleter::canEnqueue_inlock(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + string* errMsg) const { + if (isBlacklisted_inlock(ns, min, max, errMsg)) { + return false; + } + + NSMinMax toDelete(ns.toString(), min, max); + if (_deleteSet.count(&toDelete) > 0) { + *errMsg = stream() << "ns: " << ns + << ", min: " << min + << ", max: " << max + << " is already being processed for deletion."; + return false; + } + + return true; + } + + bool RangeDeleter::stopRequested() const { + scoped_lock sl(_stopMutex); + return _stopRequested; + } + +} diff --git a/src/mongo/db/range_deleter.h b/src/mongo/db/range_deleter.h new file mode 100644 index 00000000000..b8fd0789c5a --- /dev/null +++ b/src/mongo/db/range_deleter.h @@ -0,0 +1,305 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/jsobj.h" +#include "mongo/util/concurrency/mutex.h" +#include "mongo/util/concurrency/synchronization.h" + +namespace mongo { + + struct RangeDeleterEnv; + class RangeDeleterStats; + + /** + * Class for deleting documents for a given namespace and range. It contains a queue of + * jobs to be deleted. Deletions can be "immediate", in which case they are going to be put + * in front of the queue and acted on promptly, or "lazy", in which they would be acted + * upon when they get to the head of the queue. + * + * Threading assumptions: + * + * This class has (currently) one worker thread attacking the queue, one + * job at a time. If we want an immediate deletion, that job is going to + * be performed on the thread that is requesting it. + * + * All calls regarding deletion are synchronized. + * + * Life cycle: + * RangeDeleter* deleter = new RangeDeleter(new ...); + * deleter->startWorkers(); + * ... + * killCurrentOp.killAll(); // stop all deletes + * deleter->stopWorkers(); + * delete deleter; + */ + class RangeDeleter { + MONGO_DISALLOW_COPYING(RangeDeleter); + + public: + + /** + * Creates a new deleter and uses an environment object to delegate external logic like + * data deletion. Takes ownership of the environment. + */ + explicit RangeDeleter(RangeDeleterEnv* env); + + /** + * Destroys this deleter. Must make sure that no threads are working on this queue. Use + * stopWorkers to stop the internal workers, it is an error not to do so. + */ + ~RangeDeleter(); + + // + // Thread management methods + // + + /** + * Starts the background thread to work on this queue. Does nothing if the worker + * thread is already active. + * + * This call is _not_ thread safe and must be issued before any other call. + */ + void startWorkers(); + + /** + * Stops the background thread working on this queue. This will block if there are + * tasks that are being deleted, but will leave the pending tasks in the queue. + * + * Steps: + * 1. Stop accepting new queued deletes. + * 2. Stop all idle workers. + * 3. Waits for all threads to finish any task that is in progress (but see note + * below). + * + * Note: + * + * + restarting this deleter with startWorkers after stopping it is not supported. + * + * + the worker thread could be running a call in the environment. The thread is + * only going to be returned when the environment decides so. In production, + * KillCurrentOp::killAll can be used to get the thread back from the environment. + */ + void stopWorkers(); + + // + // Queue manipulation methods - can be called by anyone. + // + + /** + * Adds a new delete to the queue. + * + * If notifyDone is not NULL, it will be signaled after the delete is completed. + * Note that this will happen only if the delete was actually queued. + * + * Returns true if the task is queued and false If the given range is blacklisted, + * is already queued, or stopWorkers() was called. + */ + bool queueDelete(const std::string& ns, + const BSONObj& min, + const BSONObj& max, + const BSONObj& shardKeyPattern, + bool secondaryThrottle, + Notification* notifyDone, + std::string* errMsg); + + /** + * Removes the documents specified by the range. Unlike queueTask, this call + * blocks and the deletion is performed by the current thread. + * + * Returns true if the deletion was performed. False if the range is blacklisted, + * was already queued, or stopWorkers() was called. + */ + bool deleteNow(const std::string& ns, + const BSONObj& min, + const BSONObj& max, + const BSONObj& shardKeyPattern, + bool secondaryThrottle, + std::string* errMsg); + + /** + * Blacklist the given range for the given namespace. Use the removeFromBlackList + * method to undo this operation. + * + * Note: min is inclusive and max is exclusive. + * + * Return false if a task in the queue intersects the given range or + * if the range intersects another range that is in the black list. + */ + bool addToBlackList(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + std::string* errMsg); + + /** + * Removes the exact range from the blacklist. + * + * Returns false if range cannot be found from the black list. + */ + bool removeFromBlackList(const StringData& ns, + const BSONObj& min, + const BSONObj& max); + + // + // Introspection methods + // + + const RangeDeleterStats* getStats() const; + + // + // Methods meant to be only used for testing. Should be treated like private + // methods. + // + + /** Returns a BSON representation of the queue contents. For debugging only. */ + BSONObj toBSON() const; + + private: + struct RangeDeleteEntry; + struct NSMinMax; + + struct NSMinMaxCmp { + bool operator()(const NSMinMax* lhs, const NSMinMax* rhs) const; + }; + + typedef std::deque TaskList; // owned here + + typedef std::set NSMinMaxSet; // owned here + + /** Body of the worker thread */ + void doWork(); + + /** Returns true if range is blacklisted. Assumes _queueMutex is held */ + bool isBlacklisted_inlock(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + std::string* errMsg) const; + + /** Returns true if the range doesn't intersect with one other range */ + bool canEnqueue_inlock(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + std::string* errMsg) const; + + /** Returns true if stopWorkers() was called. This call is synchronized. */ + bool stopRequested() const; + + scoped_ptr _env; + + // Initially not active. Must be started explicitly. + scoped_ptr _worker; + + // Protects _stopRequested. + mutable mutex _stopMutex; + + // If set, no other delete taks should be accepted. + bool _stopRequested; + + // No delete is in progress. Used to make sure that there is no activity + // in this deleter, and therefore is safe to destroy it. Must be used in + // conjunction with _stopRequested. + boost::condition _nothingInProgressCV; + + // Protects all the data structure below this. + mutable mutex _queueMutex; + + // _taskQueue has a task ready to work on. + boost::condition _taskQueueNotEmptyCV; + + // Queue for storing the list of ranges that have cursors pending on it. + // + // Note: pointer life cycle is not handled here. + TaskList _notReadyQueue; + + // Queue for storing the list of ranges that are ready to be removed. + // + // Note: pointer life cycle is not handled here. + TaskList _taskQueue; + + // Set of all deletes - deletes waiting for cursors, waiting to be acted upon + // and in progress. Includes both queued and immediate deletes. + // + // queued delete life cycle: new @ queuedDelete, delete @ doWork + // deleteNow life cycle: deleteNow stack variable + NSMinMaxSet _deleteSet; + + // Keeps track of ranges that cannot be queued to _notReady. + // Invariant: should not conflict with any entry in all queues. + // + // life cycle: new @ addToBlackList, delete @ removeFromBlackList + // deleteNow life cycle: deleteNow stack variable + NSMinMaxSet _blackList; + + // Keeps track of counters regarding each of the queues. + scoped_ptr _stats; + }; + + /** + * Class for encapsulating logic used by the RangeDeleter class to perform its tasks. + */ + struct RangeDeleterEnv { + virtual ~RangeDeleterEnv() {} + + virtual void initThread() = 0; + + /** + * Deletes the documents from the given range. This method should be + * responsible for making sure that the proper contexts are setup + * to be able to perform deletions. + * + * Must be a synchronous call. Docs should be deleted after call ends. + * Must not throw Exceptions. + */ + virtual bool deleteRange(const StringData& ns, + const BSONObj& inclusiveLower, + const BSONObj& exclusiveUpper, + const BSONObj& shardKeyPattern, + bool secondaryThrottle, + std::string* errMsg) = 0; + + /** + * Gets the list of open cursors on a given namespace. The openCursors is an + * output parameter that will contain all the cursors open after this is called. + * Assume that openCursors is empty when passed in. + * + * Must be a synchronous call. CursorIds should be populated after call. + * Must not throw exception. + */ + virtual void getCursorIds(const StringData& ns, set* openCursors) = 0; + }; + +} // namespace mongo diff --git a/src/mongo/db/range_deleter_db_env.cpp b/src/mongo/db/range_deleter_db_env.cpp new file mode 100644 index 00000000000..ce7e63363bc --- /dev/null +++ b/src/mongo/db/range_deleter_db_env.cpp @@ -0,0 +1,184 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/range_deleter_db_env.h" + +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" +#include "mongo/db/dbhelpers.h" +#include "mongo/db/repl/rs.h" +#include "mongo/db/repl/write_concern.h" +#include "mongo/s/d_logic.h" + +namespace mongo { + + void RangeDeleterDBEnv::initThread() { + if ( currentClient.get() == NULL ) + Client::initThread( "RangeDeleter" ); + } + + /** + * Outline of the delete process: + * 1. Initialize the client for this thread if there is no client. This is for the worker + * threads that are attached to any of the threads servicing client requests. + * 2. Grant this thread authorization to perform deletes. + * 3. Temporarily enable mode to bypass shard version checks. TODO: Replace this hack. + * 4. Setup callback to save deletes to moveChunk directory (only if moveParanoia is true). + * 5. Delete range. + * 6. Wait until the majority of the secondaries catch up. + */ + bool RangeDeleterDBEnv::deleteRange(const StringData& ns, + const BSONObj& inclusiveLower, + const BSONObj& exclusiveUpper, + const BSONObj& keyPattern, + bool secondaryThrottle, + std::string* errMsg) { + const bool initiallyHaveClient = haveClient(); + + if (!initiallyHaveClient) { + Client::initThread("RangeDeleter"); + } + + ShardForceVersionOkModeBlock forceVersion; + { + Helpers::RemoveSaver removeSaver("moveChunk", ns.toString(), "post-cleanup"); + + // log the opId so the user can use it to cancel the delete using killOp. + unsigned int opId = cc().curop()->opNum(); + log() << "Deleter starting delete for: " << ns + << " from " << inclusiveLower + << " -> " << exclusiveUpper + << ", with opId: " << opId + << endl; + + try { + long long numDeleted = + Helpers::removeRange(KeyRange(ns.toString(), + inclusiveLower, + exclusiveUpper, + keyPattern), + false, /*maxInclusive*/ + replSet? secondaryThrottle : false, + serverGlobalParams.moveParanoia ? &removeSaver : NULL, + true, /*fromMigrate*/ + true); /*onlyRemoveOrphans*/ + + if (numDeleted < 0) { + *errMsg = "collection or index dropped before data could be cleaned"; + warning() << *errMsg << endl; + + if (!initiallyHaveClient) { + // unregister self from list of curret ops. + cc().shutdown(); + } + + return false; + } + + log() << "rangeDeleter deleted " << numDeleted + << " documents for " << ns + << " from " << inclusiveLower + << " -> " << exclusiveUpper + << endl; + } + catch (const DBException& ex) { + *errMsg = str::stream() << "Error encountered while deleting range: " + << "ns" << ns + << " from " << inclusiveLower + << " -> " << exclusiveUpper + << ", cause by:" << causedBy(ex); + + if (!initiallyHaveClient) { + // unregister self from list of curret ops. + cc().shutdown(); + } + + return false; + } + } + + if (replSet) { + Timer elapsedTime; + ReplTime lastOpApplied = cc().getLastOp().asDate(); + try { + while (!opReplicatedEnough(lastOpApplied, + BSON("w" << "majority").firstElement())) { + if (elapsedTime.seconds() >= 3600) { + *errMsg = str::stream() << "rangeDeleter timed out after " + << elapsedTime.seconds() << " seconds while " + << "waiting for deletions to be replicated " + << "to majority nodes"; + + if (!initiallyHaveClient) { + // unregister self from list of curret ops. + cc().shutdown(); + } + + return false; + } + + sleepsecs(1); + } + + LOG(elapsedTime.seconds() < 30 ? 1 : 0) + << "rangeDeleter took " << elapsedTime.seconds() << " seconds " + << " waiting for deletes to be replicated to majority nodes" << endl; + } + catch (const DBException& excep) { + *errMsg = str::stream() << "rangeDeleter encountered an exception while " + << "waiting for deletes to replicated: " << excep.toString(); + + if (!initiallyHaveClient) { + // unregister self from list of curret ops. + cc().shutdown(); + } + + return false; + } + } + + if (!initiallyHaveClient) { + // unregister self from list of curret ops. + cc().shutdown(); + } + + return true; + } + + void RangeDeleterDBEnv::getCursorIds(const StringData& ns, + std::set* openCursors) { + Client::ReadContext ctx(ns.toString()); + Collection* collection = ctx.ctx().db()->getCollection( ns ); + if ( !collection ) + return; + + collection->cursorCache()->getCursorIds( openCursors ); + } +} diff --git a/src/mongo/db/range_deleter_db_env.h b/src/mongo/db/range_deleter_db_env.h new file mode 100644 index 00000000000..b55389e3edd --- /dev/null +++ b/src/mongo/db/range_deleter_db_env.h @@ -0,0 +1,66 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/range_deleter.h" + +namespace mongo { + + /** + * This class implements the deleter methods to be used for a shard. + */ + struct RangeDeleterDBEnv : public RangeDeleterEnv { + + virtual void initThread(); + + /** + * Deletes the documents from the given range synchronously. + * + * The keyPattern will be used to determine the right index to use to perform + * the deletion and it can be a prefix of an existing index. Caller is responsible + * of making sure that both inclusiveLower and exclusiveUpper is a prefix of keyPattern. + * + * Note that secondaryThrottle will be ignored if current process is not part + * of a replica set. + * + * Does not throw Exceptions. + */ + virtual bool deleteRange(const StringData& ns, + const BSONObj& inclusiveLower, + const BSONObj& exclusiveUpper, + const BSONObj& keyPattern, + bool secondaryThrottle, + std::string* errMsg); + + /** + * Gets the list of open cursors on a given namespace. + */ + virtual void getCursorIds(const StringData& ns, std::set* openCursors); + }; +} diff --git a/src/mongo/db/range_deleter_mock_env.cpp b/src/mongo/db/range_deleter_mock_env.cpp new file mode 100644 index 00000000000..536d463417e --- /dev/null +++ b/src/mongo/db/range_deleter_mock_env.cpp @@ -0,0 +1,154 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/range_deleter_mock_env.h" + +namespace mongo { + + bool DeletedRangeCmp::operator()(const DeletedRange& lhs, + const DeletedRange& rhs) const { + const int nsComp = lhs.ns.compare(rhs.ns); + + if (nsComp < 0) { + return true; + } + + if (nsComp > 0) { + return false; + } + + return compareRanges(lhs.min, lhs.max, rhs.min, rhs.max) < 0; + } + + RangeDeleterMockEnv::RangeDeleterMockEnv(): + _deleteListMutex("delList"), + _cursorMapMutex("cursorMap"), + _pauseDeleteMutex("pauseDelete"), + _pauseDelete(false), + _pausedCount(0), + _envStatMutex("envStat"), + _getCursorsCallCount(0) { + } + + void RangeDeleterMockEnv::addCursorId(const StringData& ns, CursorId id) { + scoped_lock sl(_cursorMapMutex); + _cursorMap[ns.toString()].insert(id); + } + + void RangeDeleterMockEnv::removeCursorId(const StringData& ns, CursorId id) { + scoped_lock sl(_cursorMapMutex); + _cursorMap[ns.toString()].erase(id); + } + + void RangeDeleterMockEnv::pauseDeletes() { + scoped_lock sl(_pauseDeleteMutex); + _pauseDelete = true; + } + + void RangeDeleterMockEnv::resumeOneDelete() { + scoped_lock sl(_pauseDeleteMutex); + _pauseDelete = false; + _pausedCV.notify_one(); + } + + void RangeDeleterMockEnv::waitForNthGetCursor(uint64_t nthCall) { + scoped_lock sl(_envStatMutex); + while (_getCursorsCallCount < nthCall) { + _cursorsCallCountUpdatedCV.wait(sl.boost()); + } + } + + void RangeDeleterMockEnv::waitForNthPausedDelete(uint64_t nthPause) { + scoped_lock sl(_pauseDeleteMutex); + while(_pausedCount < nthPause) { + _pausedDeleteChangeCV.wait(sl.boost()); + } + } + + bool RangeDeleterMockEnv::deleteOccured() const { + scoped_lock sl(_deleteListMutex); + return !_deleteList.empty(); + } + + DeletedRange RangeDeleterMockEnv::getLastDelete() const { + scoped_lock sl(_deleteListMutex); + return _deleteList.back(); + } + + bool RangeDeleterMockEnv::deleteRange(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + const BSONObj& shardKeyPattern, + bool secondaryThrottle, + string* errMsg) { + + { + scoped_lock sl(_pauseDeleteMutex); + bool wasInitiallyPaused = _pauseDelete; + + if (_pauseDelete) { + _pausedCount++; + _pausedDeleteChangeCV.notify_one(); + } + + while (_pauseDelete) { + _pausedCV.wait(sl.boost()); + } + + _pauseDelete = wasInitiallyPaused; + } + + { + scoped_lock sl(_deleteListMutex); + + DeletedRange entry; + entry.ns = ns.toString(); + entry.min = min.getOwned(); + entry.max = max.getOwned(); + entry.shardKeyPattern = shardKeyPattern.getOwned(); + + _deleteList.push_back(entry); + } + + return true; + } + + void RangeDeleterMockEnv::getCursorIds(const StringData& ns, set* in) { + { + scoped_lock sl(_cursorMapMutex); + const set& _cursors = _cursorMap[ns.toString()]; + std::copy(_cursors.begin(), _cursors.end(), inserter(*in, in->begin())); + } + + { + scoped_lock sl(_envStatMutex); + _getCursorsCallCount++; + _cursorsCallCountUpdatedCV.notify_one(); + } + } +} diff --git a/src/mongo/db/range_deleter_mock_env.h b/src/mongo/db/range_deleter_mock_env.h new file mode 100644 index 00000000000..7b1cd3932e3 --- /dev/null +++ b/src/mongo/db/range_deleter_mock_env.h @@ -0,0 +1,171 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/s/range_arithmetic.h" +#include "mongo/db/range_deleter.h" + +namespace mongo { + + struct DeletedRange { + std::string ns; + BSONObj min; + BSONObj max; + BSONObj shardKeyPattern; + }; + + /** + * Comparator function object compatible with std::set. + */ + struct DeletedRangeCmp { + bool operator()(const DeletedRange& lhs, const DeletedRange& rhs) const; + }; + + /** + * Mock environment for RangeDeleter with knobs for pausing/resuming + * deletes, setting open cursors IDs per namespace and the ability to + * record the history of deletes performed through this environment. + */ + class RangeDeleterMockEnv: public mongo::RangeDeleterEnv { + public: + RangeDeleterMockEnv(); + + void initThread() {} + + // + // Environment modification methods. + // + + /** + * Adds an id to the current set of cursors in the given namespace. + */ + void addCursorId(const StringData& ns, CursorId id); + + /** + * Removes the id from the set of open cursors in the given namespace. + */ + void removeCursorId(const StringData& ns, CursorId id); + + // + // Environment synchronization methods. + // + + /** + * Blocks all new deletes from proceeding. + */ + void pauseDeletes(); + + /** + * Unblocks one paused delete. + */ + void resumeOneDelete(); + + /** + * Blocks until the getCursor method was called and terminated at least the + * specified number of times for the entire lifetime of this deleter. + */ + void waitForNthGetCursor(uint64_t nthCall); + + /** + * Blocks until the deleteRange method was called and at the same time paused + * at least the specified number of times for the entire lifetime of this deleter. + */ + void waitForNthPausedDelete(uint64_t nthPause); + + // + // Environment introspection methods. + // + + /** + * Returns true if deleteRange was called at least once. + */ + bool deleteOccured() const; + + /** + * Returns the last delete. Undefined if deleteOccured is false. + */ + DeletedRange getLastDelete() const; + + // + // Environment methods. + // + + /** + * Basic implementation of delete that matches the signature for + * RangeDeleterEnv::deleteRange. This does not actually perform the delete + * but simply keeps a record of it. Can also be paused by pauseDeletes and + * resumed with resumeDeletes. + */ + bool deleteRange(const StringData& ns, + const BSONObj& min, + const BSONObj& max, + const BSONObj& shardKeyPattern, + bool secondaryThrottle, + string* errMsg); + + /** + * Basic implementation of gathering open cursors that matches the signature for + * RangeDeleterEnv::getCursorIds. The cursors returned can be modified with + * the setCursorId and clearCursorMap methods. + */ + void getCursorIds(const StringData& ns, set* in); + + private: + // mutex acquisition ordering: + // _envStatMutex -> _pauseDeleteMutex -> _deleteListMutex -> _cursorMapMutex + + mutable mutex _deleteListMutex; + std::vector _deleteList; + + mutex _cursorMapMutex; + std::map > _cursorMap; + + // Protects _pauseDelete & _pausedCount + mutex _pauseDeleteMutex; + boost::condition _pausedCV; + bool _pauseDelete; + + // Number of times a delete gets paused. + uint64_t _pausedCount; + // _pausedCount < nthPause (used by waitForNthPausedDelete) + boost::condition _pausedDeleteChangeCV; + + // Protects all variables below this line. + mutex _envStatMutex; + + // Keeps track of the number of times getCursorIds was called. + uint64_t _getCursorsCallCount; + // _getCursorsCallCount < nthCall (used by waitForNthGetCursor) + boost::condition _cursorsCallCountUpdatedCV; + }; +} diff --git a/src/mongo/db/range_deleter_service.cpp b/src/mongo/db/range_deleter_service.cpp new file mode 100644 index 00000000000..a5393230df9 --- /dev/null +++ b/src/mongo/db/range_deleter_service.cpp @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/range_deleter_service.h" + +#include "mongo/base/init.h" +#include "mongo/db/range_deleter_db_env.h" + +namespace { + + mongo::RangeDeleter* _deleter = NULL; +} + +namespace mongo { + + MONGO_INITIALIZER(RangeDeleterInit)(InitializerContext* context) { + _deleter = new RangeDeleter(new RangeDeleterDBEnv); + return Status::OK(); + } + + RangeDeleter* getDeleter() { + return _deleter; + } +} diff --git a/src/mongo/db/range_deleter_service.h b/src/mongo/db/range_deleter_service.h new file mode 100644 index 00000000000..bea7c612307 --- /dev/null +++ b/src/mongo/db/range_deleter_service.h @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/range_deleter.h" +#include "mongo/db/range_deleter_stats.h" + +namespace mongo { + + /** + * Gets the global instance of the deleter and starts it. + */ + RangeDeleter* getDeleter(); +} diff --git a/src/mongo/db/range_deleter_stat_test.cpp b/src/mongo/db/range_deleter_stat_test.cpp new file mode 100644 index 00000000000..6de5f15d196 --- /dev/null +++ b/src/mongo/db/range_deleter_stat_test.cpp @@ -0,0 +1,322 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include +#include +#include + +#include "mongo/db/field_parser.h" +#include "mongo/db/range_deleter.h" +#include "mongo/db/range_deleter_mock_env.h" +#include "mongo/db/range_deleter_stats.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using boost::bind; + using std::string; + + using mongo::BSONObj; + using mongo::CursorId; + using mongo::FieldParser; + using mongo::Notification; + using mongo::RangeDeleter; + using mongo::RangeDeleterMockEnv; + using mongo::RangeDeleterStats; + + TEST(NoDeletes, InitialState) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + const BSONObj stats(deleter.getStats()->toBSON()); + + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, totalCount); + + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, pendingCount); + + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, inProgressCount); + + deleter.stopWorkers(); + } + + TEST(QueuedDeletes, NotReady) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + const string ns("test.user"); + deleter.startWorkers(); + + // Set cursors on NS so deletes cannot be processed immediately. + env->addCursorId(ns, 50); + + string errMsg; + Notification notifyDone; + ASSERT_TRUE(deleter.queueDelete(ns, + BSON("x" << 0), + BSON("x" << 10), + BSON("x" << 1), + true, + ¬ifyDone, + &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + env->waitForNthGetCursor(1u); + + const BSONObj stats(deleter.getStats()->toBSON()); + + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, totalCount); + + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, pendingCount); + + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, inProgressCount); + + deleter.stopWorkers(); + } + + TEST(QueuedDeletes, InProgress) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + const string ns("test.user"); + deleter.startWorkers(); + + env->pauseDeletes(); + + Notification deleteDone; + string errMsg; + ASSERT_TRUE(deleter.queueDelete(ns, + BSON("x" << 0), + BSON("x" << 10), + BSON("x" << 1), + true, + &deleteDone, + &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + env->waitForNthPausedDelete(1u); + + const BSONObj stats(deleter.getStats()->toBSON()); + + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, totalCount); + + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, inProgressCount); + + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, pendingCount); + + env->resumeOneDelete(); + deleteDone.waitToBeNotified(); + + deleter.stopWorkers(); + } + + TEST(QueuedDeletes, AfterDelete) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + const string ns("test.user"); + deleter.startWorkers(); + + string errMsg; + Notification notifyDone; + ASSERT_TRUE(deleter.queueDelete(ns, + BSON("x" << 0), + BSON("x" << 10), + BSON("x" << 1), + true, + ¬ifyDone, + &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + notifyDone.waitToBeNotified(); + + const BSONObj stats(deleter.getStats()->toBSON()); + + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, totalCount); + + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, pendingCount); + + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, inProgressCount); + + deleter.stopWorkers(); + } + + TEST(ImmediateDeletes, NotReady) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + const string ns("test.user"); + + // Set cursors on NS so deletes cannot be processed immediately. + env->addCursorId(ns, 50); + + string errMsg; + boost::thread deleterThread = boost::thread(boost::bind(&RangeDeleter::deleteNow, + &deleter, + ns, + BSON("x" << 0), + BSON("x" << 10), + BSON("x" << 1), + true, + &errMsg)); + env->waitForNthGetCursor(1u); + + const BSONObj stats(deleter.getStats()->toBSON()); + + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, totalCount); + + // Note: immediate deletes has no pending state, it goes directly to inProgress + // even while waiting for cursors. + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, pendingCount); + + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, inProgressCount); + + env->removeCursorId(ns, 50); + deleterThread.join(); + + deleter.stopWorkers(); + } + + TEST(ImmediateDeletes, InProgress) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + const string ns("test.user"); + env->pauseDeletes(); + + string errMsg; + boost::thread deleterThread = boost::thread(boost::bind(&RangeDeleter::deleteNow, + &deleter, + ns, + BSON("x" << 0), + BSON("x" << 10), + BSON("x" << 1), + true, + &errMsg)); + + env->waitForNthPausedDelete(1u); + + const BSONObj stats(deleter.getStats()->toBSON()); + + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, totalCount); + + // There is only one worker thread so you can't have inProgress > 1 + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, inProgressCount); + + // The rest should still be in pending since there is only one worker. + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, pendingCount); + + env->resumeOneDelete(); + deleterThread.join(); + + deleter.stopWorkers(); + } + + TEST(ImmediateDeletes, AfterDelete) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + const string ns("test.user"); + string errMsg; + ASSERT_TRUE(deleter.deleteNow(ns, BSON("x" << 0), BSON("x" << 10), + BSON("x" << 1), true, &errMsg)); + + const BSONObj stats(deleter.getStats()->toBSON()); + + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, totalCount); + + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, pendingCount); + + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(0, inProgressCount); + + deleter.stopWorkers(); + } + +} // unnamed namespace diff --git a/src/mongo/db/range_deleter_stats.cpp b/src/mongo/db/range_deleter_stats.cpp new file mode 100644 index 00000000000..12fb11ee25a --- /dev/null +++ b/src/mongo/db/range_deleter_stats.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/range_deleter_stats.h" + +namespace mongo { + const BSONField RangeDeleterStats::TotalDeletesField("totalDeletes"); + const BSONField RangeDeleterStats::PendingDeletesField("pendingDeletes"); + const BSONField RangeDeleterStats::InProgressDeletesField("inProgressDeletes"); + + BSONObj RangeDeleterStats::toBSON() const { + scoped_lock sl(*_lockPtr); + + BSONObjBuilder builder; + builder << TotalDeletesField(_totalDeletes); + builder << PendingDeletesField(_pendingDeletes); + builder << InProgressDeletesField(_inProgressDeletes); + + return builder.obj(); + } + + // Note: If we ever to decide to expose the other individual stats as well, we have + // to remind the caller that calling them individually is never guaranteed to have a + // consistent view of the stats. So toBSON should be used instead if the caller needs + // a snapshot view of the state on more than one property. + int RangeDeleterStats::getCurrentDeletes() const { + scoped_lock sl(*_lockPtr); + return _totalDeletes; + } +} diff --git a/src/mongo/db/range_deleter_stats.h b/src/mongo/db/range_deleter_stats.h new file mode 100644 index 00000000000..7f70884af1e --- /dev/null +++ b/src/mongo/db/range_deleter_stats.h @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/bson_field.h" +#include "mongo/db/jsobj.h" +#include "mongo/platform/cstdint.h" +#include "mongo/util/concurrency/mutex.h" + +namespace mongo { + /** + * Simple class for storing statistics for the RangeDeleter. + */ + class RangeDeleterStats { + public: + // + // BSON representation top level fields. + // + + // Total number of deletes, including deletes that are pending and in progress. + static const BSONField TotalDeletesField; + + // Total number of deletes that are yet to be worked on. + static const BSONField PendingDeletesField; + + // Total number of deletes that are currently in progress. + static const BSONField InProgressDeletesField; + + /** + * Creates a stat object given the mutex from the RangeDeleter object + * that this instance is keeping track of. + */ + RangeDeleterStats(mutex* lockPtr): + _lockPtr(lockPtr), + _totalDeletes(0), + _pendingDeletes(0), + _inProgressDeletes(0) { + } + + /** + * Returns the BSON representation of this stat object. + */ + BSONObj toBSON() const; + + // Returns the current number of active and pending deletes. + int getCurrentDeletes() const; + + // + // Setters - Should be holding mutex passed to + // the constructor when calling these methods. + // + + void incTotalDeletes_inlock() { + _totalDeletes++; + } + + void decTotalDeletes_inlock() { + _totalDeletes--; + } + + void incPendingDeletes_inlock() { + _pendingDeletes++; + } + + void decPendingDeletes_inlock() { + _pendingDeletes--; + } + + void incInProgressDeletes_inlock() { + _inProgressDeletes++; + } + + void decInProgressDeletes_inlock() { + _inProgressDeletes--; + } + + bool hasInProgress_inlock() { + return _inProgressDeletes > 0; + } + + private: + // Protects all data structures below this. Not owned here. + mutable mutex* _lockPtr; + + int _totalDeletes; + int _pendingDeletes; + int _inProgressDeletes; + }; +} diff --git a/src/mongo/db/range_deleter_test.cpp b/src/mongo/db/range_deleter_test.cpp new file mode 100644 index 00000000000..23a93da14b0 --- /dev/null +++ b/src/mongo/db/range_deleter_test.cpp @@ -0,0 +1,516 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include +#include +#include + +#include "mongo/db/field_parser.h" +#include "mongo/db/range_deleter.h" +#include "mongo/db/range_deleter_mock_env.h" +#include "mongo/db/range_deleter_stats.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using boost::bind; + using std::string; + + using mongo::BSONObj; + using mongo::CursorId; + using mongo::DeletedRange; + using mongo::FieldParser; + using mongo::Notification; + using mongo::RangeDeleter; + using mongo::RangeDeleterMockEnv; + using mongo::RangeDeleterStats; + + // Capped sleep interval is 640 mSec, Nyquist frequency is 1280 mSec => round up to 2 sec. + const int MAX_IMMEDIATE_DELETE_WAIT_SECS = 2; + + // Should not be able to queue deletes if deleter workers were not started. + TEST(QueueDelete, CantAfterStop) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + deleter.startWorkers(); + deleter.stopWorkers(); + + string errMsg; + ASSERT_FALSE(deleter.queueDelete("test.user", + BSON("x" << 120), + BSON("x" << 200), + BSON("x" << 1), + true, + NULL /* notifier not needed */, + &errMsg)); + ASSERT_FALSE(errMsg.empty()); + ASSERT_FALSE(env->deleteOccured()); + } + + // Should not start delete if the set of cursors that were open when the + // delete was queued is still open. + TEST(QueuedDelete, ShouldWaitCursor) { + const string ns("test.user"); + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + deleter.startWorkers(); + + env->addCursorId(ns, 345); + + Notification notifyDone; + ASSERT_TRUE(deleter.queueDelete(ns, BSON("x" << 0), BSON("x" << 10), BSON("x" << 1), + true, ¬ifyDone, NULL /* errMsg not needed */)); + + env->waitForNthGetCursor(1u); + + const BSONObj stats(deleter.getStats()->toBSON()); + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, pendingCount); + ASSERT_FALSE(env->deleteOccured()); + + // Set the open cursors to a totally different sets of cursorIDs. + env->addCursorId(ns, 200); + env->removeCursorId(ns, 345); + + notifyDone.waitToBeNotified(); + + ASSERT_TRUE(env->deleteOccured()); + const DeletedRange deletedChunk(env->getLastDelete()); + + ASSERT_EQUALS(ns, deletedChunk.ns); + ASSERT_TRUE(deletedChunk.min.equal(BSON("x" << 0))); + ASSERT_TRUE(deletedChunk.max.equal(BSON("x" << 10))); + + deleter.stopWorkers(); + } + + // Should terminate when stop is requested. + TEST(QueuedDelete, StopWhileWaitingCursor) { + const string ns("test.user"); + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + deleter.startWorkers(); + + env->addCursorId(ns, 345); + + Notification notifyDone; + ASSERT_TRUE(deleter.queueDelete(ns, BSON("x" << 0), BSON("x" << 10), BSON("x" << 1), + true, ¬ifyDone, NULL /* errMsg not needed */)); + + + env->waitForNthGetCursor(1u); + + deleter.stopWorkers(); + ASSERT_FALSE(env->deleteOccured()); + } + + // Should not start delete if the set of cursors that were open when the + // deleteNow method is called is still open. + TEST(ImmediateDelete, ShouldWaitCursor) { + const string ns("test.user"); + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + deleter.startWorkers(); + + env->addCursorId(ns, 345); + + string errMsg; + boost::thread deleterThread = boost::thread(boost::bind(&RangeDeleter::deleteNow, + &deleter, + ns, + BSON("x" << 0), + BSON("x" << 10), + BSON("x" << 1), + true, + &errMsg)); + + env->waitForNthGetCursor(1u); + + // Note: immediate deletes has no pending state, it goes directly to inProgress + // even while waiting for cursors. + const BSONObj stats(deleter.getStats()->toBSON()); + int inProgCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, inProgCount); + + ASSERT_FALSE(env->deleteOccured()); + + // Set the open cursors to a totally different sets of cursorIDs. + env->addCursorId(ns, 200); + env->removeCursorId(ns, 345); + + ASSERT_TRUE(deleterThread.timed_join( + boost::posix_time::seconds(MAX_IMMEDIATE_DELETE_WAIT_SECS))); + + ASSERT_TRUE(env->deleteOccured()); + const DeletedRange deletedChunk(env->getLastDelete()); + + ASSERT_EQUALS(ns, deletedChunk.ns); + ASSERT_TRUE(deletedChunk.min.equal(BSON("x" << 0))); + ASSERT_TRUE(deletedChunk.max.equal(BSON("x" << 10))); + ASSERT_TRUE(deletedChunk.shardKeyPattern.equal(BSON("x" << 1))); + + deleter.stopWorkers(); + } + + // Should terminate when stop is requested. + TEST(ImmediateDelete, StopWhileWaitingCursor) { + const string ns("test.user"); + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + deleter.startWorkers(); + + env->addCursorId(ns, 345); + + string errMsg; + boost::thread deleterThread = boost::thread(boost::bind(&RangeDeleter::deleteNow, + &deleter, + ns, + BSON("x" << 0), + BSON("x" << 10), + BSON("x" << 1), + true, + &errMsg)); + + env->waitForNthGetCursor(1u); + + // Note: immediate deletes has no pending state, it goes directly to inProgress + // even while waiting for cursors. + const BSONObj stats(deleter.getStats()->toBSON()); + int inProgCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, inProgCount); + + ASSERT_FALSE(env->deleteOccured()); + + deleter.stopWorkers(); + + ASSERT_TRUE(deleterThread.timed_join( + boost::posix_time::seconds(MAX_IMMEDIATE_DELETE_WAIT_SECS))); + + ASSERT_FALSE(env->deleteOccured()); + } + + // Tests the interaction of multiple deletes queued with different states. + // Starts by adding a new delete task, waits for the worker to work on it, + // and then adds 2 more task, one of which is ready to be deleted, while the + // other one is waiting for an open cursor. The test then makes sure that the + // deletes are performed in the right order. + TEST(MixedDeletes, MultipleDeletes) { + const string blockedNS("foo.bar"); + const string ns("test.user"); + + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + deleter.startWorkers(); + + env->addCursorId(blockedNS, 345); + env->pauseDeletes(); + + Notification notifyDone1; + ASSERT_TRUE(deleter.queueDelete(ns, + BSON("x" << 10), + BSON("x" << 20), + BSON("x" << 1), + true, + ¬ifyDone1, + NULL /* don't care errMsg */)); + + env->waitForNthPausedDelete(1u); + + // Make sure that the delete is already in progress before proceeding. + const BSONObj stats(deleter.getStats()->toBSON()); + int inProgressCount = 0; + ASSERT_TRUE(FieldParser::extract(stats, RangeDeleterStats::InProgressDeletesField, + &inProgressCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, inProgressCount); + + Notification notifyDone2; + ASSERT_TRUE(deleter.queueDelete(blockedNS, + BSON("x" << 20), + BSON("x" << 30), + BSON("x" << 1), + true, + ¬ifyDone2, + NULL /* don't care errMsg */)); + + Notification notifyDone3; + ASSERT_TRUE(deleter.queueDelete(ns, + BSON("x" << 30), + BSON("x" << 40), + BSON("x" << 1), + true, + ¬ifyDone3, + NULL /* don't care errMsg */)); + + // Now, the setup is: + // { x: 10 } => { x: 20 } in progress. + // { x: 20 } => { x: 30 } waiting for cursor id 345. + // { x: 30 } => { x: 40 } waiting to be picked up by worker. + + // Make sure that the current state matches the setup. + const BSONObj stats2(deleter.getStats()->toBSON()); + int totalCount = 0; + ASSERT_TRUE(FieldParser::extract(stats2, RangeDeleterStats::TotalDeletesField, + &totalCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(3, totalCount); + + int pendingCount = 0; + ASSERT_TRUE(FieldParser::extract(stats2, RangeDeleterStats::PendingDeletesField, + &pendingCount, NULL /* don't care errMsg */)); + ASSERT_EQUALS(2, pendingCount); + + int inProgressCount2 = 0; + ASSERT_TRUE(FieldParser::extract(stats2, RangeDeleterStats::InProgressDeletesField, + &inProgressCount2, NULL /* don't care errMsg */)); + ASSERT_EQUALS(1, inProgressCount2); + + // Let the first delete proceed. + env->resumeOneDelete(); + notifyDone1.waitToBeNotified(); + + ASSERT_TRUE(env->deleteOccured()); + + // { x: 10 } => { x: 20 } should be the first one since it is already in + // progress before the others are queued. + DeletedRange deleted1(env->getLastDelete()); + + ASSERT_EQUALS(ns, deleted1.ns); + ASSERT_TRUE(deleted1.min.equal(BSON("x" << 10))); + ASSERT_TRUE(deleted1.max.equal(BSON("x" << 20))); + ASSERT_TRUE(deleted1.shardKeyPattern.equal(BSON("x" << 1))); + + // Let the second delete proceed. + env->resumeOneDelete(); + notifyDone3.waitToBeNotified(); + + DeletedRange deleted2(env->getLastDelete()); + + // { x: 30 } => { x: 40 } should be next since there are still + // cursors open for blockedNS. + + ASSERT_EQUALS(ns, deleted2.ns); + ASSERT_TRUE(deleted2.min.equal(BSON("x" << 30))); + ASSERT_TRUE(deleted2.max.equal(BSON("x" << 40))); + ASSERT_TRUE(deleted2.shardKeyPattern.equal(BSON("x" << 1))); + + env->removeCursorId(blockedNS, 345); + // Let the last delete proceed. + env->resumeOneDelete(); + notifyDone2.waitToBeNotified(); + + DeletedRange deleted3(env->getLastDelete()); + + ASSERT_EQUALS(blockedNS, deleted3.ns); + ASSERT_TRUE(deleted3.min.equal(BSON("x" << 20))); + ASSERT_TRUE(deleted3.max.equal(BSON("x" << 30))); + ASSERT_TRUE(deleted3.shardKeyPattern.equal(BSON("x" << 1))); + + deleter.stopWorkers(); + } + + // Should not be able to delete ranges that overlaps with a black listed range. + TEST(BlackList, CantDeleteBlackListed) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + deleter.startWorkers(); + + const string ns("test.user"); + + string errMsg; + ASSERT_TRUE(deleter.addToBlackList(ns, BSON("x" << 100), BSON("x" << 200), &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + errMsg.clear(); + ASSERT_FALSE(deleter.queueDelete(ns, BSON("x" << 120), BSON("x" << 140), BSON("x" << 1), + false, NULL /* notifier not needed */, &errMsg)); + ASSERT_FALSE(errMsg.empty()); + + errMsg.clear(); + ASSERT_FALSE(deleter.deleteNow(ns, BSON("x" << 120), BSON("x" << 140), + BSON("x" << 1), false, &errMsg)); + ASSERT_FALSE(errMsg.empty()); + + ASSERT_FALSE(env->deleteOccured()); + + deleter.stopWorkers(); + } + + // Should not be able to black list a range that overlaps with a range that is + // already blacklisted. + TEST(BlackList, CantDoubleBlackList) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + const string ns("test.user"); + + string errMsg; + ASSERT_TRUE(deleter.addToBlackList(ns, BSON("x" << 100), BSON("x" << 200), &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + errMsg.clear(); + ASSERT_FALSE(deleter.addToBlackList(ns, BSON("x" << 100), BSON("x" << 200), &errMsg)); + ASSERT_FALSE(errMsg.empty()); + + errMsg.clear(); + ASSERT_FALSE(deleter.addToBlackList(ns, BSON("x" << 80), BSON("x" << 120), &errMsg)); + ASSERT_FALSE(errMsg.empty()); + + deleter.stopWorkers(); + } + + // Should not be able to black list a range that overlaps with a range that is already + // queued for deletion. + TEST(BlackList, CantBlackListQueued) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + const string ns("test.user"); + deleter.startWorkers(); + + // Set cursors on NS so deletes cannot be processed immediately. + env->addCursorId(ns, 58); + + Notification notifyDone; + deleter.queueDelete(ns, BSON("x" << 0), BSON("x" << 10), BSON("x" << 1), + false, ¬ifyDone, NULL /* errMsg not needed */); + + string errMsg; + ASSERT_FALSE(deleter.addToBlackList(ns, BSON("x" << 5), BSON("x" << 15), &errMsg)); + ASSERT_FALSE(errMsg.empty()); + + env->removeCursorId(ns, 58); + notifyDone.waitToBeNotified(); + + // But should be able to black list again once removed from the queue. + errMsg.clear(); + ASSERT_TRUE(deleter.addToBlackList(ns, BSON("x" << 5), BSON("x" << 15), &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + deleter.stopWorkers(); + } + + // Should not be able to black list a range that overlaps the range of an + // immediate delete that is currently in progress. + TEST(BlackList, CantBlackListImmediateInProgress) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + const string ns("test.user"); + + env->pauseDeletes(); + + string delErrMsg; + boost::thread deleterThread = boost::thread(boost::bind(&RangeDeleter::deleteNow, + &deleter, + ns, + BSON("x" << 64), + BSON("x" << 70), + BSON("x" << 1), + true, + &delErrMsg)); + + env->waitForNthPausedDelete(1u); + + string blErrMsg; + ASSERT_FALSE(deleter.addToBlackList(ns, BSON("x" << 10), BSON("x" << 90), &blErrMsg)); + ASSERT_FALSE(blErrMsg.empty()); + + env->resumeOneDelete(); + deleterThread.join(); + ASSERT_TRUE(delErrMsg.empty()); + + // Can blacklist again after delete completed. + blErrMsg.clear(); + ASSERT_TRUE(deleter.addToBlackList(ns, BSON("x" << 10), BSON("x" << 90), &blErrMsg)); + ASSERT_TRUE(blErrMsg.empty()); + + deleter.stopWorkers(); + } + + // Undo black list should only work if the range given exactly match with an + // existing black listed range. + TEST(BlackList, UndoShouldBeExact) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + const string ns("test.user"); + + ASSERT_TRUE(deleter.addToBlackList(ns, BSON("x" << 1234), BSON("x" << 8952), + NULL /* errMsg not needed */)); + + ASSERT_FALSE(deleter.removeFromBlackList(ns, BSON("x" << 1234), BSON("x" << 9000))); + + // Range should still be blacklisted + ASSERT_FALSE(deleter.deleteNow(ns, BSON("x" << 2000), BSON("x" << 4000), BSON("x" << 1), + false, NULL /* errMsg not needed */)); + + deleter.stopWorkers(); + } + + // Should be able to delete the range again once the black list has been undone. + TEST(BlackList, UndoBlackList) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + const string ns("test.user"); + + string errMsg; + ASSERT_TRUE(deleter.addToBlackList(ns, BSON("x" << 500), BSON("x" << 801), &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + errMsg.clear(); + ASSERT_FALSE(deleter.deleteNow(ns, BSON("x" << 600), BSON("x" << 700), + BSON("x" << 1), false, &errMsg)); + ASSERT_FALSE(errMsg.empty()); + + ASSERT_TRUE(deleter.removeFromBlackList(ns, BSON("x" << 500), BSON("x" << 801))); + + errMsg.clear(); + ASSERT_TRUE(deleter.deleteNow(ns, BSON("x" << 600), BSON("x" << 700), + BSON("x" << 1), false, &errMsg)); + ASSERT_TRUE(errMsg.empty()); + + deleter.stopWorkers(); + } + + // Black listing should only affect the specified namespace. + TEST(BlackList, NSIsolation) { + RangeDeleterMockEnv* env = new RangeDeleterMockEnv(); + RangeDeleter deleter(env); + + deleter.addToBlackList("foo.bar", BSON("x" << 100), BSON("x" << 200), + NULL /* errMsg not needed */); + + ASSERT_TRUE(deleter.deleteNow("test.user", BSON("x" << 120), BSON("x" << 140), + BSON("x" << 1), true, NULL /* errMsg not needed */)); + + deleter.stopWorkers(); + } + +} // unnamed namespace diff --git a/src/mongo/db/range_preserver.h b/src/mongo/db/range_preserver.h new file mode 100644 index 00000000000..dac32c30fd6 --- /dev/null +++ b/src/mongo/db/range_preserver.h @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include + +#include "mongo/db/clientcursor.h" + +namespace mongo { + + /** + * A RangePreserver prevents the RangeDeleter from removing any new data ranges in a collection. + * Previously queued ranges may still be deleted but the documents in those ranges will be + * filtered by CollectionMetadata::belongsToMe. + * + * TODO(greg/hk): Currently, creating a ClientCursor is how we accomplish this. This should + * change. + */ + class RangePreserver { + public: + /** + * Sharding uses the set of active cursor IDs as the current state. We add a dummy + * ClientCursor, which creates an additional cursor ID. The cursor ID lasts as long as this + * object does. The ClientCursorPin guarantees that the underlying ClientCursor is not + * deleted until this object goes out of scope. + */ + RangePreserver(const Collection* collection) { + invariant( collection ); + // Not a memory leak. Cached in a static structure by CC's ctor. + ClientCursor* cc = new ClientCursor(collection); + + // Pin keeps the CC from being deleted while it's in scope. We delete it ourselves. + _pin.reset(new ClientCursorPin(collection, cc->cursorid())); + } + + ~RangePreserver() { + _pin->deleteUnderlying(); + } + + private: + boost::scoped_ptr _pin; + }; + +} // namespace mongo diff --git a/src/mongo/db/record.cpp b/src/mongo/db/record.cpp deleted file mode 100644 index d55d6d40f68..00000000000 --- a/src/mongo/db/record.cpp +++ /dev/null @@ -1,595 +0,0 @@ -// record.cpp - -/** -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#include "pch.h" -#include "mongo/base/init.h" -#include "mongo/db/curop.h" -#include "mongo/db/databaseholder.h" -#include "mongo/db/pagefault.h" -#include "mongo/db/pdfile.h" -#include "mongo/db/record.h" -#include "mongo/db/commands/server_status.h" -#include "mongo/platform/bits.h" -#include "mongo/platform/unordered_set.h" -#include "mongo/util/net/listen.h" -#include "mongo/util/processinfo.h" -#include "mongo/util/stack_introspect.h" - -namespace mongo { - - RecordStats recordStats; - - void RecordStats::record( BSONObjBuilder& b ) { - b.appendNumber( "accessesNotInMemory" , accessesNotInMemory.load() ); - b.appendNumber( "pageFaultExceptionsThrown" , pageFaultExceptionsThrown.load() ); - - } - - void Record::appendStats( BSONObjBuilder& b ) { - recordStats.record( b ); - } - - namespace ps { - - enum State { - In , Out, Unk - }; - - enum Constants { - SliceSize = 1024 , - MaxChain = 20 , // intentionally very low - NumSlices = 10 , - RotateTimeSecs = 90 , - BigHashSize = 128 - }; - - int hash( size_t region ) { - return - abs( ( ( 7 + (int)(region & 0xFFFF) ) - * ( 11 + (int)( ( region >> 16 ) & 0xFFFF ) ) -#if defined(_WIN64) || defined(__amd64__) - * ( 13 + (int)( ( region >> 32 ) & 0xFFFF ) ) - * ( 17 + (int)( ( region >> 48 ) & 0xFFFF ) ) -#endif - ) % SliceSize ); - } - - - /** - * simple hash map for region -> status - * this constitutes a single region of time - * it does chaining, but very short chains - */ - class Slice { - - struct Entry { - size_t region; - unsigned long long value; - }; - - public: - - Slice() { - reset(); - } - - void reset() { - memset( _data , 0 , SliceSize * sizeof(Entry) ); - _lastReset = time(0); - } - - State get( int regionHash , size_t region , short offset ) { - DEV verify( hash( region ) == regionHash ); - - Entry * e = _get( regionHash , region , false ); - if ( ! e ) - return Unk; - - return ( e->value & ( 1ULL << offset ) ) ? In : Out; - } - - /** - * @return true if added, false if full - */ - bool in( int regionHash , size_t region , short offset ) { - DEV verify( hash( region ) == regionHash ); - - Entry * e = _get( regionHash , region , true ); - if ( ! e ) - return false; - - e->value |= 1ULL << offset; - return true; - } - - - void addPages( unordered_set* pages ) { - for ( int i = 0; i < SliceSize; i++ ) { - unsigned long long v = _data[i].value; - - while ( v ) { - int offset = firstBitSet( v ) - 1; - - size_t page = ( _data[i].region << 6 | offset ); - pages->insert( page ); - - v &= ~( 1ULL << offset ); - } - } - } - - time_t lastReset() const { return _lastReset; } - private: - - Entry* _get( int start , size_t region , bool add ) { - for ( int i=0; i ( 1000 * RotateTimeSecs ) ) { - _rotate(); - } - } - - for ( int i=0; i* pages, Slice* mySlices ) { - time_t oldestTimestamp = std::numeric_limits::max(); - { - // by doing this, we're in the lock only about half as long as the naive way - // that's measure with a small data set - // Assumption is that with a large data set, actually adding to set may get more costly - // so this way the time in lock should be totally constant - SimpleMutex::scoped_lock lk( _lock ); - memcpy( mySlices, _slices, NumSlices * sizeof(Slice) ); - - for ( int i = 0; i < NumSlices; i++ ) { - oldestTimestamp = std::min( oldestTimestamp, _slices[i].lastReset() ); - } - } - - for ( int i = 0; i < NumSlices; i++ ) { - mySlices[i].addPages( pages ); - } - - return oldestTimestamp; - } - private: - - void _rotate() { - _curSlice = ( _curSlice + 1 ) % NumSlices; - _slices[_curSlice].reset(); - _lastRotate = Listener::getElapsedTimeMillis(); - } - - int _curSlice; - long long _lastRotate; - Slice _slices[NumSlices]; - - SimpleMutex _lock; - }; - - Rolling* rolling = new Rolling[BigHashSize]; - - int bigHash( size_t region ) { - return hash( region ) % BigHashSize; - } - - namespace PointerTable { - - /* A "superpage" is a group of 16 contiguous pages that differ - * only in the low-order 16 bits. This means that there is - * enough room in the low-order bits to store a bitmap for each - * page in the superpage. - */ - static const size_t superpageMask = ~0xffffLL; - static const size_t superpageShift = 16; - static const size_t pageSelectorMask = 0xf000LL; // selects a page in a superpage - static const int pageSelectorShift = 12; - - // Tunables - static const int capacity = 128; // in superpages - static const int bucketSize = 4; // half cache line - static const int buckets = capacity/bucketSize; - - struct Data { - /** organized similar to a CPU cache - * bucketSize-way set associative - * least-recently-inserted replacement policy - */ - size_t _table[buckets][bucketSize]; - long long _lastReset; // time in millis - }; - - void reset(Data* data) { - memset(data->_table, 0, sizeof(data->_table)); - data->_lastReset = Listener::getElapsedTimeMillis(); - } - - inline void resetIfNeeded( Data* data ) { - const long long now = Listener::getElapsedTimeMillis(); - if (MONGO_unlikely(now - data->_lastReset > RotateTimeSecs*1000)) - reset(data); - } - - inline size_t pageBitOf(size_t ptr) { - return 1LL << ((ptr & pageSelectorMask) >> pageSelectorShift); - } - - inline size_t superpageOf(size_t ptr) { - return ptr & superpageMask; - } - - inline size_t bucketFor(size_t ptr) { - return (ptr >> superpageShift) % buckets; - } - - inline bool haveSeenPage(size_t superpage, size_t ptr) { - return superpage & pageBitOf(ptr); - } - - inline void markPageSeen(size_t& superpage, size_t ptr) { - superpage |= pageBitOf(ptr); - } - - /** call this to check a page has been seen yet. */ - inline bool seen(Data* data, size_t ptr) { - resetIfNeeded(data); - - // A bucket contains 4 superpages each containing 16 contiguous pages - // See below for a more detailed explanation of superpages - size_t* bucket = data->_table[bucketFor(ptr)]; - - for (int i = 0; i < bucketSize; i++) { - if (superpageOf(ptr) == superpageOf(bucket[i])) { - if (haveSeenPage(bucket[i], ptr)) - return true; - - markPageSeen(bucket[i], ptr); - return false; - } - } - - // superpage isn't in thread-local cache - // slide bucket forward and add new superpage at front - for (int i = bucketSize-1; i > 0; i--) - bucket[i] = bucket[i-1]; - - bucket[0] = superpageOf(ptr); - markPageSeen(bucket[0], ptr); - - return false; - } - - Data* getData(); - - }; - - void appendWorkingSetInfo( BSONObjBuilder& b ) { - boost::scoped_array mySlices( new Slice[NumSlices] ); - - unordered_set totalPages; - Timer t; - - time_t timestamp = 0; - - for ( int i = 0; i < BigHashSize; i++ ) { - time_t myOldestTimestamp = rolling[i].addPages( &totalPages, mySlices.get() ); - timestamp = std::max( timestamp, myOldestTimestamp ); - } - - b.append( "note", "thisIsAnEstimate" ); - b.appendNumber( "pagesInMemory", totalPages.size() ); - b.appendNumber( "computationTimeMicros", static_cast(t.micros()) ); - b.append( "overSeconds", static_cast( time(0) - timestamp ) ); - - } - - } - - - // These need to be outside the ps namespace due to the way they are defined -#if defined(__linux__) && defined(__GNUC__) - __thread ps::PointerTable::Data _pointerTableData; - ps::PointerTable::Data* ps::PointerTable::getData() { - return &_pointerTableData; - } -#elif defined(_WIN32) - __declspec( thread ) ps::PointerTable::Data _pointerTableData; - ps::PointerTable::Data* ps::PointerTable::getData() { - return &_pointerTableData; - } -#else - TSP_DEFINE(ps::PointerTable::Data, _pointerTableData); - ps::PointerTable::Data* ps::PointerTable::getData() { - return _pointerTableData.getMake(); - } -#endif - - bool Record::MemoryTrackingEnabled = true; - - volatile int __record_touch_dummy = 1; // this is used to make sure the compiler doesn't get too smart on us - void Record::touch( bool entireRecrd ) const { - if ( _lengthWithHeaders > HeaderSize ) { // this also makes sure lengthWithHeaders is in memory - const char * addr = _data; - const char * end = _data + _netLength(); - for ( ; addr <= end ; addr += 2048 ) { - __record_touch_dummy += addr[0]; - - break; // TODO: remove this, pending SERVER-3711 - - // note if this is a touch of a deletedrecord, we don't want to touch more than the first part. we may simply - // be updated the linked list and a deletedrecord could be gigantic. similar circumstance just less extreme - // exists for any record if we are just updating its header, say on a remove(); some sort of hints might be - // useful. - - if ( ! entireRecrd ) - break; - } - } - } - - static bool blockSupported = false; - - MONGO_INITIALIZER_WITH_PREREQUISITES(RecordBlockSupported, - ("SystemInfo"))(InitializerContext* cx) { - blockSupported = ProcessInfo::blockCheckSupported(); - return Status::OK(); - } - - void Record::appendWorkingSetInfo( BSONObjBuilder& b ) { - if ( ! blockSupported ) { - b.append( "info", "not supported" ); - return; - } - - ps::appendWorkingSetInfo( b ); - } - - bool Record::likelyInPhysicalMemory() const { - return likelyInPhysicalMemory( _data ); - } - - bool Record::likelyInPhysicalMemory( const char* data ) { - DEV { - // we don't want to do this too often as it makes DEBUG builds very slow - // at some point we might want to pass in what type of Record this is and - // then we can use that to make a more intelligent decision - int mod; - if ( Lock::isReadLocked() ) { - // we'll check read locks less often - // since its a lower probability of error - mod = 1000; - } - else if ( Lock::isLocked() ) { - // write lock's can more obviously cause issues - // check more often than reads - mod = 100; - } - else { - // no lock??? - // if we get here we should be very paranoid - mod = 50; - } - - if ( rand() % mod == 0 ) - return false; - } // end DEV test code - - if ( ! MemoryTrackingEnabled ) - return true; - - const size_t page = (size_t)data >> 12; - const size_t region = page >> 6; - const size_t offset = page & 0x3f; - - const bool seen = ps::PointerTable::seen( ps::PointerTable::getData(), reinterpret_cast(data)); - if (seen || ps::rolling[ps::bigHash(region)].access( region , offset , false ) ) { - -#ifdef _DEBUG - if ( blockSupported && ! ProcessInfo::blockInMemory(data) ) { - RARELY warning() << "we think data is in ram but system says no" << endl; - } -#endif - return true; - } - - if ( ! blockSupported ) { - // this means we don't fallback to system call - // and assume things aren't in memory - // possible we yield too much - but better than not yielding through a fault - return false; - } - - return ProcessInfo::blockInMemory( const_cast(data) ); - } - - - Record* Record::accessed() { - const bool seen = ps::PointerTable::seen( ps::PointerTable::getData(), reinterpret_cast(_data)); - if (!seen){ - const size_t page = (size_t)_data >> 12; - const size_t region = page >> 6; - const size_t offset = page & 0x3f; - ps::rolling[ps::bigHash(region)].access( region , offset , true ); - } - - return this; - } - - Record* DiskLoc::rec() const { - Record *r = DataFileMgr::getRecord(*this); - memconcept::is(r, memconcept::concept::record); - return r; - } - - void Record::_accessing() const { - if ( likelyInPhysicalMemory() ) - return; - - const Client& client = cc(); - Database* db = client.database(); - - recordStats.accessesNotInMemory.fetchAndAdd(1); - if ( db ) - db->recordStats().accessesNotInMemory.fetchAndAdd(1); - - if ( ! client.allowedToThrowPageFaultException() ) - return; - - if ( client.curop() && client.curop()->elapsedMillis() > 50 ) { - // this means we've been going too long to restart - // we should track how often this happens - return; - } - - recordStats.pageFaultExceptionsThrown.fetchAndAdd(1); - if ( db ) - db->recordStats().pageFaultExceptionsThrown.fetchAndAdd(1); - - DEV fassert( 16236 , ! inConstructorChain(true) ); - throw PageFaultException(this); - } - - void DeletedRecord::_accessing() const { - - } - - namespace { - - class WorkingSetSSS : public ServerStatusSection { - public: - WorkingSetSSS() : ServerStatusSection( "workingSet" ){} - virtual bool includeByDefault() const { return false; } - - BSONObj generateSection(const BSONElement& configElement) const { - BSONObjBuilder b; - Record::appendWorkingSetInfo( b ); - return b.obj(); - } - - } asserts; - - class RecordStats : public ServerStatusSection { - public: - RecordStats() : ServerStatusSection( "recordStats" ){} - virtual bool includeByDefault() const { return true; } - - BSONObj generateSection(const BSONElement& configElement) const { - BSONObjBuilder record; - - Record::appendStats( record ); - - set dbs; - { - Lock::DBRead read( "local" ); - dbHolder().getAllShortNames( dbs ); - } - - for ( set::iterator i = dbs.begin(); i != dbs.end(); ++i ) { - string db = *i; - Client::ReadContext ctx( db ); - BSONObjBuilder temp( record.subobjStart( db ) ); - ctx.ctx().db()->recordStats().record( temp ); - temp.done(); - } - - return record.obj(); - } - - } recordStats; - - } -} diff --git a/src/mongo/db/record.h b/src/mongo/db/record.h deleted file mode 100644 index 20b0528a16c..00000000000 --- a/src/mongo/db/record.h +++ /dev/null @@ -1,35 +0,0 @@ -// database.h - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#pragma once - -#include "mongo/bson/bsonobjbuilder.h" -#include "mongo/platform/atomic_word.h" - -namespace mongo { - - struct RecordStats { - void record( BSONObjBuilder& b ); - - AtomicInt64 accessesNotInMemory; - AtomicInt64 pageFaultExceptionsThrown; - }; - - - -} diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp new file mode 100644 index 00000000000..03d61395e61 --- /dev/null +++ b/src/mongo/db/repair_database.cpp @@ -0,0 +1,469 @@ +// repair_database.cpp + +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/repair_database.h" + +#include + +#include "mongo/db/background.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_holder.h" +#include "mongo/db/catalog/index_create.h" +#include "mongo/db/client.h" +#include "mongo/db/cloner.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/kill_current_op.h" +#include "mongo/db/structure/collection_iterator.h" +#include "mongo/util/file.h" +#include "mongo/util/file_allocator.h" + +namespace mongo { + + typedef boost::filesystem::path Path; + + // TODO SERVER-4328 + bool inDBRepair = false; + struct doingRepair { + doingRepair() { + verify( ! inDBRepair ); + inDBRepair = true; + } + ~doingRepair() { + inDBRepair = false; + } + }; + + // inheritable class to implement an operation that may be applied to all + // files in a database using _applyOpToDataFiles() + class FileOp { + public: + virtual ~FileOp() {} + // Return true if file exists and operation successful + virtual bool apply( const boost::filesystem::path &p ) = 0; + virtual const char * op() const = 0; + }; + + void _applyOpToDataFiles(const string& database, FileOp &fo, bool afterAllocator = false, + const string& path = storageGlobalParams.dbpath); + + void _deleteDataFiles(const std::string& database) { + if (storageGlobalParams.directoryperdb) { + FileAllocator::get()->waitUntilFinished(); + MONGO_ASSERT_ON_EXCEPTION_WITH_MSG( + boost::filesystem::remove_all( + boost::filesystem::path(storageGlobalParams.dbpath) / database), + "delete data files with a directoryperdb"); + return; + } + class : public FileOp { + virtual bool apply( const boost::filesystem::path &p ) { + return boost::filesystem::remove( p ); + } + virtual const char * op() const { + return "remove"; + } + } deleter; + _applyOpToDataFiles( database, deleter, true ); + } + + void boostRenameWrapper( const Path &from, const Path &to ) { + try { + boost::filesystem::rename( from, to ); + } + catch ( const boost::filesystem::filesystem_error & ) { + // boost rename doesn't work across partitions + boost::filesystem::copy_file( from, to); + boost::filesystem::remove( from ); + } + } + + // back up original database files to 'temp' dir + void _renameForBackup( const std::string& database, const Path &reservedPath ) { + Path newPath( reservedPath ); + if (storageGlobalParams.directoryperdb) + newPath /= database; + class Renamer : public FileOp { + public: + Renamer( const Path &newPath ) : newPath_( newPath ) {} + private: + const boost::filesystem::path &newPath_; + virtual bool apply( const Path &p ) { + if ( !boost::filesystem::exists( p ) ) + return false; + boostRenameWrapper( p, newPath_ / ( p.leaf().string() + ".bak" ) ); + return true; + } + virtual const char * op() const { + return "renaming"; + } + } renamer( newPath ); + _applyOpToDataFiles( database, renamer, true ); + } + + intmax_t dbSize( const string& database ) { + class SizeAccumulator : public FileOp { + public: + SizeAccumulator() : totalSize_( 0 ) {} + intmax_t size() const { + return totalSize_; + } + private: + virtual bool apply( const boost::filesystem::path &p ) { + if ( !boost::filesystem::exists( p ) ) + return false; + totalSize_ += boost::filesystem::file_size( p ); + return true; + } + virtual const char *op() const { + return "checking size"; + } + intmax_t totalSize_; + }; + SizeAccumulator sa; + _applyOpToDataFiles( database, sa ); + return sa.size(); + } + + // move temp files to standard data dir + void _replaceWithRecovered( const string& database, const char *reservedPathString ) { + Path newPath(storageGlobalParams.dbpath); + if (storageGlobalParams.directoryperdb) + newPath /= database; + class Replacer : public FileOp { + public: + Replacer( const Path &newPath ) : newPath_( newPath ) {} + private: + const boost::filesystem::path &newPath_; + virtual bool apply( const Path &p ) { + if ( !boost::filesystem::exists( p ) ) + return false; + boostRenameWrapper( p, newPath_ / p.leaf() ); + return true; + } + virtual const char * op() const { + return "renaming"; + } + } replacer( newPath ); + _applyOpToDataFiles( database, replacer, true, reservedPathString ); + } + + // generate a directory name for storing temp data files + Path uniqueReservedPath( const char *prefix ) { + Path repairPath = Path(storageGlobalParams.repairpath); + Path reservedPath; + int i = 0; + bool exists = false; + do { + stringstream ss; + ss << prefix << "_repairDatabase_" << i++; + reservedPath = repairPath / ss.str(); + MONGO_ASSERT_ON_EXCEPTION( exists = boost::filesystem::exists( reservedPath ) ); + } + while ( exists ); + return reservedPath; + } + + void _applyOpToDataFiles( const string& database, FileOp &fo, bool afterAllocator, const string& path ) { + if ( afterAllocator ) + FileAllocator::get()->waitUntilFinished(); + string c = database; + c += '.'; + boost::filesystem::path p(path); + if (storageGlobalParams.directoryperdb) + p /= database; + boost::filesystem::path q; + q = p / (c+"ns"); + bool ok = false; + MONGO_ASSERT_ON_EXCEPTION( ok = fo.apply( q ) ); + if ( ok ) { + LOG(2) << fo.op() << " file " << q.string() << endl; + } + int i = 0; + int extra = 10; // should not be necessary, this is defensive in case there are missing files + while ( 1 ) { + verify( i <= DiskLoc::MaxFiles ); + stringstream ss; + ss << c << i; + q = p / ss.str(); + MONGO_ASSERT_ON_EXCEPTION( ok = fo.apply(q) ); + if ( ok ) { + if ( extra != 10 ) { + LOG(1) << fo.op() << " file " << q.string() << endl; + log() << " _applyOpToDataFiles() warning: extra == " << extra << endl; + } + } + else if ( --extra <= 0 ) + break; + i++; + } + } + + class RepairFileDeleter { + public: + RepairFileDeleter( const string& dbName, + const string& pathString, + const Path& path ) + : _dbName( dbName ), + _pathString( pathString ), + _path( path ), + _success( false ) { + } + + ~RepairFileDeleter() { + if ( _success ) + return; + + log() << "cleaning up failed repair " + << "db: " << _dbName << " path: " << _pathString; + + try { + getDur().syncDataAndTruncateJournal(); + MongoFile::flushAll(true); // need both in case journaling is disabled + { + Client::Context tempContext( _dbName, _pathString ); + Database::closeDatabase( _dbName, _pathString ); + } + MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::remove_all( _path ) ); + } + catch ( DBException& e ) { + error() << "RepairFileDeleter failed to cleanup: " << e; + error() << "aborting"; + fassertFailed( 17402 ); + } + } + + void success() { + _success = true; + } + + private: + string _dbName; + string _pathString; + Path _path; + bool _success; + }; + + Status repairDatabase( string dbName, + bool preserveClonedFilesOnFailure, + bool backupOriginalFiles ) { + scoped_ptr repairFileDeleter; + doingRepair dr; + dbName = nsToDatabase( dbName ); + + log() << "repairDatabase " << dbName << endl; + + invariant( cc().database()->name() == dbName ); + invariant( cc().database()->path() == storageGlobalParams.dbpath ); + + BackgroundOperation::assertNoBgOpInProgForDb(dbName); + + getDur().syncDataAndTruncateJournal(); // Must be done before and after repair + + intmax_t totalSize = dbSize( dbName ); + intmax_t freeSize = File::freeSpace(storageGlobalParams.repairpath); + + if ( freeSize > -1 && freeSize < totalSize ) { + return Status( ErrorCodes::OutOfDiskSpace, + str::stream() << "Cannot repair database " << dbName + << " having size: " << totalSize + << " (bytes) because free disk space is: " << freeSize << " (bytes)" ); + } + + killCurrentOp.checkForInterrupt(); + + Path reservedPath = + uniqueReservedPath( ( preserveClonedFilesOnFailure || backupOriginalFiles ) ? + "backup" : "_tmp" ); + MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::create_directory( reservedPath ) ); + string reservedPathString = reservedPath.string(); + + if ( !preserveClonedFilesOnFailure ) + repairFileDeleter.reset( new RepairFileDeleter( dbName, + reservedPathString, + reservedPath ) ); + + { + Database* originalDatabase = dbHolder().get( dbName, storageGlobalParams.dbpath ); + if ( originalDatabase == NULL ) + return Status( ErrorCodes::NamespaceNotFound, "database does not exist to repair" ); + + Database* tempDatabase = NULL; + { + bool justCreated = false; + tempDatabase = dbHolderW().getOrCreate( dbName, reservedPathString, justCreated ); + invariant( justCreated ); + } + + map namespacesToCopy; + { + string ns = dbName + ".system.namespaces"; + Client::Context ctx( ns ); + Collection* coll = originalDatabase->getCollection( ns ); + if ( coll ) { + scoped_ptr it( coll->getIterator( DiskLoc(), + false, + CollectionScanParams::FORWARD ) ); + while ( !it->isEOF() ) { + DiskLoc loc = it->getNext(); + BSONObj obj = coll->docFor( loc ); + + string ns = obj["name"].String(); + + NamespaceString nss( ns ); + if ( nss.isSystem() ) { + if ( nss.isSystemDotIndexes() ) + continue; + if ( nss.coll() == "system.namespaces" ) + continue; + } + + if ( !nss.isNormal() ) + continue; + + CollectionOptions options; + if ( obj["options"].isABSONObj() ) { + Status status = options.parse( obj["options"].Obj() ); + if ( !status.isOK() ) + return status; + } + namespacesToCopy[ns] = options; + } + } + } + + for ( map::const_iterator i = namespacesToCopy.begin(); + i != namespacesToCopy.end(); + ++i ) { + string ns = i->first; + CollectionOptions options = i->second; + + Collection* tempCollection = NULL; + { + Client::Context tempContext( ns, tempDatabase ); + tempCollection = tempDatabase->createCollection( ns, options, true, false ); + } + + Client::Context readContext( ns, originalDatabase ); + Collection* originalCollection = originalDatabase->getCollection( ns ); + invariant( originalCollection ); + + // data + + MultiIndexBlock indexBlock( tempCollection ); + { + vector indexes; + IndexCatalog::IndexIterator ii = + originalCollection->getIndexCatalog()->getIndexIterator( false ); + while ( ii.more() ) { + IndexDescriptor* desc = ii.next(); + indexes.push_back( desc->infoObj() ); + } + + Client::Context tempContext( ns, tempDatabase ); + Status status = indexBlock.init( indexes ); + if ( !status.isOK() ) + return status; + + } + + scoped_ptr iterator( originalCollection->getIterator( DiskLoc(), + false, + CollectionScanParams::FORWARD ) ); + while ( !iterator->isEOF() ) { + DiskLoc loc = iterator->getNext(); + invariant( !loc.isNull() ); + + BSONObj doc = originalCollection->docFor( loc ); + + Client::Context tempContext( ns, tempDatabase ); + StatusWith result = tempCollection->insertDocument( doc, indexBlock ); + if ( !result.isOK() ) + return result.getStatus(); + + getDur().commitIfNeeded(); + killCurrentOp.checkForInterrupt(false); + } + + { + Client::Context tempContext( ns, tempDatabase ); + Status status = indexBlock.commit(); + if ( !status.isOK() ) + return status; + } + + } + + getDur().syncDataAndTruncateJournal(); + MongoFile::flushAll(true); // need both in case journaling is disabled + + killCurrentOp.checkForInterrupt(false); + + Client::Context tempContext( dbName, reservedPathString ); + Database::closeDatabase( dbName, reservedPathString ); + } + + // at this point if we abort, we don't want to delete new files + // as they might be the only copies + + if ( repairFileDeleter.get() ) + repairFileDeleter->success(); + + Client::Context ctx( dbName ); + Database::closeDatabase(dbName, storageGlobalParams.dbpath); + + if ( backupOriginalFiles ) { + _renameForBackup( dbName, reservedPath ); + } + else { + // first make new directory before deleting data + Path newDir = Path(storageGlobalParams.dbpath) / dbName; + MONGO_ASSERT_ON_EXCEPTION(boost::filesystem::create_directory(newDir)); + + // this deletes old files + _deleteDataFiles( dbName ); + + if ( !boost::filesystem::exists(newDir) ) { + // we deleted because of directoryperdb + // re-create + MONGO_ASSERT_ON_EXCEPTION(boost::filesystem::create_directory(newDir)); + } + } + + _replaceWithRecovered( dbName, reservedPathString.c_str() ); + + if ( !backupOriginalFiles ) + MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::remove_all( reservedPath ) ); + + return Status::OK(); + } + + +} diff --git a/src/mongo/db/repair_database.h b/src/mongo/db/repair_database.h new file mode 100644 index 00000000000..4fffe435558 --- /dev/null +++ b/src/mongo/db/repair_database.h @@ -0,0 +1,52 @@ +// repair_database.h + +/** +* Copyright (C) 2014 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include + +#include "mongo/base/status.h" +#include "mongo/platform/cstdint.h" + +namespace mongo { + + // TODO: move + intmax_t dbSize( const std::string& database ); + + // TODO: move + void _deleteDataFiles(const std::string& database); + + // must have a global lock + Status repairDatabase( std::string db, + bool preserveClonedFilesOnFailure = false, + bool backupOriginalFiles = false ); + + +} // namespace mongo diff --git a/src/mongo/db/repl.cpp b/src/mongo/db/repl.cpp deleted file mode 100644 index 06bf2a7e3cd..00000000000 --- a/src/mongo/db/repl.cpp +++ /dev/null @@ -1,1640 +0,0 @@ -// @file repl.cpp - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -/* Collections we use: - - local.sources - indicates what sources we pull from as a "slave", and the last update of each - local.oplog.$main - our op log as "master" - local.dbinfo. - no longer used??? - local.pair.startup - [deprecated] can contain a special value indicating for a pair that we have the master copy. - used when replacing other half of the pair which has permanently failed. - local.pair.sync - [deprecated] { initialsynccomplete: 1 } -*/ - -#include "pch.h" - -#include -#include -#include - -#include "mongo/db/auth/action_set.h" -#include "mongo/db/auth/action_type.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/privilege.h" -#include "jsobj.h" -#include "../util/goodies.h" -#include "repl.h" -#include "../util/net/message.h" -#include "../util/background.h" -#include "../client/connpool.h" -#include "pdfile.h" -#include "db.h" -#include "commands.h" -#include "cmdline.h" -#include "repl_block.h" -#include "repl/rs.h" -#include "replutil.h" -#include "repl/connections.h" -#include "ops/update.h" -#include "pcrecpp.h" -#include "mongo/db/commands/server_status.h" -#include "mongo/db/instance.h" -#include "mongo/db/server_parameters.h" -#include "mongo/db/queryutil.h" -#include "mongo/base/counter.h" - -namespace mongo { - - // our config from command line etc. - ReplSettings replSettings; - - /* if 1 sync() is running */ - volatile int syncing = 0; - static volatile int relinquishSyncingSome = 0; - - /* "dead" means something really bad happened like replication falling completely out of sync. - when non-null, we are dead and the string is informational - */ - const char *replAllDead = 0; - - time_t lastForcedResync = 0; - -} // namespace mongo - -namespace mongo { - - /* output by the web console */ - const char *replInfo = ""; - struct ReplInfo { - ReplInfo(const char *msg) { - replInfo = msg; - } - ~ReplInfo() { - replInfo = "?"; - } - }; - - /* operator requested resynchronization of replication (on the slave). { resync : 1 } */ - class CmdResync : public Command { - public: - virtual bool slaveOk() const { - return true; - } - virtual bool adminOnly() const { - return true; - } - virtual bool logTheOp() { return false; } - virtual bool lockGlobally() const { return true; } - virtual LockType locktype() const { return WRITE; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) { - ActionSet actions; - actions.addAction(ActionType::resync); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); - } - void help(stringstream&h) const { h << "resync (from scratch) an out of date replica slave.\nhttp://dochub.mongodb.org/core/masterslave"; } - CmdResync() : Command("resync") { } - virtual bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { - if( cmdLine.usingReplSets() ) { - errmsg = "resync command not currently supported with replica sets. See RS102 info in the mongodb documentations"; - result.append("info", "http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember"); - return false; - } - - if ( cmdObj.getBoolField( "force" ) ) { - if ( !waitForSyncToFinish( errmsg ) ) - return false; - replAllDead = "resync forced"; - } - if ( !replAllDead ) { - errmsg = "not dead, no need to resync"; - return false; - } - if ( !waitForSyncToFinish( errmsg ) ) - return false; - - ReplSource::forceResyncDead( "client" ); - result.append( "info", "triggered resync for all sources" ); - return true; - } - bool waitForSyncToFinish( string &errmsg ) const { - // Wait for slave thread to finish syncing, so sources will be be - // reloaded with new saved state on next pass. - Timer t; - while ( 1 ) { - if ( syncing == 0 || t.millis() > 30000 ) - break; - { - Lock::TempRelease t; - relinquishSyncingSome = 1; - sleepmillis(1); - } - } - if ( syncing ) { - errmsg = "timeout waiting for sync() to finish"; - return false; - } - return true; - } - } cmdResync; - - bool anyReplEnabled() { - return replSettings.slave || replSettings.master || theReplSet; - } - - bool replAuthenticate(DBClientBase *conn, bool skipAuthCheck); - - void appendReplicationInfo(BSONObjBuilder& result, int level) { - if ( replSet ) { - if( theReplSet == 0 || theReplSet->state().shunned() ) { - result.append("ismaster", false); - result.append("secondary", false); - result.append("info", ReplSet::startupStatusMsg.get()); - result.append( "isreplicaset" , true ); - } - else { - theReplSet->fillIsMaster(result); - } - return; - } - - if ( replAllDead ) { - result.append("ismaster", 0); - string s = string("dead: ") + replAllDead; - result.append("info", s); - } - else { - result.appendBool("ismaster", _isMaster() ); - } - - if ( level && replSet ) { - result.append( "info" , "is replica set" ); - } - else if ( level ) { - BSONObjBuilder sources( result.subarrayStart( "sources" ) ); - - int n = 0; - list src; - { - Client::ReadContext ctx("local.sources", dbpath); - shared_ptr c = findTableScan("local.sources", BSONObj()); - while ( c->ok() ) { - src.push_back(c->current()); - c->advance(); - } - } - - for( list::const_iterator i = src.begin(); i != src.end(); i++ ) { - BSONObj s = *i; - BSONObjBuilder bb; - bb.append( s["host"] ); - string sourcename = s["source"].valuestr(); - if ( sourcename != "main" ) - bb.append( s["source"] ); - { - BSONElement e = s["syncedTo"]; - BSONObjBuilder t( bb.subobjStart( "syncedTo" ) ); - t.appendDate( "time" , e.timestampTime() ); - t.append( "inc" , e.timestampInc() ); - t.done(); - } - - if ( level > 1 ) { - wassert( !Lock::isLocked() ); - // note: there is no so-style timeout on this connection; perhaps we should have one. - scoped_ptr conn( ScopedDbConnection::getInternalScopedDbConnection( s["host"].valuestr() ) ); - - DBClientConnection *cliConn = dynamic_cast< DBClientConnection* >( &conn->conn() ); - if ( cliConn && replAuthenticate(cliConn, false) ) { - BSONObj first = conn->get()->findOne( (string)"local.oplog.$" + sourcename, - Query().sort( BSON( "$natural" << 1 ) ) ); - BSONObj last = conn->get()->findOne( (string)"local.oplog.$" + sourcename, - Query().sort( BSON( "$natural" << -1 ) ) ); - bb.appendDate( "masterFirst" , first["ts"].timestampTime() ); - bb.appendDate( "masterLast" , last["ts"].timestampTime() ); - double lag = (double) (last["ts"].timestampTime() - s["syncedTo"].timestampTime()); - bb.append( "lagSeconds" , lag / 1000 ); - } - conn->done(); - } - - sources.append( BSONObjBuilder::numStr( n++ ) , bb.obj() ); - } - - sources.done(); - } - } - - class ReplicationInfoServerStatus : public ServerStatusSection { - public: - ReplicationInfoServerStatus() : ServerStatusSection( "repl" ){} - bool includeByDefault() const { return true; } - - BSONObj generateSection(const BSONElement& configElement) const { - if ( ! anyReplEnabled() ) - return BSONObj(); - - int level = configElement.numberInt(); - - BSONObjBuilder result; - appendReplicationInfo(result, level); - return result.obj(); - } - } replicationInfoServerStatus; - - class CmdIsMaster : public Command { - public: - virtual bool requiresAuth() { return false; } - virtual bool slaveOk() const { - return true; - } - virtual void help( stringstream &help ) const { - help << "Check if this server is primary for a replica pair/set; also if it is --master or --slave in simple master/slave setups.\n"; - help << "{ isMaster : 1 }"; - } - virtual LockType locktype() const { return NONE; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector* out) {} // No auth required - CmdIsMaster() : Command("isMaster", true, "ismaster") { } - virtual bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) { - /* currently request to arbiter is (somewhat arbitrarily) an ismaster request that is not - authenticated. - */ - appendReplicationInfo(result, 0); - - result.appendNumber("maxBsonObjectSize", BSONObjMaxUserSize); - result.appendNumber("maxMessageSizeBytes", MaxMessageSizeBytes); - result.appendDate("localTime", jsTime()); - return true; - } - } cmdismaster; - - ReplSource::ReplSource() { - nClonedThisPass = 0; - } - - ReplSource::ReplSource(BSONObj o) : nClonedThisPass(0) { - only = o.getStringField("only"); - hostName = o.getStringField("host"); - _sourceName = o.getStringField("source"); - uassert( 10118 , "'host' field not set in sources collection object", !hostName.empty() ); - uassert( 10119 , "only source='main' allowed for now with replication", sourceName() == "main" ); - BSONElement e = o.getField("syncedTo"); - if ( !e.eoo() ) { - uassert( 10120 , "bad sources 'syncedTo' field value", e.type() == Date || e.type() == Timestamp ); - OpTime tmp( e.date() ); - syncedTo = tmp; - } - - BSONObj dbsObj = o.getObjectField("dbsNextPass"); - if ( !dbsObj.isEmpty() ) { - BSONObjIterator i(dbsObj); - while ( 1 ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - addDbNextPass.insert( e.fieldName() ); - } - } - - dbsObj = o.getObjectField("incompleteCloneDbs"); - if ( !dbsObj.isEmpty() ) { - BSONObjIterator i(dbsObj); - while ( 1 ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - incompleteCloneDbs.insert( e.fieldName() ); - } - } - } - - /* Turn our C++ Source object into a BSONObj */ - BSONObj ReplSource::jsobj() { - BSONObjBuilder b; - b.append("host", hostName); - b.append("source", sourceName()); - if ( !only.empty() ) - b.append("only", only); - if ( !syncedTo.isNull() ) - b.appendTimestamp("syncedTo", syncedTo.asDate()); - - BSONObjBuilder dbsNextPassBuilder; - int n = 0; - for ( set::iterator i = addDbNextPass.begin(); i != addDbNextPass.end(); i++ ) { - n++; - dbsNextPassBuilder.appendBool(*i, 1); - } - if ( n ) - b.append("dbsNextPass", dbsNextPassBuilder.done()); - - BSONObjBuilder incompleteCloneDbsBuilder; - n = 0; - for ( set::iterator i = incompleteCloneDbs.begin(); i != incompleteCloneDbs.end(); i++ ) { - n++; - incompleteCloneDbsBuilder.appendBool(*i, 1); - } - if ( n ) - b.append("incompleteCloneDbs", incompleteCloneDbsBuilder.done()); - - return b.obj(); - } - - void ReplSource::save() { - BSONObjBuilder b; - verify( !hostName.empty() ); - b.append("host", hostName); - // todo: finish allowing multiple source configs. - // this line doesn't work right when source is null, if that is allowed as it is now: - //b.append("source", _sourceName); - BSONObj pattern = b.done(); - - BSONObj o = jsobj(); - LOG( 1 ) << "Saving repl source: " << o << endl; - - { - OpDebug debug; - Client::Context ctx("local.sources"); - UpdateResult res = updateObjects("local.sources", o, pattern, true/*upsert for pair feature*/, false,false,debug); - verify( ! res.mod ); - verify( res.num == 1 ); - } - } - - static void addSourceToList(ReplSource::SourceVector &v, ReplSource& s, ReplSource::SourceVector &old) { - if ( !s.syncedTo.isNull() ) { // Don't reuse old ReplSource if there was a forced resync. - for ( ReplSource::SourceVector::iterator i = old.begin(); i != old.end(); ) { - if ( s == **i ) { - v.push_back(*i); - old.erase(i); - return; - } - i++; - } - } - - v.push_back( shared_ptr< ReplSource >( new ReplSource( s ) ) ); - } - - /* we reuse our existing objects so that we can keep our existing connection - and cursor in effect. - */ - void ReplSource::loadAll(SourceVector &v) { - Client::Context ctx("local.sources"); - SourceVector old = v; - v.clear(); - - if ( !cmdLine.source.empty() ) { - // --source specified. - // check that no items are in sources other than that - // add if missing - shared_ptr c = findTableScan("local.sources", BSONObj()); - int n = 0; - while ( c->ok() ) { - n++; - ReplSource tmp(c->current()); - if ( tmp.hostName != cmdLine.source ) { - log() << "repl: --source " << cmdLine.source << " != " << tmp.hostName << " from local.sources collection" << endl; - log() << "repl: for instructions on changing this slave's source, see:" << endl; - log() << "http://dochub.mongodb.org/core/masterslave" << endl; - log() << "repl: terminating mongod after 30 seconds" << endl; - sleepsecs(30); - dbexit( EXIT_REPLICATION_ERROR ); - } - if ( tmp.only != cmdLine.only ) { - log() << "--only " << cmdLine.only << " != " << tmp.only << " from local.sources collection" << endl; - log() << "terminating after 30 seconds" << endl; - sleepsecs(30); - dbexit( EXIT_REPLICATION_ERROR ); - } - c->advance(); - } - uassert( 10002 , "local.sources collection corrupt?", n<2 ); - if ( n == 0 ) { - // source missing. add. - ReplSource s; - s.hostName = cmdLine.source; - s.only = cmdLine.only; - s.save(); - } - } - else { - try { - massert( 10384 , "--only requires use of --source", cmdLine.only.empty()); - } - catch ( ... ) { - dbexit( EXIT_BADOPTIONS ); - } - } - - shared_ptr c = findTableScan("local.sources", BSONObj()); - while ( c->ok() ) { - ReplSource tmp(c->current()); - if ( tmp.syncedTo.isNull() ) { - DBDirectClient c; - if ( c.exists( "local.oplog.$main" ) ) { - BSONObj op = c.findOne( "local.oplog.$main", QUERY( "op" << NE << "n" ).sort( BSON( "$natural" << -1 ) ) ); - if ( !op.isEmpty() ) { - tmp.syncedTo = op[ "ts" ].date(); - } - } - } - addSourceToList(v, tmp, old); - c->advance(); - } - } - - BSONObj opTimeQuery = fromjson("{\"getoptime\":1}"); - - bool ReplSource::throttledForceResyncDead( const char *requester ) { - if ( time( 0 ) - lastForcedResync > 600 ) { - forceResyncDead( requester ); - lastForcedResync = time( 0 ); - return true; - } - return false; - } - - void ReplSource::forceResyncDead( const char *requester ) { - if ( !replAllDead ) - return; - SourceVector sources; - ReplSource::loadAll(sources); - for( SourceVector::iterator i = sources.begin(); i != sources.end(); ++i ) { - log() << requester << " forcing resync from " << (*i)->hostName << endl; - (*i)->forceResync( requester ); - } - replAllDead = 0; - } - - void ReplSource::forceResync( const char *requester ) { - BSONObj info; - { - dbtemprelease t; - if (!oplogReader.connect(hostName)) { - msgassertedNoTrace( 14051 , "unable to connect to resync"); - } - /* todo use getDatabaseNames() method here */ - bool ok = oplogReader.conn()->runCommand( "admin", BSON( "listDatabases" << 1 ), info ); - massert( 10385 , "Unable to get database list", ok ); - } - BSONObjIterator i( info.getField( "databases" ).embeddedObject() ); - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - string name = e.embeddedObject().getField( "name" ).valuestr(); - if ( !e.embeddedObject().getBoolField( "empty" ) ) { - if ( name != "local" ) { - if ( only.empty() || only == name ) { - resyncDrop( name.c_str(), requester ); - } - } - } - } - syncedTo = OpTime(); - addDbNextPass.clear(); - save(); - } - - string ReplSource::resyncDrop( const char *db, const char *requester ) { - log() << "resync: dropping database " << db << endl; - Client::Context ctx(db); - dropDatabase(db); - return db; - } - - /* grab initial copy of a database from the master */ - void ReplSource::resync(const std::string& dbName) { - const std::string db(dbName); // need local copy of the name, we're dropping the original - string dummyNs = resyncDrop( db.c_str(), "internal" ); - Client::Context ctx( dummyNs ); - { - log() << "resync: cloning database " << db << " to get an initial copy" << endl; - ReplInfo r("resync: cloning a database"); - string errmsg; - int errCode = 0; - bool ok = Cloner::cloneFrom(hostName.c_str(), errmsg, cc().database()->name, false, /*slaveOk*/ true, /*replauth*/ true, /*snapshot*/false, /*mayYield*/true, /*mayBeInterrupted*/false, &errCode); - if ( !ok ) { - if ( errCode == DatabaseDifferCaseCode ) { - resyncDrop( db.c_str(), "internal" ); - log() << "resync: database " << db << " not valid on the master due to a name conflict, dropping." << endl; - return; - } - else { - problem() << "resync of " << db << " from " << hostName << " failed " << errmsg << endl; - throw SyncException(); - } - } - } - - log() << "resync: done with initial clone for db: " << db << endl; - - return; - } - - DatabaseIgnorer ___databaseIgnorer; - - void DatabaseIgnorer::doIgnoreUntilAfter( const string &db, const OpTime &futureOplogTime ) { - if ( futureOplogTime > _ignores[ db ] ) { - _ignores[ db ] = futureOplogTime; - } - } - - bool DatabaseIgnorer::ignoreAt( const string &db, const OpTime ¤tOplogTime ) { - if ( _ignores[ db ].isNull() ) { - return false; - } - if ( _ignores[ db ] >= currentOplogTime ) { - return true; - } else { - // The ignore state has expired, so clear it. - _ignores.erase( db ); - return false; - } - } - - bool ReplSource::handleDuplicateDbName( const BSONObj &op, const char *ns, const char *db ) { - if ( dbHolder()._isLoaded( ns, dbpath ) ) { - // Database is already present. - return true; - } - BSONElement ts = op.getField( "ts" ); - if ( ( ts.type() == Date || ts.type() == Timestamp ) && ___databaseIgnorer.ignoreAt( db, ts.date() ) ) { - // Database is ignored due to a previous indication that it is - // missing from master after optime "ts". - return false; - } - if ( Database::duplicateUncasedName( false, db, dbpath ).empty() ) { - // No duplicate database names are present. - return true; - } - - OpTime lastTime; - bool dbOk = false; - { - dbtemprelease release; - - // We always log an operation after executing it (never before), so - // a database list will always be valid as of an oplog entry generated - // before it was retrieved. - - BSONObj last = oplogReader.findOne( this->ns().c_str(), Query().sort( BSON( "$natural" << -1 ) ) ); - if ( !last.isEmpty() ) { - BSONElement ts = last.getField( "ts" ); - massert( 14032, "Invalid 'ts' in remote log", ts.type() == Date || ts.type() == Timestamp ); - lastTime = OpTime( ts.date() ); - } - - BSONObj info; - bool ok = oplogReader.conn()->runCommand( "admin", BSON( "listDatabases" << 1 ), info ); - massert( 14033, "Unable to get database list", ok ); - BSONObjIterator i( info.getField( "databases" ).embeddedObject() ); - while( i.more() ) { - BSONElement e = i.next(); - - const char * name = e.embeddedObject().getField( "name" ).valuestr(); - if ( strcasecmp( name, db ) != 0 ) - continue; - - if ( strcmp( name, db ) == 0 ) { - // The db exists on master, still need to check that no conflicts exist there. - dbOk = true; - continue; - } - - // The master has a db name that conflicts with the requested name. - dbOk = false; - break; - } - } - - if ( !dbOk ) { - ___databaseIgnorer.doIgnoreUntilAfter( db, lastTime ); - incompleteCloneDbs.erase(db); - addDbNextPass.erase(db); - return false; - } - - // Check for duplicates again, since we released the lock above. - set< string > duplicates; - Database::duplicateUncasedName( false, db, dbpath, &duplicates ); - - // The database is present on the master and no conflicting databases - // are present on the master. Drop any local conflicts. - for( set< string >::const_iterator i = duplicates.begin(); i != duplicates.end(); ++i ) { - ___databaseIgnorer.doIgnoreUntilAfter( *i, lastTime ); - incompleteCloneDbs.erase(*i); - addDbNextPass.erase(*i); - Client::Context ctx(*i); - dropDatabase(*i); - } - - massert( 14034, "Duplicate database names present after attempting to delete duplicates", - Database::duplicateUncasedName( false, db, dbpath ).empty() ); - return true; - } - - void ReplSource::applyOperation(const BSONObj& op) { - try { - bool failedUpdate = applyOperation_inlock( op ); - if (failedUpdate) { - Sync sync(hostName); - if (sync.shouldRetry(op)) { - uassert(15914, "Failure retrying initial sync update", !applyOperation_inlock(op)); - } - } - } - catch ( UserException& e ) { - log() << "sync: caught user assertion " << e << " while applying op: " << op << endl;; - } - catch ( DBException& e ) { - log() << "sync: caught db exception " << e << " while applying op: " << op << endl;; - } - - } - - /* local.$oplog.main is of the form: - { ts: ..., op: , ns: ..., o: , o2: , b: } - ... - see logOp() comments. - - @param alreadyLocked caller already put us in write lock if true - */ - void ReplSource::sync_pullOpLog_applyOperation(BSONObj& op, bool alreadyLocked) { - if( logLevel >= 6 ) // op.tostring is expensive so doing this check explicitly - LOG(6) << "processing op: " << op << endl; - - if( op.getStringField("op")[0] == 'n' ) - return; - - char clientName[MaxDatabaseNameLen]; - const char *ns = op.getStringField("ns"); - nsToDatabase(ns, clientName); - - if ( *ns == '.' ) { - problem() << "skipping bad op in oplog: " << op.toString() << endl; - return; - } - else if ( *ns == 0 ) { - /*if( op.getStringField("op")[0] != 'n' )*/ { - problem() << "halting replication, bad op in oplog:\n " << op.toString() << endl; - replAllDead = "bad object in oplog"; - throw SyncException(); - } - //ns = "local.system.x"; - //nsToDatabase(ns, clientName); - } - - if ( !only.empty() && only != clientName ) - return; - - if( cmdLine.pretouch && !alreadyLocked/*doesn't make sense if in write lock already*/ ) { - if( cmdLine.pretouch > 1 ) { - /* note: this is bad - should be put in ReplSource. but this is first test... */ - static int countdown; - verify( countdown >= 0 ); - if( countdown > 0 ) { - countdown--; // was pretouched on a prev pass - } - else { - const int m = 4; - if( tp.get() == 0 ) { - int nthr = min(8, cmdLine.pretouch); - nthr = max(nthr, 1); - tp.reset( new ThreadPool(nthr) ); - } - vector v; - oplogReader.peek(v, cmdLine.pretouch); - unsigned a = 0; - while( 1 ) { - if( a >= v.size() ) break; - unsigned b = a + m - 1; // v[a..b] - if( b >= v.size() ) b = v.size() - 1; - tp->schedule(pretouchN, v, a, b); - DEV cout << "pretouch task: " << a << ".." << b << endl; - a += m; - } - // we do one too... - pretouchOperation(op); - tp->join(); - countdown = v.size(); - } - } - else { - pretouchOperation(op); - } - } - - scoped_ptr lk( alreadyLocked ? 0 : new Lock::GlobalWrite() ); - - if ( replAllDead ) { - // hmmm why is this check here and not at top of this function? does it get set between top and here? - log() << "replAllDead, throwing SyncException: " << replAllDead << endl; - throw SyncException(); - } - - if ( !handleDuplicateDbName( op, ns, clientName ) ) { - return; - } - - Client::Context ctx( ns ); - ctx.getClient()->curop()->reset(); - - bool empty = ctx.db()->isEmpty(); - bool incompleteClone = incompleteCloneDbs.count( clientName ) != 0; - - if( logLevel >= 6 ) - LOG(6) << "ns: " << ns << ", justCreated: " << ctx.justCreated() << ", empty: " << empty << ", incompleteClone: " << incompleteClone << endl; - - // always apply admin command command - // this is a bit hacky -- the semantics of replication/commands aren't well specified - if ( strcmp( clientName, "admin" ) == 0 && *op.getStringField( "op" ) == 'c' ) { - applyOperation( op ); - return; - } - - if ( ctx.justCreated() || empty || incompleteClone ) { - // we must add to incomplete list now that setClient has been called - incompleteCloneDbs.insert( clientName ); - if ( nClonedThisPass ) { - /* we only clone one database per pass, even if a lot need done. This helps us - avoid overflowing the master's transaction log by doing too much work before going - back to read more transactions. (Imagine a scenario of slave startup where we try to - clone 100 databases in one pass.) - */ - addDbNextPass.insert( clientName ); - } - else { - if ( incompleteClone ) { - log() << "An earlier initial clone of '" << clientName << "' did not complete, now resyncing." << endl; - } - save(); - Client::Context ctx(ns); - nClonedThisPass++; - resync(ctx.db()->name); - addDbNextPass.erase(clientName); - incompleteCloneDbs.erase( clientName ); - } - save(); - } - else { - applyOperation( op ); - addDbNextPass.erase( clientName ); - } - } - - void ReplSource::syncToTailOfRemoteLog() { - string _ns = ns(); - BSONObjBuilder b; - if ( !only.empty() ) { - b.appendRegex("ns", string("^") + pcrecpp::RE::QuoteMeta( only )); - } - BSONObj last = oplogReader.findOne( _ns.c_str(), Query( b.done() ).sort( BSON( "$natural" << -1 ) ) ); - if ( !last.isEmpty() ) { - BSONElement ts = last.getField( "ts" ); - massert( 10386 , "non Date ts found: " + last.toString(), ts.type() == Date || ts.type() == Timestamp ); - syncedTo = OpTime( ts.date() ); - } - } - - class ReplApplyBatchSize : public ServerParameter { - public: - ReplApplyBatchSize() - : ServerParameter( ServerParameterSet::getGlobal(), "replApplyBatchSize" ), - _value( 1 ) { - } - - int get() const { return _value; } - - virtual void append( BSONObjBuilder& b, const string& name ) { - b.append( name, _value ); - } - - virtual Status set( const BSONElement& newValuElement ) { - return set( newValuElement.numberInt() ); - } - - virtual Status set( int b ) { - if( b < 1 || b > 1024 ) { - return Status( ErrorCodes::BadValue, - "replApplyBatchSize has to be >= 1 and < 1024" ); - } - - if ( replSettings.slavedelay != 0 && b > 1 ) { - return Status( ErrorCodes::BadValue, - "can't use a batch size > 1 with slavedelay" ); - } - if ( ! replSettings.slave ) { - return Status( ErrorCodes::BadValue, - "can't set replApplyBatchSize on a non-slave machine" ); - } - - _value = b; - return Status::OK(); - } - - virtual Status setFromString( const string& str ) { - return set( atoi( str.c_str() ) ); - } - - int _value; - - } replApplyBatchSize; - - /* slave: pull some data from the master's oplog - note: not yet in db mutex at this point. - @return -1 error - 0 ok, don't sleep - 1 ok, sleep - */ - int ReplSource::sync_pullOpLog(int& nApplied) { - int okResultCode = 1; - string ns = string("local.oplog.$") + sourceName(); - LOG(2) << "repl: sync_pullOpLog " << ns << " syncedTo:" << syncedTo.toStringLong() << '\n'; - - bool tailing = true; - oplogReader.tailCheck(); - - bool initial = syncedTo.isNull(); - - if ( !oplogReader.haveCursor() || initial ) { - if ( initial ) { - // Important to grab last oplog timestamp before listing databases. - syncToTailOfRemoteLog(); - BSONObj info; - bool ok = oplogReader.conn()->runCommand( "admin", BSON( "listDatabases" << 1 ), info ); - massert( 10389 , "Unable to get database list", ok ); - BSONObjIterator i( info.getField( "databases" ).embeddedObject() ); - while( i.moreWithEOO() ) { - BSONElement e = i.next(); - if ( e.eoo() ) - break; - string name = e.embeddedObject().getField( "name" ).valuestr(); - if ( !e.embeddedObject().getBoolField( "empty" ) ) { - if ( name != "local" ) { - if ( only.empty() || only == name ) { - LOG( 2 ) << "adding to 'addDbNextPass': " << name << endl; - addDbNextPass.insert( name ); - } - } - } - } - // obviously global isn't ideal, but non-repl set is old so - // keeping it simple - Lock::GlobalWrite lk; - save(); - } - - BSONObjBuilder gte; - gte.appendTimestamp("$gte", syncedTo.asDate()); - BSONObjBuilder query; - query.append("ts", gte.done()); - if ( !only.empty() ) { - // note we may here skip a LOT of data table scanning, a lot of work for the master. - // maybe append "\\." here? - query.appendRegex("ns", string("^") + pcrecpp::RE::QuoteMeta( only )); - } - BSONObj queryObj = query.done(); - // e.g. queryObj = { ts: { $gte: syncedTo } } - - oplogReader.tailingQuery(ns.c_str(), queryObj); - tailing = false; - } - else { - LOG(2) << "repl: tailing=true\n"; - } - - if( !oplogReader.haveCursor() ) { - problem() << "repl: dbclient::query returns null (conn closed?)" << endl; - oplogReader.resetConnection(); - return -1; - } - - // show any deferred database creates from a previous pass - { - set::iterator i = addDbNextPass.begin(); - if ( i != addDbNextPass.end() ) { - BSONObjBuilder b; - b.append("ns", *i + '.'); - b.append("op", "db"); - BSONObj op = b.done(); - sync_pullOpLog_applyOperation(op, false); - } - } - - if ( !oplogReader.more() ) { - if ( tailing ) { - LOG(2) << "repl: tailing & no new activity\n"; - if( oplogReader.awaitCapable() ) - okResultCode = 0; // don't sleep - - } - else { - log() << "repl: " << ns << " oplog is empty" << endl; - } - { - Lock::GlobalWrite lk; - save(); - } - return okResultCode; - } - - OpTime nextOpTime; - { - BSONObj op = oplogReader.next(); - BSONElement ts = op.getField("ts"); - if ( ts.type() != Date && ts.type() != Timestamp ) { - string err = op.getStringField("$err"); - if ( !err.empty() ) { - // 13051 is "tailable cursor requested on non capped collection" - if (op.getIntField("code") == 13051) { - problem() << "trying to slave off of a non-master" << '\n'; - massert( 13344 , "trying to slave off of a non-master", false ); - } - else { - problem() << "repl: $err reading remote oplog: " + err << '\n'; - massert( 10390 , "got $err reading remote oplog", false ); - } - } - else { - problem() << "repl: bad object read from remote oplog: " << op.toString() << '\n'; - massert( 10391 , "repl: bad object read from remote oplog", false); - } - } - - nextOpTime = OpTime( ts.date() ); - LOG(2) << "repl: first op time received: " << nextOpTime.toString() << '\n'; - if ( initial ) { - LOG(1) << "repl: initial run\n"; - } - if( tailing ) { - if( !( syncedTo < nextOpTime ) ) { - log() << "repl ASSERTION failed : syncedTo < nextOpTime" << endl; - log() << "repl syncTo: " << syncedTo.toStringLong() << endl; - log() << "repl nextOpTime: " << nextOpTime.toStringLong() << endl; - verify(false); - } - oplogReader.putBack( op ); // op will be processed in the loop below - nextOpTime = OpTime(); // will reread the op below - } - else if ( nextOpTime != syncedTo ) { // didn't get what we queried for - error - Nullstream& l = log(); - l << "repl: nextOpTime " << nextOpTime.toStringLong() << ' '; - if ( nextOpTime < syncedTo ) - l << ""; - - l << " syncedTo " << syncedTo.toStringLong() << '\n'; - log() << "repl: time diff: " << (nextOpTime.getSecs() - syncedTo.getSecs()) << "sec\n"; - log() << "repl: tailing: " << tailing << '\n'; - log() << "repl: data too stale, halting replication" << endl; - replInfo = replAllDead = "data too stale halted replication"; - verify( syncedTo < nextOpTime ); - throw SyncException(); - } - else { - /* t == syncedTo, so the first op was applied previously or it is the first op of initial query and need not be applied. */ - } - } - - // apply operations - { - int n = 0; - time_t saveLast = time(0); - while ( 1 ) { - - bool moreInitialSyncsPending = !addDbNextPass.empty() && n; // we need "&& n" to assure we actually process at least one op to get a sync point recorded in the first place. - - if ( moreInitialSyncsPending || !oplogReader.more() ) { - Lock::GlobalWrite lk; - - // NOTE aaron 2011-03-29 This block may be unnecessary, but I'm leaving it in place to avoid changing timing behavior. - { - dbtemprelease t; - if ( !moreInitialSyncsPending && oplogReader.more() ) { - continue; - } - // otherwise, break out of loop so we can set to completed or clone more dbs - } - - if( oplogReader.awaitCapable() && tailing ) - okResultCode = 0; // don't sleep - syncedTo = nextOpTime; - save(); // note how far we are synced up to now - log() << "repl: applied " << n << " operations" << endl; - nApplied = n; - log() << "repl: end sync_pullOpLog syncedTo: " << syncedTo.toStringLong() << endl; - break; - } - else { - } - - OCCASIONALLY if( n > 0 && ( n > 100000 || time(0) - saveLast > 60 ) ) { - // periodically note our progress, in case we are doing a lot of work and crash - Lock::GlobalWrite lk; - syncedTo = nextOpTime; - // can't update local log ts since there are pending operations from our peer - save(); - log() << "repl: checkpoint applied " << n << " operations" << endl; - log() << "repl: syncedTo: " << syncedTo.toStringLong() << endl; - saveLast = time(0); - n = 0; - } - - BSONObj op = oplogReader.next(); - - int b = replApplyBatchSize.get(); - bool justOne = b == 1; - scoped_ptr lk( justOne ? 0 : new Lock::GlobalWrite() ); - while( 1 ) { - - BSONElement ts = op.getField("ts"); - if( !( ts.type() == Date || ts.type() == Timestamp ) ) { - log() << "sync error: problem querying remote oplog record" << endl; - log() << "op: " << op.toString() << endl; - log() << "halting replication" << endl; - replInfo = replAllDead = "sync error: no ts found querying remote oplog record"; - throw SyncException(); - } - OpTime last = nextOpTime; - nextOpTime = OpTime( ts.date() ); - if ( !( last < nextOpTime ) ) { - log() << "sync error: last applied optime at slave >= nextOpTime from master" << endl; - log() << " last: " << last.toStringLong() << endl; - log() << " nextOpTime: " << nextOpTime.toStringLong() << endl; - log() << " halting replication" << endl; - replInfo = replAllDead = "sync error last >= nextOpTime"; - uassert( 10123 , "replication error last applied optime at slave >= nextOpTime from master", false); - } - if ( replSettings.slavedelay && ( unsigned( time( 0 ) ) < nextOpTime.getSecs() + replSettings.slavedelay ) ) { - verify( justOne ); - oplogReader.putBack( op ); - _sleepAdviceTime = nextOpTime.getSecs() + replSettings.slavedelay + 1; - Lock::GlobalWrite lk; - if ( n > 0 ) { - syncedTo = last; - save(); - } - log() << "repl: applied " << n << " operations" << endl; - log() << "repl: syncedTo: " << syncedTo.toStringLong() << endl; - log() << "waiting until: " << _sleepAdviceTime << " to continue" << endl; - return okResultCode; - } - - sync_pullOpLog_applyOperation(op, !justOne); - n++; - - if( --b == 0 ) - break; - // if to here, we are doing mulpile applications in a singel write lock acquisition - if( !oplogReader.moreInCurrentBatch() ) { - // break if no more in batch so we release lock while reading from the master - break; - } - op = oplogReader.next(); - - getDur().commitIfNeeded(); - } - } - } - - return okResultCode; - } - - BSONObj userReplQuery = fromjson("{\"user\":\"repl\"}"); - - /* Generally replAuthenticate will only be called within system threads to fully authenticate - * connections to other nodes in the cluster that will be used as part of internal operations. - * If a user-initiated action results in needing to call replAuthenticate, you can call it - * with skipAuthCheck set to false. Only do this if you are certain that the proper auth - * checks have already run to ensure that the user is authorized to do everything that this - * connection will be used for! - */ - bool replAuthenticate(DBClientBase *conn, bool skipAuthCheck) { - if( noauth ) { - return true; - } - if (!skipAuthCheck && !cc().getAuthorizationManager()->hasInternalAuthorization()) { - log() << "replauthenticate: requires internal authorization, failing" << endl; - return false; - } - - string u; - string p; - if (internalSecurity.pwd.length() > 0) { - u = internalSecurity.user; - p = internalSecurity.pwd; - } - else { - BSONObj user; - { - Client::ReadContext ctxt("local."); - if( !Helpers::findOne("local.system.users", userReplQuery, user) || - // try the first user in local - !Helpers::getSingleton("local.system.users", user) ) { - log() << "replauthenticate: no user in local.system.users to use for authentication" << endl; - return false; - } - } - u = user.getStringField("user"); - p = user.getStringField("pwd"); - massert( 10392 , "bad user object? [1]", !u.empty()); - massert( 10393 , "bad user object? [2]", !p.empty()); - } - - string err; - if( !conn->auth("local", u.c_str(), p.c_str(), err, false) ) { - log() << "replauthenticate: can't authenticate to master server, user:" << u << endl; - return false; - } - - return true; - } - - bool replHandshake(DBClientConnection *conn) { - string myname = getHostName(); - - BSONObj me; - { - - Lock::DBWrite l("local"); - // local.me is an identifier for a server for getLastError w:2+ - if ( ! Helpers::getSingleton( "local.me" , me ) || - ! me.hasField("host") || - me["host"].String() != myname ) { - - // clean out local.me - Helpers::emptyCollection("local.me"); - - // repopulate - BSONObjBuilder b; - b.appendOID( "_id" , 0 , true ); - b.append( "host", myname ); - me = b.obj(); - Helpers::putSingleton( "local.me" , me ); - } - } - - BSONObjBuilder cmd; - cmd.appendAs( me["_id"] , "handshake" ); - if (theReplSet) { - cmd.append("member", theReplSet->selfId()); - cmd.append("config", theReplSet->myConfig().asBson()); - } - - BSONObj res; - bool ok = conn->runCommand( "admin" , cmd.obj() , res ); - // ignoring for now on purpose for older versions - LOG( ok ? 1 : 0 ) << "replHandshake res not: " << ok << " res: " << res << endl; - return true; - } - - //number of readers created; - // this happens when the source source changes, a reconfig/network-error or the cursor dies - static Counter64 readersCreatedStats; - static ServerStatusMetricField displayReadersCreated( - "repl.network.readersCreated", - &readersCreatedStats ); - - OplogReader::OplogReader( bool doHandshake ) : - _doHandshake( doHandshake ) { - - _tailingQueryOptions = QueryOption_SlaveOk; - _tailingQueryOptions |= QueryOption_CursorTailable | QueryOption_OplogReplay; - - /* TODO: slaveOk maybe shouldn't use? */ - _tailingQueryOptions |= QueryOption_AwaitData; - - readersCreatedStats.increment(); - } - - bool OplogReader::commonConnect(const string& hostName) { - if( conn() == 0 ) { - _conn = shared_ptr(new DBClientConnection(false, - 0, - 30 /* tcp timeout */)); - string errmsg; - ReplInfo r("trying to connect to sync source"); - if ( !_conn->connect(hostName.c_str(), errmsg) || - (!noauth && !replAuthenticate(_conn.get(), true)) ) { - resetConnection(); - log() << "repl: " << errmsg << endl; - return false; - } - } - return true; - } - - bool OplogReader::connect(const std::string& hostName) { - if (conn() != 0) { - return true; - } - - if ( ! commonConnect(hostName) ) { - return false; - } - - - if ( _doHandshake && ! replHandshake(_conn.get() ) ) { - return false; - } - - return true; - } - - bool OplogReader::connect(const BSONObj& rid, const int from, const string& to) { - if (conn() != 0) { - return true; - } - if (commonConnect(to)) { - log() << "handshake between " << from << " and " << to << endl; - return passthroughHandshake(rid, from); - } - return false; - } - - bool OplogReader::passthroughHandshake(const BSONObj& rid, const int nextOnChainId) { - BSONObjBuilder cmd; - cmd.appendAs(rid["_id"], "handshake"); - if (theReplSet) { - const Member* chainedMember = theReplSet->findById(nextOnChainId); - if (chainedMember != NULL) { - cmd.append("config", chainedMember->config().asBson()); - } - } - cmd.append("member", nextOnChainId); - - BSONObj res; - return conn()->runCommand("admin", cmd.obj(), res); - } - - void OplogReader::tailingQuery(const char *ns, const BSONObj& query, const BSONObj* fields ) { - verify( !haveCursor() ); - LOG(2) << "repl: " << ns << ".find(" << query.toString() << ')' << endl; - cursor.reset( _conn->query( ns, query, 0, 0, fields, _tailingQueryOptions ).release() ); - } - - void OplogReader::tailingQueryGTE(const char *ns, OpTime optime, const BSONObj* fields ) { - BSONObjBuilder gte; - gte.appendTimestamp("$gte", optime.asDate()); - BSONObjBuilder query; - query.append("ts", gte.done()); - tailingQuery(ns, query.done(), fields); - } - - - /* note: not yet in mutex at this point. - returns >= 0 if ok. return -1 if you want to reconnect. - return value of zero indicates no sleep necessary before next call - */ - int ReplSource::sync(int& nApplied) { - _sleepAdviceTime = 0; - ReplInfo r("sync"); - if ( !cmdLine.quiet ) { - Nullstream& l = log(); - l << "repl: syncing from "; - if( sourceName() != "main" ) { - l << "source:" << sourceName() << ' '; - } - l << "host:" << hostName << endl; - } - nClonedThisPass = 0; - - // FIXME Handle cases where this db isn't on default port, or default port is spec'd in hostName. - if ( (string("localhost") == hostName || string("127.0.0.1") == hostName) && cmdLine.port == CmdLine::DefaultDBPort ) { - log() << "repl: can't sync from self (localhost). sources configuration may be wrong." << endl; - sleepsecs(5); - return -1; - } - - if ( !oplogReader.connect(hostName) ) { - LOG(4) << "repl: can't connect to sync source" << endl; - return -1; - } - - /* - // get current mtime at the server. - BSONObj o = conn->findOne("admin.$cmd", opTimeQuery); - BSONElement e = o.getField("optime"); - if( e.eoo() ) { - log() << "repl: failed to get cur optime from master" << endl; - log() << " " << o.toString() << endl; - return false; - } - uassert( 10124 , e.type() == Date ); - OpTime serverCurTime; - serverCurTime.asDate() = e.date(); - */ - return sync_pullOpLog(nApplied); - } - - /* --------------------------------------------------------------*/ - - /* - TODO: - _ source has autoptr to the cursor - _ reuse that cursor when we can - */ - - /* returns: # of seconds to sleep before next pass - 0 = no sleep recommended - 1 = special sentinel indicating adaptive sleep recommended - */ - int _replMain(ReplSource::SourceVector& sources, int& nApplied) { - { - ReplInfo r("replMain load sources"); - Lock::GlobalWrite lk; - ReplSource::loadAll(sources); - replSettings.fastsync = false; // only need this param for initial reset - } - - if ( sources.empty() ) { - /* replication is not configured yet (for --slave) in local.sources. Poll for config it - every 20 seconds. - */ - log() << "no source given, add a master to local.sources to start replication" << endl; - return 20; - } - - int sleepAdvice = 1; - for ( ReplSource::SourceVector::iterator i = sources.begin(); i != sources.end(); i++ ) { - ReplSource *s = i->get(); - int res = -1; - try { - res = s->sync(nApplied); - bool moreToSync = s->haveMoreDbsToSync(); - if( res < 0 ) { - sleepAdvice = 3; - } - else if( moreToSync ) { - sleepAdvice = 0; - } - else if ( s->sleepAdvice() ) { - sleepAdvice = s->sleepAdvice(); - } - else - sleepAdvice = res; - } - catch ( const SyncException& ) { - log() << "caught SyncException" << endl; - return 10; - } - catch ( AssertionException& e ) { - if ( e.severe() ) { - log() << "replMain AssertionException " << e.what() << endl; - return 60; - } - else { - log() << "repl: AssertionException " << e.what() << endl; - } - replInfo = "replMain caught AssertionException"; - } - catch ( const DBException& e ) { - log() << "repl: DBException " << e.what() << endl; - replInfo = "replMain caught DBException"; - } - catch ( const std::exception &e ) { - log() << "repl: std::exception " << e.what() << endl; - replInfo = "replMain caught std::exception"; - } - catch ( ... ) { - log() << "unexpected exception during replication. replication will halt" << endl; - replAllDead = "caught unexpected exception during replication"; - } - if ( res < 0 ) - s->oplogReader.resetConnection(); - } - return sleepAdvice; - } - - void replMain() { - ReplSource::SourceVector sources; - while ( 1 ) { - int s = 0; - { - Lock::GlobalWrite lk; - if ( replAllDead ) { - // throttledForceResyncDead can throw - if ( !replSettings.autoresync || !ReplSource::throttledForceResyncDead( "auto" ) ) { - log() << "all sources dead: " << replAllDead << ", sleeping for 5 seconds" << endl; - break; - } - } - verify( syncing == 0 ); // i.e., there is only one sync thread running. we will want to change/fix this. - syncing++; - } - try { - int nApplied = 0; - s = _replMain(sources, nApplied); - if( s == 1 ) { - if( nApplied == 0 ) s = 2; - else if( nApplied > 100 ) { - // sleep very little - just enough that we aren't truly hammering master - sleepmillis(75); - s = 0; - } - } - } - catch (...) { - out() << "caught exception in _replMain" << endl; - s = 4; - } - { - Lock::GlobalWrite lk; - verify( syncing == 1 ); - syncing--; - } - - if( relinquishSyncingSome ) { - relinquishSyncingSome = 0; - s = 1; // sleep before going back in to syncing=1 - } - - if ( s ) { - stringstream ss; - ss << "repl: sleep " << s << " sec before next pass"; - string msg = ss.str(); - if ( ! cmdLine.quiet ) - log() << msg << endl; - ReplInfo r(msg.c_str()); - sleepsecs(s); - } - } - } - - static void replMasterThread() { - sleepsecs(4); - Client::initThread("replmaster"); - int toSleep = 10; - while( 1 ) { - - sleepsecs( toSleep ); - /* write a keep-alive like entry to the log. this will make things like - printReplicationStatus() and printSlaveReplicationStatus() stay up-to-date - even when things are idle. - */ - { - writelocktry lk(1); - if ( lk.got() ) { - toSleep = 10; - - replLocalAuth(); - - try { - logKeepalive(); - } - catch(...) { - log() << "caught exception in replMasterThread()" << endl; - } - } - else { - LOG(5) << "couldn't logKeepalive" << endl; - toSleep = 1; - } - } - } - } - - void replSlaveThread() { - sleepsecs(1); - Client::initThread("replslave"); - - { - Lock::GlobalWrite lk; - replLocalAuth(); - } - - while ( 1 ) { - try { - replMain(); - sleepsecs(5); - } - catch ( AssertionException& ) { - ReplInfo r("Assertion in replSlaveThread(): sleeping 5 minutes before retry"); - problem() << "Assertion in replSlaveThread(): sleeping 5 minutes before retry" << endl; - sleepsecs(300); - } - catch ( DBException& e ) { - problem() << "exception in replSlaveThread(): " << e.what() - << ", sleeping 5 minutes before retry" << endl; - sleepsecs(300); - } - catch ( ... ) { - problem() << "error in replSlaveThread(): sleeping 5 minutes before retry" << endl; - sleepsecs(300); - } - } - } - - void newRepl(); - void oldRepl(); - void startReplSets(ReplSetCmdline*); - void startReplication() { - /* if we are going to be a replica set, we aren't doing other forms of replication. */ - if( !cmdLine._replSet.empty() ) { - if( replSettings.slave || replSettings.master ) { - log() << "***" << endl; - log() << "ERROR: can't use --slave or --master replication options with --replSet" << endl; - log() << "***" << endl; - } - newRepl(); - - replSet = true; - ReplSetCmdline *replSetCmdline = new ReplSetCmdline(cmdLine._replSet); - boost::thread t( boost::bind( &startReplSets, replSetCmdline) ); - - return; - } - - oldRepl(); - - if( !replSettings.slave && !replSettings.master ) - return; - - { - Lock::GlobalWrite lk; - replLocalAuth(); - } - - if ( replSettings.slave ) { - verify( replSettings.slave == SimpleSlave ); - LOG(1) << "slave=true" << endl; - boost::thread repl_thread(replSlaveThread); - } - - if ( replSettings.master ) { - LOG(1) << "master=true" << endl; - replSettings.master = true; - createOplog(); - boost::thread t(replMasterThread); - } - - while( replSettings.fastsync ) // don't allow writes until we've set up from log - sleepmillis( 50 ); - } - - void testPretouch() { - int nthr = min(8, 8); - nthr = max(nthr, 1); - int m = 8 / nthr; - ThreadPool tp(nthr); - vector v; - - BSONObj x = BSON( "ns" << "test.foo" << "o" << BSON( "_id" << 1 ) << "op" << "i" ); - - v.push_back(x); - v.push_back(x); - v.push_back(x); - - unsigned a = 0; - while( 1 ) { - if( a >= v.size() ) break; - unsigned b = a + m - 1; // v[a..b] - if( b >= v.size() ) b = v.size() - 1; - tp.schedule(pretouchN, v, a, b); - DEV cout << "pretouch task: " << a << ".." << b << endl; - a += m; - } - tp.join(); - } - - /** we allow queries to SimpleSlave's */ - void replVerifyReadsOk(const ParsedQuery* pq) { - if( replSet ) { - // todo: speed up the secondary case. as written here there are 2 mutex entries, it - // can b 1. - if( isMaster() ) return; - uassert(13435, "not master and slaveOk=false", - !pq || pq->hasOption(QueryOption_SlaveOk) || pq->hasReadPref()); - uassert(13436, - "not master or secondary; cannot currently read from this replSet member", - theReplSet && theReplSet->isSecondary() ); - } - else { - notMasterUnless(isMaster() || (!pq || pq->hasOption(QueryOption_SlaveOk)) || - replSettings.slave == SimpleSlave ); - } - } - - OpCounterServerStatusSection replOpCounterServerStatusSection( "opcountersRepl", &replOpCounters ); - -} // namespace mongo diff --git a/src/mongo/db/repl.h b/src/mongo/db/repl.h deleted file mode 100644 index 2b85a8f376c..00000000000 --- a/src/mongo/db/repl.h +++ /dev/null @@ -1,197 +0,0 @@ -// repl.h - replication - -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -/* replication data overview - - at the slave: - local.sources { host: ..., source: ..., only: ..., syncedTo: ..., localLogTs: ..., dbsNextPass: { ... }, incompleteCloneDbs: { ... } } - - at the master: - local.oplog.$ -*/ - -#pragma once - -#include "pdfile.h" -#include "db.h" -#include "dbhelpers.h" -#include "../util/optime.h" -#include "oplog.h" -#include "../util/concurrency/thread_pool.h" -#include "oplogreader.h" -#include "cloner.h" - -namespace mongo { - - /* replication slave? (possibly with slave) - --slave cmd line setting -> SimpleSlave - */ - typedef enum { NotSlave=0, SimpleSlave } SlaveTypes; - - class ReplSettings { - public: - SlaveTypes slave; - - /** true means we are master and doing replication. if we are not writing to oplog, this won't be true. */ - bool master; - - bool fastsync; - - bool autoresync; - - int slavedelay; - - set discoveredSeeds; - mutex discoveredSeeds_mx; - - BSONObj reconfig; - - ReplSettings() - : slave(NotSlave), - master(false), - fastsync(), - autoresync(false), - slavedelay(), - discoveredSeeds(), - discoveredSeeds_mx("ReplSettings::discoveredSeeds") { - } - - }; - - extern ReplSettings replSettings; - - /* A replication exception */ - class SyncException : public DBException { - public: - SyncException() : DBException( "sync exception" , 10001 ) {} - }; - - /* A Source is a source from which we can pull (replicate) data. - stored in collection local.sources. - - Can be a group of things to replicate for several databases. - - { host: ..., source: ..., only: ..., syncedTo: ..., dbsNextPass: { ... }, incompleteCloneDbs: { ... } } - - 'source' defaults to 'main'; support for multiple source names is - not done (always use main for now). - */ - class ReplSource { - shared_ptr tp; - - void resync(const std::string& dbName); - - /** @param alreadyLocked caller already put us in write lock if true */ - void sync_pullOpLog_applyOperation(BSONObj& op, bool alreadyLocked); - - /* pull some operations from the master's oplog, and apply them. - calls sync_pullOpLog_applyOperation - */ - int sync_pullOpLog(int& nApplied); - - /* we only clone one database per pass, even if a lot need done. This helps us - avoid overflowing the master's transaction log by doing too much work before going - back to read more transactions. (Imagine a scenario of slave startup where we try to - clone 100 databases in one pass.) - */ - set addDbNextPass; - - set incompleteCloneDbs; - - ReplSource(); - - // returns the dummy ns used to do the drop - string resyncDrop( const char *db, const char *requester ); - // call without the db mutex - void syncToTailOfRemoteLog(); - string ns() const { return string( "local.oplog.$" ) + sourceName(); } - unsigned _sleepAdviceTime; - - /** - * If 'db' is a new database and its name would conflict with that of - * an existing database, synchronize these database names with the - * master. - * @return true iff an op with the specified ns may be applied. - */ - bool handleDuplicateDbName( const BSONObj &op, const char *ns, const char *db ); - - public: - OplogReader oplogReader; - - void applyOperation(const BSONObj& op); - string hostName; // ip addr or hostname plus optionally, ":" - string _sourceName; // a logical source name. - string sourceName() const { return _sourceName.empty() ? "main" : _sourceName; } - string only; // only a certain db. note that in the sources collection, this may not be changed once you start replicating. - - /* the last time point we have already synced up to (in the remote/master's oplog). */ - OpTime syncedTo; - - int nClonedThisPass; - - typedef vector< shared_ptr< ReplSource > > SourceVector; - static void loadAll(SourceVector&); - explicit ReplSource(BSONObj); - - /* -1 = error */ - int sync(int& nApplied); - - void save(); // write ourself to local.sources - - // make a jsobj from our member fields of the form - // { host: ..., source: ..., syncedTo: ... } - BSONObj jsobj(); - - bool operator==(const ReplSource&r) const { - return hostName == r.hostName && sourceName() == r.sourceName(); - } - string toString() const { return sourceName() + "@" + hostName; } - - bool haveMoreDbsToSync() const { return !addDbNextPass.empty(); } - int sleepAdvice() const { - if ( !_sleepAdviceTime ) - return 0; - int wait = _sleepAdviceTime - unsigned( time( 0 ) ); - return wait > 0 ? wait : 0; - } - - static bool throttledForceResyncDead( const char *requester ); - static void forceResyncDead( const char *requester ); - void forceResync( const char *requester ); - }; - - bool anyReplEnabled(); - - /** - * Helper class used to set and query an ignore state for a named database. - * The ignore state will expire after a specified OpTime. - */ - class DatabaseIgnorer { - public: - /** Indicate that operations for 'db' should be ignored until after 'futureOplogTime' */ - void doIgnoreUntilAfter( const string &db, const OpTime &futureOplogTime ); - /** - * Query ignore state of 'db'; if 'currentOplogTime' is after the ignore - * limit, the ignore state will be cleared. - */ - bool ignoreAt( const string &db, const OpTime ¤tOplogTime ); - private: - map< string, OpTime > _ignores; - }; - -} // namespace mongo diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index f2c22151636..ab1921ec4dd 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #include "mongo/pch.h" @@ -20,13 +32,19 @@ #include "mongo/db/commands/fsync.h" #include "mongo/db/commands/server_status.h" #include "mongo/db/repl/bgsync.h" +#include "mongo/db/repl/oplog.h" #include "mongo/db/repl/rs_sync.h" +#include "mongo/db/repl/rs.h" #include "mongo/util/fail_point_service.h" #include "mongo/base/counter.h" #include "mongo/db/stats/timer_stats.h" namespace mongo { namespace replset { + + int SleepToAllowBatchingMillis = 2; + const int BatchIsSmallish = 40000; // bytes + MONGO_FP_DECLARE(rsBgSyncProduce); BackgroundSync* BackgroundSync::s_instance = 0; @@ -74,8 +92,6 @@ namespace replset { _appliedBuffer(true), _assumingPrimary(false), _currentSyncTarget(NULL), - _oplogMarkerTarget(NULL), - _oplogMarker(true /* doHandshake */), _consumedOpTime(0, 0) { } @@ -119,9 +135,9 @@ namespace replset { Client::initThread("rsSyncNotifier"); replLocalAuth(); - while (!inShutdown()) { - bool clearTarget = false; + theReplSet->syncSourceFeedback.go(); + while (!inShutdown()) { if (!theReplSet) { sleepsecs(5); continue; @@ -144,87 +160,65 @@ namespace replset { markOplog(); } catch (DBException &e) { - clearTarget = true; log() << "replset tracking exception: " << e.getInfo() << rsLog; sleepsecs(1); } catch (std::exception &e2) { - clearTarget = true; log() << "replset tracking error" << e2.what() << rsLog; sleepsecs(1); } - - if (clearTarget) { - boost::unique_lock lock(_mutex); - _oplogMarkerTarget = NULL; - } } cc().shutdown(); } void BackgroundSync::markOplog() { - LOG(3) << "replset markOplog: " << _consumedOpTime << " " << theReplSet->lastOpTimeWritten << rsLog; + LOG(3) << "replset markOplog: " << _consumedOpTime << " " + << theReplSet->lastOpTimeWritten << rsLog; + + boost::unique_lock oplogLockSSF(theReplSet->syncSourceFeedback.oplock); + if (theReplSet->syncSourceFeedback.supportsUpdater()) { + oplogLockSSF.unlock(); + _consumedOpTime = theReplSet->lastOpTimeWritten; + theReplSet->syncSourceFeedback.updateSelfInMap(theReplSet->lastOpTimeWritten); + } + else { + if (!hasCursor()) { + oplogLockSSF.unlock(); + sleepmillis(500); + return; + } - if (!hasCursor()) { - sleepsecs(1); - return; - } + if (!theReplSet->syncSourceFeedback.moreInCurrentBatch()) { + theReplSet->syncSourceFeedback.more(); + } - if (!_oplogMarker.moreInCurrentBatch()) { - _oplogMarker.more(); - } + if (!theReplSet->syncSourceFeedback.more()) { + theReplSet->syncSourceFeedback.tailCheck(); + return; + } - if (!_oplogMarker.more()) { - _oplogMarker.tailCheck(); - sleepsecs(1); - return; - } + // if this member has written the op at optime T + // we want to nextSafe up to and including T + while (_consumedOpTime < theReplSet->lastOpTimeWritten + && theReplSet->syncSourceFeedback.more()) { + BSONObj temp = theReplSet->syncSourceFeedback.nextSafe(); + _consumedOpTime = temp["ts"]._opTime(); + } - // if this member has written the op at optime T, we want to nextSafe up to and including T - while (_consumedOpTime < theReplSet->lastOpTimeWritten && _oplogMarker.more()) { - BSONObj temp = _oplogMarker.nextSafe(); - _consumedOpTime = temp["ts"]._opTime(); + // call more() to signal the sync target that we've synced T + theReplSet->syncSourceFeedback.more(); } - - // call more() to signal the sync target that we've synced T - _oplogMarker.more(); } bool BackgroundSync::hasCursor() { - { - // prevent writers from blocking readers during fsync - SimpleMutex::scoped_lock fsynclk(filesLockedFsync); - // we don't need the local write lock yet, but it's needed by OplogReader::connect - // so we take it preemptively to avoid deadlocking. - Lock::DBWrite lk("local"); - - boost::unique_lock lock(_mutex); - - if (!_oplogMarkerTarget || _currentSyncTarget != _oplogMarkerTarget) { - if (!_currentSyncTarget) { - return false; - } - - log() << "replset setting oplog notifier to " << _currentSyncTarget->fullName() << rsLog; - _oplogMarkerTarget = _currentSyncTarget; - - _oplogMarker.resetConnection(); - - if (!_oplogMarker.connect(_oplogMarkerTarget->fullName())) { - LOG(1) << "replset could not connect to " << _oplogMarkerTarget->fullName() << rsLog; - _oplogMarkerTarget = NULL; - return false; - } - } - } - - if (!_oplogMarker.haveCursor()) { + if (!theReplSet->syncSourceFeedback.haveCursor()) { BSONObj fields = BSON("ts" << 1); - _oplogMarker.tailingQueryGTE(rsoplog, theReplSet->lastOpTimeWritten, &fields); + theReplSet->syncSourceFeedback.tailingQueryGTE(rsoplog, + theReplSet->lastOpTimeWritten, &fields); } - return _oplogMarker.haveCursor(); + return theReplSet->syncSourceFeedback.haveCursor(); } void BackgroundSync::producerThread() { @@ -287,7 +281,7 @@ namespace replset { void BackgroundSync::produce() { // this oplog reader does not do a handshake because we don't want the server it's syncing // from to track how far it has synced - OplogReader r(false /* doHandshake */); + OplogReader r; OpTime lastOpTimeFetched; // find a target to sync from the last op time written getOplogReader(r); @@ -321,12 +315,25 @@ namespace replset { } while (!inShutdown()) { - if (!r.moreInCurrentBatch()) { // Check some things periodically // (whenever we run out of items in the // current cursor batch) + int bs = r.currentBatchMessageSize(); + if( bs > 0 && bs < BatchIsSmallish ) { + // on a very low latency network, if we don't wait a little, we'll be + // getting ops to write almost one at a time. this will both be expensive + // for the upstream server as well as potentially defeating our parallel + // application of batches on the secondary. + // + // the inference here is basically if the batch is really small, we are + // "caught up". + // + dassert( !Lock::isLocked() ); + sleepmillis(SleepToAllowBatchingMillis); + } + if (theReplSet->gotForceSync()) { return; } @@ -396,6 +403,8 @@ namespace replset { boost::unique_lock lock(_mutex); _lastH = o["h"].numberLong(); _lastOpTimeFetched = o["ts"]._opTime(); + LOG(3) << "replSet lastOpTimeFetched: " + << _lastOpTimeFetched.toStringPretty() << rsLog; } } } @@ -415,14 +424,6 @@ namespace replset { bool BackgroundSync::peek(BSONObj* op) { - { - boost::unique_lock lock(_mutex); - - if (_currentSyncTarget != _oplogMarkerTarget && - _currentSyncTarget != NULL) { - _oplogMarkerTarget = NULL; - } - } return _buffer.peek(*op); } @@ -448,7 +449,6 @@ namespace replset { log() << "replSet remoteOldestOp: " << remoteTs.toStringLong() << rsLog; log() << "replSet lastOpTimeFetched: " << _lastOpTimeFetched.toStringLong() << rsLog; } - LOG(3) << "replSet remoteOldestOp: " << remoteTs.toStringLong() << rsLog; { boost::unique_lock lock(_mutex); @@ -492,6 +492,7 @@ namespace replset { LOG(2) << "replSet can't connect to " << current << " to read operations" << rsLog; r.resetConnection(); theReplSet->veto(current); + sleepsecs(1); continue; } @@ -508,6 +509,9 @@ namespace replset { _currentSyncTarget = target; } + boost::unique_lock oplogLockSSF(theReplSet->syncSourceFeedback.oplock); + theReplSet->syncSourceFeedback.connect(target); + return; } diff --git a/src/mongo/db/repl/bgsync.h b/src/mongo/db/repl/bgsync.h index 8459f835b4c..15288a54e30 100644 --- a/src/mongo/db/repl/bgsync.h +++ b/src/mongo/db/repl/bgsync.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -19,7 +31,7 @@ #include #include "mongo/util/queue.h" -#include "mongo/db/oplogreader.h" +#include "mongo/db/repl/oplogreader.h" #include "mongo/db/repl/rs.h" #include "mongo/db/jsobj.h" @@ -85,8 +97,6 @@ namespace replset { boost::condition_variable _lastOpCond; boost::mutex _lastOpMutex; - const Member* _oplogMarkerTarget; - OplogReader _oplogMarker; // not locked, only used by notifier thread OpTime _consumedOpTime; // not locked, only used by notifier thread BackgroundSync(); diff --git a/src/mongo/db/repl/connections.h b/src/mongo/db/repl/connections.h index 09eac82ceda..d0ae0411971 100644 --- a/src/mongo/db/repl/connections.h +++ b/src/mongo/db/repl/connections.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. */ #pragma once @@ -21,6 +33,9 @@ #include #include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/security_key.h" +#include "mongo/db/repl/rs.h" // extern Tee* rslog namespace mongo { @@ -52,7 +67,7 @@ namespace mongo { } void reconnect() { connInfo->cc.reset(new DBClientConnection(true, 0, connInfo->getTimeout())); - connInfo->cc->_logLevel = 2; + connInfo->cc->_logLevel = logger::LogSeverity::Debug(2); connInfo->connected = false; connect(); } @@ -87,7 +102,7 @@ namespace mongo { /*replicaSet*/ 0, /*timeout*/ ReplSetConfig::DEFAULT_HB_TIMEOUT)), connected(false) { - cc->_logLevel = 2; + cc->_logLevel = logger::LogSeverity::Debug(2); } void tagPort() { @@ -126,15 +141,8 @@ namespace mongo { // or our key file has to change. if our key file has to change, we'll // be rebooting. if their file has to change, they'll be rebooted so the // connection created above will go dead, reconnect, and reauth. - if (!noauth) { - if (!connInfo->cc->auth("local", - internalSecurity.user, - internalSecurity.pwd, - err, - false)) { - log() << "could not authenticate against " << _hostport << ", " << err << rsLog; - return false; - } + if (getGlobalAuthorizationManager()->isAuthEnabled()) { + return authenticateInternalUser(connInfo->cc.get()); } return true; diff --git a/src/mongo/db/repl/consensus.cpp b/src/mongo/db/repl/consensus.cpp index f056befb605..c20a4216f3f 100644 --- a/src/mongo/db/repl/consensus.cpp +++ b/src/mongo/db/repl/consensus.cpp @@ -12,12 +12,25 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ -#include "pch.h" -#include "../commands.h" -#include "rs.h" -#include "multicmd.h" +#include "mongo/pch.h" + +#include "mongo/db/commands.h" +#include "mongo/db/repl/multicmd.h" +#include "mongo/db/repl/rs.h" namespace mongo { @@ -26,13 +39,14 @@ namespace mongo { */ class CmdReplSetFresh : public ReplSetCommand { public: + void help(stringstream& h) const { h << "internal"; } CmdReplSetFresh() : ReplSetCommand("replSetFresh") { } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { ActionSet actions; - actions.addAction(ActionType::replSetFresh); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + actions.addAction(ActionType::internal); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } private: @@ -43,6 +57,12 @@ namespace mongo { return false; } + int cfgver = cmdObj["cfgver"].Int(); + if (theReplSet->config().version < cfgver) { + // Our config is stale; do not veto. + return false; + } + unsigned id = cmdObj["id"].Int(); const Member* primary = theReplSet->box.getPrimary(); const Member* hopeful = theReplSet->findById(id); @@ -61,7 +81,9 @@ namespace mongo { return true; } - if (primary && primary->hbinfo().opTime >= hopeful->hbinfo().opTime) { + if (primary && + (hopeful->hbinfo().id() != primary->hbinfo().id()) && + (primary->hbinfo().opTime >= hopeful->hbinfo().opTime)) { // other members might be aware of more up-to-date nodes errmsg = str::stream() << hopeful->fullName() << " is trying to elect itself but " << primary->fullName() << @@ -123,13 +145,14 @@ namespace mongo { class CmdReplSetElect : public ReplSetCommand { public: + void help(stringstream& h) const { h << "internal"; } CmdReplSetElect() : ReplSetCommand("replSetElect") { } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector* out) { ActionSet actions; - actions.addAction(ActionType::replSetElect); - out->push_back(Privilege(AuthorizationManager::SERVER_RESOURCE_NAME, actions)); + actions.addAction(ActionType::internal); + out->push_back(Privilege(ResourcePattern::forClusterResource(), actions)); } private: virtual bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { @@ -251,7 +274,6 @@ namespace mongo { try { vote = yea(whoid); dassert( hopeful->id() == whoid ); - rs.relinquish(); log() << "replSet info voting yea for " << hopeful->fullName() << " (" << whoid << ')' << rsLog; } catch(VoteException&) { @@ -369,7 +391,7 @@ namespace mongo { rs.sethbmsg("",9); - if( !allUp && time(0) - started < 60 * 5 ) { + if (!allUp && time(0) - serverGlobalParams.started < 60 * 5) { /* the idea here is that if a bunch of nodes bounce all at once, we don't want to drop data if we don't have to -- we'd rather be offline and wait a little longer instead todo: make this configurable. @@ -441,6 +463,10 @@ namespace mongo { /* succeeded. */ LOG(1) << "replSet election succeeded, assuming primary role" << rsLog; success = true; + { + mutex::scoped_lock lk(OpTime::m); + setElectionTime(OpTime::now(lk)); + } rs.assumePrimary(); } } diff --git a/src/mongo/db/repl/health.cpp b/src/mongo/db/repl/health.cpp index a45809df732..00957c8c536 100644 --- a/src/mongo/db/repl/health.cpp +++ b/src/mongo/db/repl/health.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. */ #include "mongo/pch.h" @@ -23,6 +35,8 @@ #include "mongo/db/dbhelpers.h" #include "mongo/db/repl/bgsync.h" #include "mongo/db/repl/connections.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/oplogreader.h" #include "mongo/db/repl/rs.h" #include "mongo/util/background.h" #include "mongo/util/concurrency/task.h" @@ -43,7 +57,7 @@ namespace mongo { using namespace mongoutils::html; using namespace bson; - static RamLog * _rsLog = new RamLog( "rs" ); + static RamLog * _rsLog = RamLog::get("rs"); Tee *rsLog = _rsLog; extern bool replSetBlind; // for testing @@ -126,8 +140,6 @@ namespace mongo { return ""; } - extern time_t started; - // oplogdiags in web ui static void say(stringstream&ss, const bo& op) { ss << ""; @@ -177,21 +189,19 @@ namespace mongo { const bo fields; /** todo fix we might want an so timeout here */ - DBClientConnection conn(false, 0, /*timeout*/ 20); - { - string errmsg; - if( !conn.connect(m->fullName(), errmsg) ) { - ss << "couldn't connect to " << m->fullName() << ' ' << errmsg; - return; - } + OplogReader reader; + + if (reader.connect(m->fullName()) == false) { + ss << "couldn't connect to " << m->fullName(); + return; } - auto_ptr c = conn.query(rsoplog, Query().sort("$natural",1), 20, 0, &fields); - if( c.get() == 0 ) { + reader.query(rsoplog, Query().sort("$natural",1), 20, 0, &fields); + if ( !reader.haveCursor() ) { ss << "couldn't query " << rsoplog; return; } - static const char *h[] = {"ts","optime", "h","op","ns","rest",0}; + static const char *h[] = {"ts","optime","h","op","ns","rest",0}; ss << "